From a7bfe46ca8122e1a83894a2f1b698e307be7c62a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 10:52:18 +0200 Subject: [PATCH 001/362] Removed whitespace --- .../Duplicati.Library.SQLiteHelper.csproj | 2 +- .../Library/SQLiteHelper/SQLiteLoader.cs | 30 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj index 88fdf5cae..ae3314d0a 100644 --- a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj +++ b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj @@ -8,7 +8,7 @@ Duplicati.Library.SQLiteHelper Copyright © 2025 Team Duplicati, MIT license - + diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 99f15e42b..4df83428b 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From a5c783760f03dcc069616646574683a05fb2798d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 10:54:17 +0200 Subject: [PATCH 002/362] Changed the SQLiteHelper.SQLiteLoader to use Microsoft.Data.SQLite as backend. --- .../Duplicati.Library.SQLiteHelper.csproj | 12 +++------- .../Library/SQLiteHelper/SQLiteLoader.cs | 24 ++++++++++--------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj index ae3314d0a..4a92aaee8 100644 --- a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj +++ b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj @@ -22,21 +22,15 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - - + + diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 4df83428b..897c05ac2 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -46,7 +46,7 @@ namespace Duplicati.Library.SQLiteHelper /// The SQLite connection object /// The location of Duplicati's database. /// The password to use for decryption. - public static void OpenDatabase(System.Data.IDbConnection con, string databasePath, string? decryptionPassword) + public static async Task OpenDatabase(Microsoft.Data.Sqlite.SqliteConnection con, string databasePath, string? decryptionPassword) { if (!string.IsNullOrWhiteSpace(decryptionPassword) && SQLiteRC4Decrypter.IsDatabaseEncrypted(databasePath)) { @@ -85,14 +85,14 @@ namespace Duplicati.Library.SQLiteHelper /// Loads an SQLite connection instance and opening the database /// /// The SQLite connection instance. - public static System.Data.IDbConnection LoadConnection() + public static async Task LoadConnection() { - System.Data.IDbConnection? con = null; + Microsoft.Data.Sqlite.SqliteConnection? con = null; SetEnvironmentVariablesForSQLiteTempDir(); try { - con = (System.Data.IDbConnection?)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); + con = new Microsoft.Data.Sqlite.SqliteConnection(); } catch (Exception ex) { @@ -111,7 +111,7 @@ namespace Duplicati.Library.SQLiteHelper /// The connection to apply the pragmas to. /// The page cache size to set. /// The connection with the pragmas applied. - public static System.Data.IDbConnection ApplyCustomPragmas(System.Data.IDbConnection con, long pagecachesize) + public static async Task ApplyCustomPragmas(Microsoft.Data.Sqlite.SqliteConnection con, long pagecachesize) { var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI") ?? ""; if (pagecachesize > MINIMUM_SQLITE_PAGE_CACHE_SIZE) @@ -121,7 +121,9 @@ namespace Duplicati.Library.SQLiteHelper return con; using (var cmd = con.CreateCommand()) - foreach (var opt in opts.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries)) + using (var transaction = con.BeginTransaction()) + { + cmd.Transaction = transaction; { Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt); try @@ -143,7 +145,7 @@ namespace Duplicati.Library.SQLiteHelper /// The SQLite connection instance. /// The optional path to the database. /// The page cache size to set. - public static System.Data.IDbConnection LoadConnection(string targetpath, long pagecachesize) + public static async Task LoadConnection(string targetpath, long pagecachesize) { if (string.IsNullOrWhiteSpace(targetpath)) throw new ArgumentNullException(nameof(targetpath)); @@ -173,7 +175,7 @@ namespace Duplicati.Library.SQLiteHelper { get { - return typeof(System.Data.SQLite.SQLiteConnection); + return typeof(Microsoft.Data.Sqlite.SqliteConnection); } } @@ -215,7 +217,7 @@ namespace Duplicati.Library.SQLiteHelper /// Wrapper to dispose the SQLite connection /// /// The connection to close. - private static void DisposeConnection(System.Data.IDbConnection? con) + private static async Task DisposeConnection(Microsoft.Data.Sqlite.SqliteConnection? con) { if (con != null) try { con.Dispose(); } @@ -227,7 +229,7 @@ namespace Duplicati.Library.SQLiteHelper /// /// The connection to use. /// Path to the file to open, which may not exist. - private static void OpenSQLiteFile(System.Data.IDbConnection con, string path) + private static async Task OpenSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con, string path) { con.ConnectionString = "Data Source=" + path; con.Open(); @@ -248,7 +250,7 @@ namespace Duplicati.Library.SQLiteHelper /// Tests the SQLite connection, throwing an exception if the connection does not work /// /// The connection to test. - private static void TestSQLiteFile(System.Data.IDbConnection con) + private static async Task TestSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con) { // Do a dummy query to make sure we have a working db using (var cmd = con.CreateCommand()) From 1cfe154921d3055d81a8c9f7a8ce81f6e4570110 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 10:55:22 +0200 Subject: [PATCH 003/362] Made the SQLiteLoader async --- .../Library/SQLiteHelper/SQLiteLoader.cs | 37 ++++++++----------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 897c05ac2..0170fe7e0 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -23,6 +23,7 @@ using System; using System.Globalization; +using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; @@ -66,12 +67,12 @@ namespace Duplicati.Library.SQLiteHelper try { //Attempt to open in preferred state - OpenSQLiteFile(con, databasePath); - TestSQLiteFile(con); + await OpenSQLiteFile(con, databasePath); + await TestSQLiteFile(con); } catch { - try { con.Dispose(); } + try { await con.DisposeAsync(); } catch { } throw; @@ -97,7 +98,7 @@ namespace Duplicati.Library.SQLiteHelper catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, "Failed to load connection."); - DisposeConnection(con); + await DisposeConnection(con); throw; } @@ -129,13 +130,15 @@ namespace Duplicati.Library.SQLiteHelper try { cmd.CommandText = string.Format(CultureInfo.InvariantCulture, "PRAGMA {0}", opt); - cmd.ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CustomSQLiteOption", ex, @"Error setting custom SQLite option '{0}'.", opt); } } + await transaction.CommitAsync(); + } return con; } @@ -150,22 +153,22 @@ namespace Duplicati.Library.SQLiteHelper if (string.IsNullOrWhiteSpace(targetpath)) throw new ArgumentNullException(nameof(targetpath)); - var con = LoadConnection(); + var con = await LoadConnection(); try { - OpenSQLiteFile(con, targetpath); + await OpenSQLiteFile(con, targetpath); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, @"Failed to load connection with path '{0}'.", targetpath); - DisposeConnection(con); + await DisposeConnection(con); throw; } // set custom Sqlite options - return ApplyCustomPragmas(con, pagecachesize); + return await ApplyCustomPragmas(con, pagecachesize); } /// @@ -220,7 +223,7 @@ namespace Duplicati.Library.SQLiteHelper private static async Task DisposeConnection(Microsoft.Data.Sqlite.SqliteConnection? con) { if (con != null) - try { con.Dispose(); } + try { await con.DisposeAsync(); } catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "ConnectionDisposeError", ex, "Failed to dispose connection"); } } @@ -232,13 +235,7 @@ namespace Duplicati.Library.SQLiteHelper private static async Task OpenSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con, string path) { con.ConnectionString = "Data Source=" + path; - con.Open(); - if (con is System.Data.SQLite.SQLiteConnection sqlitecon && !OperatingSystem.IsMacOS()) - { - // These configuration options crash on MacOS (arm64), but the other platforms should be enough to detect incorrect SQL - sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DDL, false); - sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DML, false); - } + await con.OpenAsync(); // Make the file only accessible by the current user, unless opting out if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile))) @@ -253,11 +250,7 @@ namespace Duplicati.Library.SQLiteHelper private static async Task TestSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con) { // Do a dummy query to make sure we have a working db - using (var cmd = con.CreateCommand()) - { - cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER"; - cmd.ExecuteScalar(); - } + await cmd.ExecuteScalarAsync(); } } } From e4d2ee275c3ed490abbff8afcad9519f1374e781 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 10:56:05 +0200 Subject: [PATCH 004/362] Small changes to use more modern C# --- Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 0170fe7e0..548c01593 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -125,6 +125,7 @@ namespace Duplicati.Library.SQLiteHelper using (var transaction = con.BeginTransaction()) { cmd.Transaction = transaction; + foreach (var opt in opts.Split([';'], StringSplitOptions.RemoveEmptyEntries)) { Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt); try @@ -250,6 +251,8 @@ namespace Duplicati.Library.SQLiteHelper private static async Task TestSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con) { // Do a dummy query to make sure we have a working db + using var cmd = con.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER"; await cmd.ExecuteScalarAsync(); } } From d1b49238bdf7c3e0d9e45ad869025a9561d8699f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 10:56:22 +0200 Subject: [PATCH 005/362] Added TODOs --- Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 548c01593..a3c261915 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -114,6 +114,7 @@ namespace Duplicati.Library.SQLiteHelper /// The connection with the pragmas applied. public static async Task ApplyCustomPragmas(Microsoft.Data.Sqlite.SqliteConnection con, long pagecachesize) { + // TODO more default custom options var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI") ?? ""; if (pagecachesize > MINIMUM_SQLITE_PAGE_CACHE_SIZE) opts = string.Format(CultureInfo.InvariantCulture, "cache_size=-{0};{1}", pagecachesize / 1024L, opts); @@ -237,6 +238,13 @@ namespace Duplicati.Library.SQLiteHelper { con.ConnectionString = "Data Source=" + path; await con.OpenAsync(); + // TODO legacy configuration? + //if (con is Microsoft.Data.Sqlite.SqliteConnection sqlitecon && !OperatingSystem.IsMacOS()) + //{ + // // These configuration options crash on MacOS (arm64), but the other platforms should be enough to detect incorrect SQL + // sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DDL, false); + // sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DML, false); + //} // Make the file only accessible by the current user, unless opting out if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile))) From eeb16e3800c89287fee33d087a0f8dad0e17c474 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 16:56:41 +0200 Subject: [PATCH 006/362] Removed whitespace --- .../Library/Main/Database/LocalDatabase.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index aaa45a253..b5e159ff9 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From 6be2b3f5f58aca0c2438b0c098f72e9d57c56136 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 16:58:23 +0200 Subject: [PATCH 007/362] Updated LocalDatabase to be async using Microsoft.Data.Sqlite --- .../Library/Main/Database/LocalDatabase.cs | 1526 ++++++++--------- 1 file changed, 758 insertions(+), 768 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index b5e159ff9..ee10a3c3d 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -24,14 +24,15 @@ using System; using System.Collections.Generic; using System.Data; -using System.Linq; -using System.Text; using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; using Duplicati.Library.Modules.Builtin.ResultSerialization; using Duplicati.Library.Utility; -using System.Runtime.CompilerServices; using Duplicati.Library.Interface; - +using Microsoft.Data.Sqlite; // Expose internal classes to UnitTests, so that Database classes can be tested [assembly: InternalsVisibleTo("Duplicati.UnitTest")] @@ -51,45 +52,45 @@ namespace Duplicati.Library.Main.Database /// SQLite has a limit of 999 parameters in a single statement public const int CHUNK_SIZE = 128; - protected readonly IDbConnection m_connection; + protected readonly SqliteConnection m_connection; protected readonly long m_operationid = -1; protected readonly long m_pagecachesize; private bool m_hasExecutedVacuum; - private readonly IDbCommand m_updateremotevolumeCommand; - private readonly IDbCommand m_selectremotevolumesCommand; - private readonly IDbCommand m_selectremotevolumeCommand; - private readonly IDbCommand m_removeremotevolumeCommand; - private readonly IDbCommand m_removedeletedremotevolumeCommand; - private readonly IDbCommand m_selectremotevolumeIdCommand; - private readonly IDbCommand m_createremotevolumeCommand; - private readonly IDbCommand m_selectduplicateRemoteVolumesCommand; + private readonly SqliteCommand m_updateremotevolumeCommand; + private readonly SqliteCommand m_selectremotevolumesCommand; + private readonly SqliteCommand m_selectremotevolumeCommand; + private readonly SqliteCommand m_removeremotevolumeCommand; + private readonly SqliteCommand m_removedeletedremotevolumeCommand; + private readonly SqliteCommand m_selectremotevolumeIdCommand; + private readonly SqliteCommand m_createremotevolumeCommand; + private readonly SqliteCommand m_selectduplicateRemoteVolumesCommand; - private readonly IDbCommand m_insertlogCommand; - private readonly IDbCommand m_insertremotelogCommand; - private readonly IDbCommand m_insertIndexBlockLink; + private readonly SqliteCommand m_insertlogCommand; + private readonly SqliteCommand m_insertremotelogCommand; + private readonly SqliteCommand m_insertIndexBlockLink; - private readonly IDbCommand m_findpathprefixCommand; - private readonly IDbCommand m_insertpathprefixCommand; + private readonly SqliteCommand m_findpathprefixCommand; + private readonly SqliteCommand m_insertpathprefixCommand; public const long FOLDER_BLOCKSET_ID = -100; public const long SYMLINK_BLOCKSET_ID = -200; public DateTime OperationTimestamp { get; private set; } - internal IDbConnection Connection { get { return m_connection; } } + internal SqliteConnection Connection { get { return m_connection; } } public bool IsDisposed { get; private set; } public bool ShouldCloseConnection { get; set; } - protected static IDbConnection CreateConnection(string path, long pagecachesize) + protected static async Task CreateConnection(string path, long pagecachesize) { path = Path.GetFullPath(path); if (!Directory.Exists(Path.GetDirectoryName(path))) Directory.CreateDirectory(Path.GetDirectoryName(path) ?? throw new DirectoryNotFoundException("Path was a root folder.")); - var c = SQLiteHelper.SQLiteLoader.LoadConnection(path, pagecachesize); + var c = await SQLiteHelper.SQLiteLoader.LoadConnection(path, pagecachesize); try { @@ -98,7 +99,7 @@ namespace Duplicati.Library.Main.Database catch { //Don't leak database connections when something goes wrong - c.Dispose(); + await c.DisposeAsync(); throw; } @@ -126,7 +127,7 @@ namespace Duplicati.Library.Main.Database /// Should the connection be closed when this object is disposed /// The page cache size public LocalDatabase(string path, string operation, bool shouldclose, long pagecachesize) - : this(CreateConnection(path, pagecachesize), operation) + : this(CreateConnection(path, pagecachesize).Await(), operation) { ShouldCloseConnection = shouldclose; m_pagecachesize = pagecachesize; @@ -148,7 +149,7 @@ namespace Duplicati.Library.Main.Database /// Creates a new database instance and starts a new operation /// /// The name of the operation. If null, continues last operation - public LocalDatabase(IDbConnection connection, string operation) + public LocalDatabase(SqliteConnection connection, string operation) : this(connection) { OperationTimestamp = DateTime.UtcNow; @@ -159,28 +160,29 @@ namespace Duplicati.Library.Main.Database if (operation != null) { - using (var cmd = m_connection.CreateCommand()) - m_operationid = cmd.SetCommandAndParameters(@"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (@Description, @Timestamp); SELECT last_insert_rowid();") - .SetParameterValue("@Description", operation) - .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(OperationTimestamp)) - .ExecuteScalarInt64(-1); + using var cmd = m_connection.CreateCommand(); + using var transaction = m_connection.BeginTransaction(); + cmd.Transaction = transaction; + cmd.CommandText = @"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (@Description, @Timestamp); SELECT last_insert_rowid();"; + cmd.SetParameterValue("@Description", operation); + cmd.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(OperationTimestamp)); + m_operationid = cmd.ExecuteScalarInt64Async(-1).Await(); } else { // Get last operation - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1")) - { - if (!rd.Read()) - throw new Exception("LocalDatabase does not contain a previous operation."); + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = @"SELECT ""ID"", ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1"; + using var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1"); + if (!rd.Read()) + throw new Exception("LocalDatabase does not contain a previous operation."); - m_operationid = rd.ConvertValueToInt64(0); - OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); - } + m_operationid = rd.ConvertValueToInt64(0); + OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); } } - private LocalDatabase(IDbConnection connection) + private LocalDatabase(SqliteConnection connection) { m_connection = connection; m_insertlogCommand = connection.CreateCommand(@"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (@OperationID, @Timestamp, @Type, @Message, @Exception)"); @@ -200,31 +202,31 @@ namespace Duplicati.Library.Main.Database /// /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value - /// + /// public static DateTime ParseFromEpochSeconds(long seconds) { return Library.Utility.Utility.EPOCH.AddSeconds(seconds); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, IDbTransaction? transaction = null) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, SqliteTransaction transaction) { - UpdateRemoteVolume(name, state, size, hash, false, transaction); + await UpdateRemoteVolume(name, state, size, hash, false, transaction); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, IDbTransaction? transaction = null) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, SqliteTransaction transaction) { - UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null, transaction); + await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null, transaction); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, IDbTransaction? transaction = null) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, SqliteTransaction transaction) { m_updateremotevolumeCommand.Transaction = transaction; - var c = m_updateremotevolumeCommand.SetParameterValue("@OperationID", m_operationid) - .SetParameterValue("@State", state.ToString()) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .SetParameterValue("@Name", name) - .ExecuteNonQuery(); + m_updateremotevolumeCommand.SetParameterValue("@OperationID", m_operationid); + m_updateremotevolumeCommand.SetParameterValue("@State", state.ToString()); + m_updateremotevolumeCommand.SetParameterValue("@Hash", hash); + m_updateremotevolumeCommand.SetParameterValue("@Size", size); + m_updateremotevolumeCommand.SetParameterValue("@Name", name); + var c = await m_updateremotevolumeCommand.ExecuteNonQueryAsync(); if (c != 1) { @@ -233,52 +235,52 @@ namespace Duplicati.Library.Main.Database if (deleteGraceTime.Ticks > 0) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - c = cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = @DeleteGraceTime WHERE ""Name"" = @Name ") - .SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)) - .SetParameterValue("@Name", name) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = @"UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = @DeleteGraceTime WHERE ""Name"" = @Name "; + cmd.Transaction = transaction; + cmd.SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)); + cmd.SetParameterValue("@Name", name); + c = await cmd.ExecuteNonQueryAsync(); - if (c != 1) - throw new Exception($"Unexpected number of updates when recording remote volume updates: {c}!"); - } + if (c != 1) + throw new Exception($"Unexpected number of updates when recording remote volume updates: {c}!"); } if (setArchived.HasValue) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - c = cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""ArchiveTime"" = @ArchiveTime WHERE ""Name"" = @Name ") - .SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0) - .SetParameterValue("@Name", name) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = @"UPDATE ""RemoteVolume"" SET ""ArchiveTime"" = @ArchiveTime WHERE ""Name"" = @Name "; + cmd.Transaction = transaction; + cmd.SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0); + cmd.SetParameterValue("@Name", name); + c = await cmd.ExecuteNonQueryAsync(); - if (c != 1) - throw new Exception($"Unexpected number of updates when recording remote volume archive-time updates: {c}!"); - } + if (c != 1) + throw new Exception($"Unexpected number of updates when recording remote volume archive-time updates: {c}!"); } if (!suppressCleanup && state == RemoteVolumeState.Deleted) { - RemoveRemoteVolume(name, transaction); + await RemoveRemoteVolume(name, transaction); } } - public IEnumerable> FilesetTimes + public async IAsyncEnumerable> FilesetTimes() { - get - { - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC")) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime()); - } + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = @"SELECT ""ID"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"; + using var rd = await cmd.ExecuteReaderAsync(); + while (rd.Read()) + yield return new KeyValuePair(rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime()); } - public (string Query, Dictionary Values) GetFilelistWhereClause(DateTime time, long[] versions, IEnumerable>? filesetslist = null, bool singleTimeMatch = false) + public async Task<(string Query, Dictionary Values)> GetFilelistWhereClause(DateTime time, long[] versions, IEnumerable>? filesetslist = null, bool singleTimeMatch = false) { - var filesets = (filesetslist ?? FilesetTimes).ToArray(); + KeyValuePair[] filesets; + if (filesetslist != null) + filesets = [.. filesetslist]; + else + filesets = await FilesetTimes().ToArrayAsync(); var query = new StringBuilder(); var args = new Dictionary(); if (time.Ticks > 0 || (versions != null && versions.Length > 0)) @@ -305,7 +307,7 @@ namespace Duplicati.Library.Main.Database var argName = "@Fileset" + v; args.Add(argName, filesets[v].Key); qs.Append(argName); - qs.Append(","); + qs.Append(','); } else Logging.Log.WriteWarningMessage(LOGTAG, "SkipInvalidVersion", null, "Skipping invalid version: {0}", v); @@ -329,32 +331,31 @@ namespace Duplicati.Library.Main.Database return (query.ToString(), args); } - public long GetRemoteVolumeID(string file, IDbTransaction? transaction = null) + public async Task GetRemoteVolumeID(string file, SqliteTransaction? transaction = null) { m_selectremotevolumeIdCommand.Transaction = transaction; - return m_selectremotevolumeIdCommand.SetParameterValue("@Name", file).ExecuteScalarInt64(-1); + m_selectremotevolumeIdCommand.SetParameterValue("@Name", file); + return await m_selectremotevolumeIdCommand.ExecuteScalarInt64Async(-1); } - public IEnumerable> GetRemoteVolumeIDs(IEnumerable files, IDbTransaction? transaction = null) + public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files, SqliteTransaction? transaction = null) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - using var tmptable = new TemporaryDbValueList(m_connection, transaction, files); - cmd.SetCommandAndParameters(@"SELECT ""Name"", ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name)") - .ExpandInClauseParameter("@Name", tmptable); + using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name)"); + cmd.Transaction = transaction; + using var tmptable = new TemporaryDbValueList(m_connection, transaction, files); + cmd.ExpandInClauseParameter("@Name", tmptable); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); - } + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } - public RemoteVolumeEntry GetRemoteVolume(string file, IDbTransaction? transaction = null) + public async Task GetRemoteVolume(string file, SqliteTransaction? transaction = null) { m_selectremotevolumeCommand.Transaction = transaction; m_selectremotevolumeCommand.SetParameterValue("@Name", file); - using (var rd = m_selectremotevolumeCommand.ExecuteReader()) - if (rd.Read()) + using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync()) + if (await rd.ReadAsync()) return new RemoteVolumeEntry( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), @@ -369,9 +370,10 @@ namespace Duplicati.Library.Main.Database return RemoteVolumeEntry.Empty; } - public IEnumerable> DuplicateRemoteVolumes(IDbTransaction? transaction) + public async IAsyncEnumerable> DuplicateRemoteVolumes(SqliteTransaction? transaction) { - foreach (var rd in m_selectduplicateRemoteVolumesCommand.SetTransaction(transaction).ExecuteReaderEnumerable()) + m_selectduplicateRemoteVolumesCommand.Transaction = transaction; + await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync()) { yield return new KeyValuePair( rd.ConvertValueToString(0) ?? throw new Exception("Name was null"), @@ -380,24 +382,22 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable GetRemoteVolumes(IDbTransaction? transaction = null) + public async IAsyncEnumerable GetRemoteVolumes(SqliteTransaction? transaction = null) { m_selectremotevolumesCommand.Transaction = transaction; - using (var rd = m_selectremotevolumesCommand.ExecuteReader()) + using var rd = await m_selectremotevolumesCommand.ExecuteReaderAsync(); + while (await rd.ReadAsync()) { - while (rd.Read()) - { - yield return new RemoteVolumeEntry( - rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1), - rd.ConvertValueToString(4), - rd.ConvertValueToInt64(3, -1), - (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""), - (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""), - ParseFromEpochSeconds(rd.ConvertValueToInt64(6, 0)), - ParseFromEpochSeconds(rd.ConvertValueToInt64(7, 0)) - ); - } + yield return new RemoteVolumeEntry( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(4), + rd.ConvertValueToInt64(3, -1), + (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""), + (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""), + ParseFromEpochSeconds(rd.ConvertValueToInt64(6, 0)), + ParseFromEpochSeconds(rd.ConvertValueToInt64(7, 0)) + ); } } @@ -407,15 +407,15 @@ namespace Duplicati.Library.Main.Database /// The operation performed /// The path involved /// Any data relating to the operation - public void LogRemoteOperation(string operation, string path, string? data, IDbTransaction? transaction) + public async Task LogRemoteOperation(string operation, string path, string? data, SqliteTransaction? transaction) { - m_insertremotelogCommand - .SetParameterValue("@OperationID", m_operationid) - .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) - .SetParameterValue("@Operation", operation) - .SetParameterValue("@Path", path) - .SetParameterValue("@Data", data) - .ExecuteNonQuery(transaction); + m_insertremotelogCommand.Transaction = transaction; + m_insertremotelogCommand.SetParameterValue("@OperationID", m_operationid); + m_insertremotelogCommand.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); + m_insertremotelogCommand.SetParameterValue("@Operation", operation); + m_insertremotelogCommand.SetParameterValue("@Path", path); + m_insertremotelogCommand.SetParameterValue("@Data", data); + await m_insertremotelogCommand.ExecuteNonQueryAsync(); } /// @@ -424,86 +424,82 @@ namespace Duplicati.Library.Main.Database /// The message type /// The message /// An optional exception - public void LogMessage(string type, string message, Exception? exception, IDbTransaction? transaction) + public async Task LogMessage(string type, string message, Exception? exception, SqliteTransaction? transaction) { - m_insertlogCommand.SetParameterValue("@OperationID", m_operationid) - .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) - .SetParameterValue("@Type", type) - .SetParameterValue("@Message", message) - .SetParameterValue("@Exception", exception?.ToString()) - .ExecuteNonQuery(transaction); + m_insertlogCommand.Transaction = transaction; + m_insertlogCommand.SetParameterValue("@OperationID", m_operationid); + m_insertlogCommand.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); + m_insertlogCommand.SetParameterValue("@Type", type); + m_insertlogCommand.SetParameterValue("@Message", message); + m_insertlogCommand.SetParameterValue("@Exception", exception?.ToString()); + await m_insertlogCommand.ExecuteNonQueryAsync(); } - public void UnlinkRemoteVolume(string name, RemoteVolumeState state, IDbTransaction? transaction = null) + public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state, SqliteTransaction transaction) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - var c = cmd.SetCommandAndParameters(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = @Name AND ""State"" = @State ") - .SetParameterValue("@Name", name) - .SetParameterValue("@State", state.ToString()) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = @Name AND ""State"" = @State "); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Name", name); + cmd.SetParameterValue("@State", state.ToString()); + var c = await cmd.ExecuteNonQueryAsync(); - if (c != 1) - throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); + if (c != 1) + throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); - tr.Commit(); - } + await transaction.CommitAsync(); } - public void RemoveRemoteVolume(string name, IDbTransaction? transaction = null) + public async Task RemoveRemoteVolume(string name, SqliteTransaction transaction) { - RemoveRemoteVolumes([name], transaction); + await RemoveRemoteVolumes([name], transaction); } - public void RemoveRemoteVolumes(IEnumerable names, IDbTransaction? transaction = null) + public async Task RemoveRemoteVolumes(IEnumerable names, SqliteTransaction transaction) { if (names == null || !names.Any()) return; - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var deletecmd = m_connection.CreateCommand(tr.Parent)) - { - string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var volidstable = "DelVolSetIds-" + temptransguid; - var blocksetidstable = "DelBlockSetIds-" + temptransguid; - var filesetidstable = "DelFilesetIds-" + temptransguid; + using var deletecmd = m_connection.CreateCommand(); + string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var volidstable = "DelVolSetIds-" + temptransguid; + var blocksetidstable = "DelBlockSetIds-" + temptransguid; + var filesetidstable = "DelFilesetIds-" + temptransguid; - // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames)")) - .ExpandInClauseParameter("@VolumeNames", names.ToArray()) - .ExecuteNonQuery(); + // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. + deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames)")) + .ExpandInClauseParameter("@VolumeNames", names.ToArray()) + .ExecuteNonQuery(); - var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); - deletecmd.Parameters.Clear(); + var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); + deletecmd.Parameters.Clear(); - var bsIdsSubQuery = FormatInvariant(@$" + var bsIdsSubQuery = FormatInvariant(@$" SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" -WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) -UNION ALL +WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) +UNION ALL SELECT DISTINCT ""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))"); - // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{blocksetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery}")); - bsIdsSubQuery = FormatInvariant($@"SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" "); - deletecmd.Parameters.Clear(); + // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMP TABLE ""{blocksetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery}")); + bsIdsSubQuery = FormatInvariant($@"SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" "); + deletecmd.Parameters.Clear(); - // Create a temp table to associate metadata that is being deleted to a fileset - var metadataFilesetQuery = FormatInvariant($@"SELECT Metadataset.ID, FilesetEntry.FilesetID + // Create a temp table to associate metadata that is being deleted to a fileset + var metadataFilesetQuery = FormatInvariant($@"SELECT Metadataset.ID, FilesetEntry.FilesetID FROM Metadataset INNER JOIN FileLookup ON FileLookup.MetadataID = Metadataset.ID INNER JOIN FilesetEntry ON FilesetEntry.FileID = FileLookup.ID WHERE Metadataset.BlocksetID IN ({bsIdsSubQuery}) OR Metadataset.ID IN (SELECT MetadataID FROM FileLookup WHERE BlocksetID IN ({bsIdsSubQuery}))"); - var metadataFilesetTable = @"DelMetadataFilesetIds-" + temptransguid; - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{metadataFilesetTable}"" (MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER)")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (MetadataID, FilesetID) {metadataFilesetQuery}")); + var metadataFilesetTable = @"DelMetadataFilesetIds-" + temptransguid; + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMP TABLE ""{metadataFilesetTable}"" (MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER)")); + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (MetadataID, FilesetID) {metadataFilesetQuery}")); - // Delete FilesetEntry rows that had their metadata deleted - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry + // Delete FilesetEntry rows that had their metadata deleted + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FilesetID IN (SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"") AND FilesetEntry.FileID IN ( SELECT FilesetEntry.FileID @@ -511,161 +507,156 @@ AND FilesetEntry.FileID IN ( INNER JOIN FileLookup ON FileLookup.ID = FilesetEntry.FileID WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}""))")); - // Delete FilesetEntry rows that had their blocks deleted - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FileID IN ( + // Delete FilesetEntry rows that had their blocks deleted + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FileID IN ( SELECT ID FROM FileLookup WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}))")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")")); + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery})")); + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery})")); + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); + await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery})")); - // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetID IN (SELECT ID FROM Fileset WHERE VolumeID IN ({volIdsSubQuery}))")); + // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetID IN (SELECT ID FROM Fileset WHERE VolumeID IN ({volIdsSubQuery}))")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{filesetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - // Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""Fileset"".""ID"" IN + deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{filesetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + // Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore + deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""Fileset"".""ID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{metadataFilesetTable}"") AND ""Fileset"".""ID"" NOT IN (SELECT DISTINCT ""FilesetID"" FROM FilesetEntry)")); - // Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later - deletecmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""RemoteVolume"" SET ""State"" = @NewState WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Fileset"" WHERE ""Fileset"".""ID"" IN (SELECT ""ID"" FROM ""{filesetidstable}"")) AND ""State"" IN (@AllowedStates)")) - .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) - .ExpandInClauseParameter("@AllowedStates", [RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString()]) - .ExecuteNonQuery(); + // Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later + deletecmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""RemoteVolume"" SET ""State"" = @NewState WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Fileset"" WHERE ""Fileset"".""ID"" IN (SELECT ""ID"" FROM ""{filesetidstable}"")) AND ""State"" IN (@AllowedStates)")) + .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) + .ExpandInClauseParameter("@AllowedStates", [RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString()]) + .ExecuteNonQuery(); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Fileset"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{filesetidstable}"")")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Fileset"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{filesetidstable}"")")); - // Clean up temp tables for subqueries. We truncate content and then try to delete. - // Drop in try-block, as it fails in nested transactions (SQLite problem) - // SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{blocksetidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{metadataFilesetTable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{filesetidstable}"" ")); - try - { - deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ")); - } - catch { /* Ignore, will be deleted on close anyway. */ } - - m_removeremotevolumeCommand.Transaction = tr.Parent; - m_removeremotevolumeCommand.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); - m_removeremotevolumeCommand.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - foreach (var name in names) - { - m_removeremotevolumeCommand.SetParameterValue("@Name", name); - m_removeremotevolumeCommand.ExecuteNonQuery(); - } - - // Validate before commiting changes - var nonAttachedFiles = deletecmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); - if (nonAttachedFiles > 0) - throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); - - tr.Commit(); + // Clean up temp tables for subqueries. We truncate content and then try to delete. + // Drop in try-block, as it fails in nested transactions (SQLite problem) + // SQLite.SQLiteException (0x80004005): database table is locked + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{blocksetidstable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{metadataFilesetTable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{filesetidstable}"" ")); + try + { + deletecmd.CommandTimeout = 2; + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ")); } + catch { /* Ignore, will be deleted on close anyway. */ } + + m_removeremotevolumeCommand.Transaction = transaction; + m_removeremotevolumeCommand.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); + m_removeremotevolumeCommand.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + foreach (var name in names) + { + m_removeremotevolumeCommand.SetParameterValue("@Name", name); + await m_removeremotevolumeCommand.ExecuteNonQueryAsync(); + } + + // Validate before commiting changes + var nonAttachedFiles = await deletecmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); + if (nonAttachedFiles > 0) + throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); + + await transaction.CommitAsync(); } - public void Vacuum() + public async Task Vacuum() { m_hasExecutedVacuum = true; - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery("VACUUM"); + using var cmd = m_connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync("VACUUM"); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state) { - return RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), null); + var transaction = m_connection.BeginTransaction(); + return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), transaction); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, IDbTransaction? transaction) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, SqliteTransaction transaction) { - return RegisterRemoteVolume(name, type, state, new TimeSpan(0), transaction); + return await RegisterRemoteVolume(name, type, state, new TimeSpan(0), transaction); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, IDbTransaction? transaction) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, SqliteTransaction transaction) { - return RegisterRemoteVolume(name, type, state, -1, deleteGraceTime, transaction); + return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime, transaction); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, IDbTransaction? transaction) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, SqliteTransaction transaction) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - var r = m_createremotevolumeCommand.SetParameterValue("@OperationId", m_operationid) - .SetParameterValue("@Name", name) - .SetParameterValue("@Type", type.ToString()) - .SetParameterValue("@State", state.ToString()) - .SetParameterValue("@Size", size) - .SetParameterValue("@VerificationCount", 0) - .SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks) - .SetParameterValue("@ArchiveTime", 0) - .ExecuteScalarInt64(tr.Parent); + m_createremotevolumeCommand.Transaction = transaction; + m_createremotevolumeCommand.SetParameterValue("@OperationId", m_operationid); + m_createremotevolumeCommand.SetParameterValue("@Name", name); + m_createremotevolumeCommand.SetParameterValue("@Type", type.ToString()); + m_createremotevolumeCommand.SetParameterValue("@State", state.ToString()); + m_createremotevolumeCommand.SetParameterValue("@Size", size); + m_createremotevolumeCommand.SetParameterValue("@VerificationCount", 0); + m_createremotevolumeCommand.SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks); + m_createremotevolumeCommand.SetParameterValue("@ArchiveTime", 0); + var r = await m_createremotevolumeCommand.ExecuteScalarInt64Async(); - tr.Commit(); - return r; - } + await transaction.CommitAsync(); + + return r; } - public IEnumerable GetFilesetIDs(DateTime restoretime, long[] versions, bool singleTimeMatch = false) + public async Task> GetFilesetIDs(DateTime restoretime, long[] versions, bool singleTimeMatch = false) { if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); - (var query, var values) = GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch); + (var query, var values) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch); var res = new List(); - using (var cmd = m_connection.CreateCommand()) + using var cmd = m_connection.CreateCommand(); + using (var rd = await cmd.ExecuteReaderAsync($@"SELECT ""ID"" FROM ""Fileset"" {query} ORDER BY ""Timestamp"" DESC", values)) + while (await rd.ReadAsync()) + res.Add(rd.ConvertValueToInt64(0)); + + if (res.Count == 0) { - using (var rd = cmd.ExecuteReader($@"SELECT ""ID"" FROM ""Fileset"" {query} ORDER BY ""Timestamp"" DESC", values)) - while (rd.Read()) + cmd.Parameters.Clear(); + using (var rd = await cmd.ExecuteReaderAsync(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) + while (await rd.ReadAsync()) res.Add(rd.ConvertValueToInt64(0)); if (res.Count == 0) - { - cmd.Parameters.Clear(); - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) - while (rd.Read()) - res.Add(rd.ConvertValueToInt64(0)); - - if (res.Count == 0) - throw new Duplicati.Library.Interface.UserInformationException("No backup at the specified date", "NoBackupAtDate"); - else - Logging.Log.WriteWarningMessage(LOGTAG, "RestoreTimeNoMatch", null, "Restore time or version did not match any existing backups, selecting newest backup"); - } - - return res; + throw new Duplicati.Library.Interface.UserInformationException("No backup at the specified date", "NoBackupAtDate"); + else + Logging.Log.WriteWarningMessage(LOGTAG, "RestoreTimeNoMatch", null, "Restore time or version did not match any existing backups, selecting newest backup"); } + + return res; } - public IEnumerable FindMatchingFilesets(DateTime restoretime, long[] versions) + public async Task> FindMatchingFilesets(DateTime restoretime, long[] versions) { if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); - var tmp = GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true); - string query = tmp.Item1; - var args = tmp.Item2; + var (query, args) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true); var res = new List(); using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"" FROM ""Fileset"" " + query + @" ORDER BY ""Timestamp"" DESC", args)) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync(@"SELECT ""ID"" FROM ""Fileset"" " + query + @" ORDER BY ""Timestamp"" DESC", args)) + while (await rd.ReadAsync()) res.Add(rd.ConvertValueToInt64(0)); return res; @@ -725,15 +716,20 @@ AND ""Fileset"".""ID"" NOT IN private IEnumerable> GetDbOptionList(IDbTransaction? transaction = null) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(@"SELECT ""Key"", ""Value"" FROM ""Configuration"" ")) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? ""); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + using var rd = await cmd.ExecuteReaderAsync(@"SELECT ""Key"", ""Value"" FROM ""Configuration"" "); + while (await rd.ReadAsync()) + yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? ""); } - public IDictionary GetDbOptions(IDbTransaction? transaction = null) + public async Task> GetDbOptions(SqliteTransaction? transaction) { - return GetDbOptionList(transaction).ToDictionary(x => x.Key, x => x.Value); + var t = transaction ?? m_connection.BeginTransaction(); + var res = await GetDbOptionList(t).ToDictionaryAsync(x => x.Key, x => x.Value); + if (transaction == null) + await t.CommitAsync(); + return res; } /// @@ -741,16 +737,18 @@ AND ""Fileset"".""ID"" NOT IN /// /// The key to update /// The value to set - private void UpdateDbOption(string key, bool value) + private async Task UpdateDbOption(string key, bool value) { - var opts = GetDbOptions(); + var transaction = m_connection.BeginTransaction(); + var opts = await GetDbOptions(transaction); if (value) opts[key] = "true"; else opts.Remove(key); - SetDbOptions(opts); + await SetDbOptions(opts, transaction); + await transaction.CommitAsync(); } /// @@ -758,8 +756,8 @@ AND ""Fileset"".""ID"" NOT IN /// public bool RepairInProgress { - get => GetDbOptions().ContainsKey("repair-in-progress"); - set => UpdateDbOption("repair-in-progress", value); + get => GetDbOptions(null).Await().ContainsKey("repair-in-progress"); + set => UpdateDbOption("repair-in-progress", value).Await(); } /// @@ -767,8 +765,8 @@ AND ""Fileset"".""ID"" NOT IN /// public bool PartiallyRecreated { - get => GetDbOptions().ContainsKey("partially-recreated"); - set => UpdateDbOption("partially-recreated", value); + get => GetDbOptions(null).Await().ContainsKey("partially-recreated"); + set => UpdateDbOption("partially-recreated", value).Await(); } /// @@ -776,8 +774,8 @@ AND ""Fileset"".""ID"" NOT IN /// public bool TerminatedWithActiveUploads { - get => GetDbOptions().ContainsKey("terminated-with-active-uploads"); - set => UpdateDbOption("terminated-with-active-uploads", value); + get => GetDbOptions(null).Await().ContainsKey("terminated-with-active-uploads"); + set => UpdateDbOption("terminated-with-active-uploads", value).Await(); } /// @@ -785,28 +783,27 @@ AND ""Fileset"".""ID"" NOT IN /// /// The options to set /// An optional transaction - public void SetDbOptions(IDictionary options, IDbTransaction? transaction = null) + public async Task SetDbOptions(IDictionary options, SqliteTransaction transaction) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - cmd.ExecuteNonQuery(@"DELETE FROM ""Configuration"" "); - foreach (var kp in options) - cmd.SetCommandAndParameters(@"INSERT INTO ""Configuration"" (""Key"", ""Value"") VALUES (@Key, @Value) ") - .SetParameterValue("@Key", kp.Key) - .SetParameterValue("@Value", kp.Value) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync(@"DELETE FROM ""Configuration"" "); - tr.Commit(); + foreach (var kp in options) + { + cmd.SetCommandAndParameters(@"INSERT INTO ""Configuration"" (""Key"", ""Value"") VALUES (@Key, @Value) "); + cmd.SetParameterValue("@Key", kp.Key); + cmd.SetParameterValue("@Value", kp.Value); + await cmd.ExecuteNonQueryAsync(); } + + await transaction.CommitAsync(); } - public long GetBlocksLargerThan(long fhblocksize) + public async Task GetBlocksLargerThan(long fhblocksize) { - using (var cmd = m_connection.CreateCommand()) - return cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size") - .SetParameterValue("@Size", fhblocksize) - .ExecuteScalarInt64(-1); + using var cmd = m_connection.CreateCommand(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size"); + cmd.SetParameterValue("@Size", fhblocksize); + return await cmd.ExecuteScalarInt64Async(-1); } /// @@ -816,8 +813,8 @@ AND ""Fileset"".""ID"" NOT IN /// The hash size in byts /// Also verify filelists (can be slow) /// The transaction to run in - public void VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists, IDbTransaction transaction) - => VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false, transaction); + public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists, SqliteTransaction transaction) + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false, transaction); /// /// Verifies the consistency of the database prior to repair @@ -826,8 +823,8 @@ AND ""Fileset"".""ID"" NOT IN /// The hash size in byts /// Also verify filelists (can be slow) /// The transaction to run in - public void VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists, IDbTransaction transaction) - => VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true, transaction); + public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists, SqliteTransaction transaction) + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true, transaction); /// /// Verifies the consistency of the database @@ -837,21 +834,21 @@ AND ""Fileset"".""ID"" NOT IN /// Also verify filelists (can be slow) /// Disable verify for errors that will be fixed by repair /// The transaction to run in - private void VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair, IDbTransaction transaction) + private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - // Calculate the lengths for each blockset - var combinedLengths = @" -SELECT - ""A"".""ID"" AS ""BlocksetID"", - IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"", + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + // Calculate the lengths for each blockset + var combinedLengths = @" +SELECT + ""A"".""ID"" AS ""BlocksetID"", + IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"", ""A"".""Length"" FROM ""Blockset"" A LEFT OUTER JOIN ( - SELECT + SELECT ""BlocksetEntry"".""BlocksetID"", SUM(""Block"".""Size"") AS ""CalcLen"" FROM @@ -866,152 +863,156 @@ ON ""A"".""ID"" = ""B"".""BlocksetID"" "; - // For each blockset with wrong lengths, fetch the file path - var reportDetails = @"SELECT ""CalcLen"", ""Length"", ""A"".""BlocksetID"", ""File"".""Path"" FROM (" + combinedLengths + @") A, ""File"" WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" AND ""A"".""CalcLen"" != ""A"".""Length"" "; + // For each blockset with wrong lengths, fetch the file path + var reportDetails = @"SELECT ""CalcLen"", ""Length"", ""A"".""BlocksetID"", ""File"".""Path"" FROM (" + combinedLengths + @") A, ""File"" WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" AND ""A"".""CalcLen"" != ""A"".""Length"" "; - using (var rd = cmd.ExecuteReader(reportDetails)) - if (rd.Read()) - { - var sb = new StringBuilder(); - sb.AppendLine("Found inconsistency in the following files while validating database: "); - var c = 0; - do - { - if (c < 5) - sb.AppendFormat("{0}, actual size {1}, dbsize {2}, blocksetid: {3}{4}", rd.GetValue(3), rd.GetValue(1), rd.GetValue(0), rd.GetValue(2), Environment.NewLine); - c++; - } while (rd.Read()); - - c -= 5; - if (c > 0) - sb.AppendFormat("... and {0} more", c); - - sb.Append(". Run repair to fix it."); - throw new DatabaseInconsistencyException(sb.ToString()); - } - - var real_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM ""BlocklistHash""", 0); - var unique_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); - - if (real_count != unique_count) - throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); - - var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")"), 0); - if (itemswithnoblocklisthash != 0) - throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); - - if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Blockset"" WHERE ""Length"" > 0 AND ""ID"" NOT IN (SELECT ""BlocksetId"" FROM ""BlocksetEntry"")") != 0) - throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); - - if (cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""BlocksetID"" != @FolderBlocksetId AND ""BlocksetID"" != @SymlinkBlocksetId AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")") - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64(0) != 0) - throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!"); - - if (!laxVerifyForRepair) + using (var rd = await cmd.ExecuteReaderAsync(reportDetails)) + if (await rd.ReadAsync()) { - var filesetsMissingVolumes = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) - .ExecuteScalarInt64(0); - - if (filesetsMissingVolumes != 0) + var sb = new StringBuilder(); + sb.AppendLine("Found inconsistency in the following files while validating database: "); + var c = 0; + do { - if (filesetsMissingVolumes == 1) - using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) - .ExecuteReader()) - if (reader.Read()) - throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); + if (c < 5) + sb.AppendFormat("{0}, actual size {1}, dbsize {2}, blocksetid: {3}{4}", rd.GetValue(3), rd.GetValue(1), rd.GetValue(0), rd.GetValue(2), Environment.NewLine); + c++; + } while (await rd.ReadAsync()); - throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes"); + c -= 5; + if (c > 0) + sb.AppendFormat("... and {0} more", c); + + sb.Append(". Run repair to fix it."); + throw new DatabaseInconsistencyException(sb.ToString()); + } + + var real_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM ""BlocklistHash""", 0); + var unique_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); + + if (real_count != unique_count) + throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); + + var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")"), 0); + if (itemswithnoblocklisthash != 0) + throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); + + if (await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM ""Blockset"" WHERE ""Length"" > 0 AND ""ID"" NOT IN (SELECT ""BlocksetId"" FROM ""BlocksetEntry"")") != 0) + throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); + + cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""BlocksetID"" != @FolderBlocksetId AND ""BlocksetID"" != @SymlinkBlocksetId AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")"); + cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); + cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); + if (await cmd.ExecuteScalarInt64Async(0) != 0) + throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!"); + + if (!laxVerifyForRepair) + { + cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)"); + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + var filesetsMissingVolumes = await cmd.ExecuteScalarInt64Async(0); + + if (filesetsMissingVolumes != 0) + { + if (filesetsMissingVolumes == 1) + { + cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)"); + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + using var reader = await cmd.ExecuteReaderAsync(); + if (await reader.ReadAsync()) + throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); } - var volumesMissingFilests = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) - .ExecuteScalarInt64(0); - if (volumesMissingFilests != 0) - { - if (volumesMissingFilests == 1) - using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) - .ExecuteReader()) - if (reader.Read()) - throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); + throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes"); + } - throw new DatabaseInconsistencyException($"Detected {volumesMissingFilests} volumes with missing filesets"); + cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")"); + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + var volumesMissingFilests = await cmd.ExecuteScalarInt64Async(0); + if (volumesMissingFilests != 0) + { + if (volumesMissingFilests == 1) + { + cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")"); + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + using var reader = await cmd.ExecuteReaderAsync(); + if (await reader.ReadAsync()) + throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); + } + + throw new DatabaseInconsistencyException($"Detected {volumesMissingFilests} volumes with missing filesets"); + } + } + + var nonAttachedFiles = await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); + if (nonAttachedFiles != 0) + { + // Attempt to create a better error message by finding the first 10 fileset ids with the issue + using var filesetIdReader = await cmd.ExecuteReaderAsync(@"SELECT DISTINCT(FilesetID) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"") LIMIT 11"); + var filesetIds = new HashSet(); + var overflow = false; + while (await filesetIdReader.ReadAsync()) + { + if (filesetIds.Count >= 10) + { + overflow = true; + break; + } + filesetIds.Add(filesetIdReader.ConvertValueToInt64(0)); + } + + var pairs = FilesetTimes() + .Select((x, i) => new { FilesetId = x.Key, Version = i, Time = x.Value }) + .Where(x => filesetIds.Contains(x.FilesetId)) + .Select(x => $"Fileset {x.Version}: {x.Time} (id = {x.FilesetId})"); + + // Fall back to a generic error message if we can't find the fileset ids + if (!await pairs.AnyAsync()) + throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); + + if (overflow) + pairs = pairs.Append("... and more"); + + throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry in the following filesets:{Environment.NewLine}{string.Join(Environment.NewLine, pairs)}"); + } + + if (verifyfilelists) + { + var anyError = new List(); + using (var cmd2 = m_connection.CreateCommand()) + { + cmd2.Transaction = transaction; + await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) + { + var expandedCmd = FormatInvariant($@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FILESETS}) UNION SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FOLDERS_AND_SYMLINKS}))"); + cmd2.SetCommandAndParameters(expandedCmd); + cmd2.SetParameterValue("@FilesetId", filesetid); + cmd2.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); + cmd2.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); + var expandedlist = await cmd.ExecuteScalarInt64Async(0); + //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != @FolderBlocksetId AND ""FileLookup"".""BlocksetID"" != @SymlinkBlocksetId"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + cmd2.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId"); + cmd2.SetParameterValue("@FilesetId", filesetid); + var storedlist = await cmd2.ExecuteScalarInt64Async(0); + + if (expandedlist != storedlist) + { + var filesetname = filesetid.ToString(); + var fileset = await FilesetTimes().Zip(AsyncEnumerable.Range(0, await FilesetTimes().CountAsync()), (a, b) => new Tuple(b, a.Key, a.Value)).FirstOrDefaultAsync(x => x.Item2 == filesetid); + if (fileset != null) + filesetname = $"version {fileset.Item1}: {fileset.Item3} (database id: {fileset.Item2})"; + anyError.Add($"Unexpected difference in fileset {filesetname}, found {expandedlist} entries, but expected {storedlist}"); + } } } - var nonAttachedFiles = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); - if (nonAttachedFiles != 0) + if (anyError.Any()) { - // Attempt to create a better error message by finding the first 10 fileset ids with the issue - using var filesetIdReader = cmd.ExecuteReader(@"SELECT DISTINCT(FilesetID) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"") LIMIT 11"); - var filesetIds = new HashSet(); - var overflow = false; - while (filesetIdReader.Read()) - { - if (filesetIds.Count >= 10) - { - overflow = true; - break; - } - filesetIds.Add(filesetIdReader.ConvertValueToInt64(0)); - } - - var pairs = FilesetTimes - .Select((x, i) => new { FilesetId = x.Key, Version = i, Time = x.Value }) - .Where(x => filesetIds.Contains(x.FilesetId)) - .Select(x => $"Fileset {x.Version}: {x.Time} (id = {x.FilesetId})"); - - // Fall back to a generic error message if we can't find the fileset ids - if (!pairs.Any()) - throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); - - if (overflow) - pairs = pairs.Append("... and more"); - - throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry in the following filesets:{Environment.NewLine}{string.Join(Environment.NewLine, pairs)}"); - } - - if (verifyfilelists) - { - var anyError = new List(); - using (var cmd2 = m_connection.CreateCommand(transaction)) - { - foreach (var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) - { - var expandedCmd = FormatInvariant($@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FILESETS}) UNION SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FOLDERS_AND_SYMLINKS}))"); - var expandedlist = cmd2 - .SetCommandAndParameters(expandedCmd) - .SetParameterValue("@FilesetId", filesetid) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64(0); - //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != @FolderBlocksetId AND ""FileLookup"".""BlocksetID"" != @SymlinkBlocksetId"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); - var storedlist = cmd2.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetid) - .ExecuteScalarInt64(0); - - if (expandedlist != storedlist) - { - var filesetname = filesetid.ToString(); - var fileset = FilesetTimes.Zip(Enumerable.Range(0, FilesetTimes.Count()), (a, b) => new Tuple(b, a.Key, a.Value)).FirstOrDefault(x => x.Item2 == filesetid); - if (fileset != null) - filesetname = $"version {fileset.Item1}: {fileset.Item3} (database id: {fileset.Item2})"; - anyError.Add($"Unexpected difference in fileset {filesetname}, found {expandedlist} entries, but expected {storedlist}"); - } - } - } - if (anyError.Any()) - { - throw new DatabaseInconsistencyException(string.Join("\n\r", anyError), "FilesetDifferences"); - } + throw new DatabaseInconsistencyException(string.Join("\n\r", anyError), "FilesetDifferences"); } } } @@ -1022,26 +1023,20 @@ ON long Size { get; } } - internal class Block : IBlock + internal class Block(string hash, long size) : IBlock { - public string Hash { get; private set; } - public long Size { get; private set; } - - public Block(string hash, long size) - { - Hash = hash; - Size = size; - } + public string Hash { get; private set; } = hash; + public long Size { get; private set; } = size; } - public IEnumerable GetBlocks(long volumeid, IDbTransaction? transaction = null) + public async IAsyncEnumerable GetBlocks(long volumeid, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(transaction) - .SetCommandAndParameters(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId") - .SetParameterValue("@VolumeId", volumeid); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new Block(rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), rd.ConvertValueToInt64(1)); + using var cmd = m_connection.CreateCommand(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@VolumeId", volumeid); + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + yield return new Block(rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), rd.ConvertValueToInt64(1)); } // TODO: Replace this with an enumerable method @@ -1135,13 +1130,13 @@ ON public const string LIST_FILESETS = @" SELECT - ""L"".""Path"", - ""L"".""Lastmodified"", - ""L"".""Filelength"", - ""L"".""Filehash"", - ""L"".""Metahash"", + ""L"".""Path"", + ""L"".""Lastmodified"", + ""L"".""Filelength"", + ""L"".""Filehash"", + ""L"".""Metahash"", ""L"".""Metalength"", - ""L"".""BlocklistHash"", + ""L"".""BlocklistHash"", ""L"".""FirstBlockHash"", ""L"".""FirstBlockSize"", ""L"".""FirstMetaBlockHash"", @@ -1149,27 +1144,27 @@ SELECT ""M"".""Hash"" AS ""MetaBlocklistHash"" FROM ( - SELECT - ""J"".""Path"", - ""J"".""Lastmodified"", - ""J"".""Filelength"", - ""J"".""Filehash"", - ""J"".""Metahash"", + SELECT + ""J"".""Path"", + ""J"".""Lastmodified"", + ""J"".""Filelength"", + ""J"".""Filehash"", + ""J"".""Metahash"", ""J"".""Metalength"", - ""K"".""Hash"" AS ""BlocklistHash"", + ""K"".""Hash"" AS ""BlocklistHash"", ""J"".""FirstBlockHash"", ""J"".""FirstBlockSize"", ""J"".""FirstMetaBlockHash"", ""J"".""FirstMetaBlockSize"", ""J"".""MetablocksetID"" - FROM + FROM ( - SELECT - ""A"".""Path"" AS ""Path"", - ""D"".""Lastmodified"" AS ""Lastmodified"", - ""B"".""Length"" AS ""Filelength"", - ""B"".""FullHash"" AS ""Filehash"", - ""E"".""FullHash"" AS ""Metahash"", + SELECT + ""A"".""Path"" AS ""Path"", + ""D"".""Lastmodified"" AS ""Lastmodified"", + ""B"".""Length"" AS ""Filelength"", + ""B"".""FullHash"" AS ""Filehash"", + ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"", ""F"".""Hash"" AS ""FirstBlockHash"", @@ -1177,11 +1172,11 @@ FROM ""H"".""Hash"" AS ""FirstMetaBlockHash"", ""H"".""Size"" AS ""FirstMetaBlockSize"", ""C"".""BlocksetID"" AS ""MetablocksetID"" - FROM - ""File"" A + FROM + ""File"" A LEFT JOIN ""Blockset"" B - ON ""A"".""BlocksetID"" = ""B"".""ID"" - LEFT JOIN ""Metadataset"" C + ON ""A"".""BlocksetID"" = ""B"".""ID"" + LEFT JOIN ""Metadataset"" C ON ""A"".""MetadataID"" = ""C"".""ID"" LEFT JOIN ""FilesetEntry"" D ON ""A"".""ID"" = ""D"".""FileID"" @@ -1189,22 +1184,22 @@ FROM ON ""E"".""ID"" = ""C"".""BlocksetID"" LEFT JOIN ""BlocksetEntry"" G ON ""B"".""ID"" = ""G"".""BlocksetID"" - LEFT JOIN ""Block"" F - ON ""G"".""BlockID"" = ""F"".""ID"" + LEFT JOIN ""Block"" F + ON ""G"".""BlockID"" = ""F"".""ID"" LEFT JOIN ""BlocksetEntry"" I ON ""E"".""ID"" = ""I"".""BlocksetID"" - LEFT JOIN ""Block"" H + LEFT JOIN ""Block"" H ON ""I"".""BlockID"" = ""H"".""ID"" - WHERE + WHERE ""A"".""BlocksetId"" >= 0 AND ""D"".""FilesetID"" = @FilesetId AND - (""I"".""Index"" = 0 OR ""I"".""Index"" IS NULL) AND + (""I"".""Index"" = 0 OR ""I"".""Index"" IS NULL) AND (""G"".""Index"" = 0 OR ""G"".""Index"" IS NULL) ) J - LEFT OUTER JOIN - ""BlocklistHash"" K - ON - ""K"".""BlocksetID"" = ""J"".""BlocksetID"" + LEFT OUTER JOIN + ""BlocklistHash"" K + ON + ""K"".""BlocksetID"" = ""J"".""BlocksetID"" ORDER BY ""J"".""Path"", ""K"".""Index"" ) L @@ -1236,16 +1231,16 @@ FROM ""F"".""Hash"" AS ""FirstMetaBlockHash"", ""C"".""BlocksetID"" AS ""MetaBlocksetID"" FROM - ""FilesetEntry"" A, - ""File"" B, - ""Metadataset"" C, + ""FilesetEntry"" A, + ""File"" B, + ""Metadataset"" C, ""Blockset"" D, ""BlocksetEntry"" E, ""Block"" F - WHERE - ""A"".""FileID"" = ""B"".""ID"" - AND ""B"".""MetadataID"" = ""C"".""ID"" - AND ""C"".""BlocksetID"" = ""D"".""ID"" + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""MetadataID"" = ""C"".""ID"" + AND ""C"".""BlocksetID"" = ""D"".""ID"" AND ""E"".""BlocksetID"" = ""C"".""BlocksetID"" AND ""E"".""BlockID"" = ""F"".""ID"" AND ""E"".""Index"" = 0 @@ -1261,94 +1256,92 @@ ORDER BY "; - public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, IDbTransaction transaction) + public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - cmd.SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS) - .SetParameterValue("@FilesetId", filesetId) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS); + cmd.SetParameterValue("@FilesetId", filesetId); + cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); + cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); - string? lastpath = null; - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + string? lastpath = null; + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) + { + var blocksetID = rd.ConvertValueToInt64(0, -1); + var path = rd.ConvertValueToString(2); + var metalength = rd.ConvertValueToInt64(3, -1); + var metahash = rd.ConvertValueToString(4); + var metablockhash = rd.ConvertValueToString(6); + var metablocklisthash = rd.ConvertValueToString(7); + + if (path == lastpath) + Logging.Log.WriteWarningMessage(LOGTAG, "DuplicatePathFound", null, "Duplicate path detected: {0}", path); + + lastpath = path; + + if (blocksetID == FOLDER_BLOCKSET_ID) + filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); + else if (blocksetID == SYMLINK_BLOCKSET_ID) + filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); + } + + // TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x + + cmd.SetCommandAndParameters(LIST_FILESETS); + cmd.SetParameterValue("@FilesetId", filesetId); + + using (var rd = await cmd.ExecuteReaderAsync()) + if (await rd.ReadAsync()) + { + var more = false; + do { - var blocksetID = rd.ConvertValueToInt64(0, -1); - var path = rd.ConvertValueToString(2); - var metalength = rd.ConvertValueToInt64(3, -1); + var path = rd.ConvertValueToString(0); + var filehash = rd.ConvertValueToString(3); + var size = rd.ConvertValueToInt64(2); + var lastmodified = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); var metahash = rd.ConvertValueToString(4); - var metablockhash = rd.ConvertValueToString(6); - var metablocklisthash = rd.ConvertValueToString(7); + var metasize = rd.ConvertValueToInt64(5, -1); + var p = rd.GetValue(6); + var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd); + var blockhash = rd.ConvertValueToString(7); + var blocksize = rd.ConvertValueToInt64(8, -1); + var metablockhash = rd.ConvertValueToString(9); + //var metablocksize = rd.ConvertValueToInt64(10, -1); + var metablocklisthash = rd.ConvertValueToString(11); - if (path == lastpath) - Logging.Log.WriteWarningMessage(LOGTAG, "DuplicatePathFound", null, "Duplicate path detected: {0}", path); + if (blockhash == filehash) + blockhash = null; - lastpath = path; + if (metablockhash == metahash) + metablockhash = null; - if (blocksetID == FOLDER_BLOCKSET_ID) - filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); - else if (blocksetID == SYMLINK_BLOCKSET_ID) - filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); - } + filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); + if (blrd == null) + more = await rd.ReadAsync(); + else + more = blrd.MoreData; - // TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x - - cmd.SetCommandAndParameters(LIST_FILESETS) - .SetParameterValue("@FilesetId", filesetId); - - using (var rd = cmd.ExecuteReader()) - if (rd.Read()) - { - var more = false; - do - { - var path = rd.ConvertValueToString(0); - var filehash = rd.ConvertValueToString(3); - var size = rd.ConvertValueToInt64(2); - var lastmodified = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); - var metahash = rd.ConvertValueToString(4); - var metasize = rd.ConvertValueToInt64(5, -1); - var p = rd.GetValue(6); - var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd); - var blockhash = rd.ConvertValueToString(7); - var blocksize = rd.ConvertValueToInt64(8, -1); - var metablockhash = rd.ConvertValueToString(9); - //var metablocksize = rd.ConvertValueToInt64(10, -1); - var metablocklisthash = rd.ConvertValueToString(11); - - if (blockhash == filehash) - blockhash = null; - - if (metablockhash == metahash) - metablockhash = null; - - filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); - if (blrd == null) - more = rd.Read(); - else - more = blrd.MoreData; - - } while (more); - } - } + } while (more); + } } - public void LinkFilesetToVolume(long filesetid, long volumeid, IDbTransaction transaction) + public async Task LinkFilesetToVolume(long filesetid, long volumeid, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - var c = cmd.SetCommandAndParameters(@"UPDATE ""Fileset"" SET ""VolumeID"" = @VolumeId WHERE ""ID"" = @FilesetId") - .SetParameterValue("@VolumeId", volumeid) - .SetParameterValue("@FilesetId", filesetid) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"UPDATE ""Fileset"" SET ""VolumeID"" = @VolumeId WHERE ""ID"" = @FilesetId"); + cmd.SetParameterValue("@VolumeId", volumeid); + cmd.SetParameterValue("@FilesetId", filesetid); + var c = await cmd.ExecuteNonQueryAsync(); - if (c != 1) - throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); - } + if (c != 1) + throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); } - public void PushTimestampChangesToPreviousVersion(long filesetId, IDbTransaction transaction) + public async Task PushTimestampChangesToPreviousVersion(long filesetId, SqliteTransaction transaction) { var query = @" UPDATE FilesetEntry AS oldVersion @@ -1358,20 +1351,21 @@ WHERE oldVersion.FileID = tempVersion.FileID AND tempVersion.FilesetID = @FilesetId AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER BY Timestamp DESC LIMIT 1)"; - using (var cmd = m_connection.CreateCommand(transaction, query)) - cmd.SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(query); + cmd.Transaction = transaction; + cmd.SetParameterValue("@FilesetId", filesetId); + await cmd.ExecuteNonQueryAsync(); } /// /// Keeps a list of filenames in a temporary table with a single column Path - /// + /// public class FilteredFilenameTable : IDisposable { public string Tablename { get; private set; } - private readonly IDbConnection m_connection; + private readonly SqliteConnection m_connection; - public FilteredFilenameTable(IDbConnection connection, IFilter filter, IDbTransaction? transaction) + public FilteredFilenameTable(SqliteConnection connection, IFilter filter, SqliteTransaction transaction) { m_connection = connection; Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -1385,36 +1379,32 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER if (filter.Empty) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ")); - return; - } + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ")).Await(); + return; } if (type == FilterType.Regexp || type == FilterType.Group) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")); - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")).Await(); + cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{Tablename}"" (""Path"") VALUES (@Path)")); + using (var c2 = m_connection.CreateCommand()) + using (var rd = c2.ExecuteReaderAsync(@"SELECT DISTINCT ""Path"" FROM ""File"" ").Await()) + while (rd.ReadAsync().Await()) { - cmd.SetCommandAndParameters(tr.Parent, FormatInvariant($@"INSERT INTO ""{Tablename}"" (""Path"") VALUES (@Path)")); - using (var c2 = m_connection.CreateCommand()) - using (var rd = c2.ExecuteReader(@"SELECT DISTINCT ""Path"" FROM ""File"" ")) - while (rd.Read()) - { - var p = rd.ConvertValueToString(0); - if (FilterExpression.Matches(filter, p)) - cmd.SetParameterValue("@Path", p) - .ExecuteNonQuery(); - } - - - tr.Commit(); + var p = rd.ConvertValueToString(0); + if (FilterExpression.Matches(filter, p)) + { + cmd.SetParameterValue("@Path", p); + cmd.ExecuteNonQueryAsync().Await(); + } } - } + + transaction.CommitAsync().Await(); } else { @@ -1438,13 +1428,11 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } } - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")); - cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}"), args); - tr.Commit(); - } + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}"), args).Await(); + transaction.CommitAsync().Await(); } } @@ -1453,43 +1441,39 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER if (Tablename != null) try { - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{Tablename}"" ")); + using var cmd = m_connection.CreateCommand(); + cmd.ExecuteNonQueryAsync(FormatInvariant(@$"DROP TABLE IF EXISTS ""{Tablename}"" ")).Await(); } catch { } finally { Tablename = null!; } } } - public void RenameRemoteFile(string oldname, string newname, IDbTransaction? transaction) + public async Task RenameRemoteFile(string oldname, string newname, SqliteTransaction transaction) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - //Rename the old entry, to preserve ID links - var c = cmd.SetCommandAndParameters(@"UPDATE ""Remotevolume"" SET ""Name"" = @Newname WHERE ""Name"" = @Oldname") - .SetParameterValue("@Newname", newname) - .SetParameterValue("@Oldname", oldname) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + //Rename the old entry, to preserve ID links + cmd.SetCommandAndParameters(@"UPDATE ""Remotevolume"" SET ""Name"" = @Newname WHERE ""Name"" = @Oldname"); + cmd.SetParameterValue("@Newname", newname); + cmd.SetParameterValue("@Oldname", oldname); + var c = await cmd.ExecuteNonQueryAsync(); - if (c != 1) - throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}"); + if (c != 1) + throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}"); - // Grab the type of entry - var type = (RemoteVolumeType)Enum.Parse( - typeof(RemoteVolumeType), - cmd.SetCommandAndParameters(@"SELECT ""Type"" FROM ""Remotevolume"" WHERE ""Name"" = @Name") - .SetParameterValue("@Name", newname) - .ExecuteScalar() - ?.ToString() ?? "", - true); + // Grab the type of entry + cmd.SetCommandAndParameters(@"SELECT ""Type"" FROM ""Remotevolume"" WHERE ""Name"" = @Name"); + cmd.SetParameterValue("@Name", newname); + var type = (RemoteVolumeType)Enum.Parse( + typeof(RemoteVolumeType), (await cmd.ExecuteScalarAsync())?.ToString() ?? "", + true); - //Create a fake new entry with the old name and mark as deleting - // as this ensures we will remove it, if it shows up in some later listing - RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, tr.Parent); + //Create a fake new entry with the old name and mark as deleting + // as this ensures we will remove it, if it shows up in some later listing + await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, transaction); - tr.Commit(); - } + await transaction.CommitAsync(); } /// @@ -1498,20 +1482,20 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// The ID of the fileset volume to update /// The timestamp of the operation to create /// An optional external transaction - public virtual long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction? transaction = null) + public virtual async Task CreateFileset(long volumeid, DateTime timestamp, SqliteTransaction transaction) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - var id = cmd.SetCommandAndParameters(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"", ""IsFullBackup"") VALUES (@OperationId, @Timestamp, @VolumeId, @IsFullBackup); SELECT last_insert_rowid();") - .SetParameterValue("@OperationId", m_operationid) - .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) - .SetParameterValue("@VolumeId", volumeid) - .SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP) - .ExecuteScalarInt64(-1); - tr.Commit(); - return id; - } + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"", ""IsFullBackup"") VALUES (@OperationId, @Timestamp, @VolumeId, @IsFullBackup); SELECT last_insert_rowid();"); + cmd.SetParameterValue("@OperationId", m_operationid); + cmd.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)); + cmd.SetParameterValue("@VolumeId", volumeid); + cmd.SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP); + var id = await cmd.ExecuteScalarInt64Async(-1); + + await transaction.CommitAsync(); + + return id; } /// @@ -1540,41 +1524,42 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// The size of the hash /// An optional external transaction /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data - public IEnumerable> GetBlocklists(long volumeid, long blocksize, int hashsize, IDbTransaction? transaction = null) + public async IAsyncEnumerable> GetBlocklists(long volumeid, long blocksize, int hashsize, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - // Group subquery by hash to ensure that each blocklist hash appears only once in the result - var sql = FormatInvariant($@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + // Group subquery by hash to ensure that each blocklist hash appears only once in the result + var sql = FormatInvariant($@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM (SELECT ""BlocklistHash"".""BlocksetID"", ""Block"".""Hash"", ""BlocklistHash"".""Index"" FROM ""BlocklistHash"",""Block"" WHERE ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId GROUP BY ""Block"".""Hash"", ""Block"".""Size"") A, - ""BlocksetEntry"" B, ""Block"" C WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" AND - ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize}) AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize}) AND ""C"".""ID"" = ""B"".""BlockID"" + ""BlocksetEntry"" B, ""Block"" C WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" AND + ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize}) AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize}) AND ""C"".""ID"" = ""B"".""BlockID"" ORDER BY ""A"".""BlocksetID"", ""B"".""Index"""); - string? curHash = null; - var count = 0; - var buffer = new byte[blocksize]; + string? curHash = null; + var count = 0; + var buffer = new byte[blocksize]; - using (var rd = cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", volumeid).ExecuteReader()) - while (rd.Read()) + cmd.SetCommandAndParameters(sql); + cmd.SetParameterValue("@VolumeId", volumeid); + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) + { + var blockhash = rd.ConvertValueToString(0); + if ((blockhash != curHash && curHash != null) || count + hashsize > buffer.Length) { - var blockhash = rd.ConvertValueToString(0); - if ((blockhash != curHash && curHash != null) || count + hashsize > buffer.Length) - { - yield return new Tuple(curHash!, buffer, count); - buffer = new byte[blocksize]; - count = 0; - } - - var hash = Convert.FromBase64String(rd.ConvertValueToString(1) ?? throw new Exception("Hash is null")); - Array.Copy(hash, 0, buffer, count, hashsize); - curHash = blockhash; - count += hashsize; + yield return new Tuple(curHash!, buffer, count); + buffer = new byte[blocksize]; + count = 0; } - if (curHash != null) - yield return new Tuple(curHash, buffer, count); - } + var hash = Convert.FromBase64String(rd.ConvertValueToString(1) ?? throw new Exception("Hash is null")); + Array.Copy(hash, 0, buffer, count, hashsize); + curHash = blockhash; + count += hashsize; + } + + if (curHash != null) + yield return new Tuple(curHash, buffer, count); } /// @@ -1583,17 +1568,16 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// Existing file set to update /// Full backup state /// An optional external transaction - public void UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup, IDbTransaction? transaction = null) + public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup, SqliteTransaction transaction) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - cmd.SetCommandAndParameters(@"UPDATE ""Fileset"" SET ""IsFullBackup"" = @IsFullBackup WHERE ""ID"" = @FilesetId;") - .SetParameterValue("@FilesetId", fileSetId) - .SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP) - .ExecuteNonQuery(); - tr.Commit(); - } + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"UPDATE ""Fileset"" SET ""IsFullBackup"" = @IsFullBackup WHERE ""ID"" = @FilesetId;"); + cmd.SetParameterValue("@FilesetId", fileSetId); + cmd.SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP); + await cmd.ExecuteNonQueryAsync(); + + await transaction.CommitAsync(); } /// @@ -1601,12 +1585,13 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// /// The fileset ID to clear /// The transaction to use - public void ClearFilesetEntries(long filesetId, IDbTransaction transaction) + public async Task ClearFilesetEntries(long filesetId, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId"); + cmd.SetParameterValue("@FilesetId", filesetId); + await cmd.ExecuteNonQueryAsync(); } /// @@ -1614,11 +1599,11 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// /// The transaction to use /// The last incomplete fileset or default - public RemoteVolumeEntry GetLastIncompleteFilesetVolume(IDbTransaction transaction) + public async Task GetLastIncompleteFilesetVolume(SqliteTransaction transaction) { - var candidates = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).ToArray(); - if (candidates.Any()) - return GetRemoteVolumeFromFilesetID(candidates.Last().Key, transaction); + var candidates = GetIncompleteFilesets(transaction).OrderBy(x => x.Value); + if (await candidates.AnyAsync()) + return await GetRemoteVolumeFromFilesetID((await candidates.LastAsync()).Key, transaction); return default; } @@ -1628,17 +1613,18 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// /// An optional transaction /// A list of fileset IDs and timestamps - public IEnumerable> GetIncompleteFilesets(IDbTransaction? transaction) + public async IAsyncEnumerable> GetIncompleteFilesets(SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(FormatInvariant(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')"))) - while (rd.Read()) - { - yield return new KeyValuePair( - rd.ConvertValueToInt64(0), - ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() - ); - } + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + using var rd = await cmd.ExecuteReaderAsync(FormatInvariant(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')")); + while (await rd.ReadAsync()) + { + yield return new KeyValuePair( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() + ); + } } /// @@ -1647,53 +1633,53 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// The fileset ID /// An optional transaction /// The remote volume entry or default - public RemoteVolumeEntry GetRemoteVolumeFromFilesetID(long filesetID, IDbTransaction? transaction = null) + public async Task GetRemoteVolumeFromFilesetID(long filesetID, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.SetCommandAndParameters(@"SELECT ""RemoteVolume"".""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" FROM ""RemoteVolume"", ""Fileset"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Fileset"".""ID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetID) - .ExecuteReader()) - if (rd.Read()) - return new RemoteVolumeEntry( - rd.ConvertValueToInt64(0, -1), - rd.ConvertValueToString(1), - rd.ConvertValueToString(4), - rd.ConvertValueToInt64(3, -1), - (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""), - (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""), - ParseFromEpochSeconds(rd.ConvertValueToInt64(6)).ToLocalTime(), - ParseFromEpochSeconds(rd.ConvertValueToInt64(7)).ToLocalTime() - ); - else - return default(RemoteVolumeEntry); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"SELECT ""RemoteVolume"".""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" FROM ""RemoteVolume"", ""Fileset"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Fileset"".""ID"" = @FilesetId"); + cmd.SetParameterValue("@FilesetId", filesetID); + using var rd = await cmd.ExecuteReaderAsync(); + if (await rd.ReadAsync()) + return new RemoteVolumeEntry( + rd.ConvertValueToInt64(0, -1), + rd.ConvertValueToString(1), + rd.ConvertValueToString(4), + rd.ConvertValueToInt64(3, -1), + (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""), + (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""), + ParseFromEpochSeconds(rd.ConvertValueToInt64(6)).ToLocalTime(), + ParseFromEpochSeconds(rd.ConvertValueToInt64(7)).ToLocalTime() + ); + else + return default(RemoteVolumeEntry); } - public void PurgeLogData(DateTime threshold) + public async Task PurgeLogData(DateTime threshold) { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) - { - var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold); - cmd.SetCommandAndParameters(@"DELETE FROM ""LogData"" WHERE ""Timestamp"" < @Timestamp") - .SetParameterValue("@Timestamp", t) - .ExecuteNonQuery(); - cmd.SetCommandAndParameters(@"DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < @Timestamp") - .SetParameterValue("@Timestamp", t) - .ExecuteNonQuery(); + using var transaction = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold); + cmd.SetCommandAndParameters(@"DELETE FROM ""LogData"" WHERE ""Timestamp"" < @Timestamp"); + cmd.SetParameterValue("@Timestamp", t); + await cmd.ExecuteNonQueryAsync(); + cmd.SetCommandAndParameters(@"DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < @Timestamp"); + cmd.SetParameterValue("@Timestamp", t); + await cmd.ExecuteNonQueryAsync(); - tr.Commit(); - } + await transaction.CommitAsync(); } - public void PurgeDeletedVolumes(DateTime threshold) + public async Task PurgeDeletedVolumes(DateTime threshold) { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) - { - m_removedeletedremotevolumeCommand.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)) - .ExecuteNonQuery(tr); - tr.Commit(); - } + using var transaction = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + m_removedeletedremotevolumeCommand.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)); + await m_removedeletedremotevolumeCommand.ExecuteNonQueryAsync(); + + await transaction.CommitAsync(); } public virtual void Dispose() @@ -1707,15 +1693,16 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER { if (m_connection.State == ConnectionState.Open && !m_hasExecutedVacuum) { - using (var transaction = m_connection.BeginTransactionSafe()) - using (var command = m_connection.CreateCommand(transaction)) + using (var transaction = m_connection.BeginTransaction()) + using (var command = m_connection.CreateCommand()) { + command.Transaction = transaction; // SQLite recommends that PRAGMA optimize is run just before closing each database connection. command.ExecuteNonQuery("PRAGMA optimize"); try { - transaction.Commit(); + transaction.CommitAsync().Await(); } catch (Exception ex) { @@ -1723,7 +1710,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } } - m_connection.Close(); + m_connection.CloseAsync().Await(); } m_connection.Dispose(); @@ -1773,7 +1760,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER throw new AggregateException(exceptions); } - public void WriteResults(IBasicResults result) + public async Task WriteResults(IBasicResults result) { if (IsDisposed) return; @@ -1788,7 +1775,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } var serializer = new JsonFormatSerializer(); - LogMessage("Result", + await LogMessage("Result", serializer.SerializeResults(result), null, null @@ -1811,7 +1798,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// The path prefix ID. /// The path to get the prefix for. /// The transaction to use for insertion, or null for no transaction - public long GetOrCreatePathPrefix(string prefix, IDbTransaction? transaction) + public async Task GetOrCreatePathPrefix(string prefix, SqliteTransaction transaction) { // Ring-buffer style lookup for (var i = 0; i < m_pathPrefixLookup.Length; i++) @@ -1822,12 +1809,15 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } m_findpathprefixCommand.Transaction = transaction; - var id = m_findpathprefixCommand.SetParameterValue("@Prefix", prefix) - .ExecuteScalarInt64(transaction); + m_findpathprefixCommand.SetParameterValue("@Prefix", prefix); + var id = await m_findpathprefixCommand.ExecuteScalarInt64Async(); if (id < 0) - id = m_insertpathprefixCommand.SetParameterValue("@Prefix", prefix) - .ExecuteScalarInt64(transaction); + { + m_insertpathprefixCommand.SetParameterValue("@Prefix", prefix); + m_insertpathprefixCommand.Transaction = transaction; + id = await m_insertpathprefixCommand.ExecuteScalarInt64Async(); + } m_pathPrefixIndex = (m_pathPrefixIndex + 1) % m_pathPrefixLookup.Length; m_pathPrefixLookup[m_pathPrefixIndex] = new KeyValuePair(prefix, id); @@ -1838,10 +1828,10 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// /// The path separators on this system /// - private static readonly char[] _pathseparators = new char[] { + private static readonly char[] _pathseparators = [ Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar, - }; + ]; /// /// Helper method that splits a path on the last path separator From bb504c1a44fbcdca27ffac074cb41731178f3506 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 17:09:35 +0200 Subject: [PATCH 008/362] Post master merge fixes to LocalDatabase --- .../Library/Main/Database/LocalDatabase.cs | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index ee10a3c3d..493e46161 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -662,16 +662,17 @@ AND ""Fileset"".""ID"" NOT IN return res; } - public bool IsFilesetFullBackup(DateTime filesetTime, IDbTransaction? transaction) + public async Task IsFilesetFullBackup(DateTime filesetTime, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.SetCommandAndParameters($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = @Timestamp").SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)).ExecuteReader()) - { - if (!rd.Read()) - return false; - var isFullBackup = rd.GetInt32(0); - return isFullBackup == BackupType.FULL_BACKUP; - } + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = @Timestamp"); + cmd.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)); + using var rd = await cmd.ExecuteReaderAsync(); + if (!await rd.ReadAsync()) + return false; + var isFullBackup = rd.GetInt32(0); + return isFullBackup == BackupType.FULL_BACKUP; } // TODO: Remove this @@ -1504,16 +1505,17 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// The ID of the index volume. /// The ID of the block volume. /// An optional transaction. - public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, IDbTransaction transaction) + public async Task AddIndexBlockLink(long indexVolumeID, long blockVolumeID, SqliteTransaction transaction) { if (indexVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(indexVolumeID), "Index volume ID must be greater than 0."); if (blockVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0."); - m_insertIndexBlockLink.SetParameterValue("@IndexVolumeId", indexVolumeID) - .SetParameterValue("@BlockVolumeId", blockVolumeID) - .ExecuteNonQuery(transaction); + m_insertIndexBlockLink.Transaction = transaction; + m_insertIndexBlockLink.SetParameterValue("@IndexVolumeId", indexVolumeID); + m_insertIndexBlockLink.SetParameterValue("@BlockVolumeId", blockVolumeID); + await m_insertIndexBlockLink.ExecuteNonQueryAsync(); } /// @@ -1687,7 +1689,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER if (IsDisposed) return; - DisposeAllFields(this, false); + DisposeAllFields(this, false); if (ShouldCloseConnection && m_connection != null) { @@ -1698,7 +1700,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER { command.Transaction = transaction; // SQLite recommends that PRAGMA optimize is run just before closing each database connection. - command.ExecuteNonQuery("PRAGMA optimize"); + command.ExecuteNonQueryAsync("PRAGMA optimize").Await(); try { From 7bdb948820c8f15b6b8451b374d4d6ef1a09539d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 17:11:50 +0200 Subject: [PATCH 009/362] No need to keep the old package references --- .../SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj index 4a92aaee8..d45e3fe4e 100644 --- a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj +++ b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj @@ -22,14 +22,6 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - From 7f919b320571ce36661afaed32b3d01808b63182 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 12 May 2025 17:12:40 +0200 Subject: [PATCH 010/362] Removed deprecated nuget package. It's replaced by built-in analyzer: https://www.nuget.org/packages/Microsoft.DotNet.Analyzers.Compatibility/0.2.12-alpha --- .../SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj index d45e3fe4e..55a97fc9f 100644 --- a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj +++ b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj @@ -17,13 +17,7 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - From 19009011b2f39d2bf5a5df9e53f1a2872275cfff Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 08:33:09 +0200 Subject: [PATCH 011/362] Updated LocalRestoreDatabase to async using Microsoft.Data.Sqlite --- .../Main/Database/LocalRestoreDatabase.cs | 1118 +++++++++-------- 1 file changed, 579 insertions(+), 539 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index a4d5dfcd9..fd6db1a11 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -27,10 +27,12 @@ using System.Collections.Generic; using System.Data; using System.Linq; using System.Text; +using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Main.Operation.Restore; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -47,7 +49,7 @@ namespace Duplicati.Library.Main.Database /// protected string? m_tempfiletable; protected string? m_tempblocktable; - protected ConcurrentBag m_connection_pool = []; + protected ConcurrentBag m_connection_pool = []; protected string? m_latestblocktable; protected string? m_fileprogtable; protected string? m_totalprogtable; @@ -83,46 +85,47 @@ namespace Duplicati.Library.Main.Database /// Note: If a file is done once and then set back to a none restored state, the file is not automatically removed. /// But if it reaches a restored state later, it will be re-added (trigger will fire) /// - public void CreateProgressTracker(bool createFilesNewlyDoneTracker) + public async Task CreateProgressTracker(bool createFilesNewlyDoneTracker) { m_fileprogtable = "FileProgress-" + m_temptabsetguid; m_totalprogtable = "TotalProgress-" + m_temptabsetguid; m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; - using (var cmd = m_connection.CreateCommand()) - { - // How to handle METADATA? - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + + // How to handle METADATA? + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( ""FileId"" INTEGER PRIMARY KEY, ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL, ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL )")); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( ""TotalFiles"" INTEGER NOT NULL, ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL, ""FilesFullyRestored"" INTEGER NOT NULL, ""FilesPartiallyRestored"" INTEGER NOT NULL, ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL )")); - if (createFilesNewlyDoneTracker) - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( + if (createFilesNewlyDoneTracker) + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( ""ID"" INTEGER PRIMARY KEY )")); - } + } - try - { - // Initialize statistics with File- and Block-Data (it is valid to already have restored blocks in files) - // A rebuild with this function should be valid anytime. - // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. - string sql; + try + { + // Initialize statistics with File- and Block-Data (it is valid to already have restored blocks in files) + // A rebuild with this function should be valid anytime. + // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. + string sql; - // We use a LEFT JOIN to allow for empty files (no data Blocks) - sql = FormatInvariant($@" INSERT INTO ""{m_fileprogtable}"" (""FileId"", ""TotalBlocks"", ""TotalSize"", ""BlocksRestored"", ""SizeRestored"") + // We use a LEFT JOIN to allow for empty files (no data Blocks) + sql = FormatInvariant($@" INSERT INTO ""{m_fileprogtable}"" (""FileId"", ""TotalBlocks"", ""TotalSize"", ""BlocksRestored"", ""SizeRestored"") SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size""), 0) , IFNULL(COUNT(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""ID"" ELSE NULL END), 0) , IFNULL(SUM(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""Size"" ELSE 0 END), 0) @@ -131,10 +134,10 @@ SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size"" WHERE ""B"".""Metadata"" IS NOT 1 GROUP BY ""F"".""ID"" "); - // Will be one row per file. - cmd.ExecuteNonQuery(sql); + // Will be one row per file. + await cmd.ExecuteNonQueryAsync(sql); - sql = FormatInvariant($@"INSERT INTO ""{m_totalprogtable}"" ( + sql = FormatInvariant($@"INSERT INTO ""{m_totalprogtable}"" ( ""TotalFiles"", ""TotalBlocks"", ""TotalSize"" , ""FilesFullyRestored"", ""FilesPartiallyRestored"", ""BlocksRestored"", ""SizeRestored"" ) @@ -144,27 +147,27 @@ SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size"" , IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) FROM ""{m_fileprogtable}"" ""P"" "); - // Will result in a single line (no support to also track metadata) - cmd.ExecuteNonQuery(sql); + // Will result in a single line (no support to also track metadata) + await cmd.ExecuteNonQueryAsync(sql); - // Finally we create TRIGGERs to keep all our statistics up to date. - // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic. - // Note: We do assume that neither files nor blocks will be added or deleted during restore process - // and that the size of each block stays constant so there is no need to track that information - // with additional INSERT and DELETE triggers. + // Finally we create TRIGGERs to keep all our statistics up to date. + // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic. + // Note: We do assume that neither files nor blocks will be added or deleted during restore process + // and that the size of each block stays constant so there is no need to track that information + // with additional INSERT and DELETE triggers. - // A trigger to update the file-stat entry each time a block changes restoration state. - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" AFTER UPDATE OF ""Restored"" ON ""{m_tempblocktable}"" + // A trigger to update the file-stat entry each time a block changes restoration state. + sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" AFTER UPDATE OF ""Restored"" ON ""{m_tempblocktable}"" WHEN OLD.""Restored"" != NEW.""Restored"" AND NEW.""Metadata"" = 0 BEGIN UPDATE ""{m_fileprogtable}"" SET ""BlocksRestored"" = ""{m_fileprogtable}"".""BlocksRestored"" + (NEW.""Restored"" - OLD.""Restored"") , ""SizeRestored"" = ""{m_fileprogtable}"".""SizeRestored"" + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" ; END "); - cmd.ExecuteNonQuery(sql); + await cmd.ExecuteNonQueryAsync(sql); - // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" AFTER UPDATE ON ""{m_fileprogtable}"" + // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) + sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" AFTER UPDATE ON ""{m_fileprogtable}"" BEGIN UPDATE ""{m_totalprogtable}"" SET ""FilesFullyRestored"" = ""{m_totalprogtable}"".""FilesFullyRestored"" + (CASE WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" THEN 1 ELSE 0 END) @@ -175,156 +178,158 @@ BEGIN UPDATE ""{m_totalprogtable}"" , ""BlocksRestored"" = ""{m_totalprogtable}"".""BlocksRestored"" + NEW.""BlocksRestored"" - OLD.""BlocksRestored"" , ""SizeRestored"" = ""{m_totalprogtable}"".""SizeRestored"" + NEW.""SizeRestored"" - OLD.""SizeRestored"" ; END"); - cmd.ExecuteNonQuery(sql); + await cmd.ExecuteNonQueryAsync(sql); - if (createFilesNewlyDoneTracker) - { - // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" AFTER UPDATE OF ""BlocksRestored"", ""TotalBlocks"" ON ""{m_fileprogtable}"" + if (createFilesNewlyDoneTracker) + { + // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) + sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" AFTER UPDATE OF ""BlocksRestored"", ""TotalBlocks"" ON ""{m_fileprogtable}"" WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" BEGIN INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") VALUES (NEW.""FileId""); END "); - cmd.ExecuteNonQuery(sql); - } + await cmd.ExecuteNonQueryAsync(sql); + } - } - catch (Exception ex) - { - m_fileprogtable = null; - m_totalprogtable = null; - Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables"); - throw; - } + } + catch (Exception ex) + { + m_fileprogtable = null; + m_totalprogtable = null; + Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables"); + throw; + } + finally + { + await cmd.Transaction.CommitAsync(); } } - public Tuple PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter) + public async Task> PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter) { m_tempfiletable = "Fileset-" + m_temptabsetguid; m_tempblocktable = "Blocks-" + m_temptabsetguid; using (var cmd = m_connection.CreateCommand()) { - var filesetIds = GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToList(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + var filesetIds = (await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions)).ToList(); while (filesetIds.Count > 0) { var filesetId = filesetIds[0]; filesetIds.RemoveAt(0); - m_restoreTime = ParseFromEpochSeconds(cmd - .SetCommandAndParameters(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteScalarInt64(0)); + cmd.SetCommandAndParameters(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = @FilesetId"); + cmd.SetParameterValue("@FilesetId", filesetId); + m_restoreTime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); - var ix = FilesetTimes.Select((value, index) => new { value.Key, index }) + var ix = await FilesetTimes().Select((value, index) => new { value.Key, index }) .Where(n => n.Key == filesetId) .Select(pair => pair.index + 1) - .FirstOrDefault() - 1; + .FirstOrDefaultAsync() - 1; Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime); cmd.Parameters.Clear(); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempfiletable}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""TargetPath"" TEXT NULL, ""DataVerified"" BOOLEAN NOT NULL, ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN) ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocktable}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL, ""Metadata"" BOOLEAN NOT NULL, ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL)")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempfiletable}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""TargetPath"" TEXT NULL, ""DataVerified"" BOOLEAN NOT NULL, ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN) ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocktable}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL, ""Metadata"" BOOLEAN NOT NULL, ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL)")); // TODO: Optimize to use the path prefix if (filter == null || filter.Empty) { // Simple case, restore everything - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId")) - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); + cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId")); + cmd.SetParameterValue("@FilesetId", filesetId); + await cmd.ExecuteNonQueryAsync(); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple) { + using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset")) + await cmd.Transaction.CommitAsync(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); // If we get a list of filenames, the lookup table is faster // unfortunately we cannot do this if the filesystem is case sensitive as // SQLite only supports ASCII compares - using (var tr = m_connection.BeginTransactionSafe()) + var p = expression.GetSimpleList(); + var m_filenamestable = "Filenames-" + m_temptabsetguid; + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filenamestable}"" (""Path"" TEXT NOT NULL) ")); + cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_filenamestable}"" (""Path"") VALUES (@Path)")); + + foreach (var s in p) { - var p = expression.GetSimpleList(); - var m_filenamestable = "Filenames-" + m_temptabsetguid; - cmd.Transaction = tr; - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filenamestable}"" (""Path"" TEXT NOT NULL) ")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_filenamestable}"" (""Path"") VALUES (@Path)")); - - foreach (var s in p) - { - cmd.SetParameterValue("@Path", s) - .ExecuteNonQuery(); - } - - var c = cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{m_filenamestable}"") ")) - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); - - cmd.Parameters.Clear(); - - if (c != p.Length && c != 0) - { - var sb = new StringBuilder(); - sb.AppendLine(); - - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""Path"" FROM ""{m_filenamestable}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tempfiletable}"")"))) - while (rd.Read()) - sb.AppendLine(rd.ConvertValueToString(0)); - - var actualrestoretime = ParseFromEpochSeconds( - cmd.SetCommandAndParameters(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteScalarInt64(0)); - - Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); - cmd.Parameters.Clear(); - } - - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ")); - - using (new Logging.Timer(LOGTAG, "CommitPrepareFileset", "CommitPrepareFileset")) - tr.Commit(); + cmd.SetParameterValue("@Path", s); + await cmd.ExecuteNonQueryAsync(); } + + cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{m_filenamestable}"") ")); + cmd.SetParameterValue("@FilesetId", filesetId); + var c = await cmd.ExecuteNonQueryAsync(); + + cmd.Parameters.Clear(); + + if (c != p.Length && c != 0) + { + var sb = new StringBuilder(); + sb.AppendLine(); + + using (var rd = await cmd.ExecuteReaderAsync(FormatInvariant($@"SELECT ""Path"" FROM ""{m_filenamestable}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tempfiletable}"")"))) + while (await rd.ReadAsync()) + sb.AppendLine(rd.ConvertValueToString(0)); + + cmd.SetCommandAndParameters(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = @FilesetId"); + cmd.SetParameterValue("@FilesetId", filesetId); + var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); + + Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); + cmd.Parameters.Clear(); + } + + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ")); + + using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset")) + await cmd.Transaction.CommitAsync(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); } else { // Restore but filter elements based on the filter expression // If this is too slow, we could add a special handler for wildcard searches too - cmd.SetCommandAndParameters(@"SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId); + cmd.SetCommandAndParameters(@"SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetID"" = @FilesetId"); + cmd.SetParameterValue("@FilesetId", filesetId); object[] values = new object[4]; - using (var cmd2 = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (@ID, @Path, @BlocksetID, @MetadataID, 0)"))) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using var cmd2 = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (@ID, @Path, @BlocksetID, @MetadataID, 0)")); + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + { + rd.GetValues(values); + if (values[1] != null && values[1] != DBNull.Value && FilterExpression.Matches(filter, values[1].ToString())) { - rd.GetValues(values); - if (values[1] != null && values[1] != DBNull.Value && FilterExpression.Matches(filter, values[1].ToString())) - { - cmd2.SetParameterValue("@ID", values[0]) - .SetParameterValue("@Path", values[1]) - .SetParameterValue("@BlocksetID", values[2]) - .SetParameterValue("@MetadataID", values[3]) - .ExecuteNonQuery(); - } + cmd2.SetParameterValue("@ID", values[0]); + cmd2.SetParameterValue("@Path", values[1]); + cmd2.SetParameterValue("@BlocksetID", values[2]); + cmd2.SetParameterValue("@MetadataID", values[3]); + await cmd2.ExecuteNonQueryAsync(); } + } } //creating indexes after insertion is much faster - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_ID"" ON ""{m_tempfiletable}"" (""ID"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_TargetPath"" ON ""{m_tempfiletable}"" (""TargetPath"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_Path"" ON ""{m_tempfiletable}"" (""Path"")")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_ID"" ON ""{m_tempfiletable}"" (""ID"")")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_TargetPath"" ON ""{m_tempfiletable}"" (""TargetPath"")")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_Path"" ON ""{m_tempfiletable}"" (""Path"")")); - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), SUM(""Blockset"".""Length"") FROM ""{m_tempfiletable}"", ""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" "))) + using (var rd = await cmd.ExecuteReaderAsync(FormatInvariant($@"SELECT COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), SUM(""Blockset"".""Length"") FROM ""{m_tempfiletable}"", ""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" "))) { var filecount = 0L; var filesize = 0L; - if (rd.Read()) + if (await rd.ReadAsync()) { filecount = rd.ConvertValueToInt64(0, 0); filesize = rd.ConvertValueToInt64(1, 0); @@ -337,161 +342,167 @@ END "); } } } + + await cmd.Transaction.CommitAsync(); } return new Tuple(0, 0); } - public string? GetFirstPath() + public async Task GetFirstPath() { - using (var cmd = m_connection.CreateCommand()) - { - var v0 = cmd.ExecuteScalar(FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); - if (v0 == null || v0 == DBNull.Value) - return null; + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + var v0 = await cmd.ExecuteScalarAsync(FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); + if (v0 == null || v0 == DBNull.Value) + return null; + await cmd.Transaction.CommitAsync(); - return v0.ToString(); - } + return v0.ToString(); } - public string GetLargestPrefix() + public async Task GetLargestPrefix() { - using (var cmd = m_connection.CreateCommand()) + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + var v0 = await cmd.ExecuteScalarAsync(FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); + var maxpath = ""; + if (v0 != null && v0 != DBNull.Value) + maxpath = v0.ToString()!; + + var dirsep = Util.GuessDirSeparator(maxpath); + + var filecount = await cmd.ExecuteScalarInt64Async(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"""), -1); + var foundfiles = -1L; + + //TODO: Handle FS case-sensitive? + cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix")); + + while (filecount != foundfiles && maxpath.Length > 0) { - var v0 = cmd.ExecuteScalar(FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); - var maxpath = ""; - if (v0 != null && v0 != DBNull.Value) - maxpath = v0.ToString()!; + var mp = Util.AppendDirSeparator(maxpath, dirsep); + cmd.SetParameterValue("@PrefixLength", mp.Length); + cmd.SetParameterValue("@Prefix", mp); + foundfiles = await cmd.ExecuteScalarInt64Async(-1); - var dirsep = Util.GuessDirSeparator(maxpath); - - var filecount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"""), -1); - var foundfiles = -1L; - - //TODO: Handle FS case-sensitive? - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix")); - - while (filecount != foundfiles && maxpath.Length > 0) + if (filecount != foundfiles) { - var mp = Util.AppendDirSeparator(maxpath, dirsep); - foundfiles = cmd.SetParameterValue("@PrefixLength", mp.Length) - .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64(-1); + var oldlen = maxpath.Length; - if (filecount != foundfiles) - { - var oldlen = maxpath.Length; - - var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); - maxpath = maxpath.Substring(0, lix + 1); - if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen) - maxpath = ""; - } + var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); + maxpath = maxpath.Substring(0, lix + 1); + if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen) + maxpath = ""; } - - return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); } + + await cmd.Transaction.CommitAsync(); + + return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); } - public void SetTargetPaths(string largest_prefix, string destination) + public async Task SetTargetPaths(string largest_prefix, string destination) { - var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? GetFirstPath() : largest_prefix); + var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? await GetFirstPath() : largest_prefix); - using (var cmd = m_connection.CreateCommand()) + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + if (string.IsNullOrEmpty(destination)) { - if (string.IsNullOrEmpty(destination)) + //The string fixing here is meant to provide some non-random + // defaults when restoring cross OS, e.g. backup on Linux, restore on Windows + //This is mostly meaningless, and the user really should use --restore-path + + if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") { - //The string fixing here is meant to provide some non-random - // defaults when restoring cross OS, e.g. backup on Linux, restore on Windows - //This is mostly meaningless, and the user really should use --restore-path + // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END")); - if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") - { - // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END")); - - } - else if (OperatingSystem.IsWindows() && dirsep == "/") - { - // For Linux -> Win, we use the temporary folder's drive as the root path - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 1) == '/' THEN @Path || SUBSTR(""Path"", 2) ELSE ""Path"" END")) - .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) - .ExecuteNonQuery(); - } - else - { - // Same OS, just use the path directly - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = ""Path"" ")); - } + } + else if (OperatingSystem.IsWindows() && dirsep == "/") + { + // For Linux -> Win, we use the temporary folder's drive as the root path + cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 1) == '/' THEN @Path || SUBSTR(""Path"", 2) ELSE ""Path"" END")); + cmd.SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")); + await cmd.ExecuteNonQueryAsync(); } else { - if (string.IsNullOrEmpty(largest_prefix)) - { - //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) - - // We use the format / / - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); - - // For UNC paths, we use \\server\folder -> / / - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END")); - } - else - { - largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength)")) - .SetParameterValue("@PrefixLength", largest_prefix.Length + 1) - .ExecuteNonQuery(); - } - } - - // Cross-os path remapping support - if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") - // For Win paths on Linux - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')")); - else if (OperatingSystem.IsWindows() && dirsep == "/") - // For Linux paths on Windows - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')")); - - if (!string.IsNullOrEmpty(destination)) - { - // Paths are now relative with target-os naming system - // so we prefix them with the target path - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @Destination || ""TargetPath"" ")) - .SetParameterValue("@Destination", Util.AppendDirSeparator(destination)) - .ExecuteNonQuery(); + // Same OS, just use the path directly + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = ""Path"" ")); } } - } - - public void FindMissingBlocks(bool skipMetadata) - { - using (var cmd = m_connection.CreateCommand()) + else { - var p1 = cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ")); + if (string.IsNullOrEmpty(largest_prefix)) + { + //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) - var p2 = 0; - if (!skipMetadata) - p2 = cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ")); + // We use the format / / + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); - //creating indexes after insertion is much faster - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" ON ""{m_tempblocktable}"" (""Hash"", ""Size"")")); - // better suited to speed up commit on UpdateBlocks - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" ON ""{m_tempblocktable}"" (""FileId"", ""Index"")")); - - var size = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_tempblocktable}"" "), 0); - Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); + // For UNC paths, we use \\server\folder -> / / + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END")); + } + else + { + largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); + cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength)")); + cmd.SetParameterValue("@PrefixLength", largest_prefix.Length + 1); + await cmd.ExecuteNonQueryAsync(); + } } + + // Cross-os path remapping support + if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") + // For Win paths on Linux + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')")); + else if (OperatingSystem.IsWindows() && dirsep == "/") + // For Linux paths on Windows + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')")); + + if (!string.IsNullOrEmpty(destination)) + { + // Paths are now relative with target-os naming system + // so we prefix them with the target path + cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @Destination || ""TargetPath"" ")); + cmd.SetParameterValue("@Destination", Util.AppendDirSeparator(destination)); + await cmd.ExecuteNonQueryAsync(); + } + + await cmd.Transaction.CommitAsync(); } - public void UpdateTargetPath(long ID, string newname) + public async Task FindMissingBlocks(bool skipMetadata) { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @TargetPath WHERE ""ID"" = @ID")) - .SetParameterValue("@TargetPath", newname) - .SetParameterValue("@ID", ID); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + var p1 = await cmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ")); - cmd.ExecuteNonQuery(); + var p2 = 0; + if (!skipMetadata) + p2 = await cmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ")); + + //creating indexes after insertion is much faster + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" ON ""{m_tempblocktable}"" (""Hash"", ""Size"")")); + // better suited to speed up commit on UpdateBlocks + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" ON ""{m_tempblocktable}"" (""FileId"", ""Index"")")); + + var size = await cmd.ExecuteScalarInt64Async(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_tempblocktable}"" "), 0); + Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); + + await cmd.Transaction.CommitAsync(); + } + + public async Task UpdateTargetPath(long ID, string newname) + { + using var cmd = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @TargetPath WHERE ""ID"" = @ID")); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + cmd.SetParameterValue("@TargetPath", newname); + cmd.SetParameterValue("@ID", ID); + await cmd.ExecuteNonQueryAsync(); + await cmd.Transaction.CommitAsync(); } public interface IExistingFileBlock @@ -560,9 +571,9 @@ END "); private class ExistingFile : IExistingFile { - private readonly IDataReader m_reader; + private readonly SqliteDataReader m_reader; - public ExistingFile(IDataReader rd) { m_reader = rd; HasMore = true; } + public ExistingFile(SqliteDataReader rd) { m_reader = rd; HasMore = true; } public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } public string TargetHash { get { return m_reader.ConvertValueToString(1) ?? ""; } } @@ -573,9 +584,9 @@ END "); private class ExistingFileBlock : IExistingFileBlock { - private readonly IDataReader m_reader; + private readonly SqliteDataReader m_reader; - public ExistingFileBlock(IDataReader rd) { m_reader = rd; } + public ExistingFileBlock(SqliteDataReader rd) { m_reader = rd; } public string Hash { get { return m_reader.ConvertValueToString(4) ?? ""; } } public long Index { get { return m_reader.ConvertValueToInt64(5); } } @@ -590,33 +601,35 @@ END "); while (HasMore && p == TargetPath) { yield return new ExistingFileBlock(m_reader); - HasMore = m_reader.Read(); + HasMore = m_reader.ReadAsync().Await(); } } } - public static IEnumerable GetExistingFilesWithBlocks(IDbConnection connection, string tablename) + public static async IAsyncEnumerable GetExistingFilesWithBlocks(SqliteConnection connection, string tablename) { - using (var cmd = connection.CreateCommand(FormatInvariant($@"SELECT ""{tablename}"".""TargetPath"", ""Blockset"".""FullHash"", ""{tablename}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{tablename}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index"""))) - using (var rd = cmd.ExecuteReader()) - if (rd.Read()) + using var cmd = connection.CreateCommand(FormatInvariant($@"SELECT ""{tablename}"".""TargetPath"", ""Blockset"".""FullHash"", ""{tablename}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{tablename}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index""")); + cmd.Transaction = connection.BeginTransaction(deferred: true); + using var rd = await cmd.ExecuteReaderAsync(); + if (await rd.ReadAsync()) + { + var more = true; + while (more) { - var more = true; - while (more) - { - var f = new ExistingFile(rd); - string current = f.TargetPath; - yield return f; + var f = new ExistingFile(rd); + string current = f.TargetPath; + yield return f; - more = f.HasMore; - while (more && current == f.TargetPath) - more = rd.Read(); - } + more = f.HasMore; + while (more && current == f.TargetPath) + more = await rd.ReadAsync(); } + } + await cmd.Transaction.CommitAsync(); } } - public IEnumerable GetExistingFilesWithBlocks() + public IAsyncEnumerable GetExistingFilesWithBlocks() { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); @@ -629,16 +642,16 @@ END "); { private class BlockSource : IBlockSource { - private readonly IDataReader m_reader; - public BlockSource(IDataReader rd) { m_reader = rd; } + private readonly SqliteDataReader m_reader; + public BlockSource(SqliteDataReader rd) { m_reader = rd; } public string Path { get { return m_reader.ConvertValueToString(6) ?? ""; } } public long Offset { get { return m_reader.ConvertValueToInt64(7); } } public bool IsMetadata { get { return false; } } } - private readonly IDataReader m_reader; - public BlockDescriptor(IDataReader rd) { m_reader = rd; HasMore = true; } + private readonly SqliteDataReader m_reader; + public BlockDescriptor(SqliteDataReader rd) { m_reader = rd; HasMore = true; } private string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } @@ -661,14 +674,14 @@ END "); while (HasMore && p == TargetPath && h == Hash && s == Size) { yield return new BlockSource(m_reader); - HasMore = m_reader.Read(); + HasMore = m_reader.ReadAsync().Await(); } } } } - private readonly IDataReader m_reader; - public LocalBlockSource(IDataReader rd) { m_reader = rd; HasMore = true; } + private readonly SqliteDataReader m_reader; + public LocalBlockSource(SqliteDataReader rd) { m_reader = rd; HasMore = true; } public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } public long TargetFileID { get { return m_reader.ConvertValueToInt64(1); } } @@ -690,55 +703,55 @@ END "); HasMore = c.HasMore; while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) - HasMore = m_reader.Read(); + HasMore = m_reader.ReadAsync().Await(); } } } - public static IEnumerable GetFilesAndSourceBlocks(IDbConnection connection, string filetablename, string blocktablename, long blocksize, bool skipMetadata) + public static async IAsyncEnumerable GetFilesAndSourceBlocks(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, bool skipMetadata) { // TODO: Skip metadata as required // Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows - using (var cmd = connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {blocksize}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {blocksize}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{filetablename}"" ""A"", ""{blocktablename}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0 ORDER BY ""A"".""TargetPath"", ""B"".""Index"""))) - using (var rd = cmd.ExecuteReader()) + using var cmd = connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {blocksize}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {blocksize}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{filetablename}"" ""A"", ""{blocktablename}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0 ORDER BY ""A"".""TargetPath"", ""B"".""Index""")); + cmd.Transaction = connection.BeginTransaction(deferred: true); + using var rd = await cmd.ExecuteReaderAsync(); + var more = await rd.ReadAsync(); + while (more) { - var more = rd.Read(); - while (more) - { - var f = new LocalBlockSource(rd); - string current = f.TargetPath; - yield return f; + var f = new LocalBlockSource(rd); + string current = f.TargetPath; + yield return f; - more = f.HasMore; - while (more && current == f.TargetPath) - more = rd.Read(); - } + more = f.HasMore; + while (more && current == f.TargetPath) + more = await rd.ReadAsync(); } + await cmd.Transaction.CommitAsync(); } } - public IEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize) + public IAsyncEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize) { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); return LocalBlockSource.GetFilesAndSourceBlocks(m_connection, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata); } - public IEnumerable GetMissingVolumes() + public async IAsyncEnumerable GetMissingVolumes() { - using (var cmd = m_connection.CreateCommand()) - { - // Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore. - // If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file. - // If we manage to get the volumes in an order restoring front blocks first, this can save time. - // An optimal algorithm would build a dependency net with cycle resolution to find the best near topological - // order of volumes, but this is a bit too fancy here. - // We will just put a very simple heuristic to work, that will try to prefer volumes containing lower block indexes: - // We just order all volumes by the maximum block index they contain. This query is slow, but should be worth the effort. - // Now it is likely to restore all files from front to back. Large files will always be done last. - // One could also use like the average block number in a volume, that needs to be measured. + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + // Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore. + // If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file. + // If we manage to get the volumes in an order restoring front blocks first, this can save time. + // An optimal algorithm would build a dependency net with cycle resolution to find the best near topological + // order of volumes, but this is a bit too fancy here. + // We will just put a very simple heuristic to work, that will try to prefer volumes containing lower block indexes: + // We just order all volumes by the maximum block index they contain. This query is slow, but should be worth the effort. + // Now it is likely to restore all files from front to back. Large files will always be done last. + // One could also use like the average block number in a volume, that needs to be measured. - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT ""RV"".""Name"", ""RV"".""Hash"", ""RV"".""Size"", ""BB"".""MaxIndex"" + cmd.SetCommandAndParameters(FormatInvariant($@"SELECT ""RV"".""Name"", ""RV"".""Hash"", ""RV"".""Size"", ""BB"".""MaxIndex"" FROM ""RemoteVolume"" ""RV"" INNER JOIN (SELECT ""TB"".""VolumeID"", MAX(""TB"".""Index"") as ""MaxIndex"" FROM ""{m_tempblocktable}"" ""TB"" @@ -747,20 +760,19 @@ FROM ""RemoteVolume"" ""RV"" INNER JOIN ) as ""BB"" ON ""RV"".""ID"" = ""BB"".""VolumeID"" ORDER BY ""BB"".""MaxIndex"" ")); - using (var rd = cmd.ExecuteReader()) - { - object[] r = new object[3]; - while (rd.Read()) - { - rd.GetValues(r); - yield return new RemoteVolume( - rd.ConvertValueToString(0), - rd.ConvertValueToString(1), - rd.ConvertValueToInt64(2, -1) - ); - } - } + using var rd = await cmd.ExecuteReaderAsync(); + object[] r = new object[3]; + while (await rd.ReadAsync()) + { + rd.GetValues(r); + yield return new RemoteVolume( + rd.ConvertValueToString(0), + rd.ConvertValueToString(1), + rd.ConvertValueToInt64(2, -1) + ); } + + await cmd.Transaction.CommitAsync(); } public interface IFilesAndMetadata : IDisposable @@ -776,53 +788,58 @@ ORDER BY ""BB"".""MaxIndex"" ")); private readonly string m_blocktablename; private readonly long m_blocksize; - private readonly IDbConnection m_connection; + private readonly SqliteConnection m_connection; - public FilesAndMetadata(IDbConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + public FilesAndMetadata(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) { m_filetablename = filetablename; m_blocktablename = blocktablename; m_blocksize = blocksize; m_connection = connection; - using (var c = m_connection.CreateCommand()) + using var c = m_connection.CreateCommand(); + c.Transaction = m_connection.BeginTransaction(deferred: true); + m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + c.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tmptable}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )")).Await(); + + c.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tmptable}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); + foreach (var s in curvolume.Blocks) { - m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - c.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tmptable}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )")); - - c.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tmptable}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); - foreach (var s in curvolume.Blocks) - { - c.SetParameterValue("@Hash", s.Key); - c.SetParameterValue("@Size", s.Value); - c.ExecuteNonQuery(); - } - - // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks + c.SetParameterValue("@Hash", s.Key); + c.SetParameterValue("@Size", s.Value); + c.ExecuteNonQueryAsync().Await(); } + + // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks + + c.Transaction.CommitAsync().Await(); } public void Dispose() { if (m_tmptable != null) - using (var c = m_connection.CreateCommand()) - c.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tmptable}""")); + { + using var c = m_connection.CreateCommand(); + c.Transaction = m_connection.BeginTransaction(deferred: true); + c.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tmptable}""")); + c.Transaction.CommitAsync().Await(); + } } private class VolumePatch : IVolumePatch { private class PatchBlock : IPatchBlock { - private readonly IDataReader m_reader; - public PatchBlock(IDataReader rd) { m_reader = rd; } + private readonly SqliteDataReader m_reader; + public PatchBlock(SqliteDataReader rd) { m_reader = rd; } public long Offset { get { return m_reader.ConvertValueToInt64(2); } } public long Size { get { return m_reader.ConvertValueToInt64(3); } } public string Key { get { return m_reader.ConvertValueToString(4) ?? ""; } } } - private readonly IDataReader m_reader; - public VolumePatch(IDataReader rd) { m_reader = rd; HasMore = true; } + private readonly SqliteDataReader m_reader; + public VolumePatch(SqliteDataReader rd) { m_reader = rd; HasMore = true; } public string Path { get { return m_reader.ConvertValueToString(0) ?? ""; } } public long FileID { get { return m_reader.ConvertValueToInt64(1); } } @@ -836,7 +853,7 @@ ORDER BY ""BB"".""MaxIndex"" ")); while (HasMore && p == Path) { yield return new PatchBlock(m_reader); - HasMore = m_reader.Read(); + HasMore = m_reader.ReadAsync().Await(); } } } @@ -846,32 +863,30 @@ ORDER BY ""BB"".""MaxIndex"" ")); { get { - using (var cmd = m_connection.CreateCommand()) - { - // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! + cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"0"} AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); - using (var rd = cmd.ExecuteReader()) + using var rd = cmd.ExecuteReaderAsync().Await(); + if (rd.ReadAsync().Await()) + { + var more = true; + while (more) { - if (rd.Read()) - { - var more = true; - while (more) - { - var f = new VolumePatch(rd); - var current = f.Path; - yield return f; + var f = new VolumePatch(rd); + var current = f.Path; + yield return f; - more = f.HasMore; - while (more && current == f.Path) - more = rd.Read(); - } - } + more = f.HasMore; + while (more && current == f.Path) + more = rd.ReadAsync().Await(); } } + cmd.Transaction.CommitAsync().Await(); } } @@ -879,30 +894,27 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); { get { - using (var cmd = m_connection.CreateCommand()) - { - // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! + cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"1"} AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); - using (var rd = cmd.ExecuteReader()) + using var rd = cmd.ExecuteReaderAsync().Await(); + if (rd.ReadAsync().Await()) + { + var more = true; + while (more) { - if (rd.Read()) - { - var more = true; - while (more) - { - var f = new VolumePatch(rd); - string current = f.Path; - yield return f; + var f = new VolumePatch(rd); + string current = f.Path; + yield return f; - more = f.HasMore; - while (more && current == f.Path) - more = rd.Read(); - } - } + more = f.HasMore; + while (more && current == f.Path) + more = rd.ReadAsync().Await(); } } } @@ -920,19 +932,15 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// Returns a connection from the connection pool. /// /// A connection from the connection pool. - public IDbConnection GetConnectionFromPool() + public async Task GetConnectionFromPool() { if (!m_connection_pool.TryTake(out var connection)) { - connection = SQLiteHelper.SQLiteLoader.LoadConnection(); + connection = await SQLiteHelper.SQLiteLoader.LoadConnection(); connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; connection.Open(); - SQLiteHelper.SQLiteLoader.ApplyCustomPragmas(connection, m_pagecachesize); - - using var cmd = connection.CreateCommand(); - cmd.ExecuteNonQuery("PRAGMA journal_mode = WAL;"); - cmd.ExecuteNonQuery("PRAGMA read_uncommitted = true;"); + await SQLiteHelper.SQLiteLoader.ApplyCustomPragmas(connection, m_pagecachesize); } return connection; @@ -952,47 +960,56 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); } } - public IEnumerable GetFilesToRestore(bool onlyNonVerified) + public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified) { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT ""{m_tempfiletable}"".""ID"", ""{m_tempfiletable}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{m_tempfiletable}"",""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified")) - .SetParameterValue("@Verified", !onlyNonVerified); + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT ""{m_tempfiletable}"".""ID"", ""{m_tempfiletable}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{m_tempfiletable}"",""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified")); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + cmd.SetParameterValue("@Verified", !onlyNonVerified); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new FileToRestore( - rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + yield return new FileToRestore( + rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); + + await cmd.Transaction.CommitAsync(); } /// /// Returns a list of files and symlinks to restore. /// /// A list of files and symlinks to restore. - public IEnumerable GetFilesAndSymlinksToRestore() + public async IAsyncEnumerable GetFilesAndSymlinksToRestore() { using var cmd = m_connection.CreateCommand(FormatInvariant($@" SELECT F.ID, F.Path, F.TargetPath, IFNULL(B.FullHash, ''), IFNULL(B.Length, 0), F.BlocksetID FROM ""{m_tempfiletable}"" F LEFT JOIN Blockset B ON F.BlocksetID = B.ID WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID}")); - using var rd = cmd.ExecuteReader(); - while (rd.Read()) + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); + + await cmd.Transaction.CommitAsync(); } /// /// Returns a list of folders to restore. Used to restore folder metadata. /// /// A list of folders to restore. - public IEnumerable GetFolderMetadataToRestore() + public async IAsyncEnumerable GetFolderMetadataToRestore() { using var cmd = m_connection.CreateCommand(); - using var rd = cmd.ExecuteReader(FormatInvariant($@" + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + using var rd = await cmd.ExecuteReaderAsync(FormatInvariant($@" SELECT F.ID, '', F.TargetPath, '', 0, {FOLDER_BLOCKSET_ID} FROM ""{m_tempfiletable}"" F WHERE F.BlocksetID = {FOLDER_BLOCKSET_ID} AND F.MetadataID IS NOT NULL AND F.MetadataID >= 0")); - while (rd.Read()) + while (await rd.ReadAsync()) yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); + + await cmd.Transaction.CommitAsync(); } /// @@ -1000,10 +1017,11 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// /// Flag indicating whether the returned blocks should exclude the metadata blocks. /// A list of tuples containing the block ID and the volume ID of the block. - public IEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) + public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) { using var cmd = Connection.CreateCommand(); - using var reader = cmd.ExecuteReader(FormatInvariant($@" + cmd.Transaction = Connection.BeginTransaction(deferred: true); + using var reader = await cmd.ExecuteReaderAsync(FormatInvariant($@" SELECT ""Block"".""ID"", ""Block"".""VolumeID"" FROM ""BlocksetEntry"" INNER JOIN ""{m_tempfiletable}"" ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" @@ -1017,8 +1035,10 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); INNER JOIN ""BlocksetEntry"" ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "))); - while (reader.Read()) + while (await reader.ReadAsync()) yield return (reader.ConvertValueToInt64(0), reader.ConvertValueToInt64(1)); + + await cmd.Transaction.CommitAsync(); } /// @@ -1026,20 +1046,23 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// /// The BlocksetID of the file. /// A list of needed to restore the given file. - public IEnumerable GetBlocksFromFile(long blocksetID) + public async IAsyncEnumerable GetBlocksFromFile(long blocksetID) { - var connection = GetConnectionFromPool(); + var connection = await GetConnectionFromPool(); using var cmd = connection.CreateCommand(FormatInvariant(@$" SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""BlocksetEntry"" INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID")) - .SetParameterValue("@BlocksetID", blocksetID); + WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID")); + cmd.SetParameterValue("@BlocksetID", blocksetID); + cmd.Transaction = connection.BeginTransaction(deferred: true); - using var reader = cmd.ExecuteReader(); - for (long i = 0; reader.Read(); i++) + using var reader = await cmd.ExecuteReaderAsync(); + for (long i = 0; await reader.ReadAsync(); i++) yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); + await cmd.Transaction.CommitAsync(); + // Return the connection to the pool m_connection_pool.Add(connection); } @@ -1049,9 +1072,9 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// /// The ID of the file. /// A list of needed to restore the metadata of the given file. - public IEnumerable GetMetadataBlocksFromFile(long fileID) + public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID) { - var connection = GetConnectionFromPool(); + var connection = await GetConnectionFromPool(); using var cmd = connection.CreateCommand(FormatInvariant($@" SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""File"" @@ -1059,15 +1082,18 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); INNER JOIN ""BlocksetEntry"" ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" WHERE ""File"".""ID"" = @FileID - ")) - .SetParameterValue("@FileID", fileID); + ")); + cmd.Transaction = connection.BeginTransaction(deferred: true); + cmd.SetParameterValue("@FileID", fileID); - using var reader = cmd.ExecuteReader(); - for (long i = 0; reader.Read(); i++) + using var reader = await cmd.ExecuteReaderAsync(); + for (long i = 0; await reader.ReadAsync(); i++) { yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); } + await cmd.Transaction.CommitAsync(); + // Return the connection to the pool m_connection_pool.Add(connection); } @@ -1077,78 +1103,80 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// /// The ID of the volume. /// A tuple containing the name, size, and hash of the volume. - public IEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) + public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) { - using var cmd = m_connection.CreateCommand("SELECT Name, Size, Hash FROM RemoteVolume WHERE ID = @VolumeID") - .SetParameterValue("@VolumeID", VolumeID); - using var reader = cmd.ExecuteReader(); - while (reader.Read()) + using var cmd = m_connection.CreateCommand("SELECT Name, Size, Hash FROM RemoteVolume WHERE ID = @VolumeID"); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + cmd.SetParameterValue("@VolumeID", VolumeID); + using var reader = await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) yield return (reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), reader.ConvertValueToString(2) ?? ""); + await cmd.Transaction.CommitAsync(); } - public void DropRestoreTable() + public async Task DropRestoreTable() { - using (var cmd = m_connection.CreateCommand()) - { - if (m_tempfiletable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_tempfiletable = null; } + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + if (m_tempfiletable != null) + try + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""")); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_tempfiletable = null; } - if (m_tempblocktable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_tempblocktable = null; } + if (m_tempblocktable != null) + try + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""")); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_tempblocktable = null; } - if (m_latestblocktable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_latestblocktable = null; } + if (m_latestblocktable != null) + try + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""")); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_latestblocktable = null; } - if (m_fileprogtable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_fileprogtable = null; } + if (m_fileprogtable != null) + try + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""")); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_fileprogtable = null; } - if (m_totalprogtable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_totalprogtable = null; } + if (m_totalprogtable != null) + try + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""")); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_totalprogtable = null; } - if (m_filesnewlydonetable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_filesnewlydonetable = null; } + if (m_filesnewlydonetable != null) + try + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""")); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_filesnewlydonetable = null; } - } + await cmd.Transaction.CommitAsync(); } public interface IBlockMarker : IDisposable { - void SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata); - void SetAllBlocksMissing(long targetfileid); - void SetAllBlocksRestored(long targetfileid, bool includeMetadata); - void SetFileDataVerified(long targetfileid); - void Commit(); - void UpdateProcessed(IOperationProgressUpdater writer); + Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata); + Task SetAllBlocksMissing(long targetfileid); + Task SetAllBlocksRestored(long targetfileid, bool includeMetadata); + Task SetFileDataVerified(long targetfileid); + Task CommitAsync(); + Task UpdateProcessed(IOperationProgressUpdater writer); } /// @@ -1158,110 +1186,113 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// private class DirectBlockMarker : IBlockMarker { - private IDbCommand m_insertblockCommand; - private IDbCommand m_resetfileCommand; - private IDbCommand m_updateAsRestoredCommand; - private IDbCommand m_updateFileAsDataVerifiedCommand; - private IDbCommand m_statUpdateCommand; - private IDbTransaction m_transaction; + private readonly SqliteCommand m_insertblockCommand; + private readonly SqliteCommand m_resetfileCommand; + private readonly SqliteCommand m_updateAsRestoredCommand; + private readonly SqliteCommand m_updateFileAsDataVerifiedCommand; + private readonly SqliteCommand m_statUpdateCommand; + private SqliteTransaction m_transaction; private bool m_hasUpdates = false; private readonly string m_blocktablename; private readonly string m_filetablename; - public DirectBlockMarker(IDbConnection connection, string blocktablename, string filetablename, string statstablename) + public DirectBlockMarker(SqliteConnection connection, string blocktablename, string filetablename, string statstablename) { - m_transaction = connection.BeginTransactionSafe(); + m_transaction = connection.BeginTransaction(deferred: true); m_blocktablename = blocktablename; m_filetablename = filetablename; - m_insertblockCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 + m_insertblockCommand = connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND ""Size"" = @Size AND ""Metadata"" = @Metadata AND ""Restored"" = 0 ")); + m_insertblockCommand.Transaction = m_transaction; - m_resetfileCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = @TargetFileId ")); - m_updateAsRestoredCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 WHERE ""FileID"" = @TargetFileId AND ""Metadata"" <= @Metadata ")); - m_updateFileAsDataVerifiedCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = @TargetFileId")); + m_resetfileCommand = connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = @TargetFileId ")); + m_resetfileCommand.Transaction = m_transaction; + + m_updateAsRestoredCommand = connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 WHERE ""FileID"" = @TargetFileId AND ""Metadata"" <= @Metadata ")); + m_updateAsRestoredCommand.Transaction = m_transaction; + + m_updateFileAsDataVerifiedCommand = connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = @TargetFileId")); + m_updateFileAsDataVerifiedCommand.Transaction = m_transaction; if (statstablename != null) { // Fields in Stats: TotalFiles, TotalBlocks, TotalSize // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored - m_statUpdateCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" ")); + m_statUpdateCommand = connection.CreateCommand(FormatInvariant($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" ")); } else // very slow fallback if stats tables were not created - m_statUpdateCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{m_blocktablename}"" WHERE ""Restored"" = 1 ")); + m_statUpdateCommand = connection.CreateCommand(FormatInvariant($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{m_blocktablename}"" WHERE ""Restored"" = 1 ")); + m_statUpdateCommand.Transaction = m_transaction; } - public void UpdateProcessed(IOperationProgressUpdater updater) + public async Task UpdateProcessed(IOperationProgressUpdater updater) { if (!m_hasUpdates) return; m_hasUpdates = false; - using (var rd = m_statUpdateCommand.ExecuteReader()) + using var rd = await m_statUpdateCommand.ExecuteReaderAsync(); + var filesprocessed = 0L; + var processedsize = 0L; + + if (rd.Read()) { - var filesprocessed = 0L; - var processedsize = 0L; - - if (rd.Read()) - { - filesprocessed += rd.ConvertValueToInt64(0, 0); - processedsize += rd.ConvertValueToInt64(1, 0); - } - - updater.UpdatefilesProcessed(filesprocessed, processedsize); + filesprocessed += rd.ConvertValueToInt64(0, 0); + processedsize += rd.ConvertValueToInt64(1, 0); } + + updater.UpdatefilesProcessed(filesprocessed, processedsize); } - public void SetAllBlocksMissing(long targetfileid) + public async Task SetAllBlocksMissing(long targetfileid) { m_hasUpdates = true; - var r = m_resetfileCommand.SetParameterValue("@TargetFileId", targetfileid) - .ExecuteNonQuery(); + m_resetfileCommand.SetParameterValue("@TargetFileId", targetfileid); + var r = await m_resetfileCommand.ExecuteNonQueryAsync(); if (r <= 0) throw new Exception("Unexpected reset result"); } - public void SetAllBlocksRestored(long targetfileid, bool includeMetadata) + public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata) { m_hasUpdates = true; - var r = m_updateAsRestoredCommand.SetParameterValue("@TargetFileId", targetfileid) - .SetParameterValue("@Metadata", includeMetadata ? 1 : 0) - .ExecuteNonQuery(); + m_updateAsRestoredCommand.SetParameterValue("@TargetFileId", targetfileid); + m_updateAsRestoredCommand.SetParameterValue("@Metadata", includeMetadata ? 1 : 0); + var r = await m_updateAsRestoredCommand.ExecuteNonQueryAsync(); if (r <= 0) throw new Exception("Unexpected reset result"); } - public void SetFileDataVerified(long targetfileid) + public async Task SetFileDataVerified(long targetfileid) { m_hasUpdates = true; - var r = m_updateFileAsDataVerifiedCommand.SetParameterValue("@TargetFileId", targetfileid) - .ExecuteNonQuery(); + m_updateFileAsDataVerifiedCommand.SetParameterValue("@TargetFileId", targetfileid); + var r = await m_updateFileAsDataVerifiedCommand.ExecuteNonQueryAsync(); if (r != 1) throw new Exception("Unexpected result when marking file as verified."); } - public void SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata) + public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata) { m_hasUpdates = true; - var r = m_insertblockCommand.SetParameterValue("@TargetFileId", targetfileid) - .SetParameterValue("@Index", index) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .SetParameterValue("@Metadata", metadata) - .ExecuteNonQuery(); + m_insertblockCommand.SetParameterValue("@TargetFileId", targetfileid); + m_insertblockCommand.SetParameterValue("@Index", index); + m_insertblockCommand.SetParameterValue("@Hash", hash); + m_insertblockCommand.SetParameterValue("@Size", size); + m_insertblockCommand.SetParameterValue("@Metadata", metadata); + var r = await m_insertblockCommand.ExecuteNonQueryAsync(); if (r != 1) throw new Exception("Unexpected result when marking block."); } - public void Commit() + public async Task CommitAsync() { - m_insertblockCommand.Dispose(); - m_insertblockCommand = null!; using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) - m_transaction.Commit(); - m_transaction.Dispose(); + await m_transaction.CommitAsync(); + await m_transaction.DisposeAsync(); m_transaction = null!; } @@ -1293,17 +1324,19 @@ WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND connection.Dispose(); } m_connection_pool.Clear(); - DropRestoreTable(); + DropRestoreTable().Await(); base.Dispose(); } - public IEnumerable GetTargetFolders() + public async IAsyncEnumerable GetTargetFolders() { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT ""TargetPath"" FROM ""{m_tempfiletable}"" WHERE ""BlocksetID"" == @BlocksetID")) - .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return rd.ConvertValueToString(0) ?? ""; + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT ""TargetPath"" FROM ""{m_tempfiletable}"" WHERE ""BlocksetID"" == @BlocksetID")); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + cmd.SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + yield return rd.ConvertValueToString(0) ?? ""; + await cmd.Transaction.CommitAsync(); } public interface IFastSource @@ -1326,18 +1359,18 @@ WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND { private class BlockEntry : IBlockEntry { - private readonly IDataReader m_rd; + private readonly SqliteDataReader m_rd; private readonly long m_blocksize; - public BlockEntry(IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } + public BlockEntry(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } } public long Index { get { return m_rd.ConvertValueToInt64(3); } } public long Size { get { return m_rd.ConvertValueToInt64(5); } } public string Hash { get { return m_rd.ConvertValueToString(4) ?? ""; } } } - private readonly IDataReader m_rd; + private readonly SqliteDataReader m_rd; private readonly long m_blocksize; - public FastSource(IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } + public FastSource(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } public bool MoreData { get; private set; } public string TargetPath { get { return m_rd.ConvertValueToString(0) ?? ""; } } public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } @@ -1352,28 +1385,30 @@ WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND do { yield return new BlockEntry(m_rd, m_blocksize); - } while ((MoreData = m_rd.Read()) && tid == TargetFileID); + } while ((MoreData = m_rd.ReadAsync().Await()) && tid == TargetFileID); } } } - public IEnumerable GetFilesAndSourceBlocksFast(long blocksize) + public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize) { - using (var transaction = m_connection.BeginTransactionSafe()) + using (var transaction = m_connection.BeginTransaction(deferred: true)) using (var cmdReader = m_connection.CreateCommand()) - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand()) { + cmdReader.Transaction = transaction; + cmd.Transaction = transaction; cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""LocalSourceExists"" = 1 WHERE Path = @Path")); - using (var rd = cmdReader.ExecuteReader(FormatInvariant($@"SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"""))) + using (var rd = await cmdReader.ExecuteReaderAsync(FormatInvariant($@"SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"""))) { - while (rd.Read()) + while (await rd.ReadAsync()) { var sourcepath = rd.ConvertValueToString(0); if (SystemIO.IO_OS.FileExists(sourcepath)) { cmd.SetParameterValue("@Path", sourcepath); - cmd.ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync(); } else { @@ -1383,8 +1418,8 @@ WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND } //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"")")); - transaction.Commit(); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"")")); + await transaction.CommitAsync(); } m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; @@ -1417,13 +1452,15 @@ WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND {whereclause}) GROUP BY ""File"".""Path"""); + using var transaction2 = m_connection.BeginTransaction(deferred: true); using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_latestblocktable}_path"" ON ""{m_latestblocktable}"" (""Path"")")); + cmd.Transaction = transaction2; + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_latestblocktable}_path"" ON ""{m_latestblocktable}"" (""Path"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET LatestBlocksetId = (SELECT BlocksetId FROM ""{m_latestblocktable}"" WHERE Path = ""{m_tempfiletable}"".Path)")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET LatestBlocksetId = (SELECT BlocksetId FROM ""{m_latestblocktable}"" WHERE Path = ""{m_tempfiletable}"".Path)")); } var sources = FormatInvariant($@" @@ -1447,9 +1484,10 @@ WHERE ORDER BY ""{m_tempfiletable}"".""ID"", ""{m_tempblocktable}"".""Index"" "); using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(sources)) { - if (rd.Read()) + cmd.Transaction = transaction2; + using var rd = await cmd.ExecuteReaderAsync(sources); + if (await rd.ReadAsync()) { bool more; do @@ -1460,11 +1498,13 @@ ORDER BY ""{m_tempfiletable}"".""ID"", ""{m_tempblocktable}"".""Index"" "); more = n.MoreData; while (more && n.TargetFileID == tid) - more = rd.Read(); + more = await rd.ReadAsync(); } while (more); } } + + await transaction2.CommitAsync(); } } From ca4295b37cc00cb28797d9e44653364809aefb45 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 08:40:07 +0200 Subject: [PATCH 012/362] Found additional references to old way of accessing the database --- Duplicati/Library/Main/Database/LocalDatabase.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 493e46161..8e6e9cc87 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1045,13 +1045,13 @@ ON { private class BlocklistHashEnumerator : IEnumerator { - private readonly IDataReader m_reader; + private readonly SqliteDataReader m_reader; private readonly BlocklistHashEnumerable m_parent; private string? m_path = null; private bool m_first = true; private string? m_current = null; - public BlocklistHashEnumerator(BlocklistHashEnumerable parent, IDataReader reader) + public BlocklistHashEnumerator(BlocklistHashEnumerable parent, SqliteDataReader reader) { m_reader = reader; m_parent = parent; @@ -1080,7 +1080,7 @@ ON if (m_current == null) return false; - if (!m_reader.Read()) + if (!m_reader.ReadAsync().Await()) { m_current = null; m_parent.MoreData = false; @@ -1108,9 +1108,9 @@ ON } } - private readonly IDataReader m_reader; + private readonly SqliteDataReader m_reader; - public BlocklistHashEnumerable(IDataReader reader) + public BlocklistHashEnumerable(SqliteDataReader reader) { m_reader = reader; MoreData = true; From f443331cca13dbfbd188305a9d3fb58ec0f4c8e8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 08:40:58 +0200 Subject: [PATCH 013/362] Removed import to ensure old way of accessing database isn't hit --- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index fd6db1a11..f790a4b1d 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -24,7 +24,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Data; using System.Linq; using System.Text; using System.Threading.Tasks; From f3b102dc6695773b702c95b09474ac165ee40c66 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 08:41:17 +0200 Subject: [PATCH 014/362] Removed whitespace --- .../Main/Database/LocalRecreateDatabase.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 16900bd2e..341cf74c5 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From 1c1c83b09338d53bf1e19d9c8a06d6624e1b4358 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 08:46:24 +0200 Subject: [PATCH 015/362] Updated LocalRecreateDatabase to be async using Microsoft.Data.Sqlilte --- .../Main/Database/LocalRecreateDatabase.cs | 588 +++++++++--------- 1 file changed, 298 insertions(+), 290 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 341cf74c5..6eea3579a 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -22,8 +22,10 @@ #nullable enable using System; -using System.Data; using System.Collections.Generic; +using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -38,7 +40,7 @@ namespace Duplicati.Library.Main.Database { private struct KeyValueComparer : IComparer> { - public int Compare(KeyValuePair x, KeyValuePair y) + public readonly int Compare(KeyValuePair x, KeyValuePair y) { return x.Key == y.Key ? (x.Value == y.Value ? @@ -49,25 +51,29 @@ namespace Duplicati.Library.Main.Database } } - private readonly IDbCommand m_insertFileCommand; - private readonly IDbCommand m_insertFilesetEntryCommand; - private readonly IDbCommand m_insertMetadatasetCommand; - private readonly IDbCommand m_insertBlocksetCommand; - private readonly IDbCommand m_insertBlocklistHashCommand; - private readonly IDbCommand m_updateBlockVolumeCommand; - private readonly IDbCommand m_insertTempBlockListHash; - private readonly IDbCommand m_insertSmallBlockset; - private readonly IDbCommand m_findBlocksetCommand; - private readonly IDbCommand m_findMetadatasetCommand; - private readonly IDbCommand m_findFilesetCommand; - private readonly IDbCommand m_findTempBlockListHashCommand; - private readonly IDbCommand m_findHashBlockCommand; - private readonly IDbCommand m_insertBlockCommand; - private readonly IDbCommand m_insertDuplicateBlockCommand; + private readonly SqliteCommand m_insertFileCommand; + private readonly SqliteCommand m_insertFilesetEntryCommand; + private readonly SqliteCommand m_insertMetadatasetCommand; + private readonly SqliteCommand m_insertBlocksetCommand; + private readonly SqliteCommand m_insertBlocklistHashCommand; + private readonly SqliteCommand m_updateBlockVolumeCommand; + private readonly SqliteCommand m_insertTempBlockListHash; + private readonly SqliteCommand m_insertSmallBlockset; + private readonly SqliteCommand m_findBlocksetCommand; + private readonly SqliteCommand m_findMetadatasetCommand; + private readonly SqliteCommand m_findFilesetCommand; + private readonly SqliteCommand m_findTempBlockListHashCommand; + private readonly SqliteCommand m_findHashBlockCommand; + private readonly SqliteCommand m_insertBlockCommand; + private readonly SqliteCommand m_insertDuplicateBlockCommand; private string m_tempblocklist; private string m_tempsmalllist; + private new readonly SqliteConnection m_connection; + + //public new bool RepairInProgress { get; set; } = false; + /// /// A lookup table that prevents multiple downloads of the same volume /// @@ -80,7 +86,7 @@ namespace Duplicati.Library.Main.Database // {2} --> Temp-Table // {3} --> FullBlocklist-BlockCount [equals ({0} / {1}), if SQLite pays respect to ints] private static string SELECT_BLOCKLIST_ENTRIES(long blocksize, long blockhashsize, string temptable, long fullBlockListBlockCount) => - FormatInvariant($@" + FormatInvariant($@" SELECT ""E"".""BlocksetID"", ""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount}) AS ""FullIndex"", @@ -93,16 +99,16 @@ namespace Duplicati.Library.Main.Database ( SELECT * FROM ( - SELECT + SELECT ""A"".""BlocksetID"", ""A"".""Index"" AS ""BlocklistIndex"", MIN({fullBlockListBlockCount} * {blockhashsize}, (((""B"".""Length"" + {blocksize} - 1) / {blocksize}) - (""A"".""Index"" * ({fullBlockListBlockCount}))) * {blockhashsize}) AS ""BlocklistSize"", ""A"".""Hash"" AS ""BlocklistHash"", ""B"".""Length"" - FROM + FROM ""BlocklistHash"" A, ""Blockset"" B - WHERE + WHERE ""B"".""ID"" = ""A"".""BlocksetID"" ) C, ""Block"" D @@ -114,7 +120,7 @@ namespace Duplicati.Library.Main.Database ""{temptable}"" F WHERE ""F"".""BlocklistHash"" = ""E"".""Hash"" - ORDER BY + ORDER BY ""E"".""BlocksetID"", ""FullIndex"" "); @@ -125,15 +131,16 @@ namespace Duplicati.Library.Main.Database m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand()) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""Index_{m_tempblocklist}"" ON ""{m_tempblocklist}"" (""BlockListHash"");")); + if (m_connection == null) + throw new Exception("Connection is null"); + using var cmd = m_connection.CreateCommand(); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_File_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""FileHash"", ""BlockSize"");")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_Block_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");")); - } + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""Index_{m_tempblocklist}"" ON ""{m_tempblocklist}"" (""BlockListHash"");")).Await(); + + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_File_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""FileHash"", ""BlockSize"");")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_Block_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");")).Await(); m_insertFileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (@PrefixId,@Path,@BlocksetId,@MetadataId); SELECT last_insert_rowid();"); m_insertFilesetEntryCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId,@FileId,@LastModified)"); @@ -152,68 +159,67 @@ namespace Duplicati.Library.Main.Database m_insertDuplicateBlockCommand = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size), @VolumeId)"); } - public void FindMissingBlocklistHashes(long hashsize, long blocksize, IDbTransaction transaction) + public async Task FindMissingBlocklistHashes(long hashsize, long blocksize, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - //Update all small blocklists and matching blocks + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + //Update all small blocklists and matching blocks - var selectSmallBlocks = FormatInvariant($@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""); + var selectSmallBlocks = FormatInvariant($@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""); - var selectBlockHashes = FormatInvariant($@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} )"); + var selectBlockHashes = FormatInvariant($@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} )"); - var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; + var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; - var selectNewBlocks = FormatInvariant($@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" + var selectNewBlocks = FormatInvariant($@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({selectAllBlocks}) A LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" ) WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); - var insertBlocksCommand = - @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + - selectNewBlocks; + var insertBlocksCommand = + @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + + selectNewBlocks; - // Insert all known blocks into block table with volumeid = -1 - cmd.ExecuteNonQuery(insertBlocksCommand); + // Insert all known blocks into block table with volumeid = -1 + await cmd.ExecuteNonQueryAsync(insertBlocksCommand); - var selectBlocklistBlocksetEntries = FormatInvariant( - // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati - // this could be used instead - //@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" "; - $@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" " - ); + var selectBlocklistBlocksetEntries = FormatInvariant( + // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati + // this could be used instead + //@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" "; + $@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" " + ); - var selectBlocksetEntries = FormatInvariant($@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{m_tempsmalllist}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {blocksize} "); + var selectBlocksetEntries = FormatInvariant($@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{m_tempsmalllist}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {blocksize} "); - var selectAllBlocksetEntries = - selectBlocklistBlocksetEntries + - @" UNION " + - selectBlocksetEntries; + var selectAllBlocksetEntries = + selectBlocklistBlocksetEntries + + @" UNION " + + selectBlocksetEntries; - var selectFiltered = - @"SELECT DISTINCT ""H"".""BlocksetID"", ""H"".""Index"", ""H"".""BlockID"" FROM (" + - selectAllBlocksetEntries + - @") H WHERE (""H"".""BlocksetID"" || ':' || ""H"".""Index"") NOT IN (SELECT (""ExistingBlocksetEntries"".""BlocksetID"" || ':' || ""ExistingBlocksetEntries"".""Index"") FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" )"; + var selectFiltered = + @"SELECT DISTINCT ""H"".""BlocksetID"", ""H"".""Index"", ""H"".""BlockID"" FROM (" + + selectAllBlocksetEntries + + @") H WHERE (""H"".""BlocksetID"" || ':' || ""H"".""Index"") NOT IN (SELECT (""ExistingBlocksetEntries"".""BlocksetID"" || ':' || ""ExistingBlocksetEntries"".""Index"") FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" )"; - var insertBlocksetEntriesCommand = - @"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered; + var insertBlocksetEntriesCommand = + @"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered; - try + try + { + await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "BlocksetInsertFailed", ex, "Blockset insert failed, committing temporary tables for trace purposes"); + + using (var fixcmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(insertBlocksetEntriesCommand); + await fixcmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); + await fixcmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "BlocksetInsertFailed", ex, "Blockset insert failed, committing temporary tables for trace purposes"); - using (var fixcmd = m_connection.CreateCommand()) - { - fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); - fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); - } - - throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); - } + throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); } } @@ -233,160 +239,162 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); /// temp small list structure: filehash, blockhash, blocksize: as the small files are defined /// by the fact that they are contained in a single block, blockhash is the same as the filehash, /// and blocksize can vary from 0 to the configured block size for the backup - public void AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, IDbTransaction transaction, bool hashOnly = false) + public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, SqliteTransaction transaction, bool hashOnly = false) { - - using (var cmd = m_connection.CreateCommand(transaction)) - { - var insertBlocksCommand = FormatInvariant($@"INSERT INTO BLOCK (Hash, Size, VolumeID) -SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + var insertBlocksCommand = FormatInvariant($@"INSERT INTO BLOCK (Hash, Size, VolumeID) +SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM ( -SELECT NB.BlockHash, -MIN({blocksize}, BS.Length - ((NB.""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS BlockSize +SELECT NB.BlockHash, +MIN({blocksize}, BS.Length - ((NB.""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS BlockSize FROM ( - SELECT TBL.BlockListHash, TBL.BlockHash, TBL.""Index"" FROM {m_tempblocklist} TBL - LEFT OUTER JOIN Block B ON (B.Hash = TBL.BlockHash) - WHERE B.Hash IS NULL - ) NB -JOIN BlocklistHash BH ON (BH.Hash = NB.BlocklistHash) + SELECT TBL.BlockListHash, TBL.BlockHash, TBL.""Index"" FROM {m_tempblocklist} TBL + LEFT OUTER JOIN Block B ON (B.Hash = TBL.BlockHash) + WHERE B.Hash IS NULL + ) NB +JOIN BlocklistHash BH ON (BH.Hash = NB.BlocklistHash) JOIN Blockset BS ON (BS.ID = BH.Blocksetid) "); - if (!hashOnly) - { - insertBlocksCommand += FormatInvariant($@" UNION -SELECT TS.BlockHash, TS.BlockSize FROM -{m_tempsmalllist} TS -WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE - B.Hash = TS.BlockHash AND - B.Size = TS.BlockSize) + if (!hashOnly) + { + insertBlocksCommand += FormatInvariant($@" UNION +SELECT TS.BlockHash, TS.BlockSize FROM +{m_tempsmalllist} TS +WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE + B.Hash = TS.BlockHash AND + B.Size = TS.BlockSize) )"); - } + } - var insertBlocksetEntriesCommand = FormatInvariant($@"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) -SELECT DISTINCT BH.blocksetid, (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" as FullIndex, BK.ID AS BlockID -FROM {m_tempblocklist} TBL - JOIN blocklisthash BH ON (BH.hash = TBL.blocklisthash) - JOIN block BK ON (BK.Hash = TBL.BlockHash) - LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlockSetID = BH.BlocksetID AND BE.""Index"" = (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"") -WHERE + var insertBlocksetEntriesCommand = FormatInvariant($@"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) +SELECT DISTINCT BH.blocksetid, (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" as FullIndex, BK.ID AS BlockID +FROM {m_tempblocklist} TBL + JOIN blocklisthash BH ON (BH.hash = TBL.blocklisthash) + JOIN block BK ON (BK.Hash = TBL.BlockHash) + LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlockSetID = BH.BlocksetID AND BE.""Index"" = (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"") +WHERE BE.BlockSetID IS NULL "); - if (!hashOnly) - { - insertBlocksetEntriesCommand += FormatInvariant($@" UNION -SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID -FROM {m_tempsmalllist} TS - JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND - BS.Length = TS.BlockSize AND - BS.Length <= {blocksize}) - JOIN Block BL ON (BL.Hash = TS.BlockHash AND - BL.Size = TS.BlockSize) - LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) -WHERE + if (!hashOnly) + { + insertBlocksetEntriesCommand += FormatInvariant($@" UNION +SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID +FROM {m_tempsmalllist} TS + JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND + BS.Length = TS.BlockSize AND + BS.Length <= {blocksize}) + JOIN Block BL ON (BL.Hash = TS.BlockHash AND + BL.Size = TS.BlockSize) + LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) +WHERE BE.BlocksetID IS NULL "); - } + } - try + try + { + // Insert discovered new blocks into block table with volumeid = -1 + await cmd.ExecuteNonQueryAsync(insertBlocksCommand); + // Insert corresponding entries into blockset + await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "BlockOrBlocksetInsertFailed", ex, "Block or Blockset insert failed, committing temporary tables for trace purposes"); + + using (var fixcmd = m_connection.CreateCommand()) { - // Insert discovered new blocks into block table with volumeid = -1 - cmd.ExecuteNonQuery(insertBlocksCommand); - // Insert corresponding entries into blockset - cmd.ExecuteNonQuery(insertBlocksetEntriesCommand); + await fixcmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); + await fixcmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "BlockOrBlocksetInsertFailed", ex, "Block or Blockset insert failed, committing temporary tables for trace purposes"); - using (var fixcmd = m_connection.CreateCommand()) - { - fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); - fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); - } - - throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); - } + throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); } } - public void AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, IDbTransaction transaction) + public async Task AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, SqliteTransaction transaction) { - AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction); } - public void AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, IDbTransaction transaction) + public async Task AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, SqliteTransaction transaction) { - AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); } - public void AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, IDbTransaction transaction) + public async Task AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, SqliteTransaction transaction) { - AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction); } - private void AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, IDbTransaction transaction) + private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, SqliteTransaction transaction) { - var fileid = m_findFilesetCommand.SetTransaction(transaction) - .SetParameterValue("@PrefixId", pathprefixid) - .SetParameterValue("@Path", path) - .SetParameterValue("@BlocksetId", blocksetid) - .SetParameterValue("@MetadataId", metadataid) - .ExecuteScalarInt64(-1); + m_findFilesetCommand.Transaction = transaction; + m_findFilesetCommand.Parameters["@PrefixId"].Value = pathprefixid; + m_findFilesetCommand.Parameters["@Path"].Value = path; + m_findFilesetCommand.Parameters["@BlocksetId"].Value = blocksetid; + m_findFilesetCommand.Parameters["@MetadataId"].Value = metadataid; + var fileid = await m_findFilesetCommand.ExecuteScalarInt64Async(-1); if (fileid < 0) { - fileid = m_insertFileCommand.SetTransaction(transaction) - .SetParameterValue("@PrefixId", pathprefixid) - .SetParameterValue("@Path", path) - .SetParameterValue("@BlocksetId", blocksetid) - .SetParameterValue("@MetadataId", metadataid) - .ExecuteScalarInt64(-1); + m_insertFileCommand.Transaction = transaction; + m_insertFileCommand.Parameters["@PrefixId"].Value = pathprefixid; + m_insertFileCommand.Parameters["@Path"].Value = path; + m_insertFileCommand.Parameters["@BlocksetId"].Value = blocksetid; + m_insertFileCommand.Parameters["@MetadataId"].Value = metadataid; + fileid = await m_insertFileCommand.ExecuteScalarInt64Async(-1); } - m_insertFilesetEntryCommand.SetTransaction(transaction) - .SetParameterValue("@FilesetId", filesetid) - .SetParameterValue("@FileId", fileid) - .SetParameterValue("@LastModified", time.ToUniversalTime().Ticks) - .ExecuteNonQuery(); + m_insertFilesetEntryCommand.Transaction = transaction; + m_insertFilesetEntryCommand.Parameters["@FilesetId"].Value = filesetid; + m_insertFilesetEntryCommand.Parameters["@FileId"].Value = fileid; + m_insertFilesetEntryCommand.Parameters["@LastModified"].Value = time.ToUniversalTime().Ticks; + await m_insertFilesetEntryCommand.ExecuteNonQueryAsync(); } - public long AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, IDbTransaction transaction) + public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, SqliteTransaction transaction) { var metadataid = -1L; if (metahash == null) return metadataid; - metadataid = m_findMetadatasetCommand.SetTransaction(transaction) - .SetParameterValue("@FullHash", metahash) - .SetParameterValue("@Length", metahashsize) - .ExecuteScalarInt64(-1); + m_findMetadatasetCommand.Transaction = transaction; + m_findMetadatasetCommand.Parameters["@FullHash"].Value = metahash; + m_findMetadatasetCommand.Parameters["@Length"].Value = metahashsize; + metadataid = await m_findMetadatasetCommand.ExecuteScalarInt64Async(-1); + if (metadataid != -1) return metadataid; - var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction); + var blocksetid = await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction); - return m_insertMetadatasetCommand.SetTransaction(transaction) - .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64(-1); + m_insertMetadatasetCommand.Transaction = transaction; + m_insertMetadatasetCommand.Parameters["@BlocksetId"].Value = blocksetid; + metadataid = await m_insertMetadatasetCommand.ExecuteScalarInt64Async(-1); + + return metadataid; } - public long AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, IDbTransaction transaction) + public async Task AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, SqliteTransaction transaction) { - var blocksetid = m_findBlocksetCommand.SetTransaction(transaction) - .SetParameterValue("@Length", size) - .SetParameterValue("@FullHash", fullhash) - .ExecuteScalarInt64(-1); + m_findBlocksetCommand.Transaction = transaction; + m_findBlocksetCommand.Parameters["@Length"].Value = size; + m_findBlocksetCommand.Parameters["@FullHash"].Value = fullhash; + var blocksetid = await m_findBlocksetCommand.ExecuteScalarInt64Async(-1); + if (blocksetid != -1) return blocksetid; - blocksetid = m_insertBlocksetCommand.SetTransaction(transaction) - .SetParameterValue("@Length", size) - .SetParameterValue("@FullHash", fullhash) - .ExecuteScalarInt64(-1); + m_insertBlocksetCommand.Transaction = transaction; + m_insertBlocksetCommand.Parameters["@Length"].Value = size; + m_insertBlocksetCommand.Parameters["@FullHash"].Value = fullhash; + blocksetid = await m_insertBlocksetCommand.ExecuteScalarInt64Async(-1); long c = 0; if (blocklisthashes != null) { var index = 0L; - m_insertBlocklistHashCommand.SetTransaction(transaction) - .SetParameterValue("@BlocksetId", blocksetid); + m_insertBlocklistHashCommand.Transaction = transaction; + m_insertBlocklistHashCommand.Parameters["@BlocksetId"].Value = blocksetid; foreach (var hash in blocklisthashes) { @@ -395,9 +403,9 @@ BE.BlocksetID IS NULL "); c++; if (c <= expectedblocklisthashes) { - m_insertBlocklistHashCommand.SetParameterValue("@Index", index++) - .SetParameterValue("@Hash", hash) - .ExecuteNonQuery(); + m_insertBlocklistHashCommand.Parameters["@Index"].Value = index++; + m_insertBlocklistHashCommand.Parameters["@Hash"].Value = hash; + await m_insertBlocklistHashCommand.ExecuteNonQueryAsync(); } } } @@ -409,97 +417,100 @@ BE.BlocksetID IS NULL "); return blocksetid; } - public bool UpdateBlock(string hash, long size, long volumeID, IDbTransaction transaction, ref bool anyChange) + public async Task<(bool, bool)> UpdateBlock(string hash, long size, long volumeID, SqliteTransaction transaction) { - var currentVolumeId = m_findHashBlockCommand.SetTransaction(transaction) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteScalarInt64(-2); + var anyChange = false; + m_findHashBlockCommand.Transaction = transaction; + m_findHashBlockCommand.Parameters["@Hash"].Value = hash; + m_findHashBlockCommand.Parameters["@Size"].Value = size; + var currentVolumeId = await m_findHashBlockCommand.ExecuteScalarInt64Async(-2); if (currentVolumeId == volumeID) - return false; + return (anyChange, false); anyChange = true; if (currentVolumeId == -2) { //Insert - m_insertBlockCommand.SetTransaction(transaction) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .SetParameterValue("@VolumeId", volumeID) - .ExecuteNonQuery(); + m_insertBlockCommand.Transaction = transaction; + m_insertBlockCommand.Parameters["@Hash"].Value = hash; + m_insertBlockCommand.Parameters["@Size"].Value = size; + m_insertBlockCommand.Parameters["@VolumeId"].Value = volumeID; + await m_insertBlockCommand.ExecuteNonQueryAsync(); - return true; + return (anyChange, true); } else if (currentVolumeId == -1) { //Update - var c = m_updateBlockVolumeCommand.SetTransaction(transaction) - .SetParameterValue("@VolumeId", volumeID) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + m_updateBlockVolumeCommand.Transaction = transaction; + m_updateBlockVolumeCommand.Parameters["@VolumeId"].Value = volumeID; + m_updateBlockVolumeCommand.Parameters["@Hash"].Value = hash; + m_updateBlockVolumeCommand.Parameters["@Size"].Value = size; + var c = await m_updateBlockVolumeCommand.ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Failed to update table, found {c} entries for key {hash} with size {size}"); - return true; + return (anyChange, true); } else { - m_insertDuplicateBlockCommand.SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .SetParameterValue("@VolumeId", volumeID) - .ExecuteNonQuery(transaction); + m_insertDuplicateBlockCommand.Transaction = transaction; + m_insertDuplicateBlockCommand.Parameters["@Hash"].Value = hash; + m_insertDuplicateBlockCommand.Parameters["@Size"].Value = size; + m_insertDuplicateBlockCommand.Parameters["@VolumeId"].Value = volumeID; + await m_insertDuplicateBlockCommand.ExecuteNonQueryAsync(); - return false; + return (anyChange, false); } } - public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, IDbTransaction transaction) + public async Task AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, SqliteTransaction transaction) { - m_insertSmallBlockset.SetParameterValue("@FileHash", filehash) - .SetParameterValue("@BlockHash", blockhash) - .SetParameterValue("@BlockSize", blocksize) - .ExecuteNonQuery(transaction); + m_insertSmallBlockset.Transaction = transaction; + m_insertSmallBlockset.Parameters["@FileHash"].Value = filehash; + m_insertSmallBlockset.Parameters["@BlockHash"].Value = blockhash; + m_insertSmallBlockset.Parameters["@BlockSize"].Value = blocksize; + await m_insertSmallBlockset.ExecuteNonQueryAsync(); } - public bool AddTempBlockListHash(string hash, IEnumerable blocklisthashes, IDbTransaction transaction) + public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes, SqliteTransaction transaction) { - var r = m_findTempBlockListHashCommand.SetTransaction(transaction) - .SetParameterValue("@BlocklistHash", hash) - .ExecuteScalar(); + m_findTempBlockListHashCommand.Transaction = transaction; + m_findTempBlockListHashCommand.Parameters["@BlocklistHash"].Value = hash; + var r = await m_findTempBlockListHashCommand.ExecuteScalarAsync(); + if (r != null && r != DBNull.Value) return false; - m_insertTempBlockListHash.SetTransaction(transaction) - .SetParameterValue("@BlocklistHash", hash); + m_insertTempBlockListHash.Transaction = transaction; + m_insertTempBlockListHash.Parameters["@BlocklistHash"].Value = hash; var index = 0L; foreach (var s in blocklisthashes) { - m_insertTempBlockListHash.SetParameterValue("@BlockHash", s) - .SetParameterValue("@Index", index++) - .ExecuteNonQuery(); + m_insertTempBlockListHash.Parameters["@BlockHash"].Value = s; + m_insertTempBlockListHash.Parameters["@Index"].Value = index++; + await m_insertTempBlockListHash.ExecuteNonQueryAsync(); } return true; } - public IEnumerable GetBlockLists(long volumeid) + public async IAsyncEnumerable GetBlockLists(long volumeid) { - using (var cmd = m_connection.CreateCommand(@"SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId")) - { - cmd.SetParameterValue("@VolumeId", volumeid); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return rd.ConvertValueToString(0) ?? ""; - } + using var cmd = m_connection.CreateCommand(@"SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId"); + + cmd.Parameters.AddWithValue("@VolumeId", volumeid); + using var rd = await cmd.ExecuteReaderAsync(); + while (rd.Read()) + yield return rd.ConvertValueToString(0) ?? ""; } - public IEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) + public async IAsyncEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) { using (var cmd = m_connection.CreateCommand()) { @@ -521,7 +532,7 @@ BE.BlocksetID IS NULL "); { // On the first pass, we select all the volumes we know we need, // which may be an empty list - cmd.SetCommandAndParameters(selectCommand + FormatInvariant($@" WHERE ""ID"" IN ({missingBlocklistVolumes})")); + cmd.CommandText = FormatInvariant($@"{selectCommand} WHERE ""ID"" IN ({missingBlocklistVolumes})"); // Reset the list m_proccessedVolumes.Clear(); @@ -529,7 +540,9 @@ BE.BlocksetID IS NULL "); else { //On anything but the first pass, we check if we are done - var r = cmd.ExecuteScalarInt64(countMissingInformation, 0); + cmd.CommandText = countMissingInformation; + cmd.Parameters.Clear(); + var r = await cmd.ExecuteScalarInt64Async(0); if (r == 0 && !forceBlockUse) yield break; @@ -540,42 +553,41 @@ BE.BlocksetID IS NULL "); var mentionedVolumes = @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; - cmd.SetCommandAndParameters(selectCommand + FormatInvariant($@" WHERE ""ID"" NOT IN ({mentionedVolumes}) AND ""Type"" = @Type ")) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + cmd.CommandText = FormatInvariant($@"{selectCommand} WHERE ""ID"" IN ({mentionedVolumes}) AND ""Type"" = @Type "); + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@Type", RemoteVolumeType.Blocks.ToString()); + } else { // On the final pass, we select all volumes // the filter will ensure that we do not download anything twice - cmd.SetCommandAndParameters(selectCommand + @" WHERE ""Type"" = @Type ") - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + cmd.CommandText = FormatInvariant($@"{selectCommand} WHERE ""Type"" = @Type"); + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@Type", RemoteVolumeType.Blocks.ToString()); } } - using (var rd = cmd.ExecuteReader()) + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) { - while (rd.Read()) + var volumeID = rd.ConvertValueToInt64(3); + + // Guard against multiple downloads of the same file + if (m_proccessedVolumes.TryAdd(volumeID, volumeID)) { - var volumeID = rd.ConvertValueToInt64(3); - - // Guard against multiple downloads of the same file - if (!m_proccessedVolumes.ContainsKey(volumeID)) - { - m_proccessedVolumes.Add(volumeID, volumeID); - - yield return new RemoteVolume( - rd.ConvertValueToString(0), - rd.ConvertValueToString(1), - rd.ConvertValueToInt64(2, -1) - ); - } + yield return new RemoteVolume( + rd.ConvertValueToString(0), + rd.ConvertValueToString(1), + rd.ConvertValueToInt64(2, -1) + ); } } } } - public void CleanupMissingVolumes() + public async Task CleanupMissingVolumes() { var tablename = "SwapBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -587,36 +599,36 @@ BE.BlocksetID IS NULL "); // and marks the index files that pointed to them, such that they will be removed later on var sql = FormatInvariant($@" CREATE TEMPORARY TABLE ""{tablename}"" AS -SELECT - ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" -FROM - (SELECT - ""Block"".""ID"", ""Block"".""VolumeID"", ""Remotevolume"".""State"" - FROM ""Block"", ""Remotevolume"" +SELECT + ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" +FROM + (SELECT + ""Block"".""ID"", ""Block"".""VolumeID"", ""Remotevolume"".""State"" + FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Temporary}' - ) A, - (SELECT - ""DuplicateBlock"".""BlockID"", MIN(""DuplicateBlock"".""VolumeID"") AS ""VolumeID"", ""Remotevolume"".""State"" - FROM ""DuplicateBlock"", ""Remotevolume"" - WHERE ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Verified}' - GROUP BY ""DuplicateBlock"".""BlockID"", ""Remotevolume"".""State"" - ) B + ) A, + (SELECT + ""DuplicateBlock"".""BlockID"", MIN(""DuplicateBlock"".""VolumeID"") AS ""VolumeID"", ""Remotevolume"".""State"" + FROM ""DuplicateBlock"", ""Remotevolume"" + WHERE ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Verified}' + GROUP BY ""DuplicateBlock"".""BlockID"", ""Remotevolume"".""State"" + ) B WHERE ""A"".""ID"" = ""B"".""BlockID""; -UPDATE ""Block"" - SET ""VolumeID"" = (SELECT - ""TargetVolumeID"" FROM ""{tablename}"" +UPDATE ""Block"" + SET ""VolumeID"" = (SELECT + ""TargetVolumeID"" FROM ""{tablename}"" WHERE ""Block"".""ID"" = ""{tablename}"".""BlockID"" - ) + ) WHERE ""Block"".""ID"" IN (SELECT ""BlockID"" FROM ""{tablename}""); -UPDATE ""DuplicateBlock"" - SET ""VolumeID"" = (SELECT - ""SourceVolumeID"" FROM ""{tablename}"" +UPDATE ""DuplicateBlock"" + SET ""VolumeID"" = (SELECT + ""SourceVolumeID"" FROM ""{tablename}"" WHERE ""DuplicateBlock"".""BlockID"" = ""{tablename}"".""BlockID"" - ) -WHERE - (""DuplicateBlock"".""BlockID"", ""DuplicateBlock"".""VolumeID"") + ) +WHERE + (""DuplicateBlock"".""BlockID"", ""DuplicateBlock"".""VolumeID"") IN (SELECT ""BlockID"", ""TargetVolumeID"" FROM ""{tablename}""); DROP TABLE ""{tablename}""; @@ -633,13 +645,13 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" using (var cmd = m_connection.CreateCommand()) { - var cnt = cmd.ExecuteScalarInt64(countsql); + var cnt = await cmd.ExecuteScalarInt64Async(countsql); if (cnt > 0) { try { - cmd.ExecuteNonQuery(sql); - var cnt2 = cmd.ExecuteScalarInt64(countsql); + await cmd.ExecuteNonQueryAsync(sql); + var cnt2 = await cmd.ExecuteScalarInt64Async(countsql); Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); } catch (Exception ex) @@ -655,30 +667,26 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" /// Move blocks that are not referenced by any files to DeletedBlock table. /// /// Needs to be called after the last FindMissingBlocklistHashes, otherwise the tables are not up to date. - public void CleanupDeletedBlocks(IDbTransaction transaction) + public async Task CleanupDeletedBlocks(SqliteTransaction transaction) { // Find out which blocks are deleted and move them into DeletedBlock, so that compact notices these blocks are empty // Deleted blocks do not appear in the BlocksetEntry and not in the BlocklistHash table var tmptablename = "DeletedBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - // 1. Select blocks not used by any file and not as a blocklist into temporary table - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmptablename}"" + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(); + // 1. Select blocks not used by any file and not as a blocklist into temporary table + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmptablename}"" AS SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""ID"" NOT IN (SELECT ""BlocksetEntry"".""BlockID"" FROM ""BlocksetEntry"") AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")")); - // 2. Insert blocks into DeletedBlock table - cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{tmptablename}""")); - // 3. Remove blocks from Block table - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{tmptablename}"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmptablename}""")); - tr.Commit(); - } - - + // 2. Insert blocks into DeletedBlock table + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{tmptablename}""")); + // 3. Remove blocks from Block table + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{tmptablename}"")")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmptablename}""")); + await cmd.Transaction.CommitAsync(); } public override void Dispose() @@ -688,7 +696,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempblocklist != null) try { - cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""")); + cmd.ExecuteNonQueryAsync(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""")).Wait(); } catch (Exception ex) { @@ -702,7 +710,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempsmalllist != null) try { - cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""")); + cmd.ExecuteNonQueryAsync(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""")).Wait(); } catch (Exception ex) { @@ -718,4 +726,4 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" base.Dispose(); } } -} +} \ No newline at end of file From 9fcad3205844c517d4a531682da9d7cb5d60fdcc Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 09:00:09 +0200 Subject: [PATCH 016/362] Updated RecreateDatabaseHandler to utilize the new async interface --- .../Main/Operation/RecreateDatabaseHandler.cs | 143 ++++++++++-------- 1 file changed, 76 insertions(+), 67 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index e0cdd836e..32ae9fa74 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -1,26 +1,26 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using System.Data; +//using System.Data; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -28,6 +28,7 @@ using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Operation { @@ -68,7 +69,7 @@ namespace Duplicati.Library.Main.Operation await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); // Ensure database is consistent after the recreate - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); } } @@ -85,7 +86,7 @@ namespace Duplicati.Library.Main.Operation using (var db = new LocalDatabase(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache)) { - if (db.FindMatchingFilesets(m_options.Time, m_options.Version).Any()) + if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version)).Any()) { if (m_options.IgnoreUpdateIfVersionExists) { @@ -192,7 +193,7 @@ namespace Duplicati.Library.Main.Operation // If we are updating, all files should be accounted for foreach (var fl in remotefiles) - volumeIds[fl.File.Name] = updating ? restoredb.GetRemoteVolumeID(fl.File.Name) : restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded); + volumeIds[fl.File.Name] = updating ? await restoredb.GetRemoteVolumeID(fl.File.Name) : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded); var hasUpdatedOptions = false; @@ -209,7 +210,7 @@ namespace Duplicati.Library.Main.Operation { foreach (var n in filelists) if (volumeIds[n.File.Name] == -1) - volumeIds[n.File.Name] = restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), tr); + volumeIds[n.File.Name] = await restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), tr); } var isFirstFilelist = true; @@ -241,7 +242,7 @@ namespace Duplicati.Library.Main.Operation isFirstFilelist = false; if (!string.IsNullOrWhiteSpace(hash) && size > 0) - restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, tr); + await restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, tr); var parsed = VolumeBase.ParseFilename(entry.Name); @@ -257,9 +258,9 @@ namespace Duplicati.Library.Main.Operation } // Create timestamped operations based on the file timestamp - var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); + var filesetid = await restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); - RecreateFilesetFromRemoteList(restoredb, tr, compressor, filesetid, m_options, filter); + await RecreateFilesetFromRemoteList(restoredb, tr, compressor, filesetid, m_options, filter); } } catch (Exception ex) @@ -288,7 +289,8 @@ namespace Duplicati.Library.Main.Operation Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, tr); using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) - tr.Commit(); + if (tr.Connection != null) + await tr.CommitAsync(); } // do we stop after just handling the dlist files ? @@ -330,18 +332,18 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) { if (!string.IsNullOrWhiteSpace(hash) && size > 0) - restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, tr); + await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, tr); using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) { foreach (var a in svr.Volumes) { var filename = a.Filename; - var volumeID = restoredb.GetRemoteVolumeID(filename); + var volumeID = await restoredb.GetRemoteVolumeID(filename, tr); // No such file if (volumeID < 0) - volumeID = ProbeForMatchingFilename(ref filename, restoredb); + (volumeID, filename) = await ProbeForMatchingFilename(filename, restoredb); var missing = false; // Still broken, register a missing item @@ -352,16 +354,18 @@ namespace Duplicati.Library.Main.Operation throw new Exception(string.Format("Unable to parse filename: {0}", filename)); Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); missing = true; - volumeID = restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, tr); + volumeID = await restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, tr); } bool anyChange = false; //Add all block/volume mappings foreach (var b in a.Blocks) - restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr, ref anyChange); + { + anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr)).Item1; + } - restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, tr); - restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(name), volumeID, tr); + await restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, tr); + await restoredb.AddIndexBlockLink(await restoredb.GetRemoteVolumeID(name, tr), volumeID, tr); } //If there are blocklists in the index file, add them to the temp blocklist hashes table @@ -375,7 +379,7 @@ namespace Duplicati.Library.Main.Operation // We need to instantiate the list to ensure the verification is // done before we add it to the database, since we do not have nested transactions var list = b.Blocklist.ToList(); - restoredb.AddTempBlockListHash(b.Hash, list, tr); + await restoredb.AddTempBlockListHash(b.Hash, list, tr); } catch (System.IO.InvalidDataException e) { @@ -408,19 +412,19 @@ namespace Duplicati.Library.Main.Operation using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) tr.Commit(); - // TODO: In some cases, we can avoid downloading all index files, + // TODO: In some cases, we can avoid downloading all index files, // if we are lucky and pick the right ones } - restoredb.CleanupMissingVolumes(); + await restoredb.CleanupMissingVolumes(); // Update the real tables from the temp tables if (expRecreateDb) // add missing blocks and blocksetentry data (at this point // we have not yet anything in the blocksetentry table) - restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, null); + await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, null); else - restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, null); + await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, null); // We have now grabbed as much information as possible, // if we are still missing data, we must now fetch block files @@ -428,7 +432,7 @@ namespace Duplicati.Library.Main.Operation for (var i = 0; i < 3; i++) { // Grab the list matching the pass type - var lst = restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse).ToList(); + var lst = await restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse).ToListAsync(); if (lst.Count > 0) { var fullist = ": " + string.Join(", ", lst.Select(x => x.Name)); @@ -469,21 +473,23 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f)); Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); - var volumeid = restoredb.GetRemoteVolumeID(name); + var volumeid = await restoredb.GetRemoteVolumeID(name); - restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, tr); + await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, tr); bool anyChange = false; // Update the block table so we know about the block/volume map foreach (var h in rd.Blocks) - restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr, ref anyChange); + { + anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr)).Item1; + } // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes // to the needed list blocks in the volume, so grab them from the database // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table - foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) + await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) { - if (restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), tr)) + if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), tr)) { anyChange = true; } @@ -501,9 +507,9 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name); } if (expRecreateDb) - restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, tr, false); + await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, tr, false); else - restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, tr); + await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, tr); } using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) @@ -529,12 +535,12 @@ namespace Duplicati.Library.Main.Operation if (!m_options.RepairOnlyPaths) { // All blocks are collected and added into the Block table - // Find out which blocks are deleted and move them into DeletedBlock, + // Find out which blocks are deleted and move them into DeletedBlock, // so that compact can calculate the unused space - restoredb.CleanupDeletedBlocks(null); + await restoredb.CleanupDeletedBlocks(null); } - restoredb.CleanupMissingVolumes(); + await restoredb.CleanupMissingVolumes(); if (m_options.RepairOnlyPaths) { @@ -556,7 +562,7 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"), "DatabaseIsBrokenConsiderPurge"); } - restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); + await restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); @@ -567,7 +573,7 @@ namespace Duplicati.Library.Main.Operation } } - public static void RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, IDbTransaction transaction, ICompression compressor, long filesetid, Options options, IFilter filter) + public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, SqliteTransaction transaction, ICompression compressor, long filesetid, Options options, IFilter filter) { var blocksize = options.Blocksize; var hashes_pr_block = blocksize / options.BlockhashSize; @@ -576,10 +582,11 @@ namespace Duplicati.Library.Main.Operation var filesetData = VolumeReaderBase.GetFilesetData(compressor, options); // update fileset using filesetData - restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, transaction); + await restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, transaction); + transaction = restoredb.BeginTransaction(); // clear any existing fileset entries - restoredb.ClearFilesetEntries(filesetid, transaction); + await restoredb.ClearFilesetEntries(filesetid, transaction); using (var filelistreader = new FilesetVolumeReader(compressor, options)) foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) @@ -592,37 +599,37 @@ namespace Duplicati.Library.Main.Operation var metadataid = long.MinValue; var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path); - var prefixid = restoredb.GetOrCreatePathPrefix(split.Key, transaction); + var prefixid = await restoredb.GetOrCreatePathPrefix(split.Key, transaction); switch (fe.Type) { case FilelistEntryType.Folder: - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); + metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); + await restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); break; case FilelistEntryType.File: var expectedblocks = (fe.Size + blocksize - 1) / blocksize; var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block; if (expectedblocks <= 1) expectedblocklisthashes = 0; - var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, transaction); - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, transaction); + var blocksetid = await restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, transaction); + metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); + await restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, transaction); if (fe.Size <= blocksize) { if (!string.IsNullOrWhiteSpace(fe.Blockhash)) - restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, transaction); + await restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, transaction); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) - restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, transaction); + await restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, transaction); else if (fe.Size > 0) Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); } break; case FilelistEntryType.Symlink: - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); + metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); + await restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); break; default: Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path); @@ -632,9 +639,9 @@ namespace Duplicati.Library.Main.Operation if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink)) { if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) - restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, transaction); + await restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, transaction); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) - restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, transaction); + await restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, transaction); else Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path); } @@ -644,6 +651,8 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "FileEntryProcessingFailed", ex, "Failed to process file-entry: {0}", fe.Path); } } + + await transaction.CommitAsync(); } /// @@ -652,7 +661,7 @@ namespace Duplicati.Library.Main.Operation /// The volume id of the item /// The filename read and written /// The database to query - public long ProbeForMatchingFilename(ref string filename, LocalRestoreDatabase restoredb) + public async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb) { var p = VolumeBase.ParseFilename(filename); if (p != null) @@ -661,17 +670,17 @@ namespace Duplicati.Library.Main.Operation foreach (var encmodule in Library.DynamicLoader.EncryptionLoader.Keys.Union(new string[] { "" })) { var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule); - var tvid = restoredb.GetRemoteVolumeID(testfilename); + var tvid = await restoredb.GetRemoteVolumeID(testfilename); if (tvid >= 0) { Logging.Log.WriteWarningMessage(LOGTAG, "RewritingFilenameMapping", null, "Unable to find volume {0}, but mapping to matching file {1}", filename, testfilename); filename = testfilename; - return tvid; + return (tvid, filename); } } } - return -1; + return (-1, filename); } public void Dispose() From 45902c2c2dbc185534831c64efb834563a928d85 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 09:01:32 +0200 Subject: [PATCH 017/362] Removed the BeginTransaction wrapper function from LocalDatabase --- .../Library/Main/Operation/RecreateDatabaseHandler.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 32ae9fa74..8192096b6 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -198,7 +198,7 @@ namespace Duplicati.Library.Main.Operation var hasUpdatedOptions = false; //Record all blocksets and files needed - using (var tr = restoredb.BeginTransaction()) + using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) { var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count); @@ -301,7 +301,7 @@ namespace Duplicati.Library.Main.Operation //Grab all index files, and update the block table using (var hashalg = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - using (var tr = restoredb.BeginTransaction()) + using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) { hashsize = hashalg.HashSize / 8; @@ -460,7 +460,7 @@ namespace Duplicati.Library.Main.Operation { using (tmpfile) using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - using (var tr = restoredb.BeginTransaction()) + using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { @@ -583,7 +583,7 @@ namespace Duplicati.Library.Main.Operation // update fileset using filesetData await restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, transaction); - transaction = restoredb.BeginTransaction(); + transaction = restoredb.Connection.BeginTransaction(deferred: true); // clear any existing fileset entries await restoredb.ClearFilesetEntries(filesetid, transaction); From fb2c9d9b9a569bc6ef27c0bd1689a10ea9490cf0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 09:04:51 +0200 Subject: [PATCH 018/362] Simplification and syntactic changes --- .../Main/Operation/RecreateDatabaseHandler.cs | 773 +++++++++--------- 1 file changed, 384 insertions(+), 389 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 8192096b6..42e66abeb 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -64,13 +64,12 @@ namespace Duplicati.Library.Main.Operation if (File.Exists(path)) throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); - using (var db = new LocalDatabase(path, "Recreate", true, m_options.SqlitePageCache)) - { - await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); + using var db = new LocalDatabase(path, "Recreate", true, m_options.SqlitePageCache); - // Ensure database is consistent after the recreate - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); - } + await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); + + // Ensure database is consistent after the recreate + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); } /// @@ -84,31 +83,29 @@ namespace Duplicati.Library.Main.Operation if (!m_options.RepairOnlyPaths) throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); - using (var db = new LocalDatabase(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache)) + using var db = new LocalDatabase(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache); + if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version)).Any()) { - if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version)).Any()) + if (m_options.IgnoreUpdateIfVersionExists) { - if (m_options.IgnoreUpdateIfVersionExists) - { - Logging.Log.WriteInformationMessage(LOGTAG, "UpdateVersionAlreadyExists", "The version(s) being updated to already exists, ignoring update request"); - return; - } - - throw new UserInformationException("The version(s) being updated to, already exists", "UpdateVersionAlreadyExists"); + Logging.Log.WriteInformationMessage(LOGTAG, "UpdateVersionAlreadyExists", "The version(s) being updated to already exists, ignoring update request"); + return; } - // Mark as incomplete - db.PartiallyRecreated = true; - - var preexistingOptionsInDatabase = Utility.ContainsOptionsForVerification(db); - Utility.UpdateOptionsFromDb(db, m_options, null); - - // Make sure the options have not changed between calls, unless there are no previous options - if (preexistingOptionsInDatabase) - Utility.VerifyOptionsAndUpdateDatabase(db, m_options, null); - - await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); + throw new UserInformationException("The version(s) being updated to, already exists", "UpdateVersionAlreadyExists"); } + + // Mark as incomplete + db.PartiallyRecreated = true; + + var preexistingOptionsInDatabase = Utility.ContainsOptionsForVerification(db); + Utility.UpdateOptionsFromDb(db, m_options, null); + + // Make sure the options have not changed between calls, unless there are no previous options + if (preexistingOptionsInDatabase) + Utility.VerifyOptionsAndUpdateDatabase(db, m_options, null); + + await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); } /// @@ -124,99 +121,197 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running); //We build a local database in steps. - using (var restoredb = new LocalRecreateDatabase(dbparent, m_options)) + using var restoredb = new LocalRecreateDatabase(dbparent, m_options); + restoredb.RepairInProgress = true; + var expRecreateDb = false; // experimental recreate db code flag + var volumeIds = new Dictionary(); + + if (string.Equals(Environment.GetEnvironmentVariable("EXPERIMENTAL_RECREATEDB_DUPLICATI") ?? string.Empty, "1")) { - restoredb.RepairInProgress = true; - var expRecreateDb = false; // experimental recreate db code flag - var volumeIds = new Dictionary(); + expRecreateDb = true; + } - if (string.Equals(Environment.GetEnvironmentVariable("EXPERIMENTAL_RECREATEDB_DUPLICATI") ?? string.Empty, "1")) + var rawlist = await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + //First step is to examine the remote storage to see what + // kind of data we can find + var remotefiles = + (from x in rawlist + let n = VolumeBase.ParseFilename(x) + where + n != null + && + n.Prefix == m_options.Prefix + select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times + + if (remotefiles.Length == 0) + { + if (!rawlist.Any()) + throw new UserInformationException("No files were found at the remote location, perhaps the target url is incorrect?", "EmptyRemoteLocation"); + else { - expRecreateDb = true; - } - - var rawlist = await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); - - //First step is to examine the remote storage to see what - // kind of data we can find - var remotefiles = + var tmp = (from x in rawlist let n = VolumeBase.ParseFilename(x) where n != null - && - n.Prefix == m_options.Prefix - select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times + select n.Prefix).ToArray(); - if (remotefiles.Length == 0) - { - if (rawlist.Count() == 0) - throw new UserInformationException("No files were found at the remote location, perhaps the target url is incorrect?", "EmptyRemoteLocation"); + var types = tmp.Distinct().ToArray(); + if (tmp.Length == 0) + throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count()), "EmptyRemoteLocation"); + else if (types.Length == 1) + throw new UserInformationException(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup prefix?", tmp.Length, types[0]), "EmptyRemoteLocationWithPrefix"); else - { - var tmp = - (from x in rawlist - let n = VolumeBase.ParseFilename(x) - where - n != null - select n.Prefix).ToArray(); + throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count(), string.Join(", ", types)), "EmptyRemoteLocationWithPrefix"); + } + } - var types = tmp.Distinct().ToArray(); - if (tmp.Length == 0) - throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count()), "EmptyRemoteLocation"); - else if (types.Length == 1) - throw new UserInformationException(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup prefix?", tmp.Length, types[0]), "EmptyRemoteLocationWithPrefix"); + if (string.IsNullOrWhiteSpace(m_options.Passphrase) && remotefiles.Any(x => !string.IsNullOrWhiteSpace(x.EncryptionModule))) + throw new UserInformationException("The remote files are encrypted, but no passphrase was provided", "MissingPassphrase"); + + //Then we select the filelist we should work with, + // and create the filelist table to fit + IEnumerable filelists = + from n in remotefiles + where n.FileType == RemoteVolumeType.Files + orderby n.Time descending + select n; + + if (!filelists.Any()) + throw new UserInformationException("No filelists found on the remote destination", "EmptyRemoteLocation"); + + if (filelistfilter != null) + filelists = filelistfilter(filelists).Select(x => x.Value).ToArray(); + + if (!filelists.Any()) + throw new UserInformationException("No filelists", "NoMatchingRemoteFilelists"); + + // If we are updating, all files should be accounted for + foreach (var fl in remotefiles) + volumeIds[fl.File.Name] = updating ? await restoredb.GetRemoteVolumeID(fl.File.Name) : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded); + + var hasUpdatedOptions = false; + + //Record all blocksets and files needed + using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) + { + var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); + Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count); + + var progress = 0; + + // Register the files we are working with, if not already updated + if (updating) + { + foreach (var n in filelists) + if (volumeIds[n.File.Name] == -1) + volumeIds[n.File.Name] = await restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), tr); + } + + var isFirstFilelist = true; + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + var entry = new RemoteVolume(name, hash, size); + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + m_result.EndTime = DateTime.UtcNow; + return; + } + + progress++; + if (filelistWork.Count == 1 && m_options.RepairOnlyPaths) + { + m_result.OperationProgressUpdater.UpdateProgress(0.5f); + } else - throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count(), string.Join(", ", types)), "EmptyRemoteLocationWithPrefix"); + { + m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f)); + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count); + } + + using (tmpfile) + { + isFirstFilelist = false; + + if (!string.IsNullOrWhiteSpace(hash) && size > 0) + await restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, tr); + + var parsed = VolumeBase.ParseFilename(entry.Name); + + using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + + if (!hasUpdatedOptions) + { + VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options); + hasUpdatedOptions = true; + } + + // Create timestamped operations based on the file timestamp + var filesetid = await restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); + + await RecreateFilesetFromRemoteList(restoredb, tr, compressor, filesetid, m_options, filter); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FileProcessingFailed", ex, "Failed to process file: {0}", entry.Name); + if (ex.IsAbortException()) + { + m_result.EndTime = DateTime.UtcNow; + throw; + } + + if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException) + { + m_result.EndTime = DateTime.UtcNow; + throw; + } + + if (m_options.UnittestMode) + throw; + } } - if (string.IsNullOrWhiteSpace(m_options.Passphrase) && remotefiles.Any(x => !string.IsNullOrWhiteSpace(x.EncryptionModule))) - throw new UserInformationException("The remote files are encrypted, but no passphrase was provided", "MissingPassphrase"); + //Make sure we write the config if it has been read from a manifest + if (hasUpdatedOptions) + Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, tr); - //Then we select the filelist we should work with, - // and create the filelist table to fit - IEnumerable filelists = - from n in remotefiles - where n.FileType == RemoteVolumeType.Files - orderby n.Time descending - select n; + using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) + if (tr.Connection != null) + await tr.CommitAsync(); + } - if (!filelists.Any()) - throw new UserInformationException("No filelists found on the remote destination", "EmptyRemoteLocation"); + // do we stop after just handling the dlist files ? + // (if yes, we never will be able to do a backup !) + if (!m_options.RepairOnlyPaths) + { + var hashsize = 0; + //Grab all index files, and update the block table - if (filelistfilter != null) - filelists = filelistfilter(filelists).Select(x => x.Value).ToArray(); - - if (!filelists.Any()) - throw new UserInformationException("No filelists", "NoMatchingRemoteFilelists"); - - // If we are updating, all files should be accounted for - foreach (var fl in remotefiles) - volumeIds[fl.File.Name] = updating ? await restoredb.GetRemoteVolumeID(fl.File.Name) : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded); - - var hasUpdatedOptions = false; - - //Record all blocksets and files needed + using (var hashalg = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) { - var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); - Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count); + hashsize = hashalg.HashSize / 8; + + var indexfiles = ( + from n in remotefiles + where n.FileType == RemoteVolumeType.Index + select new RemoteVolume(n.File) as IRemoteVolume).ToList(); + + Logging.Log.WriteInformationMessage(LOGTAG, "FilelistsRestored", "Filelists restored, downloading {0} index files", indexfiles.Count); var progress = 0; - // Register the files we are working with, if not already updated - if (updating) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - foreach (var n in filelists) - if (volumeIds[n.File.Name] == -1) - volumeIds[n.File.Name] = await restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), tr); - } - - var isFirstFilelist = true; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - { - var entry = new RemoteVolume(name, hash, size); try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) @@ -227,96 +322,141 @@ namespace Duplicati.Library.Main.Operation } progress++; - if (filelistWork.Count == 1 && m_options.RepairOnlyPaths) - { - m_result.OperationProgressUpdater.UpdateProgress(0.5f); - } - else - { - m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f)); - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count); - } + m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f); + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingIndexlistVolumes", "Processing indexlist volume {0} of {1}", progress, indexfiles.Count); using (tmpfile) { - isFirstFilelist = false; - if (!string.IsNullOrWhiteSpace(hash) && size > 0) - await restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, tr); + await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, tr); - var parsed = VolumeBase.ParseFilename(entry.Name); - - using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); - using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - - if (!hasUpdatedOptions) + using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) { - VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options); - hasUpdatedOptions = true; + foreach (var a in svr.Volumes) + { + var filename = a.Filename; + var volumeID = await restoredb.GetRemoteVolumeID(filename, tr); + + // No such file + if (volumeID < 0) + (volumeID, filename) = await ProbeForMatchingFilename(filename, restoredb); + + var missing = false; + // Still broken, register a missing item + if (volumeID < 0) + { + var p = VolumeBase.ParseFilename(filename); + if (p == null) + throw new Exception(string.Format("Unable to parse filename: {0}", filename)); + Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); + missing = true; + volumeID = await restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, tr); + } + + bool anyChange = false; + //Add all block/volume mappings + foreach (var b in a.Blocks) + { + anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr)).Item1; + } + + await restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, tr); + await restoredb.AddIndexBlockLink(await restoredb.GetRemoteVolumeID(name, tr), volumeID, tr); + } + + //If there are blocklists in the index file, add them to the temp blocklist hashes table + int wrongHashes = 0; + foreach (var b in svr.BlockLists) + { + // Compact might have created undetected invalid blocklist entries in index files due to broken LocalDatabase.GetBlocklists + // If the hash is wrong, recreate will download the dblock volume with the correct file + try + { + // We need to instantiate the list to ensure the verification is + // done before we add it to the database, since we do not have nested transactions + var list = b.Blocklist.ToList(); + await restoredb.AddTempBlockListHash(b.Hash, list, tr); + } + catch (System.IO.InvalidDataException e) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", name); + ++wrongHashes; + } + } + if (wrongHashes != 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", name); + } } - - // Create timestamped operations based on the file timestamp - var filesetid = await restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); - - await RecreateFilesetFromRemoteList(restoredb, tr, compressor, filesetid, m_options, filter); } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(LOGTAG, "FileProcessingFailed", ex, "Failed to process file: {0}", entry.Name); + //Not fatal + Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", name); if (ex.IsAbortException()) { m_result.EndTime = DateTime.UtcNow; throw; } - if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException) - { - m_result.EndTime = DateTime.UtcNow; - throw; - } - if (m_options.UnittestMode) throw; - } } - //Make sure we write the config if it has been read from a manifest - if (hasUpdatedOptions) - Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, tr); + using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) + tr.Commit(); - using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) - if (tr.Connection != null) - await tr.CommitAsync(); + // TODO: In some cases, we can avoid downloading all index files, + // if we are lucky and pick the right ones } - // do we stop after just handling the dlist files ? - // (if yes, we never will be able to do a backup !) - if (!m_options.RepairOnlyPaths) + await restoredb.CleanupMissingVolumes(); + + // Update the real tables from the temp tables + if (expRecreateDb) + // add missing blocks and blocksetentry data (at this point + // we have not yet anything in the blocksetentry table) + await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, null); + else + await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, null); + + // We have now grabbed as much information as possible, + // if we are still missing data, we must now fetch block files + //We do this in three passes + for (var i = 0; i < 3; i++) { - var hashsize = 0; - //Grab all index files, and update the block table - - using (var hashalg = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) + // Grab the list matching the pass type + var lst = await restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse).ToListAsync(); + if (lst.Count > 0) { - hashsize = hashalg.HashSize / 8; - - var indexfiles = ( - from n in remotefiles - where n.FileType == RemoteVolumeType.Index - select new RemoteVolume(n.File) as IRemoteVolume).ToList(); - - Logging.Log.WriteInformationMessage(LOGTAG, "FilelistsRestored", "Filelists restored, downloading {0} index files", indexfiles.Count); - - var progress = 0; - - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + var fullist = ": " + string.Join(", ", lst.Select(x => x.Name)); + switch (i) { - try + case 0: + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, fullist); + Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); + break; + case 1: + Logging.Log.WriteVerboseMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, fullist); + Logging.Log.WriteInformationMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); + break; + default: + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, fullist); + Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); + break; + } + } + + var progress = 0; + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + try + { + using (tmpfile) + using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) + using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { @@ -326,251 +466,106 @@ namespace Duplicati.Library.Main.Operation } progress++; - m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f); - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingIndexlistVolumes", "Processing indexlist volume {0} of {1}", progress, indexfiles.Count); + m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f)); + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); - using (tmpfile) + var volumeid = await restoredb.GetRemoteVolumeID(name); + + await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, tr); + + bool anyChange = false; + // Update the block table so we know about the block/volume map + foreach (var h in rd.Blocks) { - if (!string.IsNullOrWhiteSpace(hash) && size > 0) - await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, tr); + anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr)).Item1; + } - using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) + // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes + // to the needed list blocks in the volume, so grab them from the database + // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table + await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) + { + if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), tr)) { - foreach (var a in svr.Volumes) - { - var filename = a.Filename; - var volumeID = await restoredb.GetRemoteVolumeID(filename, tr); - - // No such file - if (volumeID < 0) - (volumeID, filename) = await ProbeForMatchingFilename(filename, restoredb); - - var missing = false; - // Still broken, register a missing item - if (volumeID < 0) - { - var p = VolumeBase.ParseFilename(filename); - if (p == null) - throw new Exception(string.Format("Unable to parse filename: {0}", filename)); - Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); - missing = true; - volumeID = await restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, tr); - } - - bool anyChange = false; - //Add all block/volume mappings - foreach (var b in a.Blocks) - { - anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr)).Item1; - } - - await restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, tr); - await restoredb.AddIndexBlockLink(await restoredb.GetRemoteVolumeID(name, tr), volumeID, tr); - } - - //If there are blocklists in the index file, add them to the temp blocklist hashes table - int wrongHashes = 0; - foreach (var b in svr.BlockLists) - { - // Compact might have created undetected invalid blocklist entries in index files due to broken LocalDatabase.GetBlocklists - // If the hash is wrong, recreate will download the dblock volume with the correct file - try - { - // We need to instantiate the list to ensure the verification is - // done before we add it to the database, since we do not have nested transactions - var list = b.Blocklist.ToList(); - await restoredb.AddTempBlockListHash(b.Hash, list, tr); - } - catch (System.IO.InvalidDataException e) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", name); - ++wrongHashes; - } - } - if (wrongHashes != 0) - { - Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", name); - } + anyChange = true; } } - } - catch (Exception ex) - { - //Not fatal - Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", name); - if (ex.IsAbortException()) + + // Update tables if necessary (if no block or hash have been changed by a data volume + // there is no need to run expensive queries - most data volumes have been + // managed successfully by correct index volumes), so we know if we are done + // if any change, add to the block and blocksetentry tables the references found in + // the block lists of the volume saved in the temp blocklisthash table by AddTempBLockListHash + if (anyChange) { - m_result.EndTime = DateTime.UtcNow; - throw; + if (i == 2) + { + Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name); + } + if (expRecreateDb) + await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, tr, false); + else + await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, tr); } - if (m_options.UnittestMode) - throw; + using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) + tr.Commit(); + + //At this point we can patch files with data from the block volume + if (blockprocessor != null) + blockprocessor(name, rd); } } - - using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) - tr.Commit(); - - // TODO: In some cases, we can avoid downloading all index files, - // if we are lucky and pick the right ones - } - - await restoredb.CleanupMissingVolumes(); - - // Update the real tables from the temp tables - if (expRecreateDb) - // add missing blocks and blocksetentry data (at this point - // we have not yet anything in the blocksetentry table) - await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, null); - else - await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, null); - - // We have now grabbed as much information as possible, - // if we are still missing data, we must now fetch block files - //We do this in three passes - for (var i = 0; i < 3; i++) - { - // Grab the list matching the pass type - var lst = await restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse).ToListAsync(); - if (lst.Count > 0) + catch (Exception ex) { - var fullist = ": " + string.Join(", ", lst.Select(x => x.Name)); - switch (i) - { - case 0: - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, fullist); - Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); - break; - case 1: - Logging.Log.WriteVerboseMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, fullist); - Logging.Log.WriteInformationMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); - break; - default: - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, fullist); - Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); - break; - } - } - - var progress = 0; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - { - try - { - using (tmpfile) - using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - m_result.EndTime = DateTime.UtcNow; - return; - } - - progress++; - m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f)); - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); - - var volumeid = await restoredb.GetRemoteVolumeID(name); - - await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, tr); - - bool anyChange = false; - // Update the block table so we know about the block/volume map - foreach (var h in rd.Blocks) - { - anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr)).Item1; - } - - // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes - // to the needed list blocks in the volume, so grab them from the database - // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table - await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) - { - if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), tr)) - { - anyChange = true; - } - } - - // Update tables if necessary (if no block or hash have been changed by a data volume - // there is no need to run expensive queries - most data volumes have been - // managed successfully by correct index volumes), so we know if we are done - // if any change, add to the block and blocksetentry tables the references found in - // the block lists of the volume saved in the temp blocklisthash table by AddTempBLockListHash - if (anyChange) - { - if (i == 2) - { - Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name); - } - if (expRecreateDb) - await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, tr, false); - else - await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, tr); - } - - using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) - tr.Commit(); - - //At this point we can patch files with data from the block volume - if (blockprocessor != null) - blockprocessor(name, rd); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message); - if (m_options.UnittestMode) - throw; - } + Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message); + if (m_options.UnittestMode) + throw; } } } - - await backendManager.WaitForEmptyAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - - if (!m_options.RepairOnlyPaths) - { - // All blocks are collected and added into the Block table - // Find out which blocks are deleted and move them into DeletedBlock, - // so that compact can calculate the unused space - await restoredb.CleanupDeletedBlocks(null); - } - - await restoredb.CleanupMissingVolumes(); - - if (m_options.RepairOnlyPaths) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RecreateOrUpdateOnly", "Recreate/path-update completed, not running consistency checks"); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompletedCheckingDatabase", "Recreate completed, verifying the database consistency"); - - //All done, we must verify that we have all blocklist fully intact - // if this fails, the db will not be deleted, so it can be used, - // except to continue a backup - m_result.EndTime = DateTime.UtcNow; - - using (var lbfdb = new LocalListBrokenFilesDatabase(restoredb)) - { - var broken = lbfdb.GetBrokenFilesets(new DateTime(0), null, null).Count(); - if (broken != 0) - throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"), "DatabaseIsBrokenConsiderPurge"); - } - - await restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); - - Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); - - restoredb.RepairInProgress = false; - } - - m_result.EndTime = DateTime.UtcNow; } + + await backendManager.WaitForEmptyAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + if (!m_options.RepairOnlyPaths) + { + // All blocks are collected and added into the Block table + // Find out which blocks are deleted and move them into DeletedBlock, + // so that compact can calculate the unused space + await restoredb.CleanupDeletedBlocks(null); + } + + await restoredb.CleanupMissingVolumes(); + + if (m_options.RepairOnlyPaths) + { + Logging.Log.WriteInformationMessage(LOGTAG, "RecreateOrUpdateOnly", "Recreate/path-update completed, not running consistency checks"); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompletedCheckingDatabase", "Recreate completed, verifying the database consistency"); + + //All done, we must verify that we have all blocklist fully intact + // if this fails, the db will not be deleted, so it can be used, + // except to continue a backup + m_result.EndTime = DateTime.UtcNow; + + using (var lbfdb = new LocalListBrokenFilesDatabase(restoredb)) + { + var broken = lbfdb.GetBrokenFilesets(new DateTime(0), null, null).Count(); + if (broken != 0) + throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"), "DatabaseIsBrokenConsiderPurge"); + } + + await restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); + + Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); + + restoredb.RepairInProgress = false; + } + + m_result.EndTime = DateTime.UtcNow; } public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, SqliteTransaction transaction, ICompression compressor, long filesetid, Options options, IFilter filter) @@ -661,13 +656,13 @@ namespace Duplicati.Library.Main.Operation /// The volume id of the item /// The filename read and written /// The database to query - public async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb) + public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb) { var p = VolumeBase.ParseFilename(filename); if (p != null) { foreach (var compmodule in Library.DynamicLoader.CompressionLoader.Keys) - foreach (var encmodule in Library.DynamicLoader.EncryptionLoader.Keys.Union(new string[] { "" })) + foreach (var encmodule in Library.DynamicLoader.EncryptionLoader.Keys.Union([""])) { var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule); var tvid = await restoredb.GetRemoteVolumeID(testfilename); From 922d1dd01275e120bbe347510c340a9bcd83fcd4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 09:05:32 +0200 Subject: [PATCH 019/362] Removed whitespace --- .../Library/Main/Operation/RepairHandler.cs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 61b5d6ab5..5500e7139 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -1,23 +1,24 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System; using System.Collections.Generic; using System.Data; From a2757281befed1e7d02234d6ccf4274fb95308af Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 11:25:17 +0200 Subject: [PATCH 020/362] Updated LocalRepairDatabase to async using Microsoft.Data.Sqlite --- .../Main/Database/LocalRepairDatabase.cs | 635 ++++++++++-------- 1 file changed, 341 insertions(+), 294 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 5e1aaf437..352b0c322 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1,30 +1,32 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.Data; +//using System.Data; using System.Collections.Generic; +using System.Threading.Tasks; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -52,15 +54,16 @@ namespace Duplicati.Library.Main.Database /// The remote name of the fileset /// An optional transaction /// >The fileset ID - public (long FilesetId, DateTime Time, bool IsFullBackup) GetFilesetFromRemotename(string filelist, IDbTransaction? transaction) + public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist, SqliteTransaction? transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand()) { - var rd = cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name") - .SetParameterValue("@Name", filelist) - .ExecuteReader(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name"); + cmd.SetParameterValue("@Name", filelist); + var rd = await cmd.ExecuteReaderAsync(); - if (!rd.Read()) + if (!await rd.ReadAsync()) throw new Exception($"No such remote file: {filelist}"); return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); @@ -73,7 +76,7 @@ namespace Duplicati.Library.Main.Database /// Current fileset ID /// Source fileset ID /// The transaction - public void MoveFilesFromFileset(long filesetid, long prevFilesetId, IDbTransaction transaction) + public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId, SqliteTransaction transaction) { if (filesetid <= 0) throw new ArgumentException("filesetid must be > 0"); @@ -81,11 +84,12 @@ namespace Duplicati.Library.Main.Database if (prevFilesetId <= 0) throw new ArgumentException("prevId must be > 0"); - using var cmd = m_connection.CreateCommand(transaction); - cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId ") - .SetParameterValue("@CurrentFilesetId", filesetid) - .SetParameterValue("@PreviousFilesetId", prevFilesetId) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(); + cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId "); + cmd.Transaction = transaction; + cmd.SetParameterValue("@CurrentFilesetId", filesetid); + cmd.SetParameterValue("@PreviousFilesetId", prevFilesetId); + await cmd.ExecuteNonQueryAsync(); } /// @@ -93,13 +97,14 @@ namespace Duplicati.Library.Main.Database /// /// The block file ID /// >A list of index file IDs - public IEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, IDbTransaction? transaction) + public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(transaction, @"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type") - .SetParameterValue("@BlockFileId", blockfileid) - .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); + using var cmd = m_connection.CreateCommand(@"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@BlockFileId", blockfileid); + cmd.SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); } @@ -108,17 +113,18 @@ namespace Duplicati.Library.Main.Database /// /// An optional transaction /// A list of fileset IDs and timestamps - public IEnumerable> GetFilesetsWithMissingFiles(IDbTransaction transaction) + public async IAsyncEnumerable> GetFilesetsWithMissingFiles(SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(@"SELECT ID, Timestamp FROM Fileset WHERE ID IN (SELECT FilesetID FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup""))")) - while (rd.Read()) - { - yield return new KeyValuePair( - rd.ConvertValueToInt64(0), - ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() - ); - } + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + using var rd = await cmd.ExecuteReaderAsync(@"SELECT ID, Timestamp FROM Fileset WHERE ID IN (SELECT FilesetID FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup""))"); + while (await rd.ReadAsync()) + { + yield return new KeyValuePair( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() + ); + } } /// @@ -127,12 +133,13 @@ namespace Duplicati.Library.Main.Database /// The fileset ID /// An optional transaction /// The number of deleted entries - public int DeleteFilesetEntries(long filesetid, IDbTransaction? transaction = null) + public async Task DeleteFilesetEntries(long filesetid, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - return cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetid) - .ExecuteNonQuery(); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId"); + cmd.SetParameterValue("@FilesetId", filesetid); + return await cmd.ExecuteNonQueryAsync(); } private class RemoteVolume : IRemoteVolume @@ -149,12 +156,13 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable GetBlockVolumesFromIndexName(string indexName) + public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) ") - .SetParameterValue("@Name", indexName); + using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) "); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Name", indexName); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } @@ -204,39 +212,39 @@ namespace Duplicati.Library.Main.Database /// The block size /// The volume ID of the new target volume /// The restored blocks - bool SetBlockRestored(string hash, long size, long volumeId); + Task SetBlockRestored(string hash, long size, long volumeId); /// /// Gets the list of files that contains missing blocks /// /// The blocksize setting /// A list of files with missing blocks - IEnumerable GetSourceFilesWithBlocks(long blocksize); + IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize); /// /// Gets a list for filesystem entries that contain missing blocks in metadata /// /// A list of filesystem entries with missing blocks - IEnumerable GetSourceItemsWithMetadataBlocks(); + IAsyncEnumerable GetSourceItemsWithMetadataBlocks(); /// /// Gets missing blocklist hashes /// /// The number of hashes for each block /// >A list of blocklist hashes - IEnumerable GetBlocklistHashes(long hashesPerBlock); + IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock); /// /// Gets the number of missing blocks /// /// >The number of missing blocks - long GetMissingBlockCount(); + Task GetMissingBlockCount(); /// /// Gets all the filesets that are affected by missing blocks /// /// >A list of filesets - IEnumerable GetFilesetsUsingMissingBlocks(); + IAsyncEnumerable GetFilesetsUsingMissingBlocks(); /// /// Gets a list of remote files that may contain missing blocks /// /// >A list of remote files - IEnumerable GetMissingBlockSources(); + IAsyncEnumerable GetMissingBlockSources(); } /// @@ -247,7 +255,7 @@ namespace Duplicati.Library.Main.Database /// /// The connection to the database /// - private readonly IDbConnection m_connection; + private readonly SqliteConnection m_connection; /// /// The transaction to use /// @@ -255,15 +263,15 @@ namespace Duplicati.Library.Main.Database /// /// The insert command to use for restoring blocks /// - private readonly IDbCommand m_insertCommand; + private readonly SqliteCommand m_insertCommand; /// /// The command to copy blocks into the duplicate block table /// - private readonly IDbCommand m_copyIntoDuplicatedBlocks; + private readonly SqliteCommand m_copyIntoDuplicatedBlocks; /// /// The command to assign blocks to a new volume /// - private readonly IDbCommand m_assignBlocksToNewVolume; + private readonly SqliteCommand m_assignBlocksToNewVolume; /// /// The name of the temporary table /// @@ -288,25 +296,26 @@ namespace Duplicati.Library.Main.Database /// The name of the volume with missing blocks /// The connection to the database /// The transaction to use - public MissingBlockList(string volumename, IDbConnection connection, ReusableTransaction transaction) + public MissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) { m_connection = connection; m_transaction = transaction; m_volumename = volumename; var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand(m_transaction.Transaction)) + using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ")); + cmd.Transaction = transaction.Transaction; + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ")).Await(); m_tablename = tablename; - var blockCount = cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = @Name ")) - .SetParameterValue("@Name", volumename) - .ExecuteNonQuery(); + cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = @Name ")); + cmd.SetParameterValue("@Name", volumename); + var blockCount = cmd.ExecuteNonQueryAsync().Await(); if (blockCount == 0) throw new Exception($"Unexpected empty block volume: {0}"); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")")); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")")).Await(); } m_insertCommand = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue ")); @@ -316,26 +325,28 @@ namespace Duplicati.Library.Main.Database } /// - public bool SetBlockRestored(string hash, long size, long targetVolumeId) + public async Task SetBlockRestored(string hash, long size, long targetVolumeId) { - var restored = m_insertCommand.SetTransaction(m_transaction.Transaction).SetParameterValue("@NewRestoredValue", 1) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .SetParameterValue("@PreviousRestoredValue", 0) - .ExecuteNonQuery() == 1; + //m_insertCommand.SetTransaction(m_transaction.Transaction); + m_insertCommand.Transaction = m_transaction.Transaction; + m_insertCommand.SetParameterValue("@NewRestoredValue", 1); + m_insertCommand.SetParameterValue("@Hash", hash); + m_insertCommand.SetParameterValue("@Size", size); + m_insertCommand.SetParameterValue("@PreviousRestoredValue", 0); + var restored = await m_insertCommand.ExecuteNonQueryAsync() == 1; if (restored) { - m_copyIntoDuplicatedBlocks.SetTransaction(m_transaction.Transaction) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + m_copyIntoDuplicatedBlocks.Transaction = m_transaction.Transaction; + m_copyIntoDuplicatedBlocks.SetParameterValue("@Hash", hash); + m_copyIntoDuplicatedBlocks.SetParameterValue("@Size", size); + await m_copyIntoDuplicatedBlocks.ExecuteNonQueryAsync(); - var c = m_assignBlocksToNewVolume.SetTransaction(m_transaction.Transaction) - .SetParameterValue("@TargetVolumeId", targetVolumeId) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + m_assignBlocksToNewVolume.Transaction = m_transaction.Transaction; + m_assignBlocksToNewVolume.SetParameterValue("@TargetVolumeId", targetVolumeId); + m_assignBlocksToNewVolume.SetParameterValue("@Hash", hash); + m_assignBlocksToNewVolume.SetParameterValue("@Size", size); + var c = await m_assignBlocksToNewVolume.ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Unexpected number of updated blocks: {c} != 1"); @@ -345,12 +356,13 @@ namespace Duplicati.Library.Main.Database } /// - public IEnumerable GetSourceFilesWithBlocks(long blocksize) + public async IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize) { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}")) - .SetParameterValue("@Restored", 0); + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}")); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); var size = rd.ConvertValueToInt64(1); @@ -362,12 +374,13 @@ namespace Duplicati.Library.Main.Database } /// - public IEnumerable GetSourceItemsWithMetadataBlocks() + public async IAsyncEnumerable GetSourceItemsWithMetadataBlocks() { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path""")) - .SetParameterValue("@Restored", 0); + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path""")); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); var size = rd.ConvertValueToInt64(1); @@ -378,19 +391,20 @@ namespace Duplicati.Library.Main.Database } /// - public IEnumerable GetBlocklistHashes(long hashesPerBlock) + public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock) { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_transaction.Transaction; var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; try { // We need to create a snapshot as we will be updating the m_tablename table during enumeration cmd.SetCommandAndParameters(FormatInvariant($@" CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS - SELECT - b.""Hash"" AS ""BlockHash"", + SELECT + b.""Hash"" AS ""BlockHash"", bs.""Id"" AS ""BlocksetId"", - bs.""Length"" AS ""BlocksetLength"", + bs.""Length"" AS ""BlocksetLength"", bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", ( SELECT blh.""Hash"" @@ -400,11 +414,11 @@ namespace Duplicati.Library.Main.Database LIMIT 1 ) AS ""BlocklistHashHash"", bse.""Index"" AS ""BlocksetEntryIndex"" - FROM + FROM ""BlocksetEntry"" bse JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" - WHERE + WHERE EXISTS ( SELECT 1 FROM ""BlocklistHash"" blh @@ -412,23 +426,23 @@ namespace Duplicati.Library.Main.Database WHERE blh.""BlocksetID"" = bs.""ID"" AND mt.""Restored"" = @Restored ) - ")) - .SetParameterValue("@HashesPerBlock", hashesPerBlock) - .SetParameterValue("@Restored", 0) - .ExecuteNonQuery(); + ")); + cmd.SetParameterValue("@HashesPerBlock", hashesPerBlock); + cmd.SetParameterValue("@Restored", 0); + await cmd.ExecuteNonQueryAsync(); cmd.SetCommandAndParameters(FormatInvariant($@" - SELECT - ""BlockHash"", - ""BlocksetId"", - ""BlocksetLength"", - ""BlocklistHashIndex"", - ""BlocklistHashHash"", - ""BlocksetEntryIndex"" - FROM ""{blocklistTableName}"" + SELECT + ""BlockHash"", + ""BlocksetId"", + ""BlocksetLength"", + ""BlocklistHashIndex"", + ""BlocklistHashHash"", + ""BlocksetEntryIndex"" + FROM ""{blocklistTableName}"" ORDER BY ""BlocksetId"", ""BlocklistHashIndex"", ""BlocksetEntryIndex""")); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); var blocksetId = rd.ConvertValueToInt64(1); @@ -442,41 +456,43 @@ namespace Duplicati.Library.Main.Database } finally { - try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); } + try { await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); } catch { } } } /// - public long GetMissingBlockCount() + public async Task GetMissingBlockCount() { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})")) - .SetParameterValue("@Restored", 0); + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})")); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); - return cmd.ExecuteScalarInt64(0); + return await cmd.ExecuteScalarInt64Async(0); } /// /// Gets the list of missing blocks /// /// >A list of missing blocks - public IEnumerable<(string Hash, long Size)> GetMissingBlocks() + public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks() { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, m_missingBlocksQuery) - .SetParameterValue("@Restored", 0); + using var cmd = m_connection.CreateCommand(m_missingBlocksQuery); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } /// - public long MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, IDbTransaction? transaction = null) + public async Task MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId) { if (targetVolumeId <= 0) throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); // Move the source blocks into the DuplicateBlock table - using var cmd = m_connection.CreateCommand(transaction, FormatInvariant($@" + using var cmd = m_connection.CreateCommand(FormatInvariant($@" INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT b.""ID"", b.""VolumeID"" FROM ""Block"" b @@ -486,54 +502,56 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tablename}"" WHERE ""Restored"" = @Restored ) - ")) - .SetParameterValue("@SourceVolumeId", sourceVolumeId) - .SetParameterValue("@Restored", 1); + ")); + cmd.SetParameterValue("@SourceVolumeId", sourceVolumeId); + cmd.SetParameterValue("@Restored", 1); - var moved = cmd.ExecuteNonQuery(); + var moved = await cmd.ExecuteNonQueryAsync(); // Then update the blocks table to point to the new volume cmd.SetCommandAndParameters(FormatInvariant($@" - UPDATE ""Block"" - SET ""VolumeID"" = @TargetVolumeId - WHERE ""VolumeID"" = @SourceVolumeId + UPDATE ""Block"" + SET ""VolumeID"" = @TargetVolumeId + WHERE ""VolumeID"" = @SourceVolumeId AND (""Hash"", ""Size"") IN ( - SELECT ""Hash"", ""Size"" + SELECT ""Hash"", ""Size"" FROM ""{m_tablename}"" WHERE ""Restored"" = @Restored ) - ")) - .SetParameterValue("@TargetVolumeId", targetVolumeId) - .SetParameterValue("@SourceVolumeId", sourceVolumeId) - .SetParameterValue("@Restored", 1); + ")); + cmd.SetParameterValue("@TargetVolumeId", targetVolumeId); + cmd.SetParameterValue("@SourceVolumeId", sourceVolumeId); + cmd.SetParameterValue("@Restored", 1); - var updated = cmd.ExecuteNonQuery(); + var updated = await cmd.ExecuteNonQueryAsync(); if (updated != moved) throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); return updated; } - public IEnumerable GetFilesetsUsingMissingBlocks() + public async IAsyncEnumerable GetFilesetsUsingMissingBlocks() { var blocks = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = @Type AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))"); - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, cmdtxt) - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + using var cmd = m_connection.CreateCommand(cmdtxt); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } - public IEnumerable GetMissingBlockSources() + public async IAsyncEnumerable GetMissingBlockSources() { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")) - .SetParameterValue("@Restored", 0) - .SetParameterValue("@Name", m_volumename); + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); + cmd.SetParameterValue("@Name", m_volumename); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } @@ -546,8 +564,12 @@ namespace Duplicati.Library.Main.Database try { if (m_tablename != null) - using (var cmd = m_connection.CreateCommand(m_transaction.Transaction)) - cmd.SetTransaction(m_transaction.Transaction).ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")); + { + + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_transaction.Transaction; + cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")).Await(); + } } catch { } @@ -561,11 +583,12 @@ namespace Duplicati.Library.Main.Database return new MissingBlockList(volumename, m_connection, transaction); } - public void FixDuplicateMetahash() + public async Task FixDuplicateMetahash() { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand()) { + cmd.Transaction = tr; var sql_count = @"SELECT COUNT(*) FROM (" + @" SELECT DISTINCT c1 FROM (" + @@ -573,143 +596,160 @@ namespace Duplicati.Library.Main.Database @")" + @")"; - var x = cmd.ExecuteScalarInt64(sql_count, 0); + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); if (x > 1) { Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashes", "Found duplicate metadatahashes, repairing"); var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; - using (var c2 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand()) { + c2.Transaction = tr; c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""MetadataID"" = @MetadataId WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId)") + @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId AND ""ID"" != @MetadataId"); - using (var rd = cmd.ExecuteReader(sql)) - while (rd.Read()) - c2.SetParameterValue("@MetadataId", rd.GetValue(0)) - .SetParameterValue("@BlocksetId", rd.GetValue(1)) - .ExecuteNonQuery(); + using (var rd = await cmd.ExecuteReaderAsync(sql)) + while (await rd.ReadAsync()) + { + c2.SetParameterValue("@MetadataId", rd.GetValue(0)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(1)); + await c2.ExecuteNonQueryAsync(); + } } sql = FormatInvariant($@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") + SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID"""); - using (var c2 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand()) { + c2.Transaction = tr; c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId)") + FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId")); - foreach (var rd in cmd.ExecuteReaderEnumerable(sql)) - c2.SetParameterValue("@FileId", rd.GetValue(0)) - .SetParameterValue("@Path", rd.GetValue(1)) - .SetParameterValue("@BlocksetId", rd.GetValue(2)) - .SetParameterValue("@MetadataId", rd.GetValue(3)) - .ExecuteNonQuery(); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql)) + { + c2.SetParameterValue("@FileId", rd.GetValue(0)); + c2.SetParameterValue("@Path", rd.GetValue(1)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(2)); + c2.SetParameterValue("@MetadataId", rd.GetValue(3)); + await c2.ExecuteNonQueryAsync(); + } } - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); - x = cmd.ExecuteScalarInt64(sql_count, 0); + x = await cmd.ExecuteScalarInt64Async(sql_count, 0); if (x > 1) throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); - tr.Commit(); + await tr.CommitAsync(); } } } - public void FixDuplicateFileentries() + public async Task FixDuplicateFileentries() { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand()) { + cmd.Transaction = tr; var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; - var x = cmd.ExecuteScalarInt64(sql_count, 0); + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); if (x > 0) { Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") + SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID"""; - using (var c2 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand()) { + c2.Transaction = tr; c2.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetatadataId)" + @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId"); + cmd.SetCommandAndParameters(sql); - foreach (var rd in cmd.SetCommandAndParameters(sql).ExecuteReaderEnumerable()) - c2.SetParameterValue("@FileId", rd.GetValue(0)) - .SetParameterValue("@PrefixId", rd.GetValue(1)) - .SetParameterValue("@Path", rd.GetValue(2)) - .SetParameterValue("@BlocksetId", rd.GetValue(3)) - .SetParameterValue("@MetadataId", rd.GetValue(4)) - .ExecuteNonQuery(); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + { + c2.SetParameterValue("@FileId", rd.GetValue(0)); + c2.SetParameterValue("@PrefixId", rd.GetValue(1)); + c2.SetParameterValue("@Path", rd.GetValue(2)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(3)); + c2.SetParameterValue("@MetadataId", rd.GetValue(4)); + await c2.ExecuteNonQueryAsync(); + } } - x = cmd.ExecuteScalarInt64(sql_count, 0); + x = await cmd.ExecuteScalarInt64Async(sql_count, 0); if (x > 1) throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); - tr.Commit(); + await tr.CommitAsync(); } } } - public void FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize) + public async Task FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize) { var blocklistbuffer = new byte[blocksize]; - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand()) using (var blockhasher = HashFactory.CreateHasher(blockhashalgorithm)) { + cmd.Transaction = tr; var hashsize = blockhasher.HashSize / 8; var sql = FormatInvariant($@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"""); var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; - var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); + var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); if (itemswithnoblocklisthash != 0) { Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); - using (var c2 = m_connection.CreateCommand(tr)) - using (var c3 = m_connection.CreateCommand(tr)) - using (var c4 = m_connection.CreateCommand(tr)) - using (var c5 = m_connection.CreateCommand(tr)) - using (var c6 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand()) + using (var c3 = m_connection.CreateCommand()) + using (var c4 = m_connection.CreateCommand()) + using (var c5 = m_connection.CreateCommand()) + using (var c6 = m_connection.CreateCommand()) { + c2.Transaction = tr; + c3.Transaction = tr; + c4.Transaction = tr; + c5.Transaction = tr; + c6.Transaction = tr; c3.SetCommandAndParameters(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash) "); c4.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); c5.SetCommandAndParameters(@"SELECT ""ID"" FROM ""DeletedBlock"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND (""State"" IN (@State1, @State2)))"); c6.SetCommandAndParameters(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1; DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId;"); - foreach (var e in cmd.ExecuteReaderEnumerable(sql)) + await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql)) { var blocksetid = e.ConvertValueToInt64(0); var ix = 0L; int blocklistoffset = 0; - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId") - .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteNonQuery(); + c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId"); + c2.SetParameterValue("@BlocksetId", blocksetid); + await c2.ExecuteNonQueryAsync(); - c2.SetCommandAndParameters(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = @BlocksetId ORDER BY ""B"".""Index""") - .SetParameterValue("@BlocksetId", blocksetid); + c2.SetCommandAndParameters(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = @BlocksetId ORDER BY ""B"".""Index"""); + c2.SetParameterValue("@BlocksetId", blocksetid); - foreach (var h in c2.ExecuteReaderEnumerable()) + await foreach (var h in c2.ExecuteReaderEnumerableAsync()) { var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); if (blocklistbuffer.Length - blocklistoffset < tmp.Length) @@ -717,25 +757,25 @@ namespace Duplicati.Library.Main.Database var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); // Check if the block exists in "blocks" - var existingBlockId = c4.SetParameterValue("@Hash", blkey) - .SetParameterValue("@Size", blocklistoffset) - .ExecuteScalarInt64(-1); + c4.SetParameterValue("@Hash", blkey); + c4.SetParameterValue("@Size", blocklistoffset); + var existingBlockId = await c4.ExecuteScalarInt64Async(-1); if (existingBlockId <= 0) { - var deletedBlockId = c5.SetParameterValue("@Hash", blkey) - .SetParameterValue("@Size", blocklistoffset) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .ExecuteScalarInt64(-1); + c5.SetParameterValue("@Hash", blkey); + c5.SetParameterValue("@Size", blocklistoffset); + c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); + c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); + var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); if (deletedBlockId <= 0) throw new Exception($"Missing block for blocklisthash: {blkey}"); else { - var rc = c6.SetParameterValue("@DeletedBlockId", deletedBlockId) - .ExecuteNonQuery(); + c6.SetParameterValue("@DeletedBlockId", deletedBlockId); + var rc = await c6.ExecuteNonQueryAsync(); if (rc != 2) throw new Exception($"Unexpected update count: {rc}"); @@ -743,10 +783,10 @@ namespace Duplicati.Library.Main.Database } // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid) - .SetParameterValue("@Index", ix) - .SetParameterValue("@Hash", blkey) - .ExecuteNonQuery(); + c3.SetParameterValue("@BlocksetId", blocksetid); + c3.SetParameterValue("@Index", ix); + c3.SetParameterValue("@Hash", blkey); + await c3.ExecuteNonQueryAsync(); ix++; blocklistoffset = 0; @@ -762,35 +802,35 @@ namespace Duplicati.Library.Main.Database var blkeyfinal = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); // Ensure that the block exists in "blocks" - var existingBlockId = c4.SetParameterValue("@Hash", blkeyfinal) - .SetParameterValue("@Size", blocklistoffset) - .ExecuteScalarInt64(-1); + c4.SetParameterValue("@Hash", blkeyfinal); + c4.SetParameterValue("@Size", blocklistoffset); + var existingBlockId = await c4.ExecuteScalarInt64Async(-1); if (existingBlockId <= 0) { - var deletedBlockId = c5.SetParameterValue("@Hash", blkeyfinal) - .SetParameterValue("@Size", blocklistoffset) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .ExecuteScalarInt64(-1); + c5.SetParameterValue("@Hash", blkeyfinal); + c5.SetParameterValue("@Size", blocklistoffset); + c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); + c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); + var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); if (deletedBlockId == 0) throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); else { - var rc = c6.SetParameterValue("@DeletedBlockId", deletedBlockId) - .ExecuteNonQuery(); + c6.SetParameterValue("@DeletedBlockId", deletedBlockId); + var rc = await c6.ExecuteNonQueryAsync(); if (rc != 2) throw new Exception($"Unexpected update count: {rc}"); } } // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid) - .SetParameterValue("@Index", ix) - .SetParameterValue("@Hash", blkeyfinal) - .ExecuteNonQuery(); + c3.SetParameterValue("@BlocksetId", blocksetid); + c3.SetParameterValue("@Index", ix); + c3.SetParameterValue("@Hash", blkeyfinal); + await c3.ExecuteNonQueryAsync(); } } } @@ -807,50 +847,52 @@ namespace Duplicati.Library.Main.Database } - public void FixDuplicateBlocklistHashes(long blocksize, long hashsize) + public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize) { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand()) { + cmd.Transaction = tr; var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1"; var sql_count = @"SELECT COUNT(*) FROM (" + dup_sql + ")"; - var x = cmd.ExecuteScalarInt64(sql_count, 0); + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); if (x > 0) { Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashes", "Found duplicate blocklisthash entries, repairing"); - var unique_count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); + var unique_count = await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); - using (var c2 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand()) { + c2.Transaction = tr; c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId AND ""Index"" = @Index LIMIT @Limit)"); - foreach (var rd in cmd.ExecuteReaderEnumerable(dup_sql)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql)) { var expected = rd.GetInt32(2) - 1; - var actual = c2.SetParameterValue("@BlocksetId", rd.GetValue(0)) - .SetParameterValue("@Index", rd.GetValue(1)) - .SetParameterValue("@Limit", expected) - .ExecuteNonQuery(); + c2.SetParameterValue("@BlocksetId", rd.GetValue(0)); + c2.SetParameterValue("@Index", rd.GetValue(1)); + c2.SetParameterValue("@Limit", expected); + var actual = await c2.ExecuteNonQueryAsync(); if (actual != expected) throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); } } - x = cmd.ExecuteScalarInt64(sql_count); + x = await cmd.ExecuteScalarInt64Async(sql_count); if (x > 1) throw new Exception("Repair failed, there are still duplicate file entries!"); - var real_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM ""BlocklistHash""", 0); + var real_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM ""BlocklistHash""", 0); if (real_count != unique_count) throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); try { - VerifyConsistency(blocksize, hashsize, true, tr); + await VerifyConsistency(blocksize, hashsize, true, tr); } catch (Exception ex) { @@ -858,88 +900,90 @@ namespace Duplicati.Library.Main.Database } Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); - tr.Commit(); + await tr.CommitAsync(); } } } - public void CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, IDbTransaction transaction) + public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(transaction); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); foreach (var kp in blocks) { cmd.SetParameterValue("@Hash", kp.Key); cmd.SetParameterValue("@Size", kp.Value); - cmd.ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync(); } - var id = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name") - .SetParameterValue("@Name", filename) - .ExecuteScalarInt64(-1); + cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name"); + cmd.SetParameterValue("@Name", filename); + var id = await cmd.ExecuteScalarInt64Async(-1); - var aliens = cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{tablename}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != @VolumeId ")) - .SetParameterValue("@VolumeId", id) - .ExecuteScalarInt64(0); + cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{tablename}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != @VolumeId ")); + cmd.SetParameterValue("@VolumeId", id); + var aliens = await cmd.ExecuteScalarInt64Async(0); if (aliens != 0) throw new Exception($"Not all blocks were found in {filename}"); } finally { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); } } - public void CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, IDbTransaction? transaction) + public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand()) { + cmd.Transaction = transaction; var query = FormatInvariant($@" -SELECT - ""C"".""Hash"", +SELECT + ""C"".""Hash"", ""C"".""Size"" -FROM - ""BlocksetEntry"" A, +FROM + ""BlocksetEntry"" A, ( - SELECT + SELECT ""Y"".""BlocksetID"", ""Y"".""Hash"" AS ""BlocklistHash"", ""Y"".""Index"" AS ""BlocklistHashIndex"", ""Z"".""Size"" AS ""BlocklistSize"", - ""Z"".""ID"" AS ""BlocklistHashBlockID"" - FROM + ""Z"".""ID"" AS ""BlocklistHashBlockID"" + FROM ""BlocklistHash"" Y, - ""Block"" Z - WHERE + ""Block"" Z + WHERE ""Y"".""Hash"" = ""Z"".""Hash"" AND ""Y"".""Hash"" = @Hash AND ""Z"".""Size"" = @Size LIMIT 1 ) B, ""Block"" C -WHERE - ""A"".""BlocksetID"" = ""B"".""BlocksetID"" - AND +WHERE + ""A"".""BlocksetID"" = ""B"".""BlocksetID"" + AND ""A"".""BlockID"" = ""C"".""ID"" AND ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({blocksize} / {blockhashlength}) AND ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({blocksize} / {blockhashlength}) -ORDER BY +ORDER BY ""A"".""Index"" "); using (var en = blocklist.GetEnumerator()) { - cmd.SetCommandAndParameters(query) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", length); - foreach (var r in cmd.ExecuteReaderEnumerable()) + cmd.SetCommandAndParameters(query); + cmd.SetParameterValue("@Hash", hash); + cmd.SetParameterValue("@Size", length); + await foreach (var r in cmd.ExecuteReaderEnumerableAsync()) { if (!en.MoveNext()) throw new Exception($"Too few entries in source blocklist with hash {hash}"); @@ -953,33 +997,36 @@ ORDER BY } } - public IEnumerable MissingLocalFilesets() + public async IAsyncEnumerable MissingLocalFilesets(SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); + using var cmd = m_connection.CreateCommand(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return rd.ConvertValueToString(0) ?? ""; } - public IEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets() + public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets(SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(@"SELECT ""ID"", ""Timestamp"", ""IsFullBackup"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); + using var cmd = m_connection.CreateCommand(@"SELECT ""ID"", ""Timestamp"", ""IsFullBackup"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return (rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP); } - public IEnumerable EmptyIndexFiles() + public async IAsyncEnumerable EmptyIndexFiles(SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ID"" NOT IN (SELECT ""IndexVolumeId"" FROM ""IndexBlockLink"")") - .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ID"" NOT IN (SELECT ""IndexVolumeId"" FROM ""IndexBlockLink"")"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); + cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } } From 3eeeaf350bf69d824746fd3b1dc45d470498d15b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 11:27:47 +0200 Subject: [PATCH 021/362] Simplified using statements where applicable --- .../Main/Database/LocalRepairDatabase.cs | 524 +++++++++--------- 1 file changed, 257 insertions(+), 267 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 352b0c322..18c7155dd 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -22,7 +22,6 @@ #nullable enable using System; -//using System.Data; using System.Collections.Generic; using System.Threading.Tasks; using Duplicati.Library.Utility; @@ -56,18 +55,16 @@ namespace Duplicati.Library.Main.Database /// >The fileset ID public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist, SqliteTransaction? transaction) { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name"); - cmd.SetParameterValue("@Name", filelist); - var rd = await cmd.ExecuteReaderAsync(); + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name"); + cmd.SetParameterValue("@Name", filelist); + var rd = await cmd.ExecuteReaderAsync(); - if (!await rd.ReadAsync()) - throw new Exception($"No such remote file: {filelist}"); + if (!await rd.ReadAsync()) + throw new Exception($"No such remote file: {filelist}"); - return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); - } + return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); } /// @@ -585,119 +582,117 @@ namespace Duplicati.Library.Main.Database public async Task FixDuplicateMetahash() { - using (var tr = m_connection.BeginTransaction()) - using (var cmd = m_connection.CreateCommand()) + using var tr = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + + cmd.Transaction = tr; + var sql_count = + @"SELECT COUNT(*) FROM (" + + @" SELECT DISTINCT c1 FROM (" + + @"SELECT COUNT(*) AS ""C1"" FROM (SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") UNION SELECT COUNT(*) AS ""C1"" FROM ""Metadataset"" " + + @")" + + @")"; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 1) { - cmd.Transaction = tr; - var sql_count = - @"SELECT COUNT(*) FROM (" + - @" SELECT DISTINCT c1 FROM (" + - @"SELECT COUNT(*) AS ""C1"" FROM (SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") UNION SELECT COUNT(*) AS ""C1"" FROM ""Metadataset"" " + - @")" + - @")"; + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashes", "Found duplicate metadatahashes, repairing"); - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 1) + var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); + + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); + + var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; + + using (var c2 = m_connection.CreateCommand()) { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashes", "Found duplicate metadatahashes, repairing"); + c2.Transaction = tr; + c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""MetadataID"" = @MetadataId WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId)") + + @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId AND ""ID"" != @MetadataId"); + using (var rd = await cmd.ExecuteReaderAsync(sql)) + while (await rd.ReadAsync()) + { + c2.SetParameterValue("@MetadataId", rd.GetValue(0)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(1)); + await c2.ExecuteNonQueryAsync(); + } + } - var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); - - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); - - var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; - - using (var c2 = m_connection.CreateCommand()) - { - c2.Transaction = tr; - c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""MetadataID"" = @MetadataId WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId)") - + @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId AND ""ID"" != @MetadataId"); - using (var rd = await cmd.ExecuteReaderAsync(sql)) - while (await rd.ReadAsync()) - { - c2.SetParameterValue("@MetadataId", rd.GetValue(0)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(1)); - await c2.ExecuteNonQueryAsync(); - } - } - - sql = FormatInvariant($@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( + sql = FormatInvariant($@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID"""); - using (var c2 = m_connection.CreateCommand()) + using (var c2 = m_connection.CreateCommand()) + { + c2.Transaction = tr; + c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId)") + + FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId")); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql)) { - c2.Transaction = tr; - c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId)") - + FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId")); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql)) - { - c2.SetParameterValue("@FileId", rd.GetValue(0)); - c2.SetParameterValue("@Path", rd.GetValue(1)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(2)); - c2.SetParameterValue("@MetadataId", rd.GetValue(3)); - await c2.ExecuteNonQueryAsync(); - } + c2.SetParameterValue("@FileId", rd.GetValue(0)); + c2.SetParameterValue("@Path", rd.GetValue(1)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(2)); + c2.SetParameterValue("@MetadataId", rd.GetValue(3)); + await c2.ExecuteNonQueryAsync(); } - - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); - - x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 1) - throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); - await tr.CommitAsync(); } + + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); + + x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 1) + throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); + await tr.CommitAsync(); } } public async Task FixDuplicateFileentries() { - using (var tr = m_connection.BeginTransaction()) - using (var cmd = m_connection.CreateCommand()) + using var tr = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + + cmd.Transaction = tr; + var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 0) { - cmd.Transaction = tr; - var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); - - var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( + var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID"""; - using (var c2 = m_connection.CreateCommand()) + using (var c2 = m_connection.CreateCommand()) + { + c2.Transaction = tr; + c2.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetatadataId)" + + @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId"); + cmd.SetCommandAndParameters(sql); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { - c2.Transaction = tr; - c2.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetatadataId)" - + @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId"); - cmd.SetCommandAndParameters(sql); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - { - c2.SetParameterValue("@FileId", rd.GetValue(0)); - c2.SetParameterValue("@PrefixId", rd.GetValue(1)); - c2.SetParameterValue("@Path", rd.GetValue(2)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(3)); - c2.SetParameterValue("@MetadataId", rd.GetValue(4)); - await c2.ExecuteNonQueryAsync(); - } + c2.SetParameterValue("@FileId", rd.GetValue(0)); + c2.SetParameterValue("@PrefixId", rd.GetValue(1)); + c2.SetParameterValue("@Path", rd.GetValue(2)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(3)); + c2.SetParameterValue("@MetadataId", rd.GetValue(4)); + await c2.ExecuteNonQueryAsync(); } - - x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 1) - throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); - await tr.CommitAsync(); } + + x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 1) + throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); + await tr.CommitAsync(); } } @@ -706,121 +701,77 @@ namespace Duplicati.Library.Main.Database { var blocklistbuffer = new byte[blocksize]; - using (var tr = m_connection.BeginTransaction()) - using (var cmd = m_connection.CreateCommand()) - using (var blockhasher = HashFactory.CreateHasher(blockhashalgorithm)) + using var tr = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + using var blockhasher = HashFactory.CreateHasher(blockhashalgorithm); + + cmd.Transaction = tr; + var hashsize = blockhasher.HashSize / 8; + + var sql = FormatInvariant($@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"""); + var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; + + var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); + if (itemswithnoblocklisthash != 0) { - cmd.Transaction = tr; - var hashsize = blockhasher.HashSize / 8; - - var sql = FormatInvariant($@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"""); - var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; - - var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); - if (itemswithnoblocklisthash != 0) + Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); + using (var c2 = m_connection.CreateCommand()) + using (var c3 = m_connection.CreateCommand()) + using (var c4 = m_connection.CreateCommand()) + using (var c5 = m_connection.CreateCommand()) + using (var c6 = m_connection.CreateCommand()) { - Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); - using (var c2 = m_connection.CreateCommand()) - using (var c3 = m_connection.CreateCommand()) - using (var c4 = m_connection.CreateCommand()) - using (var c5 = m_connection.CreateCommand()) - using (var c6 = m_connection.CreateCommand()) + c2.Transaction = tr; + c3.Transaction = tr; + c4.Transaction = tr; + c5.Transaction = tr; + c6.Transaction = tr; + c3.SetCommandAndParameters(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash) "); + c4.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + c5.SetCommandAndParameters(@"SELECT ""ID"" FROM ""DeletedBlock"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND (""State"" IN (@State1, @State2)))"); + c6.SetCommandAndParameters(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1; DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId;"); + + await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql)) { - c2.Transaction = tr; - c3.Transaction = tr; - c4.Transaction = tr; - c5.Transaction = tr; - c6.Transaction = tr; - c3.SetCommandAndParameters(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash) "); - c4.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - c5.SetCommandAndParameters(@"SELECT ""ID"" FROM ""DeletedBlock"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND (""State"" IN (@State1, @State2)))"); - c6.SetCommandAndParameters(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1; DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId;"); + var blocksetid = e.ConvertValueToInt64(0); + var ix = 0L; + int blocklistoffset = 0; - await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql)) + c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId"); + c2.SetParameterValue("@BlocksetId", blocksetid); + await c2.ExecuteNonQueryAsync(); + + c2.SetCommandAndParameters(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = @BlocksetId ORDER BY ""B"".""Index"""); + c2.SetParameterValue("@BlocksetId", blocksetid); + + await foreach (var h in c2.ExecuteReaderEnumerableAsync()) { - var blocksetid = e.ConvertValueToInt64(0); - var ix = 0L; - int blocklistoffset = 0; - - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId"); - c2.SetParameterValue("@BlocksetId", blocksetid); - await c2.ExecuteNonQueryAsync(); - - c2.SetCommandAndParameters(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = @BlocksetId ORDER BY ""B"".""Index"""); - c2.SetParameterValue("@BlocksetId", blocksetid); - - await foreach (var h in c2.ExecuteReaderEnumerableAsync()) + var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); + if (blocklistbuffer.Length - blocklistoffset < tmp.Length) { - var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); - if (blocklistbuffer.Length - blocklistoffset < tmp.Length) - { - var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); + var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); - // Check if the block exists in "blocks" - c4.SetParameterValue("@Hash", blkey); - c4.SetParameterValue("@Size", blocklistoffset); - var existingBlockId = await c4.ExecuteScalarInt64Async(-1); - - if (existingBlockId <= 0) - { - c5.SetParameterValue("@Hash", blkey); - c5.SetParameterValue("@Size", blocklistoffset); - c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); - c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); - var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); - - if (deletedBlockId <= 0) - throw new Exception($"Missing block for blocklisthash: {blkey}"); - else - { - c6.SetParameterValue("@DeletedBlockId", deletedBlockId); - var rc = await c6.ExecuteNonQueryAsync(); - - if (rc != 2) - throw new Exception($"Unexpected update count: {rc}"); - } - } - - // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid); - c3.SetParameterValue("@Index", ix); - c3.SetParameterValue("@Hash", blkey); - await c3.ExecuteNonQueryAsync(); - - ix++; - blocklistoffset = 0; - } - - Array.Copy(tmp, 0, blocklistbuffer, blocklistoffset, tmp.Length); - blocklistoffset += tmp.Length; - - } - - if (blocklistoffset != 0) - { - var blkeyfinal = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); - - // Ensure that the block exists in "blocks" - c4.SetParameterValue("@Hash", blkeyfinal); + // Check if the block exists in "blocks" + c4.SetParameterValue("@Hash", blkey); c4.SetParameterValue("@Size", blocklistoffset); var existingBlockId = await c4.ExecuteScalarInt64Async(-1); if (existingBlockId <= 0) { - c5.SetParameterValue("@Hash", blkeyfinal); + c5.SetParameterValue("@Hash", blkey); c5.SetParameterValue("@Size", blocklistoffset); c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); - if (deletedBlockId == 0) - throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); + if (deletedBlockId <= 0) + throw new Exception($"Missing block for blocklisthash: {blkey}"); else { c6.SetParameterValue("@DeletedBlockId", deletedBlockId); var rc = await c6.ExecuteNonQueryAsync(); + if (rc != 2) throw new Exception($"Unexpected update count: {rc}"); } @@ -829,79 +780,120 @@ namespace Duplicati.Library.Main.Database // Add to table c3.SetParameterValue("@BlocksetId", blocksetid); c3.SetParameterValue("@Index", ix); - c3.SetParameterValue("@Hash", blkeyfinal); + c3.SetParameterValue("@Hash", blkey); await c3.ExecuteNonQueryAsync(); + + ix++; + blocklistoffset = 0; } + + Array.Copy(tmp, 0, blocklistbuffer, blocklistoffset, tmp.Length); + blocklistoffset += tmp.Length; + + } + + if (blocklistoffset != 0) + { + var blkeyfinal = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); + + // Ensure that the block exists in "blocks" + c4.SetParameterValue("@Hash", blkeyfinal); + c4.SetParameterValue("@Size", blocklistoffset); + var existingBlockId = await c4.ExecuteScalarInt64Async(-1); + + if (existingBlockId <= 0) + { + c5.SetParameterValue("@Hash", blkeyfinal); + c5.SetParameterValue("@Size", blocklistoffset); + c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); + c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); + var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); + + if (deletedBlockId == 0) + throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); + else + { + c6.SetParameterValue("@DeletedBlockId", deletedBlockId); + var rc = await c6.ExecuteNonQueryAsync(); + if (rc != 2) + throw new Exception($"Unexpected update count: {rc}"); + } + } + + // Add to table + c3.SetParameterValue("@BlocksetId", blocksetid); + c3.SetParameterValue("@Index", ix); + c3.SetParameterValue("@Hash", blkeyfinal); + await c3.ExecuteNonQueryAsync(); } } - - - itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); - if (itemswithnoblocklisthash != 0) - throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); - tr.Commit(); } + + itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); + if (itemswithnoblocklisthash != 0) + throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); + tr.Commit(); } } public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize) { - using (var tr = m_connection.BeginTransaction()) - using (var cmd = m_connection.CreateCommand()) + using var tr = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + + cmd.Transaction = tr; + var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1"; + + var sql_count = @"SELECT COUNT(*) FROM (" + dup_sql + ")"; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 0) { - cmd.Transaction = tr; - var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1"; + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashes", "Found duplicate blocklisthash entries, repairing"); - var sql_count = @"SELECT COUNT(*) FROM (" + dup_sql + ")"; + var unique_count = await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 0) + using (var c2 = m_connection.CreateCommand()) { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashes", "Found duplicate blocklisthash entries, repairing"); - - var unique_count = await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); - - using (var c2 = m_connection.CreateCommand()) + c2.Transaction = tr; + c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId AND ""Index"" = @Index LIMIT @Limit)"); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql)) { - c2.Transaction = tr; - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId AND ""Index"" = @Index LIMIT @Limit)"); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql)) - { - var expected = rd.GetInt32(2) - 1; - c2.SetParameterValue("@BlocksetId", rd.GetValue(0)); - c2.SetParameterValue("@Index", rd.GetValue(1)); - c2.SetParameterValue("@Limit", expected); - var actual = await c2.ExecuteNonQueryAsync(); + var expected = rd.GetInt32(2) - 1; + c2.SetParameterValue("@BlocksetId", rd.GetValue(0)); + c2.SetParameterValue("@Index", rd.GetValue(1)); + c2.SetParameterValue("@Limit", expected); + var actual = await c2.ExecuteNonQueryAsync(); - if (actual != expected) - throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); - } + if (actual != expected) + throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); } - - x = await cmd.ExecuteScalarInt64Async(sql_count); - if (x > 1) - throw new Exception("Repair failed, there are still duplicate file entries!"); - - var real_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM ""BlocklistHash""", 0); - - if (real_count != unique_count) - throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); - - try - { - await VerifyConsistency(blocksize, hashsize, true, tr); - } - catch (Exception ex) - { - throw new Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); - } - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); - await tr.CommitAsync(); } + + x = await cmd.ExecuteScalarInt64Async(sql_count); + if (x > 1) + throw new Exception("Repair failed, there are still duplicate file entries!"); + + var real_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM ""BlocklistHash""", 0); + + if (real_count != unique_count) + throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); + + try + { + await VerifyConsistency(blocksize, hashsize, true, tr); + } + catch (Exception ex) + { + throw new Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); + } + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); + await tr.CommitAsync(); } } @@ -941,10 +933,10 @@ namespace Duplicati.Library.Main.Database public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, SqliteTransaction transaction) { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = transaction; - var query = FormatInvariant($@" + using var cmd = m_connection.CreateCommand(); + + cmd.Transaction = transaction; + var query = FormatInvariant($@" SELECT ""C"".""Hash"", ""C"".""Size"" @@ -978,23 +970,21 @@ ORDER BY "); - using (var en = blocklist.GetEnumerator()) - { - cmd.SetCommandAndParameters(query); - cmd.SetParameterValue("@Hash", hash); - cmd.SetParameterValue("@Size", length); - await foreach (var r in cmd.ExecuteReaderEnumerableAsync()) - { - if (!en.MoveNext()) - throw new Exception($"Too few entries in source blocklist with hash {hash}"); - if (en.Current != r.ConvertValueToString(0)) - throw new Exception($"Mismatch in blocklist with hash {hash}"); - } + using var en = blocklist.GetEnumerator(); - if (en.MoveNext()) - throw new Exception($"Too many source blocklist entries in {hash}"); - } + cmd.SetCommandAndParameters(query); + cmd.SetParameterValue("@Hash", hash); + cmd.SetParameterValue("@Size", length); + await foreach (var r in cmd.ExecuteReaderEnumerableAsync()) + { + if (!en.MoveNext()) + throw new Exception($"Too few entries in source blocklist with hash {hash}"); + if (en.Current != r.ConvertValueToString(0)) + throw new Exception($"Mismatch in blocklist with hash {hash}"); } + + if (en.MoveNext()) + throw new Exception($"Too many source blocklist entries in {hash}"); } public async IAsyncEnumerable MissingLocalFilesets(SqliteTransaction transaction) From 15b812ee79616ece40850818cc76b0476ed7f462 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 11:28:23 +0200 Subject: [PATCH 022/362] Removed whitespace --- .../Main/Database/ReusableTransaction.cs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 4f681deec..f2c76dc98 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -1,23 +1,24 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System; using System.Data; From f9eda3118f8052f1f79155966d111ca5eb6ac9f1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 11:28:48 +0200 Subject: [PATCH 023/362] Updated ReusableTransaction to use Microsoft.Data.Sqlite --- .../Main/Database/ReusableTransaction.cs | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index f2c76dc98..550962931 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -20,7 +20,8 @@ // DEALINGS IN THE SOFTWARE. using System; -using System.Data; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; #nullable enable @@ -43,22 +44,22 @@ internal class ReusableTransaction : IDisposable /// /// The current transaction /// - private IDbTransaction? m_transaction; + private SqliteTransaction? m_transaction; /// /// Creates a new reusable transaction /// /// The database to use - public ReusableTransaction(LocalDatabase db, IDbTransaction? transaction = null) + public ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) { m_db = db; - m_transaction = transaction ?? db.BeginTransaction(); + m_transaction = transaction ?? db.Connection.BeginTransaction(); } /// /// The current transaction /// - public IDbTransaction Transaction => m_transaction ?? throw new InvalidOperationException("Transaction is disposed"); + public SqliteTransaction Transaction => m_transaction ?? throw new InvalidOperationException("Transaction is disposed"); /// /// Commits the current transaction and optionally restarts it @@ -79,7 +80,24 @@ internal class ReusableTransaction : IDisposable } if (restart) - m_transaction = m_db.BeginTransaction(); + m_transaction = m_db.Connection.BeginTransaction(); + } + + public async Task CommitAsync(string? message, bool restart = true) + { + if (m_transaction == null) + throw new InvalidOperationException("Transaction is already disposed"); + + if (m_transaction != null) + { + using (var timer = string.IsNullOrWhiteSpace(message) ? null : new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) + await m_transaction.CommitAsync(); + await m_transaction.DisposeAsync(); + m_transaction = null; + } + + if (restart) + m_transaction = m_db.Connection.BeginTransaction(); } /// From 8cf43ab6798f70d4406f0eb007a50c84f98b265b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 11:36:12 +0200 Subject: [PATCH 024/362] Updated RepairHandler to async using Microsoft.Data.Sqlite --- .../Library/Main/Operation/RepairHandler.cs | 169 +++++++++--------- 1 file changed, 86 insertions(+), 83 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 5500e7139..600342b9c 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -57,7 +57,7 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) { await RunRepairLocalAsync(backendManager, filter).ConfigureAwait(false); - RunRepairCommon(); + await RunRepairCommon(); m_result.EndTime = DateTime.UtcNow; return; } @@ -66,7 +66,7 @@ namespace Duplicati.Library.Main.Operation try { using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - knownRemotes = db.GetRemoteVolumes().Count(); + knownRemotes = await db.GetRemoteVolumes().CountAsync(); } catch (Exception ex) { @@ -91,11 +91,11 @@ namespace Duplicati.Library.Main.Operation } await RunRepairLocalAsync(backendManager, filter).ConfigureAwait(false); - RunRepairCommon(); + await RunRepairCommon(); } else { - RunRepairCommon(); + await RunRepairCommon(); await RunRepairBrokenFilesets(backendManager).ConfigureAwait(false); await RunRepairRemoteAsync(backendManager, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } @@ -141,24 +141,24 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); // Ensure the database is consistent before we start fixing the remote - db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, rtr.Transaction); + await db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, rtr.Transaction); // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist - var lastTempFilelist = db.GetLastIncompleteFilesetVolume(rtr.Transaction); + var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(rtr.Transaction); var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, rtr.Transaction, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); rtr.Commit("CommitRemoteListAnalysisTransaction"); var buffer = new byte[m_options.Blocksize]; var hashsize = m_options.BlockhashSize; - var missingRemoteFilesets = db.MissingRemoteFilesets().ToList(); - var missingLocalFilesets = db.MissingLocalFilesets().ToList(); - var emptyIndexFiles = db.EmptyIndexFiles().ToList(); + var missingRemoteFilesets = await db.MissingRemoteFilesets(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + var missingLocalFilesets = await db.MissingLocalFilesets(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + var emptyIndexFiles = await db.EmptyIndexFiles(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); var progress = 0; var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; - var mostRecentLocal = db.FilesetTimes.Select(x => x.Value.ToLocalTime()).Append(DateTime.MinValue).Max(); + var mostRecentLocal = await db.FilesetTimes().Select(x => x.Value.ToLocalTime()).Append(DateTime.MinValue).MaxAsync(cancellationToken: cancellationToken); var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); if (mostRecentLocal < DateTime.UnixEpoch) throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); @@ -206,7 +206,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); KeyValuePair>> res; - (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); + var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); using (tf) res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1); @@ -216,7 +216,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); - db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash); + await db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); } } @@ -258,7 +258,7 @@ namespace Duplicati.Library.Main.Operation { foreach (var rv in ifr.Volumes) { - var entry = db.GetRemoteVolume(rv.Filename); + var entry = await db.GetRemoteVolume(rv.Filename); if (entry.ID < 0) throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); @@ -268,23 +268,23 @@ namespace Duplicati.Library.Main.Operation if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); - db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, rtr.Transaction); + await db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, rtr.Transaction); } var blocksize = m_options.Blocksize; foreach (var ixb in ifr.BlockLists) - db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, rtr.Transaction); + await db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, rtr.Transaction); // Register the new index file and link it to the block files - var selfid = db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), rtr.Transaction); + var selfid = await db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), rtr.Transaction); foreach (var rv in ifr.Volumes) { // Guard against unknown block files - long id = db.GetRemoteVolumeID(rv.Filename, rtr.Transaction); + long id = await db.GetRemoteVolumeID(rv.Filename, rtr.Transaction); if (id == -1) Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); else - db.AddIndexBlockLink(selfid, id, rtr.Transaction); + await db.AddIndexBlockLink(selfid, id, rtr.Transaction); } if (!m_options.Dryrun) rtr.Commit("CommitIndexFileTransaction"); @@ -292,7 +292,7 @@ namespace Duplicati.Library.Main.Operation // All checks fine, we accept the new index file Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); - db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); + await db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); continue; } catch (Exception rex) @@ -303,7 +303,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { - db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); + await db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); } else @@ -343,12 +343,12 @@ namespace Duplicati.Library.Main.Operation fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); fsw.CreateFilesetFile(isfull); - db.WriteFileset(fsw, filesetId, null); + await db.WriteFileset(fsw, filesetId, null); fsw.Close(); - fsw.VolumeID = db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); - db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); + fsw.VolumeID = await db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); + await db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); if (m_options.Dryrun) { fsw.Dispose(); @@ -368,7 +368,7 @@ namespace Duplicati.Library.Main.Operation foreach (var volumename in missingLocalFilesets) { - var remoteVolume = db.GetRemoteVolume(volumename); + var remoteVolume = await db.GetRemoteVolume(volumename); using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) { var parsed = VolumeBase.ParseFilename(remoteVolume.Name); @@ -380,7 +380,8 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); var filesetid = db.CreateFileset(remoteVolume.ID, parsed.Time, rtr.Transaction); - RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); + // TODO :) recreate rewrite in progress + //RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); if (!m_options.Dryrun) rtr.Commit("CommitRecreateFilesetTransaction"); } @@ -441,7 +442,7 @@ namespace Duplicati.Library.Main.Operation try { // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = db.GetRemoteVolume(n.Name, rtr.Transaction).State; + var currentState = (await db.GetRemoteVolume(n.Name, rtr.Transaction)).State; if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) continue; @@ -527,25 +528,25 @@ namespace Duplicati.Library.Main.Operation /// A task representing the asynchronous operation private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) { - volumeWriter.VolumeID = db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, rtr.Transaction); - (var prevFilesetId, var _, var isPrevFull) = db.GetFilesetFromRemotename(originalVolume.Name, rtr.Transaction); + volumeWriter.VolumeID = await db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, rtr.Transaction); + (var prevFilesetId, var _, var isPrevFull) = await db.GetFilesetFromRemotename(originalVolume.Name, rtr.Transaction); if (!string.IsNullOrEmpty(m_options.ControlFiles)) foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) volumeWriter.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); volumeWriter.CreateFilesetFile(isPrevFull); - var newFilesetID = db.CreateFileset(volumeWriter.VolumeID, filesetTime, rtr.Transaction); - db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull); + var newFilesetID = await db.CreateFileset(volumeWriter.VolumeID, filesetTime, rtr.Transaction); + await db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull, rtr.Transaction); - db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, rtr.Transaction); - db.MoveFilesFromFileset(newFilesetID, prevFilesetId, rtr.Transaction); + await db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, rtr.Transaction); + await db.MoveFilesFromFileset(newFilesetID, prevFilesetId, rtr.Transaction); - db.WriteFileset(volumeWriter, newFilesetID, rtr.Transaction); + await db.WriteFileset(volumeWriter, newFilesetID, rtr.Transaction); volumeWriter.Close(); - db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); - db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + await db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) @@ -575,23 +576,23 @@ namespace Duplicati.Library.Main.Operation private async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IndexVolumeWriter indexWriter, RemoteVolumeEntry originalVolume, CancellationToken cancellationToken) { - indexWriter.VolumeID = db.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); + indexWriter.VolumeID = await db.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); var blockvolumeids = new List(); using var h = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); - foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name)) + await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name, rtr.Transaction)) { indexWriter.StartVolume(blockvolume.Name); - var volumeid = db.GetRemoteVolumeID(blockvolume.Name, rtr.Transaction); + var volumeid = await db.GetRemoteVolumeID(blockvolume.Name, rtr.Transaction); - foreach (var b in db.GetBlocks(volumeid)) + await foreach (var b in db.GetBlocks(volumeid, rtr.Transaction)) indexWriter.AddBlock(b.Hash, b.Size); indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size); blockvolumeids.Add(volumeid); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, m_options.BlockhashSize)) + await foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, m_options.BlockhashSize, rtr.Transaction)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); if (bh != b.Item1) @@ -602,7 +603,7 @@ namespace Duplicati.Library.Main.Operation } foreach (var blockvolumeid in blockvolumeids) - db.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, rtr.Transaction); + await db.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, rtr.Transaction); indexWriter.Close(); @@ -610,7 +611,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(indexWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } @@ -671,7 +672,7 @@ namespace Duplicati.Library.Main.Operation m_options = options; m_maxVolumeSize = options.VolumeSize - m_options.Blocksize; m_activeWriter = new BlockVolumeWriter(options); - m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); + m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction).Await(); } /// @@ -713,13 +714,13 @@ namespace Duplicati.Library.Main.Operation m_activeWriter.AddBlock(hash, buffer, offset, size, hint); m_anyData = true; if (IsFull) - StartNewVolume(); + StartNewVolume().Await(); } /// /// Starts a new volume for the current operation /// - public void StartNewVolume() + public async Task StartNewVolume() { if (!m_anyData) return; @@ -727,7 +728,7 @@ namespace Duplicati.Library.Main.Operation m_completedWriters.Add(m_activeWriter); m_activeWriter = new BlockVolumeWriter(m_options); - m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); + m_activeWriter.VolumeID = await m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); m_anyData = false; } } @@ -774,12 +775,12 @@ namespace Duplicati.Library.Main.Operation // If we have any data in the volume, complete it so it will be uploaded if (currentVolume.AnyData) - currentVolume.StartNewVolume(); + await currentVolume.StartNewVolume(); // Complete any pending uploads await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); // After reset, we need to remove the temporary volume from the database - db.RemoveRemoteVolume(currentVolume.RemoteFilename, rtr.Transaction); + await db.RemoveRemoteVolume(currentVolume.RemoteFilename, rtr.Transaction); } /// @@ -795,7 +796,7 @@ namespace Duplicati.Library.Main.Operation { foreach (var completedVolume in activeVolume.CompletedWriters) { - db.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + await db.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); // Create a new index file that points to the new volume IndexVolumeWriter newvolindex = null; @@ -803,16 +804,16 @@ namespace Duplicati.Library.Main.Operation if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); + newvolindex.VolumeID = await db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); newvolindex.StartVolume(completedVolume.RemoteFilename); - foreach (var b in db.GetBlocks(completedVolume.VolumeID)) + await foreach (var b in db.GetBlocks(completedVolume.VolumeID, rtr.Transaction)) newvolindex.AddBlock(b.Hash, b.Size); - db.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, rtr.Transaction); + await db.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, rtr.Transaction); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - indexVolumeFinished = () => + indexVolumeFinished = async () => { - foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) + await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize, rtr.Transaction)) newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); }; } @@ -833,11 +834,11 @@ namespace Duplicati.Library.Main.Operation // Prepare for deleting the old stuff foreach (var vol in activeVolume.VolumesToDelete) - db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); // Persist desired state prior to deleting the old files if (!m_options.Dryrun) - rtr.Commit("PostRepairBlockVolume"); + await rtr.CommitAsync("PostRepairBlockVolume"); // Delete the old files foreach (var vol in activeVolume.VolumesToDelete) @@ -849,7 +850,7 @@ namespace Duplicati.Library.Main.Operation // All done, the new dblocks are now in place await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("PostRepairBlockVolume"); + await rtr.CommitAsync("PostRepairBlockVolume"); activeVolume.CompletedWriters.Clear(); activeVolume.VolumesToDelete.Clear(); @@ -872,13 +873,13 @@ namespace Duplicati.Library.Main.Operation // We attempt to recover all three source parts in the steps below using (var mbl = db.CreateBlockList(originalVolume.Name, rtr)) { - var originalMissingBlockCount = mbl.GetMissingBlockCount(); + var originalMissingBlockCount = await mbl.GetMissingBlockCount(); var recoveredSourceBlocks = 0L; // First we grab all known blocks from local files string lastRestoredHash = null; long lastRestoredSize = -1; - foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) + await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) { if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) continue; @@ -914,7 +915,7 @@ namespace Duplicati.Library.Main.Operation // Found it, no need to look again lastRestoredHash = block.Hash; lastRestoredSize = block.Size; - if (mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) { recoveredSourceBlocks++; pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); @@ -927,7 +928,7 @@ namespace Duplicati.Library.Main.Operation } // Then grab all blocks with metadata - foreach (var block in mbl.GetSourceItemsWithMetadataBlocks()) + await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks()) { if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) continue; @@ -963,7 +964,7 @@ namespace Duplicati.Library.Main.Operation // Found it, no need to look again lastRestoredHash = block.Hash; lastRestoredSize = block.Size; - if (mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) { recoveredSourceBlocks++; pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); @@ -987,7 +988,7 @@ namespace Duplicati.Library.Main.Operation var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; // Helper function to emit a blocklist hash - void EmitBlockListBlock() + async Task EmitBlockListBlock() { blocklistHash.Position = 0; using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); @@ -1000,7 +1001,7 @@ namespace Duplicati.Library.Main.Operation if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) { - if (mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) + if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); } else @@ -1011,12 +1012,12 @@ namespace Duplicati.Library.Main.Operation blocklistHash.SetLength(0); } - foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) + await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) { lastBlocklist ??= blocklist; if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) - EmitBlockListBlock(); + await EmitBlockListBlock(); var data = Convert.FromBase64String(blocklist.Hash); blocklistHash.Write(data, 0, data.Length); @@ -1025,10 +1026,10 @@ namespace Duplicati.Library.Main.Operation // Handle any trailing blocklist hash if (blocklistHash.Length > 0) - EmitBlockListBlock(); + await EmitBlockListBlock(); //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources().ToListAsync(cancellationToken: cancellationToken), cancellationToken).ConfigureAwait(false)) { try { @@ -1036,7 +1037,7 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) foreach (var b in f.Blocks) - if (mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) + if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) if (f.ReadBlock(b.Key, buffer) == b.Value) pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); } @@ -1047,12 +1048,12 @@ namespace Duplicati.Library.Main.Operation } } - var missingBlocks = mbl.GetMissingBlockCount(); + var missingBlocks = await mbl.GetMissingBlockCount(); var recoveredBlocks = originalMissingBlockCount - missingBlocks; if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) { Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); - foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) + await foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); @@ -1072,15 +1073,15 @@ namespace Duplicati.Library.Main.Operation if (missingBlocks == 0) { pendingVolume.VolumesToDelete.Add(originalVolume); - var oldIndexFiles = db.GetIndexFilesReferencingBlockFile(originalVolume.ID, rtr.Transaction).ToList(); + var oldIndexFiles = await db.GetIndexFilesReferencingBlockFile(originalVolume.ID, rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); // Find all index files that point to the old volume foreach (var oldIndexFile in oldIndexFiles) { - var oldVolume = db.GetRemoteVolume(oldIndexFile, rtr.Transaction); + var oldVolume = await db.GetRemoteVolume(oldIndexFile, rtr.Transaction); if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) { - var blockVolumesReferenced = db.GetBlockVolumesFromIndexName(oldIndexFile).ToList(); + var blockVolumesReferenced = await db.GetBlockVolumesFromIndexName(oldIndexFile, rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); else @@ -1104,7 +1105,7 @@ namespace Duplicati.Library.Main.Operation using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) using (var tr = new ReusableTransaction(db)) { - var sets = db.GetFilesetsWithMissingFiles(null).ToList(); + var sets = await db.GetFilesetsWithMissingFiles(null).ToListAsync(); if (sets.Count == 0) return; @@ -1114,7 +1115,7 @@ namespace Duplicati.Library.Main.Operation { ix++; Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); - var volume = db.GetRemoteVolumeFromFilesetID(entry.Key, tr.Transaction); + var volume = await db.GetRemoteVolumeFromFilesetID(entry.Key, tr.Transaction); var parsed = VolumeBase.ParseFilename(volume.Name); using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); @@ -1123,9 +1124,11 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); // Clear out the old fileset - db.DeleteFilesetEntries(entry.Key, tr.Transaction); + await db.DeleteFilesetEntries(entry.Key, tr.Transaction); using (var rdb = new LocalRecreateDatabase(db, m_options)) - RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, tr.Transaction, compressor, entry.Key, m_options, new FilterExpression()); + { + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, tr.Transaction, compressor, entry.Key, m_options, new FilterExpression()); + } tr.Commit("PostRepairFileset"); } @@ -1133,7 +1136,7 @@ namespace Duplicati.Library.Main.Operation } } - public void RunRepairCommon() + public async Task RunRepairCommon() { if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); @@ -1147,10 +1150,10 @@ namespace Duplicati.Library.Main.Operation if (db.RepairInProgress || db.PartiallyRecreated) Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); - db.FixDuplicateMetahash(); - db.FixDuplicateFileentries(); - db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize); - db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize); + await db.FixDuplicateMetahash(); + await db.FixDuplicateFileentries(); + await db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize); + await db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize); } } } From 0a9266842c46d163df6f9a59442c7dff6d640881 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 11:39:46 +0200 Subject: [PATCH 025/362] Common syntactic rewrites --- .../Library/Main/Operation/RepairHandler.cs | 1078 ++++++++--------- 1 file changed, 536 insertions(+), 542 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 600342b9c..cab0b4c50 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -65,8 +65,8 @@ namespace Duplicati.Library.Main.Operation long knownRemotes = -1; try { - using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - knownRemotes = await db.GetRemoteVolumes().CountAsync(); + using var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); + knownRemotes = await db.GetRemoteVolumes().CountAsync(); } catch (Exception ex) { @@ -128,116 +128,75 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var rtr = new ReusableTransaction(db)) + using var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); + using var rtr = new ReusableTransaction(db); + + Utility.UpdateOptionsFromDb(db, m_options); + Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + + if (db.PartiallyRecreated) + throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); + + if (db.RepairInProgress) + throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); + + // Ensure the database is consistent before we start fixing the remote + await db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, rtr.Transaction); + + // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist + var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(rtr.Transaction); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, rtr.Transaction, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + rtr.Commit("CommitRemoteListAnalysisTransaction"); + + var buffer = new byte[m_options.Blocksize]; + var hashsize = m_options.BlockhashSize; + + var missingRemoteFilesets = await db.MissingRemoteFilesets(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + var missingLocalFilesets = await db.MissingLocalFilesets(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + var emptyIndexFiles = await db.EmptyIndexFiles(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + + var progress = 0; + var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; + + var mostRecentLocal = await db.FilesetTimes().Select(x => x.Value.ToLocalTime()).Append(DateTime.MinValue).MaxAsync(cancellationToken: cancellationToken); + var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); + if (mostRecentLocal < DateTime.UnixEpoch) + throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); + if (mostRecentRemote > mostRecentLocal) { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + if (m_options.RepairIgnoreOutdatedDatabase) + Logging.Log.WriteWarningMessage(LOGTAG, "RemoteFilesNewerThanLocalDatabase", null, "The remote files are newer ({0}) than the local database ({1}), this is likely because the database is outdated. Continuing as the options force ignoring this.", mostRecentRemote, mostRecentLocal); + else + throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again. If this is expected, set the option \"--repair-ignore-outdated-database\" ", "RemoteFilesNewerThanLocalDatabase"); + } - if (db.PartiallyRecreated) - throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - - if (db.RepairInProgress) - throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); - - // Ensure the database is consistent before we start fixing the remote - await db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, rtr.Transaction); - - // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist - var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(rtr.Transaction); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, rtr.Transaction, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); - rtr.Commit("CommitRemoteListAnalysisTransaction"); - - var buffer = new byte[m_options.Blocksize]; - var hashsize = m_options.BlockhashSize; - - var missingRemoteFilesets = await db.MissingRemoteFilesets(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); - var missingLocalFilesets = await db.MissingLocalFilesets(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); - var emptyIndexFiles = await db.EmptyIndexFiles(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); - - var progress = 0; - var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; - - var mostRecentLocal = await db.FilesetTimes().Select(x => x.Value.ToLocalTime()).Append(DateTime.MinValue).MaxAsync(cancellationToken: cancellationToken); - var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); - if (mostRecentLocal < DateTime.UnixEpoch) - throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); - if (mostRecentRemote > mostRecentLocal) + if (m_options.Dryrun) + { + if (!tp.ParsedVolumes.Any() && tp.OtherVolumes.Any()) { - if (m_options.RepairIgnoreOutdatedDatabase) - Logging.Log.WriteWarningMessage(LOGTAG, "RemoteFilesNewerThanLocalDatabase", null, "The remote files are newer ({0}) than the local database ({1}), this is likely because the database is outdated. Continuing as the options force ignoring this.", mostRecentRemote, mostRecentLocal); + if (tp.BackupPrefixes.Length == 1) + throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", m_options.Prefix, tp.BackupPrefixes[0]), "RemoteFolderEmptyWithPrefix"); else - throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again. If this is expected, set the option \"--repair-ignore-outdated-database\" ", "RemoteFilesNewerThanLocalDatabase"); + throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)), "RemoteFolderEmptyWithPrefix"); } - - if (m_options.Dryrun) + else if (!tp.ParsedVolumes.Any() && tp.ExtraVolumes.Any()) { - if (!tp.ParsedVolumes.Any() && tp.OtherVolumes.Any()) - { - if (tp.BackupPrefixes.Length == 1) - throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", m_options.Prefix, tp.BackupPrefixes[0]), "RemoteFolderEmptyWithPrefix"); - else - throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)), "RemoteFolderEmptyWithPrefix"); - } - else if (!tp.ParsedVolumes.Any() && tp.ExtraVolumes.Any()) - { - throw new UserInformationException(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()), "NoRemoteFilesMissing"); - } + throw new UserInformationException(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()), "NoRemoteFilesMissing"); } + } - if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any() || missingRemoteFilesets.Any() || missingLocalFilesets.Any() || emptyIndexFiles.Any()) + if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any() || missingRemoteFilesets.Any() || missingLocalFilesets.Any() || emptyIndexFiles.Any()) + { + if (tp.VerificationRequiredVolumes.Any()) { - if (tp.VerificationRequiredVolumes.Any()) - { - using (var testdb = new LocalTestDatabase(db)) - { - foreach (var n in tp.VerificationRequiredVolumes) - try - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(testdb, rtr.Transaction, cancellationToken).ConfigureAwait(false); - if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); - return; - } + using var testdb = new LocalTestDatabase(db); - progress++; - m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - - KeyValuePair>> res; - var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); - using (tf) - res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1); - - if (res.Value.Any()) - throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); - - if (!m_options.Dryrun) - { - Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); - await db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); - } - - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileVerificationError", ex, "Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message); - if (ex.IsAbortException()) - throw; - } - - rtr.Commit("CommitVerificationTransaction"); - } - } - - // TODO: It is actually possible to use the extra files if we parse them - foreach (var n in tp.ExtraVolumes) + foreach (var n in tp.VerificationRequiredVolumes) try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(testdb, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) rtr.Commit("CommitEarlyExit", false); return; @@ -246,82 +205,34 @@ namespace Duplicati.Library.Main.Operation progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - // If this is a new index file, we can accept it if it matches our local data - // This makes it possible to augment the remote store with new index data - if (n.FileType == RemoteVolumeType.Index && m_options.IndexfilePolicy != Options.IndexFileStrategy.None) - { - try - { - (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.File.Name, null, n.File.Size, cancellationToken).ConfigureAwait(false); - using (tf) - using (var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize)) - { - foreach (var rv in ifr.Volumes) - { - var entry = await db.GetRemoteVolume(rv.Filename); - if (entry.ID < 0) - throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); + KeyValuePair>> res; + var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); + using (tf) + res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1); - if (!new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) - throw new Exception(string.Format("Volume {0} has local state {1}", rv.Filename, entry.State)); - - if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) - throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); - - await db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, rtr.Transaction); - } - - var blocksize = m_options.Blocksize; - foreach (var ixb in ifr.BlockLists) - await db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, rtr.Transaction); - - // Register the new index file and link it to the block files - var selfid = await db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), rtr.Transaction); - foreach (var rv in ifr.Volumes) - { - // Guard against unknown block files - long id = await db.GetRemoteVolumeID(rv.Filename, rtr.Transaction); - if (id == -1) - Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); - else - await db.AddIndexBlockLink(selfid, id, rtr.Transaction); - } - if (!m_options.Dryrun) - rtr.Commit("CommitIndexFileTransaction"); - } - - // All checks fine, we accept the new index file - Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); - await db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); - continue; - } - catch (Exception rex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FailedNewIndexFile", rex, "Failed to accept new index file: {0}, message: {1}", n.File.Name, rex.Message); - } - } + if (res.Value.Any()) + throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); if (!m_options.Dryrun) { - await db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); - await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); + Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); + await db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); } - else - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "would delete file {0}", n.File.Name); + } catch (Exception ex) { - Logging.Log.WriteErrorMessage(LOGTAG, "FailedExtraFileCleanup", ex, "Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message); + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileVerificationError", ex, "Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message); if (ex.IsAbortException()) throw; } - var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks); - if (!m_options.RebuildMissingDblockFiles && missingDblocks.Count() > 0) - throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); + rtr.Commit("CommitVerificationTransaction"); + } - var anyDlistUploads = false; - foreach (var (filesetId, timestamp, isfull) in missingRemoteFilesets) + // TODO: It is actually possible to use the extra files if we parse them + foreach (var n in tp.ExtraVolumes) + try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { @@ -333,185 +244,272 @@ namespace Duplicati.Library.Main.Operation progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); - var fsw = new FilesetVolumeWriter(m_options, fileTime); - Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); - - if (!string.IsNullOrEmpty(m_options.ControlFiles)) - foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) - fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); - - fsw.CreateFilesetFile(isfull); - await db.WriteFileset(fsw, filesetId, null); - fsw.Close(); - - - fsw.VolumeID = await db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); - await db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); - if (m_options.Dryrun) + // If this is a new index file, we can accept it if it matches our local data + // This makes it possible to augment the remote store with new index data + if (n.FileType == RemoteVolumeType.Index && m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { - fsw.Dispose(); - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}", fsw.RemoteFilename); - continue; + try + { + (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.File.Name, null, n.File.Size, cancellationToken).ConfigureAwait(false); + using (tf) + using (var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize)) + { + foreach (var rv in ifr.Volumes) + { + var entry = await db.GetRemoteVolume(rv.Filename); + if (entry.ID < 0) + throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); + + if (!new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) + throw new Exception(string.Format("Volume {0} has local state {1}", rv.Filename, entry.State)); + + if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) + throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); + + await db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, rtr.Transaction); + } + + var blocksize = m_options.Blocksize; + foreach (var ixb in ifr.BlockLists) + await db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, rtr.Transaction); + + // Register the new index file and link it to the block files + var selfid = await db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), rtr.Transaction); + foreach (var rv in ifr.Volumes) + { + // Guard against unknown block files + long id = await db.GetRemoteVolumeID(rv.Filename, rtr.Transaction); + if (id == -1) + Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); + else + await db.AddIndexBlockLink(selfid, id, rtr.Transaction); + } + if (!m_options.Dryrun) + rtr.Commit("CommitIndexFileTransaction"); + } + + // All checks fine, we accept the new index file + Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); + await db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); + continue; + } + catch (Exception rex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FailedNewIndexFile", rex, "Failed to accept new index file: {0}, message: {1}", n.File.Name, rex.Message); + } } - await backendManager.PutAsync(fsw, null, null, false, null, cancellationToken).ConfigureAwait(false); + + if (!m_options.Dryrun) + { + await db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); + await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); + } + else + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "would delete file {0}", n.File.Name); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FailedExtraFileCleanup", ex, "Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message); + if (ex.IsAbortException()) + throw; } - if (anyDlistUploads) + var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks); + if (!m_options.RebuildMissingDblockFiles && missingDblocks.Count() > 0) + throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); + + var anyDlistUploads = false; + foreach (var (filesetId, timestamp, isfull) in missingRemoteFilesets) + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitFilesetTransaction"); + rtr.Commit("CommitEarlyExit", false); + return; } - foreach (var volumename in missingLocalFilesets) - { - var remoteVolume = await db.GetRemoteVolume(volumename); - using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) - { - var parsed = VolumeBase.ParseFilename(remoteVolume.Name); - using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) - using (var recreatedb = new LocalRecreateDatabase(db, m_options)) - { - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + progress++; + m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); - var filesetid = db.CreateFileset(remoteVolume.ID, parsed.Time, rtr.Transaction); - // TODO :) recreate rewrite in progress - //RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); - if (!m_options.Dryrun) - rtr.Commit("CommitRecreateFilesetTransaction"); - } + var fsw = new FilesetVolumeWriter(m_options, fileTime); + Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); + + if (!string.IsNullOrEmpty(m_options.ControlFiles)) + foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); + + fsw.CreateFilesetFile(isfull); + await db.WriteFileset(fsw, filesetId, null); + fsw.Close(); + + + fsw.VolumeID = await db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); + await db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); + if (m_options.Dryrun) + { + fsw.Dispose(); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}", fsw.RemoteFilename); + continue; + } + await backendManager.PutAsync(fsw, null, null, false, null, cancellationToken).ConfigureAwait(false); + } + + if (anyDlistUploads) + { + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitFilesetTransaction"); + } + + foreach (var volumename in missingLocalFilesets) + { + var remoteVolume = await db.GetRemoteVolume(volumename); + using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) + { + var parsed = VolumeBase.ParseFilename(remoteVolume.Name); + using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) + using (var recreatedb = new LocalRecreateDatabase(db, m_options)) + { + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + + var filesetid = await db.CreateFileset(remoteVolume.ID, parsed.Time, rtr.Transaction); + + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); + if (!m_options.Dryrun) + rtr.Commit("CommitRecreateFilesetTransaction"); } } + } - if (!m_options.Dryrun && tp.MissingVolumes.Any()) - db.TerminatedWithActiveUploads = true; + if (!m_options.Dryrun && tp.MissingVolumes.Any()) + db.TerminatedWithActiveUploads = true; - if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) - throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); + if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) + throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); - // Process each of the missing volumes in the order of blocks, files and index - // It is important that we process the blocks first, as the index files are derived from the blocks + // Process each of the missing volumes in the order of blocks, files and index + // It is important that we process the blocks first, as the index files are derived from the blocks - Action incrementProgress = () => + void incrementProgress() + { + progress++; + m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + } + + // Blocks are recreated with the entire list of missing files to handle cases where partial recreation is needed + await RunRepairDblocks(backendManager, db, rtr, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); + + // Filesets are recreated one at a time + foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Files)) + { + FilesetVolumeWriter newEntry = null; + incrementProgress(); + + try { + var timestamp = VolumeBase.ParseFilename(n.Name).Time; + var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); + var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); + + await RunRepairDlist(backendManager, db, rtr, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (newEntry != null) + try { newEntry?.Dispose(); } + catch { } + + Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); + + if (ex.IsAbortException()) + throw; + } + } + + // Index files are recreated one at a time, as they are derived from the blocks + foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Index)) + { + IndexVolumeWriter newEntry = null; + incrementProgress(); + + try + { + // Check if the index file has already been deleted, beause the dblock was recreated + var currentState = (await db.GetRemoteVolume(n.Name, rtr.Transaction)).State; + if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) + continue; + + var w = newEntry = new IndexVolumeWriter(m_options); + await RunRepairDindex(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (newEntry != null) + try { newEntry?.Dispose(); } + catch { } + + Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); + + if (ex.IsAbortException()) + throw; + } + } + + foreach (var emptyIndexFile in emptyIndexFiles) + { + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + return; + } + progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - }; - // Blocks are recreated with the entire list of missing files to handle cases where partial recreation is needed - await RunRepairDblocks(backendManager, db, rtr, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); - - // Filesets are recreated one at a time - foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Files)) - { - FilesetVolumeWriter newEntry = null; - incrementProgress(); - - try + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteEmptyIndexFile", "would delete empty index file {0}", emptyIndexFile.Name); + else { - var timestamp = VolumeBase.ParseFilename(n.Name).Time; - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); - var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); - - await RunRepairDlist(backendManager, db, rtr, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - if (newEntry != null) - try { newEntry?.Dispose(); } - catch { } - - Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); - - if (ex.IsAbortException()) - throw; - } - } - - // Index files are recreated one at a time, as they are derived from the blocks - foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Index)) - { - IndexVolumeWriter newEntry = null; - incrementProgress(); - - try - { - // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = (await db.GetRemoteVolume(n.Name, rtr.Transaction)).State; - if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) - continue; - - var w = newEntry = new IndexVolumeWriter(m_options); - await RunRepairDindex(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - if (newEntry != null) - try { newEntry?.Dispose(); } - catch { } - - Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); - - if (ex.IsAbortException()) - throw; - } - } - - foreach (var emptyIndexFile in emptyIndexFiles) - { - try - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (emptyIndexFile.Size > 2048) { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); - return; + Logging.Log.WriteWarningMessage(LOGTAG, "LargeEmptyIndexFile", null, "The empty index file {0} is larger than expected ({1} bytes), choosing not to delete it", emptyIndexFile.Name, emptyIndexFile.Size); } - - progress++; - m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteEmptyIndexFile", "would delete empty index file {0}", emptyIndexFile.Name); else { - if (emptyIndexFile.Size > 2048) - { - Logging.Log.WriteWarningMessage(LOGTAG, "LargeEmptyIndexFile", null, "The empty index file {0} is larger than expected ({1} bytes), choosing not to delete it", emptyIndexFile.Name, emptyIndexFile.Size); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); - await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); - await backendManager.FlushPendingMessagesAsync(db, null, cancellationToken).ConfigureAwait(false); - } + Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); + await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, null, cancellationToken).ConfigureAwait(false); } } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "CleanupEmptyIndexFileError", ex, "Failed to perform cleanup for empty index file: {0}, message: {1}", emptyIndexFile.Name, ex.Message); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "CleanupEmptyIndexFileError", ex, "Failed to perform cleanup for empty index file: {0}, message: {1}", emptyIndexFile.Name, ex.Message); - if (ex.IsAbortException()) - throw; - } + if (ex.IsAbortException()) + throw; } } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "DatabaseIsSynchronized", "Destination and database are synchronized, not making any changes"); - } + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "DatabaseIsSynchronized", "Destination and database are synchronized, not making any changes"); + } - m_result.OperationProgressUpdater.UpdateProgress(1); - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); - if (!m_options.Dryrun) - { - db.TerminatedWithActiveUploads = false; - rtr.Commit("CommitRepairTransaction", false); - } + m_result.OperationProgressUpdater.UpdateProgress(1); + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + { + db.TerminatedWithActiveUploads = false; + rtr.Commit("CommitRepairTransaction", false); } } @@ -871,226 +869,225 @@ namespace Duplicati.Library.Main.Operation // The dblock files are the most complex to recreate // as data can be either file contents, metadata or blocklist hashes // We attempt to recover all three source parts in the steps below - using (var mbl = db.CreateBlockList(originalVolume.Name, rtr)) + using var mbl = db.CreateBlockList(originalVolume.Name, rtr); + + var originalMissingBlockCount = await mbl.GetMissingBlockCount(); + var recoveredSourceBlocks = 0L; + + // First we grab all known blocks from local files + string lastRestoredHash = null; + long lastRestoredSize = -1; + await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) { - var originalMissingBlockCount = await mbl.GetMissingBlockCount(); - var recoveredSourceBlocks = 0L; + if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + continue; - // First we grab all known blocks from local files - string lastRestoredHash = null; - long lastRestoredSize = -1; - await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) + var size = (int)block.Size; + + try { - if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + if (!File.Exists(block.File)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileNotFound", null, "File not found: {0}", block.File); continue; + } - var size = (int)block.Size; + using var f = File.OpenRead(block.File); + f.Position = block.Offset; + var buffer = new byte[m_options.Blocksize]; + if (size != Library.Utility.Utility.ForceStreamRead(f, buffer, size)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileLengthMismatch", null, "Block length mismatch on {0}, expected {1} but got {2}", block.File, size, f.Length); + continue; + } + + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); + if (newhash != block.Hash) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileHashMismatch", null, "Block hash mismatch on {0}, expected {1} but got {2}", block.File, block.Hash, newhash); + continue; + } + + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + { + recoveredSourceBlocks++; + pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.File); + } + } + + // Then grab all blocks with metadata + await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks()) + { + if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + continue; + + var size = (int)block.Size; + + try + { + var isFile = File.Exists(block.Path); + var isDir = Directory.Exists(block.Path); + if (!isFile && !isDir) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "EntryNotFound", null, "Entry not found: {0}", block.Path); + continue; + } + + using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], true, true); + var entry = snapshot.GetFilesystemEntry(block.Path, isDir); + if (entry == null) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", null, "Entry not found: {0}", block.Path); + continue; + } + + var metadata = MetadataGenerator.GenerateMetadata(entry, entry.Attributes, m_options); + var metahash = Utility.WrapMetadata(metadata, m_options); + if (metahash.FileHash != block.Hash || metahash.Blob.Length != size) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "MetadataHashMismatch", null, "Metadata block hash mismatch on {0}, expected {1} but got {2}", block.Path, block.Hash, metahash.FileHash); + continue; + } + + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + { + recoveredSourceBlocks++; + pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.Path); + } + } + + // If we did not find any source blocks, + // we only have the database contents and duplicated blocks to work with + // so no point in creating a new dblock volume for storing this data + if (recoveredSourceBlocks != 0) + { + // Then restore any blocklists that are missing + using var blocklistHash = new MemoryStream(); + LocalRepairDatabase.BlocklistHashesEntry lastBlocklist = null; + var blockhashsize = m_options.BlockhashSize; + var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; + + // Helper function to emit a blocklist hash + async Task EmitBlockListBlock() + { + blocklistHash.Position = 0; + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var resulthash = Convert.ToBase64String(blockhasher.ComputeHash(blocklistHash)); + var totalHashes = (lastBlocklist.BlocklistHashLength + m_options.Blocksize - 1) / m_options.Blocksize; + var hashesInLastBlock = totalHashes % hashesPerBlock; + var targetsize = lastBlocklist.Index >= totalHashes - 1 + ? hashesInLastBlock * blockhashsize + : hashesPerBlock * blockhashsize; + + if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) + { + if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) + pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); + } + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "BlocklistHashMismatch", null, "Internal consistency issue: blocklist hash mismatch on {0} / {1}, expected {2} but got {3}", lastBlocklist.BlocklistHash, lastBlocklist.Index, lastBlocklist.Hash, resulthash); + } + + blocklistHash.SetLength(0); + } + + await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) + { + lastBlocklist ??= blocklist; + + if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) + await EmitBlockListBlock(); + + var data = Convert.FromBase64String(blocklist.Hash); + blocklistHash.Write(data, 0, data.Length); + lastBlocklist = blocklist; + } + + // Handle any trailing blocklist hash + if (blocklistHash.Length > 0) + await EmitBlockListBlock(); + + //Then we grab all remote volumes that have the missing blocks + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources().ToListAsync(cancellationToken: cancellationToken), cancellationToken).ConfigureAwait(false)) + { try { - if (!File.Exists(block.File)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FileNotFound", null, "File not found: {0}", block.File); - continue; - } - - using var f = File.OpenRead(block.File); - f.Position = block.Offset; - var buffer = new byte[m_options.Blocksize]; - if (size != Library.Utility.Utility.ForceStreamRead(f, buffer, size)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FileLengthMismatch", null, "Block length mismatch on {0}, expected {1} but got {2}", block.File, size, f.Length); - continue; - } - - using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); - var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); - if (newhash != block.Hash) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FileHashMismatch", null, "Block hash mismatch on {0}, expected {1} but got {2}", block.File, block.Hash, newhash); - continue; - } - - // Found it, no need to look again - lastRestoredHash = block.Hash; - lastRestoredSize = block.Size; - if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) - { - recoveredSourceBlocks++; - pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); - } + using (tmpfile) + using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) + foreach (var b in f.Blocks) + if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) + if (f.ReadBlock(b.Key, buffer) == b.Value) + pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); } catch (Exception ex) { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.File); + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); } } + } - // Then grab all blocks with metadata - await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks()) + var missingBlocks = await mbl.GetMissingBlockCount(); + var recoveredBlocks = originalMissingBlockCount - missingBlocks; + if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) + { + Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); + await foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) + Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); + + var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); + var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); + + Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); + throw new UserInformationException(logmsg, "RepairIsNotPossible"); + } + else if (recoveredBlocks > 0) + { + if (missingBlocks > 0) + Logging.Log.WriteWarningMessage(LOGTAG, "RepairMissingBlocks", null, "Repair acquired {0} blocks for volume {1}, but {2} blocks are still missing. If you want to continue working with the database, you can use the \"{3}\" and \"{4}\" commands to purge the missing data from the database and the remote storage.", recoveredBlocks, originalVolume.Name, missingBlocks, "list-broken-files", "purge-broken-files"); + + // If we cannot fully recover the blocks, we will not be able to delete the old files. + // They will instead be registered with duplicated blocks, so the purge-broken-files command + // can remove them cleanly later + if (missingBlocks == 0) { - if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) - continue; + pendingVolume.VolumesToDelete.Add(originalVolume); + var oldIndexFiles = await db.GetIndexFilesReferencingBlockFile(originalVolume.ID, rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); - var size = (int)block.Size; - - try + // Find all index files that point to the old volume + foreach (var oldIndexFile in oldIndexFiles) { - var isFile = File.Exists(block.Path); - var isDir = Directory.Exists(block.Path); - if (!isFile && !isDir) + var oldVolume = await db.GetRemoteVolume(oldIndexFile, rtr.Transaction); + if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) { - Logging.Log.WriteVerboseMessage(LOGTAG, "EntryNotFound", null, "Entry not found: {0}", block.Path); - continue; - } - - using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], true, true); - var entry = snapshot.GetFilesystemEntry(block.Path, isDir); - if (entry == null) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", null, "Entry not found: {0}", block.Path); - continue; - } - - var metadata = MetadataGenerator.GenerateMetadata(entry, entry.Attributes, m_options); - var metahash = Utility.WrapMetadata(metadata, m_options); - if (metahash.FileHash != block.Hash || metahash.Blob.Length != size) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "MetadataHashMismatch", null, "Metadata block hash mismatch on {0}, expected {1} but got {2}", block.Path, block.Hash, metahash.FileHash); - continue; - } - - // Found it, no need to look again - lastRestoredHash = block.Hash; - lastRestoredSize = block.Size; - if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) - { - recoveredSourceBlocks++; - pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); - } - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.Path); - } - } - - // If we did not find any source blocks, - // we only have the database contents and duplicated blocks to work with - // so no point in creating a new dblock volume for storing this data - if (recoveredSourceBlocks != 0) - { - // Then restore any blocklists that are missing - using var blocklistHash = new MemoryStream(); - LocalRepairDatabase.BlocklistHashesEntry lastBlocklist = null; - var blockhashsize = m_options.BlockhashSize; - var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; - - // Helper function to emit a blocklist hash - async Task EmitBlockListBlock() - { - blocklistHash.Position = 0; - using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); - var resulthash = Convert.ToBase64String(blockhasher.ComputeHash(blocklistHash)); - var totalHashes = (lastBlocklist.BlocklistHashLength + m_options.Blocksize - 1) / m_options.Blocksize; - var hashesInLastBlock = totalHashes % hashesPerBlock; - var targetsize = lastBlocklist.Index >= totalHashes - 1 - ? hashesInLastBlock * blockhashsize - : hashesPerBlock * blockhashsize; - - if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) - { - if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) - pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); + var blockVolumesReferenced = await db.GetBlockVolumesFromIndexName(oldIndexFile, rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); + else + pendingVolume.VolumesToDelete.Add(oldVolume); } else { - Logging.Log.WriteWarningMessage(LOGTAG, "BlocklistHashMismatch", null, "Internal consistency issue: blocklist hash mismatch on {0} / {1}, expected {2} but got {3}", lastBlocklist.BlocklistHash, lastBlocklist.Index, lastBlocklist.Hash, resulthash); - } - - blocklistHash.SetLength(0); - } - - await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) - { - lastBlocklist ??= blocklist; - - if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) - await EmitBlockListBlock(); - - var data = Convert.FromBase64String(blocklist.Hash); - blocklistHash.Write(data, 0, data.Length); - lastBlocklist = blocklist; - } - - // Handle any trailing blocklist hash - if (blocklistHash.Length > 0) - await EmitBlockListBlock(); - - //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources().ToListAsync(cancellationToken: cancellationToken), cancellationToken).ConfigureAwait(false)) - { - try - { - var buffer = new byte[m_options.Blocksize]; - using (tmpfile) - using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - foreach (var b in f.Blocks) - if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) - if (f.ReadBlock(b.Key, buffer) == b.Value) - pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); - } - } - } - - var missingBlocks = await mbl.GetMissingBlockCount(); - var recoveredBlocks = originalMissingBlockCount - missingBlocks; - if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); - await foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) - Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); - - var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); - var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); - - Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); - throw new UserInformationException(logmsg, "RepairIsNotPossible"); - } - else if (recoveredBlocks > 0) - { - if (missingBlocks > 0) - Logging.Log.WriteWarningMessage(LOGTAG, "RepairMissingBlocks", null, "Repair acquired {0} blocks for volume {1}, but {2} blocks are still missing. If you want to continue working with the database, you can use the \"{3}\" and \"{4}\" commands to purge the missing data from the database and the remote storage.", recoveredBlocks, originalVolume.Name, missingBlocks, "list-broken-files", "purge-broken-files"); - - // If we cannot fully recover the blocks, we will not be able to delete the old files. - // They will instead be registered with duplicated blocks, so the purge-broken-files command - // can remove them cleanly later - if (missingBlocks == 0) - { - pendingVolume.VolumesToDelete.Add(originalVolume); - var oldIndexFiles = await db.GetIndexFilesReferencingBlockFile(originalVolume.ID, rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); - - // Find all index files that point to the old volume - foreach (var oldIndexFile in oldIndexFiles) - { - var oldVolume = await db.GetRemoteVolume(oldIndexFile, rtr.Transaction); - if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) - { - var blockVolumesReferenced = await db.GetBlockVolumesFromIndexName(oldIndexFile, rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); - if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) - Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); - else - pendingVolume.VolumesToDelete.Add(oldVolume); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} is not in a state to be deleted, skipping", oldVolume.Name); - } + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} is not in a state to be deleted, skipping", oldVolume.Name); } } } @@ -1102,37 +1099,35 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var tr = new ReusableTransaction(db)) + using var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); + using var tr = new ReusableTransaction(db); + + var sets = await db.GetFilesetsWithMissingFiles(null).ToListAsync(); + if (sets.Count == 0) + return; + + Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFilesets", "Repairing {0} broken filesets", sets.Count); + var ix = 0; + foreach (var entry in sets) { - var sets = await db.GetFilesetsWithMissingFiles(null).ToListAsync(); - if (sets.Count == 0) - return; + ix++; + Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); + var volume = await db.GetRemoteVolumeFromFilesetID(entry.Key, tr.Transaction); + var parsed = VolumeBase.ParseFilename(volume.Name); + using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); + using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFilesets", "Repairing {0} broken filesets", sets.Count); - var ix = 0; - foreach (var entry in sets) + // Clear out the old fileset + await db.DeleteFilesetEntries(entry.Key, tr.Transaction); + using (var rdb = new LocalRecreateDatabase(db, m_options)) { - ix++; - Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); - var volume = await db.GetRemoteVolumeFromFilesetID(entry.Key, tr.Transaction); - var parsed = VolumeBase.ParseFilename(volume.Name); - using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); - using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); - using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - - // Clear out the old fileset - await db.DeleteFilesetEntries(entry.Key, tr.Transaction); - using (var rdb = new LocalRecreateDatabase(db, m_options)) - { - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, tr.Transaction, compressor, entry.Key, m_options, new FilterExpression()); - } - - tr.Commit("PostRepairFileset"); + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, tr.Transaction, compressor, entry.Key, m_options, new FilterExpression()); } + tr.Commit("PostRepairFileset"); } } @@ -1143,18 +1138,17 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - { - Utility.UpdateOptionsFromDb(db, m_options); + using var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); - if (db.RepairInProgress || db.PartiallyRecreated) - Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); + Utility.UpdateOptionsFromDb(db, m_options); - await db.FixDuplicateMetahash(); - await db.FixDuplicateFileentries(); - await db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize); - await db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize); - } + if (db.RepairInProgress || db.PartiallyRecreated) + Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); + + await db.FixDuplicateMetahash(); + await db.FixDuplicateFileentries(); + await db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize); + await db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize); } } } From 24ac9dad1c4c502a6198a6da24fcb10a59bcff8b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 11:42:03 +0200 Subject: [PATCH 026/362] Fund some remaining synchronous commits --- .../Library/Main/Operation/RepairHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index cab0b4c50..49c4d9f7e 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -146,7 +146,7 @@ namespace Duplicati.Library.Main.Operation // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(rtr.Transaction); var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, rtr.Transaction, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); - rtr.Commit("CommitRemoteListAnalysisTransaction"); + await rtr.CommitAsync("CommitRemoteListAnalysisTransaction"); var buffer = new byte[m_options.Blocksize]; var hashsize = m_options.BlockhashSize; @@ -198,7 +198,7 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(testdb, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); + await rtr.CommitAsync("CommitEarlyExit", false); return; } @@ -227,7 +227,7 @@ namespace Duplicati.Library.Main.Operation throw; } - rtr.Commit("CommitVerificationTransaction"); + await rtr.CommitAsync("CommitVerificationTransaction"); } // TODO: It is actually possible to use the extra files if we parse them @@ -238,7 +238,7 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); + await rtr.CommitAsync("CommitEarlyExit", false); return; } @@ -286,7 +286,7 @@ namespace Duplicati.Library.Main.Operation await db.AddIndexBlockLink(selfid, id, rtr.Transaction); } if (!m_options.Dryrun) - rtr.Commit("CommitIndexFileTransaction"); + await rtr.CommitAsync("CommitIndexFileTransaction"); } // All checks fine, we accept the new index file @@ -326,7 +326,7 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); + await rtr.CommitAsync("CommitEarlyExit", false); return; } @@ -362,7 +362,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitFilesetTransaction"); + await rtr.CommitAsync("CommitFilesetTransaction"); } foreach (var volumename in missingLocalFilesets) @@ -382,7 +382,7 @@ namespace Duplicati.Library.Main.Operation await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); if (!m_options.Dryrun) - rtr.Commit("CommitRecreateFilesetTransaction"); + await rtr.CommitAsync("CommitRecreateFilesetTransaction"); } } } @@ -509,7 +509,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { db.TerminatedWithActiveUploads = false; - rtr.Commit("CommitRepairTransaction", false); + await rtr.CommitAsync("CommitRepairTransaction", false); } } @@ -551,13 +551,13 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - rtr.Commit("CommitPriorToFilesetUpload"); + await rtr.CommitAsync("CommitPriorToFilesetUpload"); await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitFilesetTransaction"); + await rtr.CommitAsync("CommitFilesetTransaction"); } /// @@ -616,7 +616,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitRepairTransaction"); + await rtr.CommitAsync("CommitRepairTransaction"); } /// @@ -753,7 +753,7 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); + await rtr.CommitAsync("CommitEarlyExit", false); return; } @@ -822,7 +822,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReplaceBlockFile", "would upload new block file {0}", completedVolume.RemoteFilename); else { - rtr.Commit("PostRepairPreUploadBlockVolume"); + await rtr.CommitAsync("PostRepairPreUploadBlockVolume"); await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); } } @@ -1127,7 +1127,7 @@ namespace Duplicati.Library.Main.Operation await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, tr.Transaction, compressor, entry.Key, m_options, new FilterExpression()); } - tr.Commit("PostRepairFileset"); + await tr.CommitAsync("PostRepairFileset"); } } From 3f0d835937d532ec756ca75e08cc812f6a040934 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 16:15:53 +0200 Subject: [PATCH 027/362] Rewrote the logic of ReusableTransaction. The main idea was to remove internal use of nullable, and to only have one implementation; which is why the synchronous versions call the asynchronous versions. --- .../Main/Database/ReusableTransaction.cs | 57 ++++++++----------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 550962931..51c09ba42 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -21,6 +21,7 @@ using System; using System.Threading.Tasks; +using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; #nullable enable @@ -30,7 +31,7 @@ namespace Duplicati.Library.Main.Database; /// /// Wraps a transaction so it can be comitted and restarted /// -internal class ReusableTransaction : IDisposable +internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) : IDisposable { /// /// The tag used for logging @@ -40,26 +41,20 @@ internal class ReusableTransaction : IDisposable /// /// The database to use /// - private readonly LocalDatabase m_db; + private readonly LocalDatabase m_db = db; /// /// The current transaction /// - private SqliteTransaction? m_transaction; - + private SqliteTransaction m_transaction = transaction ?? db.Connection.BeginTransaction(); /// /// Creates a new reusable transaction /// - /// The database to use - public ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) - { - m_db = db; - m_transaction = transaction ?? db.Connection.BeginTransaction(); - } + private bool m_disposed = false; /// /// The current transaction /// - public SqliteTransaction Transaction => m_transaction ?? throw new InvalidOperationException("Transaction is disposed"); + public SqliteTransaction Transaction => m_disposed ? m_transaction : throw new InvalidOperationException("Transaction is disposed"); /// /// Commits the current transaction and optionally restarts it @@ -68,42 +63,36 @@ internal class ReusableTransaction : IDisposable /// True if the transaction should be restarted public void Commit(string? message, bool restart = true) { - if (m_transaction == null) - throw new InvalidOperationException("Transaction is already disposed"); - - if (m_transaction != null) - { - using (var timer = string.IsNullOrWhiteSpace(message) ? null : new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) - m_transaction.Commit(); - m_transaction.Dispose(); - m_transaction = null; - } - - if (restart) - m_transaction = m_db.Connection.BeginTransaction(); + CommitAsync(message, restart).Await(); } public async Task CommitAsync(string? message, bool restart = true) { - if (m_transaction == null) + if (m_disposed) throw new InvalidOperationException("Transaction is already disposed"); - if (m_transaction != null) - { - using (var timer = string.IsNullOrWhiteSpace(message) ? null : new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) - await m_transaction.CommitAsync(); - await m_transaction.DisposeAsync(); - m_transaction = null; - } + using (var timer = string.IsNullOrWhiteSpace(message) ? null : new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) + await m_transaction.CommitAsync(); + await m_transaction.DisposeAsync(); if (restart) m_transaction = m_db.Connection.BeginTransaction(); + else + m_disposed = true; } /// public void Dispose() { - m_transaction?.Dispose(); - m_transaction = null; + DisposeAsync().Await(); + } + + public async Task DisposeAsync() + { + if (!m_disposed) + { + m_disposed = true; + await m_transaction.DisposeAsync(); + } } } From 4c294ea5d8f06384246728b998207c25e35c42e3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 13 May 2025 16:16:06 +0200 Subject: [PATCH 028/362] Fix docstrings in ReusableTransaction --- .../Main/Database/ReusableTransaction.cs | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 51c09ba42..7052b1a6f 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -29,43 +29,59 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database; /// -/// Wraps a transaction so it can be comitted and restarted +/// Wraps a transaction so it can be comitted and restarted. /// +/// +/// Creates a new reusable transaction. +/// +/// The database to use. +/// The transaction to use. If null, a new transaction is created. internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) : IDisposable { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ReusableTransaction)); /// - /// The database to use + /// The database to use. /// private readonly LocalDatabase m_db = db; /// - /// The current transaction + /// The current transaction. /// private SqliteTransaction m_transaction = transaction ?? db.Connection.BeginTransaction(); /// - /// Creates a new reusable transaction + /// True if the transaction is disposed. /// private bool m_disposed = false; /// - /// The current transaction + /// The current transaction. /// public SqliteTransaction Transaction => m_disposed ? m_transaction : throw new InvalidOperationException("Transaction is disposed"); /// - /// Commits the current transaction and optionally restarts it + /// Commits the current transaction and optionally restarts it. /// - /// The log message to use - /// True if the transaction should be restarted + /// + /// Calls the Async version of this method and awaits it. + /// + /// The log message to use. + /// True if the transaction should be restarted. + /// If the transaction is already Disposed. public void Commit(string? message, bool restart = true) { CommitAsync(message, restart).Await(); } + /// + /// Async version of Commit: + /// + /// The log message to use. + /// True if the transaction should be restarted. + /// An awaitable task. + /// If the transaction is already Disposed. public async Task CommitAsync(string? message, bool restart = true) { if (m_disposed) @@ -82,11 +98,18 @@ internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transact } /// + /// + /// Calls the Async version of this method and awaits it. + /// public void Dispose() { DisposeAsync().Await(); } + /// + /// Async version of Dispose: + /// + /// An awaitable task. public async Task DisposeAsync() { if (!m_disposed) From c060a214d89968c6abdefbe50a455bf164e990a1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 14 May 2025 10:36:44 +0200 Subject: [PATCH 029/362] Changed from using constructors to a factory-like pattern --- .../Library/Main/Database/LocalDatabase.cs | 232 ++++++++++-------- 1 file changed, 134 insertions(+), 98 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 8e6e9cc87..6e66e875a 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -52,26 +52,27 @@ namespace Duplicati.Library.Main.Database /// SQLite has a limit of 999 parameters in a single statement public const int CHUNK_SIZE = 128; - protected readonly SqliteConnection m_connection; - protected readonly long m_operationid = -1; - protected readonly long m_pagecachesize; + // All of the required fields have been set to null! to ignore the compiler warning, as they will be initialized in the Create* factory functions. + protected SqliteConnection m_connection = null!; + protected long m_operationid = -1; + protected long m_pagecachesize; private bool m_hasExecutedVacuum; - private readonly SqliteCommand m_updateremotevolumeCommand; - private readonly SqliteCommand m_selectremotevolumesCommand; - private readonly SqliteCommand m_selectremotevolumeCommand; - private readonly SqliteCommand m_removeremotevolumeCommand; - private readonly SqliteCommand m_removedeletedremotevolumeCommand; - private readonly SqliteCommand m_selectremotevolumeIdCommand; - private readonly SqliteCommand m_createremotevolumeCommand; - private readonly SqliteCommand m_selectduplicateRemoteVolumesCommand; + private SqliteCommand m_updateremotevolumeCommand = null!; + private SqliteCommand m_selectremotevolumesCommand = null!; + private SqliteCommand m_selectremotevolumeCommand = null!; + private SqliteCommand m_removeremotevolumeCommand = null!; + private SqliteCommand m_removedeletedremotevolumeCommand = null!; + private SqliteCommand m_selectremotevolumeIdCommand = null!; + private SqliteCommand m_createremotevolumeCommand = null!; + private SqliteCommand m_selectduplicateRemoteVolumesCommand = null!; - private readonly SqliteCommand m_insertlogCommand; - private readonly SqliteCommand m_insertremotelogCommand; - private readonly SqliteCommand m_insertIndexBlockLink; + private SqliteCommand m_insertlogCommand = null!; + private SqliteCommand m_insertremotelogCommand = null!; + private SqliteCommand m_insertIndexBlockLink = null!; - private readonly SqliteCommand m_findpathprefixCommand; - private readonly SqliteCommand m_insertpathprefixCommand; + private SqliteCommand m_findpathprefixCommand = null!; + private SqliteCommand m_insertpathprefixCommand = null!; public const long FOLDER_BLOCKSET_ID = -100; public const long SYMLINK_BLOCKSET_ID = -200; @@ -84,7 +85,22 @@ namespace Duplicati.Library.Main.Database public bool ShouldCloseConnection { get; set; } - protected static async Task CreateConnection(string path, long pagecachesize) + // Constructor is private to force use of CreateLocalDatabaseAsync + private LocalDatabase() { } + + // Arguments are not used, but are required to match the constructor signatures + [Obsolete("Calling the constructor will throw an exception. Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead")] + public LocalDatabase(object? _ignore1 = null, object? _ignore2 = null, object? _ignore3 = null, object? _ignore4 = null) + { + throw new NotImplementedException("Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead"); + } + + protected static SqliteConnection CreateConnection(string path, long pagecachesize) + { + return CreateConnectionAsync(path, pagecachesize).Await(); + } + + protected static async Task CreateConnectionAsync(string path, long pagecachesize) { path = Path.GetFullPath(path); if (!Directory.Exists(Path.GetDirectoryName(path))) @@ -106,6 +122,107 @@ namespace Duplicati.Library.Main.Database return c; } + public static LocalDatabase CreateLocalDatabase(string path, string operation, bool shouldclose, long pagecachesize) + { + return CreateLocalDatabaseAsync(path, operation, shouldclose, pagecachesize).Await(); + } + + public static LocalDatabase CreateLocalDatabase(LocalDatabase db) + { + return CreateLocalDatabaseAsync(db).Await(); + } + + public static LocalDatabase CreateLocalDatabase(SqliteConnection connection, string operation) + { + return CreateLocalDatabaseAsync(connection, operation).Await(); + } + + private static LocalDatabase CreateLocalDatabase(SqliteConnection connection) + { + return CreateLocalDatabaseAsync(connection).Await(); + } + + public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize) + { + var connection = await CreateConnectionAsync(path, pagecachesize); + var db = await CreateLocalDatabaseAsync(connection, operation); + + db.ShouldCloseConnection = shouldclose; + db.m_pagecachesize = pagecachesize; + + return db; + } + + public static async Task CreateLocalDatabaseAsync(LocalDatabase db) + { + var new_db = await CreateLocalDatabaseAsync(db.m_connection); + + new_db.OperationTimestamp = db.OperationTimestamp; + new_db.m_connection = db.m_connection; + new_db.m_operationid = db.m_operationid; + new_db.m_pagecachesize = db.m_pagecachesize; + + return new_db; + } + + public static async Task CreateLocalDatabaseAsync(SqliteConnection connection, string operation) + { + var db = await CreateLocalDatabaseAsync(connection); + + db.OperationTimestamp = DateTime.UtcNow; + + if (db.m_connection.State != ConnectionState.Open) + await db.m_connection.OpenAsync(); + + using var cmd = db.m_connection.CreateCommand(); + using var transaction = db.m_connection.BeginTransaction(); + cmd.Transaction = transaction; + if (operation != null) + { + cmd.CommandText = @"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (@Description, @Timestamp); SELECT last_insert_rowid();"; + cmd.SetParameterValue("@Description", operation); + cmd.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(db.OperationTimestamp)); + db.m_operationid = await cmd.ExecuteScalarInt64Async(-1); + } + else + { + // Get last operation + using var rd = await cmd.ExecuteReaderAsync(@"SELECT ""ID"", ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1"); + if (!await rd.ReadAsync()) + throw new Exception("LocalDatabase does not contain a previous operation."); + + db.m_operationid = rd.ConvertValueToInt64(0); + db.OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); + } + + return db; + } + + private static async Task CreateLocalDatabaseAsync(SqliteConnection connection) + { + var selectremotevolumes_sql = @"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" FROM ""Remotevolume"""; + + var db = new LocalDatabase + { + m_connection = connection, + m_insertlogCommand = await connection.CreateCommandAsync(@"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (@OperationID, @Timestamp, @Type, @Message, @Exception)"), + m_insertremotelogCommand = await connection.CreateCommandAsync(@"INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") VALUES (@OperationID, @Timestamp, @Operation, @Path, @Data)"), + m_updateremotevolumeCommand = await connection.CreateCommandAsync(@"UPDATE ""Remotevolume"" SET ""OperationID"" = @OperationID, ""State"" = @State, ""Hash"" = @Hash, ""Size"" = @Size WHERE ""Name"" = @Name"), + m_selectremotevolumesCommand = await connection.CreateCommandAsync(selectremotevolumes_sql), + m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$"{selectremotevolumes_sql} WHERE ""Name"" = @Name"), + m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync(FormatInvariant($@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')")), + m_removeremotevolumeCommand = await connection.CreateCommandAsync(@"DELETE FROM ""Remotevolume"" WHERE ""Name"" = @Name AND (""DeleteGraceTime"" < @Now OR ""State"" != @State)"), + m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync(FormatInvariant($@"DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted.ToString()}' AND (""DeleteGraceTime"" < @Now OR LENGTH(""DeleteGraceTime"") > 12) ")), // >12 is to handle removal of old records that were in ticks + m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = @Name"), + m_createremotevolumeCommand = await connection.CreateCommandAsync(@"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"", ""ArchiveTime"") VALUES (@OperationID, @Name, @Type, @State, @Size, @VerificationCount, @DeleteGraceTime, @ArchiveTime); SELECT last_insert_rowid();"), + m_insertIndexBlockLink = await connection.CreateCommandAsync(@"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (@IndexVolumeId, @BlockVolumeId)"), + m_findpathprefixCommand = await connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = @Prefix"), + m_insertpathprefixCommand = await connection.CreateCommandAsync(@"INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (@Prefix); SELECT last_insert_rowid(); ") + }; + + return db; + } + /// /// Formats the string using the invariant culture /// @@ -119,87 +236,6 @@ namespace Duplicati.Library.Main.Database return File.Exists(path); } - /// - /// Creates a new database instance and starts a new operation - /// - /// The path to the database - /// The name of the operation. If null, continues last operation - /// Should the connection be closed when this object is disposed - /// The page cache size - public LocalDatabase(string path, string operation, bool shouldclose, long pagecachesize) - : this(CreateConnection(path, pagecachesize).Await(), operation) - { - ShouldCloseConnection = shouldclose; - m_pagecachesize = pagecachesize; - } - - /// - /// Creates a new database instance and starts a new operation - /// - public LocalDatabase(LocalDatabase db) - : this(db.m_connection) - { - OperationTimestamp = db.OperationTimestamp; - m_connection = db.m_connection; - m_operationid = db.m_operationid; - m_pagecachesize = db.m_pagecachesize; - } - - /// - /// Creates a new database instance and starts a new operation - /// - /// The name of the operation. If null, continues last operation - public LocalDatabase(SqliteConnection connection, string operation) - : this(connection) - { - OperationTimestamp = DateTime.UtcNow; - m_connection = connection; - - if (m_connection.State != ConnectionState.Open) - m_connection.Open(); - - if (operation != null) - { - using var cmd = m_connection.CreateCommand(); - using var transaction = m_connection.BeginTransaction(); - cmd.Transaction = transaction; - cmd.CommandText = @"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (@Description, @Timestamp); SELECT last_insert_rowid();"; - cmd.SetParameterValue("@Description", operation); - cmd.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(OperationTimestamp)); - m_operationid = cmd.ExecuteScalarInt64Async(-1).Await(); - } - else - { - // Get last operation - using var cmd = m_connection.CreateCommand(); - cmd.CommandText = @"SELECT ""ID"", ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1"; - using var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1"); - if (!rd.Read()) - throw new Exception("LocalDatabase does not contain a previous operation."); - - m_operationid = rd.ConvertValueToInt64(0); - OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); - } - } - - private LocalDatabase(SqliteConnection connection) - { - m_connection = connection; - m_insertlogCommand = connection.CreateCommand(@"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (@OperationID, @Timestamp, @Type, @Message, @Exception)"); - m_insertremotelogCommand = connection.CreateCommand(@"INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") VALUES (@OperationID, @Timestamp, @Operation, @Path, @Data)"); - m_updateremotevolumeCommand = connection.CreateCommand(@"UPDATE ""Remotevolume"" SET ""OperationID"" = @OperationID, ""State"" = @State, ""Hash"" = @Hash, ""Size"" = @Size WHERE ""Name"" = @Name"); - m_selectremotevolumesCommand = connection.CreateCommand(@"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" FROM ""Remotevolume"""); - m_selectremotevolumeCommand = connection.CreateCommand(m_selectremotevolumesCommand.CommandText + @" WHERE ""Name"" = @Name"); - m_selectduplicateRemoteVolumesCommand = connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')")); - m_removeremotevolumeCommand = connection.CreateCommand(@"DELETE FROM ""Remotevolume"" WHERE ""Name"" = @Name AND (""DeleteGraceTime"" < @Now OR ""State"" != @State)"); - m_removedeletedremotevolumeCommand = connection.CreateCommand(FormatInvariant($@"DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted.ToString()}' AND (""DeleteGraceTime"" < @Now OR LENGTH(""DeleteGraceTime"") > 12) ")); // >12 is to handle removal of old records that were in ticks - m_selectremotevolumeIdCommand = connection.CreateCommand(@"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = @Name"); - m_createremotevolumeCommand = connection.CreateCommand(@"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"", ""ArchiveTime"") VALUES (@OperationID, @Name, @Type, @State, @Size, @VerificationCount, @DeleteGraceTime, @ArchiveTime); SELECT last_insert_rowid();"); - m_insertIndexBlockLink = connection.CreateCommand(@"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (@IndexVolumeId, @BlockVolumeId)"); - m_findpathprefixCommand = connection.CreateCommand(@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = @Prefix"); - m_insertpathprefixCommand = connection.CreateCommand(@"INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (@Prefix); SELECT last_insert_rowid(); "); - } - /// /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value /// From 3c6dc243903da228872dba2a1bee601f716cc393 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 14 May 2025 10:44:24 +0200 Subject: [PATCH 030/362] Named the async functions in the SQLiteLoader explicitly, and added synchronous versions. --- .../Library/SQLiteHelper/SQLiteLoader.cs | 72 +++++++++++++++---- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index a3c261915..d32420dba 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -26,6 +26,7 @@ using System.Globalization; using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; namespace Duplicati.Library.SQLiteHelper { @@ -41,13 +42,18 @@ namespace Duplicati.Library.SQLiteHelper /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(SQLiteLoader)); + public static void OpenDatabase(Microsoft.Data.Sqlite.SqliteConnection con, string databasePath, string? decryptionPassword) + { + OpenDatabaseAsync(con, databasePath, decryptionPassword).Await(); + } + /// /// Helper method with logic to handle opening a database in possibly encrypted format /// /// The SQLite connection object /// The location of Duplicati's database. /// The password to use for decryption. - public static async Task OpenDatabase(Microsoft.Data.Sqlite.SqliteConnection con, string databasePath, string? decryptionPassword) + public static async Task OpenDatabaseAsync(Microsoft.Data.Sqlite.SqliteConnection con, string databasePath, string? decryptionPassword) { if (!string.IsNullOrWhiteSpace(decryptionPassword) && SQLiteRC4Decrypter.IsDatabaseEncrypted(databasePath)) { @@ -67,8 +73,8 @@ namespace Duplicati.Library.SQLiteHelper try { //Attempt to open in preferred state - await OpenSQLiteFile(con, databasePath); - await TestSQLiteFile(con); + await OpenSQLiteFileAsync(con, databasePath); + await TestSQLiteFileAsync(con); } catch { @@ -82,11 +88,16 @@ namespace Duplicati.Library.SQLiteHelper throw new UserInformationException("Failed to open database for unknown reason, check the logs to see error messages", "DatabaseOpenFailed"); } + public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection() + { + return LoadConnectionAsync().Await(); + } + /// /// Loads an SQLite connection instance and opening the database /// /// The SQLite connection instance. - public static async Task LoadConnection() + public static async Task LoadConnectionAsync() { Microsoft.Data.Sqlite.SqliteConnection? con = null; SetEnvironmentVariablesForSQLiteTempDir(); @@ -98,7 +109,7 @@ namespace Duplicati.Library.SQLiteHelper catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, "Failed to load connection."); - await DisposeConnection(con); + await DisposeConnectionAsync(con); throw; } @@ -106,15 +117,28 @@ namespace Duplicati.Library.SQLiteHelper return con ?? throw new InvalidOperationException("Failed to load connection"); } + public static Microsoft.Data.Sqlite.SqliteConnection ApplyCustomPragmas(Microsoft.Data.Sqlite.SqliteConnection con, long pagecachesize) + { + return ApplyCustomPragmasAsync(con, pagecachesize).Await(); + } + /// /// Applies user-supplied custom pragmas to the SQLite connection /// /// The connection to apply the pragmas to. /// The page cache size to set. /// The connection with the pragmas applied. - public static async Task ApplyCustomPragmas(Microsoft.Data.Sqlite.SqliteConnection con, long pagecachesize) + public static async Task ApplyCustomPragmasAsync(Microsoft.Data.Sqlite.SqliteConnection con, long pagecachesize) { // TODO more default custom options + //await cmd.ExecuteNonQueryAsync("PRAGMA synchronous = NORMAL;"); + //await cmd.ExecuteNonQueryAsync("PRAGMA temp_store = MEMORY;"); + //await cmd.ExecuteNonQueryAsync("PRAGMA journal_mode = WAL;"); + //await cmd.ExecuteNonQueryAsync("PRAGMA cache_size = -512000;"); + //await cmd.ExecuteNonQueryAsync("PRAGMA threads = 8;"); + //await cmd.ExecuteNonQueryAsync("PRAGMA read_uncommitted = 1;"); + //await cmd.ExecuteNonQueryAsync("PRAGMA mmap_size = 41943040;"); + //await cmd.ExecuteNonQueryAsync("PRAGMA shared_cache = true;"); var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI") ?? ""; if (pagecachesize > MINIMUM_SQLITE_PAGE_CACHE_SIZE) opts = string.Format(CultureInfo.InvariantCulture, "cache_size=-{0};{1}", pagecachesize / 1024L, opts); @@ -144,33 +168,38 @@ namespace Duplicati.Library.SQLiteHelper return con; } + public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection(string targetpath, long pagecachesize) + { + return LoadConnectionAsync(targetpath, pagecachesize).Await(); + } + /// /// Loads an SQLite connection instance and opening the database /// /// The SQLite connection instance. /// The optional path to the database. /// The page cache size to set. - public static async Task LoadConnection(string targetpath, long pagecachesize) + public static async Task LoadConnectionAsync(string targetpath, long pagecachesize) { if (string.IsNullOrWhiteSpace(targetpath)) throw new ArgumentNullException(nameof(targetpath)); - var con = await LoadConnection(); + var con = await LoadConnectionAsync(); try { - await OpenSQLiteFile(con, targetpath); + await OpenSQLiteFileAsync(con, targetpath); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, @"Failed to load connection with path '{0}'.", targetpath); - await DisposeConnection(con); + await DisposeConnectionAsync(con); throw; } // set custom Sqlite options - return await ApplyCustomPragmas(con, pagecachesize); + return await ApplyCustomPragmasAsync(con, pagecachesize); } /// @@ -218,23 +247,33 @@ namespace Duplicati.Library.SQLiteHelper Environment.SetEnvironmentVariable("TEMP", Utility.TempFolder.SystemTempPath); } + private static void DisposeConnection(Microsoft.Data.Sqlite.SqliteConnection? con) + { + DisposeConnectionAsync(con).Await(); + } + /// /// Wrapper to dispose the SQLite connection /// /// The connection to close. - private static async Task DisposeConnection(Microsoft.Data.Sqlite.SqliteConnection? con) + private static async Task DisposeConnectionAsync(Microsoft.Data.Sqlite.SqliteConnection? con) { if (con != null) try { await con.DisposeAsync(); } catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "ConnectionDisposeError", ex, "Failed to dispose connection"); } } + private static void OpenSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con, string path) + { + OpenSQLiteFileAsync(con, path).Await(); + } + /// /// Opens the SQLite file in the given connection, creating the file if required /// /// The connection to use. /// Path to the file to open, which may not exist. - private static async Task OpenSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con, string path) + private static async Task OpenSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con, string path) { con.ConnectionString = "Data Source=" + path; await con.OpenAsync(); @@ -252,11 +291,16 @@ namespace Duplicati.Library.SQLiteHelper catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "SQLiteFilePermissionError", ex, "Failed to set permissions on SQLite file '{0}'", path); } } + private static void TestSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con) + { + TestSQLiteFileAsync(con).Await(); + } + /// /// Tests the SQLite connection, throwing an exception if the connection does not work /// /// The connection to test. - private static async Task TestSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con) + private static async Task TestSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con) { // Do a dummy query to make sure we have a working db using var cmd = con.CreateCommand(); From ca52d413475fdda03e9996ad3c6545bb0bf73241 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 14 May 2025 11:12:04 +0200 Subject: [PATCH 031/362] Ensured more of the calls in LocalDatabase is calling the async versions --- Duplicati/Library/Main/Database/LocalDatabase.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 6e66e875a..cb54fd6cf 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -106,7 +106,7 @@ namespace Duplicati.Library.Main.Database if (!Directory.Exists(Path.GetDirectoryName(path))) Directory.CreateDirectory(Path.GetDirectoryName(path) ?? throw new DirectoryNotFoundException("Path was a root folder.")); - var c = await SQLiteHelper.SQLiteLoader.LoadConnection(path, pagecachesize); + var c = await SQLiteHelper.SQLiteLoader.LoadConnectionAsync(path, pagecachesize); try { @@ -376,7 +376,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files, SqliteTransaction? transaction = null) { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name)"); + using var cmd = await m_connection.CreateCommandAsync(@"SELECT ""Name"", ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name)"); cmd.Transaction = transaction; using var tmptable = new TemporaryDbValueList(m_connection, transaction, files); cmd.ExpandInClauseParameter("@Name", tmptable); @@ -473,7 +473,7 @@ namespace Duplicati.Library.Main.Database public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = @Name AND ""State"" = @State "); + using var cmd = await m_connection.CreateCommandAsync(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = @Name AND ""State"" = @State "); cmd.Transaction = transaction; cmd.SetParameterValue("@Name", name); cmd.SetParameterValue("@State", state.ToString()); @@ -838,7 +838,7 @@ AND ""Fileset"".""ID"" NOT IN public async Task GetBlocksLargerThan(long fhblocksize) { - using var cmd = m_connection.CreateCommand(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size"); + using var cmd = await m_connection.CreateCommandAsync(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size"); cmd.SetParameterValue("@Size", fhblocksize); return await cmd.ExecuteScalarInt64Async(-1); } @@ -1068,7 +1068,7 @@ ON public async IAsyncEnumerable GetBlocks(long volumeid, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId"); + using var cmd = await m_connection.CreateCommandAsync(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId"); cmd.Transaction = transaction; cmd.SetParameterValue("@VolumeId", volumeid); using var rd = await cmd.ExecuteReaderAsync(); @@ -1388,7 +1388,7 @@ WHERE oldVersion.FileID = tempVersion.FileID AND tempVersion.FilesetID = @FilesetId AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER BY Timestamp DESC LIMIT 1)"; - using var cmd = m_connection.CreateCommand(query); + using var cmd = await m_connection.CreateCommandAsync(query); cmd.Transaction = transaction; cmd.SetParameterValue("@FilesetId", filesetId); await cmd.ExecuteNonQueryAsync(); From 46aacbaa4b835f54f2915e97e8af200429dc3bfb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 15 May 2025 12:32:52 +0200 Subject: [PATCH 032/362] Restructured some of the queries in LocalDatabase for readability. Also moved some of the calls to utility functions into the extension methods. --- .../Library/Main/Database/LocalDatabase.cs | 122 ++++++++++++++---- 1 file changed, 96 insertions(+), 26 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index cb54fd6cf..31d225d91 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -179,15 +179,25 @@ namespace Duplicati.Library.Main.Database cmd.Transaction = transaction; if (operation != null) { - cmd.CommandText = @"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (@Description, @Timestamp); SELECT last_insert_rowid();"; - cmd.SetParameterValue("@Description", operation); - cmd.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(db.OperationTimestamp)); - db.m_operationid = await cmd.ExecuteScalarInt64Async(-1); + db.m_operationid = await cmd.SetCommandAndParameters(@" + INSERT INTO ""Operation"" (""Description"", ""Timestamp"") + VALUES (@Description, @Timestamp); + SELECT last_insert_rowid(); + ") + .SetParameterValue("@Description", operation) + .SetParameterValue("@Timestamp", db.OperationTimestamp) + .ExecuteScalarInt64Async(-1); } else { // Get last operation - using var rd = await cmd.ExecuteReaderAsync(@"SELECT ""ID"", ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1"); + using var rd = await cmd.ExecuteReaderAsync(@" + SELECT ""ID"", ""Timestamp"" + FROM ""Operation"" + ORDER BY ""Timestamp"" + DESC LIMIT 1 + "); + if (!await rd.ReadAsync()) throw new Exception("LocalDatabase does not contain a previous operation."); @@ -200,37 +210,97 @@ namespace Duplicati.Library.Main.Database private static async Task CreateLocalDatabaseAsync(SqliteConnection connection) { - var selectremotevolumes_sql = @"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" FROM ""Remotevolume"""; + var selectremotevolumes_sql = @" + SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" + FROM ""Remotevolume"" + "; var db = new LocalDatabase { m_connection = connection, - m_insertlogCommand = await connection.CreateCommandAsync(@"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (@OperationID, @Timestamp, @Type, @Message, @Exception)"), - m_insertremotelogCommand = await connection.CreateCommandAsync(@"INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") VALUES (@OperationID, @Timestamp, @Operation, @Path, @Data)"), - m_updateremotevolumeCommand = await connection.CreateCommandAsync(@"UPDATE ""Remotevolume"" SET ""OperationID"" = @OperationID, ""State"" = @State, ""Hash"" = @Hash, ""Size"" = @Size WHERE ""Name"" = @Name"), + m_insertlogCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") + VALUES (@OperationID, @Timestamp, @Type, @Message, @Exception) + "), + + m_insertremotelogCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") + VALUES (@OperationID, @Timestamp, @Operation, @Path, @Data) + "), + + m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" + UPDATE ""Remotevolume"" + SET ""OperationID"" = @OperationID, ""State"" = @State, ""Hash"" = @Hash, ""Size"" = @Size + WHERE ""Name"" = @Name + "), + m_selectremotevolumesCommand = await connection.CreateCommandAsync(selectremotevolumes_sql), - m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$"{selectremotevolumes_sql} WHERE ""Name"" = @Name"), - m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync(FormatInvariant($@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')")), - m_removeremotevolumeCommand = await connection.CreateCommandAsync(@"DELETE FROM ""Remotevolume"" WHERE ""Name"" = @Name AND (""DeleteGraceTime"" < @Now OR ""State"" != @State)"), - m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync(FormatInvariant($@"DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted.ToString()}' AND (""DeleteGraceTime"" < @Now OR LENGTH(""DeleteGraceTime"") > 12) ")), // >12 is to handle removal of old records that were in ticks - m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = @Name"), - m_createremotevolumeCommand = await connection.CreateCommandAsync(@"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"", ""ArchiveTime"") VALUES (@OperationID, @Name, @Type, @State, @Size, @VerificationCount, @DeleteGraceTime, @ArchiveTime); SELECT last_insert_rowid();"), - m_insertIndexBlockLink = await connection.CreateCommandAsync(@"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (@IndexVolumeId, @BlockVolumeId)"), - m_findpathprefixCommand = await connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = @Prefix"), - m_insertpathprefixCommand = await connection.CreateCommandAsync(@"INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (@Prefix); SELECT last_insert_rowid(); ") + + m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$" + {selectremotevolumes_sql} + WHERE ""Name"" = @Name + "), + + m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" + SELECT DISTINCT ""Name"", ""State"" + FROM ""Remotevolume"" + WHERE ""Name"" IN ( + SELECT ""Name"" + FROM ""Remotevolume"" + WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') + ) + AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') + "), + + m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" + DELETE FROM ""Remotevolume"" + WHERE ""Name"" = @Name + AND ( + ""DeleteGraceTime"" < @Now OR ""State"" != @State + ) + "), + + m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@" + DELETE FROM ""Remotevolume"" + WHERE ""State"" == '{RemoteVolumeState.Deleted}' + AND ( + ""DeleteGraceTime"" < @Now OR LENGTH(""DeleteGraceTime"") > 12 + ) + "), // >12 is to handle removal of old records that were in ticks + + m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Remotevolume"" + WHERE ""Name"" = @Name + "), + + m_createremotevolumeCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"", ""ArchiveTime"") + VALUES (@OperationID, @Name, @Type, @State, @Size, @VerificationCount, @DeleteGraceTime, @ArchiveTime); + SELECT last_insert_rowid(); + "), + + m_insertIndexBlockLink = await connection.CreateCommandAsync(@" + INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") + VALUES (@IndexVolumeId, @BlockVolumeId) + "), + + m_findpathprefixCommand = await connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""PathPrefix"" + WHERE ""Prefix"" = @Prefix + "), + + m_insertpathprefixCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""PathPrefix"" (""Prefix"") + VALUES (@Prefix); + SELECT last_insert_rowid(); + ") }; return db; } - /// - /// Formats the string using the invariant culture - /// - /// The formattable string - /// The formatted string - public static string FormatInvariant(FormattableString formattable) - => Library.Utility.Utility.FormatInvariant(formattable); - public static bool Exists(string path) { return File.Exists(path); From e00218feb69ee5a4240a1ee4472ea5ef99cf6804 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 15 May 2025 12:33:04 +0200 Subject: [PATCH 033/362] Removed whitespace --- .../Library/Main/Database/ExtensionMethods.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index a294c4c04..1ff0a1405 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; From 2817f05c930bbf5ca8ae3b40e18e6cf7a3dac656 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 15 May 2025 12:33:30 +0200 Subject: [PATCH 034/362] First WIP changes to extension methods --- .../Library/Main/Database/ExtensionMethods.cs | 1241 ++++++++++------- 1 file changed, 760 insertions(+), 481 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 1ff0a1405..c81fc4772 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -23,6 +23,8 @@ using System.Collections.Generic; using System.Data; using System.Linq; using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; #nullable enable @@ -31,397 +33,11 @@ namespace Duplicati.Library.Main.Database; /// /// Extension method for /// -public static class ExtensionMethods +public static partial class ExtensionMethods { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ExtensionMethods)); + // TODO parameter types and sizes - /// - /// Adds a parameter to the command with the given value. - /// - /// The type of the command - /// The command to add the parameter to - /// The name of the parameter - /// The optional value of the parameter - /// The command with the parameter added - public static T AddNamedParameter(this T self, string name, object? value = null) - where T : IDbCommand - { - var p = self.CreateParameter(); - p.ParameterName = name; - if (value != null) - p.Value = value; - self.Parameters.Add(p); - return self; - } - - /// - /// Sets the parameter values for the command, the parameters must already be added. - /// - /// The type of the command - /// The command to set the parameter values for - /// The values to set the parameters to - /// The command with the parameter values set - public static T SetParameterValues(this T self, Dictionary values) - where T : IDbCommand - { - foreach (var kvp in values) - ((IDataParameter)self.Parameters[kvp.Key]!).Value = kvp.Value; - return self; - } - - /// - /// Sets the parameter value for the command at the given index. - /// - /// The type of the command - /// The command to set the parameter value for - /// The name of the parameter to set the value for - /// The value to set the parameter to - /// The command with the parameter value set - public static T SetParameterValue(this T self, string name, object? value) - where T : IDbCommand - { -#if DEBUG - if (value is not null && value is System.Collections.IEnumerable && value is not string) - throw new ArgumentException($"Cannot set parameter '{name}' to an array or enumerable type, as the SQLite bindings does not support it.", nameof(value)); -#endif - ((IDataParameter)self.Parameters[name]!).Value = value; - return self; - } - - /// - /// Sets the transaction for the command. - /// - /// The type of the command - /// The command to set the transaction for - /// The transaction to set for the command - /// >The command with the transaction set - public static T SetTransaction(this T self, IDbTransaction? transaction) - where T : IDbCommand - { - self.Transaction = transaction; - return self; - } - - /// - /// Gets the printable command text for the given command. - /// The command to get the printable command text for - /// The printable command text - public static string GetPrintableCommandText(this IDbCommand self) - { - var txt = self.CommandText; - - foreach (var p in self.Parameters.Cast()) - { - var ix = txt.IndexOf('?'); - if (ix >= 0) - { - string v; - if (p.Value is string) - v = string.Format("\"{0}\"", p.Value); - else if (p.Value == null) - v = "NULL"; - else - v = string.Format("{0}", p.Value); - - txt = txt.Substring(0, ix) + v + txt.Substring(ix + 1); - } - } - - return txt; - } - - /// - /// Executes the command and returns the number of rows affected. - /// - /// The command instance to execute on - /// Whether to write a log entry - /// The number of rows affected - public static int ExecuteNonQuery(this IDbCommand self, bool writeLog) - { - return ExecuteNonQuery(self, writeLog, null); - } - - /// - /// Executes the command and returns the number of rows affected. - /// - /// The command instance to execute on - /// The command string to execute - /// The values to use as parameters. The parameters must already be added. - /// The number of rows affected - public static int ExecuteNonQuery(this IDbCommand self, string cmd) - { - return ExecuteNonQuery(self, true, cmd); - } - - /// - /// Executes the command and returns the number of rows affected. - /// - /// The command instance to execute on - /// The command string to execute - /// The values to use as parameters. The parameters must already be added. - /// The number of rows affected - public static int ExecuteNonQuery(this IDbCommand self, string cmd, Dictionary values) - { - return ExecuteNonQuery(self, true, cmd, values); - } - - /// - /// Executes the command and returns the number of rows affected. - /// - /// The command instance to execute on - /// Whether to write a log entry - /// The command string to execute - /// The values to use as parameters. The parameters must already be added. - /// The number of rows affected - public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd, Dictionary values) - { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - if (values != null && values.Count > 0) - self.SetParameterValues(values); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) - return self.ExecuteNonQuery(); - } - - /// - /// Executes the command and returns the number of rows affected. - /// - /// The command instance to execute on - /// Whether to write a log entry - /// The command string to execute - /// The number of rows affected - public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd) - { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) - return self.ExecuteNonQuery(); - } - - /// - /// Executes the command and returns the number of rows affected. - /// - /// The command instance to execute on - /// The transaction to use for the command - /// The number of rows affected - public static int ExecuteNonQuery(this IDbCommand self, IDbTransaction? transaction) - { - self.Transaction = transaction; - return self.ExecuteNonQuery(); - } - - /// - /// Executes the command and returns the scalar value of the first row. - /// - /// The command instance to execute on - /// The command string to execute - /// The scalar value of the first row - public static object? ExecuteScalar(this IDbCommand self, string cmd) - { - return ExecuteScalar(self, true, cmd); - } - - /// - /// Executes the command and returns the scalar value of the first row. - /// - /// The command instance to execute on - /// Whether to write a log entry - /// The command string to execute - /// The scalar value of the first row - public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string? cmd) - { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalar", string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())) : null) - return self.ExecuteScalar(); - } - - /// - /// Executes the command and returns the scalar int64 value of the first row as a string. - /// - /// The command instance to execute on - /// Whether to write a log entry - /// The default value to return if no value is found - /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, long defaultvalue = -1) - { - return ExecuteScalarInt64(self, writeLog, null, defaultvalue); - } - - /// - /// Executes the command and returns the scalar int64 value of the first row as a string. - /// - /// The command instance to execute on - /// The default value to return if no value is found - /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this IDbCommand self, long defaultvalue = -1) - { - return ExecuteScalarInt64(self, true, null, defaultvalue); - } - - /// - /// Executes the command and returns the scalar int64 value of the first row as a string. - /// - /// The command instance to execute on - /// The transaction to use for the command - /// The default value to return if no value is found - /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this IDbCommand self, IDbTransaction? transaction, long defaultvalue = -1) - { - self.Transaction = transaction; - return ExecuteScalarInt64(self, true, null, defaultvalue); - } - - /// - /// Executes the command and returns the scalar int64 value of the first row as a string. - /// - /// The command instance to execute on - /// The command string to execute - /// The default value to return if no value is found - /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this IDbCommand self, string? cmd, long defaultvalue = -1) - { - return ExecuteScalarInt64(self, true, cmd, defaultvalue); - } - - /// - /// Executes the command and returns the scalar int64 value of the first row as a string. - /// - /// The command instance to execute on - /// Whether to write a log entry - /// The command string to execute - /// The default value to return if no value is found - /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, string? cmd, long defaultvalue) - { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())) : null) - using (var rd = self.ExecuteReader()) - if (rd.Read()) - return ConvertValueToInt64(rd, 0, defaultvalue); - - return defaultvalue; - } - - /// - /// Executes the command and returns a data reader. - /// - /// The command instance to execute on - /// The command string to execute - /// The values to use as parameters. The parameters must already be added. - /// A instance - public static IDataReader ExecuteReader(this IDbCommand self, string cmd, Dictionary? values) - { - return ExecuteReader(self, true, cmd, values); - } - - - /// - /// Executes the command and returns a data reader. - /// - /// The command instance to execute on - /// The command string to execute - /// A instance - public static IDataReader ExecuteReader(this IDbCommand self, string cmd) - { - return ExecuteReader(self, true, cmd); - } - - /// - /// Executes the command and returns a data reader. - /// - /// The command instance to execute on - /// Whether to write a log entry - /// The command string to execute - /// A instance - public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd) - { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) - return self.ExecuteReader(); - } - - /// - /// Executes the command and returns a data reader. - /// - /// The command instance to execute on - /// Whether to write a log entry - /// The command string to execute - /// The values to use as parameters. The parameters must already be added. - /// A instance - public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd, Dictionary? values) - { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - if (values != null && values.Count > 0) - self.SetParameterValues(values); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) - return self.ExecuteReader(); - } - - /// - /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. - /// - /// The database command to execute on. - /// The command string to execute. - /// - public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self, string cmd) - { - using var rd = ExecuteReader(self, cmd); - while (rd.Read()) - yield return rd; - } - - /// - /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. - /// - /// The database command to execute on. - /// The command string to execute. - /// - public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self) - { - using var rd = self.ExecuteReader(); - while (rd.Read()) - yield return rd; - } - - /// - /// Converts the value at the given index of the given data reader to a string. - /// - /// The data reader to convert the value from. - /// The index of the value to convert. - /// The value at the given index as a string. - public static string? ConvertValueToString(this IDataReader reader, int index) - { - var v = reader.GetValue(index); - if (v == null || v == DBNull.Value) - return null; - - return v.ToString(); - } - - /// - /// Converts the value at the given index of the given data reader to a long. - /// - /// The data reader to convert the value from. - /// The index of the value to convert. - /// The default value to return if the value is null or cannot be converted. - /// The value at the given index as a long. - public static long ConvertValueToInt64(this IDataReader reader, int index, long defaultvalue = -1) + public static long ConvertValueToInt64(this SqliteDataReader reader, int index, long defaultvalue = -1) { try { @@ -435,101 +51,162 @@ public static class ExtensionMethods return defaultvalue; } - /// - /// Creates a command with the given transaction. - /// - /// The connection to create the command on. - /// The transaction to use for the command. - /// The command string to create the command with. - /// A new command with the given transaction. - public static IDbCommand CreateCommand(this IDbConnection self, IDbTransaction? transaction, string? cmdtext = null) + public static string? ConvertValueToString(this SqliteDataReader reader, int index) + { + var v = reader.GetValue(index); + if (v == null || v == DBNull.Value) + return null; + + return v.ToString(); + } + + // public static SqliteCommand CreateCommand(this SqliteConnection self, string cmdtext) + // { + // var cmd = self.CreateCommand(); + // cmd.SetCommandAndParameters(cmdtext); + // cmd.PrepareAsync().Wait(); + + // return cmd; + // } + + public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext) { var cmd = self.CreateCommand(); - cmd.Transaction = transaction; - if (!string.IsNullOrEmpty(cmdtext)) - cmd.SetCommandAndParameters(cmdtext); - return cmd; - } - - /// - /// Sets the command text and adds parameters to the command. - /// - /// The command to set the command text and add parameters to. - /// The transaction to use for the command. - /// The command text to set. - /// The command with the command text set and parameters added. - public static IDbCommand SetCommandAndParameters(this IDbCommand cmd, IDbTransaction transaction, string cmdtext) - { - cmd.Transaction = transaction; - return cmd.SetCommandAndParameters(cmdtext); - } - - /// - /// Sets the command text and adds parameters to the command. - /// - /// The command to set the command text and add parameters to. - /// The command text to set. - /// The command with the command text set and parameters added. - public static IDbCommand SetCommandAndParameters(this IDbCommand cmd, string cmdtext) - { - cmd.CommandText = cmdtext; - cmd.Parameters.Clear(); - -#if DEBUG - if (cmd.CommandText.Contains("?", StringComparison.OrdinalIgnoreCase)) - throw new ArgumentException("Command text cannot contain '?' as a parameter placeholder, use '@' instead.", nameof(cmdtext)); -#endif - - var parameters = Regex.Matches(cmdtext, @"@\w+"); - var found = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (Match match in parameters) - { - if (found.Contains(match.Value)) - continue; - found.Add(match.Value); - cmd.AddNamedParameter(match.Value); - } + cmd.SetCommandAndParameters(Library.Utility.Utility.FormatInvariant(cmdtext)); + await cmd.PrepareAsync(); return cmd; } - /// - /// Creates the transaction for the given connection and executes a "BEGIN IMMEDIATE" command to ensure that the transaction can be comitted or rolled back. - /// This works around a quirk in SQLite where the transaction is not initialized until the first command is executed. - /// This means that if the transaction is rolled back or committed with no commands executed, it will fail with an exception. - /// - /// The connection to create the transaction on - /// The transaction - public static IDbTransaction BeginTransactionSafe(this IDbConnection self) + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd) { - var transaction = self.BeginTransaction(); - // using (var cmd = self.CreateCommand(transaction, "BEGIN IMMEDIATE;")) - // try { cmd.ExecuteNonQuery(); } - // catch { /* ignore */ } + self.CommandText = cmd; + self.Parameters.Clear(); - return transaction; + // TODO "late format string-ing" + using (new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", $"ExecuteNonQueryAsync: {self.CommandText}")) + return await self.ExecuteNonQueryAsync().ConfigureAwait(false); } + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary values) + { + return await ExecuteNonQueryAsync(self, true, cmd, values); + } - /// - /// Creates a command with the given command string, and adds parameters to fit the input - /// - /// The connection to create the command on - /// The command string to create the command with - /// The command with the parameters added - public static IDbCommand CreateCommand(this IDbConnection self, string cmdtext) - => CreateCommand(self, null, cmdtext); + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary values) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); - /// - /// Expands the given parameter name to a list of parameters for an IN clause. - /// - /// The type of the command - /// The type of the values - /// The command to expand the parameter for - /// The original parameter name to expand - /// The values to expand the parameter for - public static TCommand ExpandInClauseParameter(this TCommand cmd, string originalParamName, IEnumerable values) - where TCommand : IDbCommand + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) + return await self.ExecuteNonQueryAsync(); + } + + //public static SqliteDataReader ExecuteReader(this SqliteCommand self, string cmd) + //{ + // return ExecuteReader(self, true, cmd); + //} + + //public static SqliteDataReader ExecuteReader(this SqliteCommand self, bool writeLog, string? cmd) + //{ + // if (cmd != null) + // self.SetCommandAndParameters(cmd); + // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) + // return self.ExecuteReader(); + //} + + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmd) + { + self.CommandText = cmd; + self.Parameters.Clear(); + + // TODO "late format string-ing" + using (new Logging.Timer(LOGTAG, "ExecuteReaderAsync", $"ExecuteReaderAsync: {self.CommandText}")) + return await self.ExecuteReaderAsync().ConfigureAwait(false); + } + + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmd, Dictionary? values) + { + return await ExecuteReaderAsync(self, true, cmd, values); + } + + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) + return await self.ExecuteReaderAsync(); + } + + public static async IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self) + { + using var rd = await self.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + yield return rd; + } + + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmd) + { + self.CommandText = cmd; + return ExecuteReaderEnumerableAsync(self); + } + + public static async Task ExecuteScalarAsync(this SqliteCommand self) + { + return await self.ExecuteScalarAsync().ConfigureAwait(false); + } + + public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmd) + { + self.CommandText = cmd; + self.Parameters.Clear(); + + return await ExecuteScalarAsync(self).ConfigureAwait(false); + } + + //public static long ExecuteScalarInt64(this SqliteCommand self, long defaultvalue = -1) + //{ + // return ExecuteScalarInt64(self, true, null, defaultvalue); + //} + + //public static long ExecuteScalarInt64(this SqliteCommand self, bool writeLog, string? cmd, long defaultvalue) + //{ + // if (cmd != null) + // self.SetCommandAndParameters(cmd); + // + // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())) : null) + // using (var rd = self.ExecuteReader()) + // if (rd.Read()) + // return ConvertValueToInt64(rd, 0, defaultvalue); + // + // return defaultvalue; + //} + + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue = -1) + { + using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) + return ConvertValueToInt64(rd, 0, defaultvalue); + + return defaultvalue; + } + + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string cmd, long defaultvalue = -1) + { + self.CommandText = cmd; + self.Parameters.Clear(); + + return await ExecuteScalarInt64Async(self, defaultvalue).ConfigureAwait(false); + } + + public static SqliteCommand ExpandInClauseParameter(this SqliteCommand cmd, string originalParamName, IEnumerable values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); @@ -557,15 +234,7 @@ public static class ExtensionMethods return cmd; } - /// - /// Expands the given parameter name to a list of parameters for an IN clause. - /// - /// The type of the command - /// The command to expand the parameter for - /// The original parameter name to expand - /// The values to expand the parameter for - public static T ExpandInClauseParameter(this T cmd, string originalParamName, TemporaryDbValueList values) - where T : IDbCommand + public static SqliteCommand ExpandInClauseParameter(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); @@ -577,4 +246,614 @@ public static class ExtensionMethods cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(), StringComparison.OrdinalIgnoreCase); return cmd; } + + public static SqliteCommand SetCommandAndParameters(this SqliteCommand cmd, string cmdtext) + { + cmd.CommandText = cmdtext; + cmd.Parameters.Clear(); + +#if DEBUG + if (cmd.CommandText.Contains('?', StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("Command text cannot contain '?' as a parameter placeholder, use '@' instead.", nameof(cmdtext)); +#endif + + var parameters = MyRegex().Matches(cmdtext); + var found = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in parameters) + { + if (found.Contains(match.Value)) + continue; + found.Add(match.Value); + cmd.AddNamedParameter(match.Value); + } + + return cmd; + } + + /// + /// New edition + /// + /// + /// + /// + /// + public static SqliteCommand SetParameterValue(this SqliteCommand self, string name, object? value) + { +#if DEBUG + if (value is not null && value is System.Collections.IEnumerable && value is not string) + throw new ArgumentException($"Cannot set parameter '{name}' to an array or enumerable type, as the SQLite bindings does not support it.", nameof(value)); +#endif + self.Parameters[name].Value = value; + + return self; + } + + // Special case for DateTime, as we need to convert it to a long + public static SqliteCommand SetParameterValue(this SqliteCommand self, string name, DateTime value) + { + self.Parameters[name].Value = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(value); + + return self; + } + + public static SqliteCommand SetParameterValues(this SqliteCommand self, Dictionary values) + { + foreach (var kvp in values) + self.Parameters[kvp.Key].Value = kvp.Value; + + return self; + } + + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ExtensionMethods)); + + [GeneratedRegex(@"@\w+")] + private static partial Regex MyRegex(); + + // TODO commennt from here on + + /// + /// Adds a parameter to the command with the given value. + /// + /// The type of the command + /// The command to add the parameter to + /// The name of the parameter + /// The optional value of the parameter + /// The command with the parameter added + public static SqliteCommand AddNamedParameter(this SqliteCommand self, string name, object? value = null) + { + var p = self.CreateParameter(); + p.ParameterName = name; + if (value != null) + p.Value = value; + self.Parameters.Add(p); + return self; + } + + // /// + // /// Sets the parameter values for the command, the parameters must already be added. + // /// + // /// The type of the command + // /// The command to set the parameter values for + // /// The values to set the parameters to + // /// The command with the parameter values set + // public static T SetParameterValues(this T self, Dictionary values) + // where T : IDbCommand + // { + // foreach (var kvp in values) + // ((IDataParameter)self.Parameters[kvp.Key]!).Value = kvp.Value; + // return self; + // } + + // /// + // /// Sets the parameter value for the command at the given index. + // /// + // /// The type of the command + // /// The command to set the parameter value for + // /// The name of the parameter to set the value for + // /// The value to set the parameter to + // /// The command with the parameter value set + // public static T SetParameterValue(this T self, string name, object? value) + // where T : IDbCommand + // { + // #if DEBUG + // if (value is not null && value is System.Collections.IEnumerable && value is not string) + // throw new ArgumentException($"Cannot set parameter '{name}' to an array or enumerable type, as the SQLite bindings does not support it.", nameof(value)); + // #endif + // ((IDataParameter)self.Parameters[name]!).Value = value; + // return self; + // } + + // /// + // /// Sets the transaction for the command. + // /// + // /// The type of the command + // /// The command to set the transaction for + // /// The transaction to set for the command + // /// >The command with the transaction set + // public static T SetTransaction(this T self, IDbTransaction? transaction) + // where T : IDbCommand + // { + // self.Transaction = transaction; + // return self; + // } + + /// + /// Gets the printable command text for the given command. + /// The command to get the printable command text for + /// The printable command text + public static string GetPrintableCommandText(this SqliteCommand self) + { + var txt = self.CommandText; + // TODO not adjusted to named parameters - it's using positional parameters. + + foreach (var p in self.Parameters.Cast()) + { + var ix = txt.IndexOf('?'); + if (ix >= 0) + { + string v; + if (p.Value is string) + v = string.Format("\"{0}\"", p.Value); + else if (p.Value == null) + v = "NULL"; + else + v = string.Format("{0}", p.Value); + + txt = string.Concat(txt.AsSpan(0, ix), v, txt.AsSpan(ix + 1)); + } + } + + return txt; + } + + // /// + // /// Executes the command and returns the number of rows affected. + // /// + // /// The command instance to execute on + // /// Whether to write a log entry + // /// The number of rows affected + // public static int ExecuteNonQuery(this IDbCommand self, bool writeLog) + // { + // return ExecuteNonQuery(self, writeLog, null); + // } + + // /// + // /// Executes the command and returns the number of rows affected. + // /// + // /// The command instance to execute on + // /// The command string to execute + // /// The values to use as parameters. The parameters must already be added. + // /// The number of rows affected + // public static int ExecuteNonQuery(this IDbCommand self, string cmd) + // { + // return ExecuteNonQuery(self, true, cmd); + // } + + // /// + // /// Executes the command and returns the number of rows affected. + // /// + // /// The command instance to execute on + // /// The command string to execute + // /// The values to use as parameters. The parameters must already be added. + // /// The number of rows affected + // public static int ExecuteNonQuery(this IDbCommand self, string cmd, Dictionary values) + // { + // return ExecuteNonQuery(self, true, cmd, values); + // } + + // /// + // /// Executes the command and returns the number of rows affected. + // /// + // /// The command instance to execute on + // /// Whether to write a log entry + // /// The command string to execute + // /// The values to use as parameters. The parameters must already be added. + // /// The number of rows affected + // public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd, Dictionary values) + // { + // if (cmd != null) + // self.SetCommandAndParameters(cmd); + + // if (values != null && values.Count > 0) + // self.SetParameterValues(values); + + // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) + // return self.ExecuteNonQuery(); + // } + + // /// + // /// Executes the command and returns the number of rows affected. + // /// + // /// The command instance to execute on + // /// Whether to write a log entry + // /// The command string to execute + // /// The number of rows affected + // public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd) + // { + // if (cmd != null) + // self.SetCommandAndParameters(cmd); + + // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) + // return self.ExecuteNonQuery(); + // } + + // /// + // /// Executes the command and returns the number of rows affected. + // /// + // /// The command instance to execute on + // /// The transaction to use for the command + // /// The number of rows affected + // public static int ExecuteNonQuery(this IDbCommand self, IDbTransaction? transaction) + // { + // self.Transaction = transaction; + // return self.ExecuteNonQuery(); + // } + + // /// + // /// Executes the command and returns the scalar value of the first row. + // /// + // /// The command instance to execute on + // /// The command string to execute + // /// The scalar value of the first row + // public static object? ExecuteScalar(this IDbCommand self, string cmd) + // { + // return ExecuteScalar(self, true, cmd); + // } + + // /// + // /// Executes the command and returns the scalar value of the first row. + // /// + // /// The command instance to execute on + // /// Whether to write a log entry + // /// The command string to execute + // /// The scalar value of the first row + // public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string? cmd) + // { + // if (cmd != null) + // self.SetCommandAndParameters(cmd); + + // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalar", string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())) : null) + // return self.ExecuteScalar(); + // } + + // /// + // /// Executes the command and returns the scalar int64 value of the first row as a string. + // /// + // /// The command instance to execute on + // /// Whether to write a log entry + // /// The default value to return if no value is found + // /// The scalar int64 value of the first row as a string + // public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, long defaultvalue = -1) + // { + // return ExecuteScalarInt64(self, writeLog, null, defaultvalue); + // } + + // /// + // /// Executes the command and returns the scalar int64 value of the first row as a string. + // /// + // /// The command instance to execute on + // /// The default value to return if no value is found + // /// The scalar int64 value of the first row as a string + // public static long ExecuteScalarInt64(this IDbCommand self, long defaultvalue = -1) + // { + // return ExecuteScalarInt64(self, true, null, defaultvalue); + // } + + // /// + // /// Executes the command and returns the scalar int64 value of the first row as a string. + // /// + // /// The command instance to execute on + // /// The transaction to use for the command + // /// The default value to return if no value is found + // /// The scalar int64 value of the first row as a string + // public static long ExecuteScalarInt64(this IDbCommand self, IDbTransaction? transaction, long defaultvalue = -1) + // { + // self.Transaction = transaction; + // return ExecuteScalarInt64(self, true, null, defaultvalue); + // } + + // /// + // /// Executes the command and returns the scalar int64 value of the first row as a string. + // /// + // /// The command instance to execute on + // /// The command string to execute + // /// The default value to return if no value is found + // /// The scalar int64 value of the first row as a string + // public static long ExecuteScalarInt64(this IDbCommand self, string? cmd, long defaultvalue = -1) + // { + // return ExecuteScalarInt64(self, true, cmd, defaultvalue); + // } + + // /// + // /// Executes the command and returns the scalar int64 value of the first row as a string. + // /// + // /// The command instance to execute on + // /// Whether to write a log entry + // /// The command string to execute + // /// The default value to return if no value is found + // /// The scalar int64 value of the first row as a string + // public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, string? cmd, long defaultvalue) + // { + // if (cmd != null) + // self.SetCommandAndParameters(cmd); + + // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())) : null) + // using (var rd = self.ExecuteReader()) + // if (rd.Read()) + // return ConvertValueToInt64(rd, 0, defaultvalue); + + // return defaultvalue; + // } + + // /// + // /// Executes the command and returns a data reader. + // /// + // /// The command instance to execute on + // /// The command string to execute + // /// The values to use as parameters. The parameters must already be added. + // /// A instance + // public static IDataReader ExecuteReader(this IDbCommand self, string cmd, Dictionary? values) + // { + // return ExecuteReader(self, true, cmd, values); + // } + + + // /// + // /// Executes the command and returns a data reader. + // /// + // /// The command instance to execute on + // /// The command string to execute + // /// A instance + // public static IDataReader ExecuteReader(this IDbCommand self, string cmd) + // { + // return ExecuteReader(self, true, cmd); + // } + + // /// + // /// Executes the command and returns a data reader. + // /// + // /// The command instance to execute on + // /// Whether to write a log entry + // /// The command string to execute + // /// A instance + // public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd) + // { + // if (cmd != null) + // self.SetCommandAndParameters(cmd); + + // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) + // return self.ExecuteReader(); + // } + + // /// + // /// Executes the command and returns a data reader. + // /// + // /// The command instance to execute on + // /// Whether to write a log entry + // /// The command string to execute + // /// The values to use as parameters. The parameters must already be added. + // /// A instance + // public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd, Dictionary? values) + // { + // if (cmd != null) + // self.SetCommandAndParameters(cmd); + + // if (values != null && values.Count > 0) + // self.SetParameterValues(values); + + // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) + // return self.ExecuteReader(); + // } + + // /// + // /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. + // /// + // /// The database command to execute on. + // /// The command string to execute. + // /// + // public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self, string cmd) + // { + // using var rd = ExecuteReader(self, cmd); + // while (rd.Read()) + // yield return rd; + // } + + // /// + // /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. + // /// + // /// The database command to execute on. + // /// The command string to execute. + // /// + // public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self) + // { + // using var rd = self.ExecuteReader(); + // while (rd.Read()) + // yield return rd; + // } + + // /// + // /// Converts the value at the given index of the given data reader to a string. + // /// + // /// The data reader to convert the value from. + // /// The index of the value to convert. + // /// The value at the given index as a string. + // public static string? ConvertValueToString(this IDataReader reader, int index) + // { + // var v = reader.GetValue(index); + // if (v == null || v == DBNull.Value) + // return null; + + // return v.ToString(); + // } + + // /// + // /// Converts the value at the given index of the given data reader to a long. + // /// + // /// The data reader to convert the value from. + // /// The index of the value to convert. + // /// The default value to return if the value is null or cannot be converted. + // /// The value at the given index as a long. + // public static long ConvertValueToInt64(this IDataReader reader, int index, long defaultvalue = -1) + // { + // try + // { + // if (!reader.IsDBNull(index)) + // return reader.GetInt64(index); + // } + // catch + // { + // } + + // return defaultvalue; + // } + + // /// + // /// Creates a command with the given transaction. + // /// + // /// The connection to create the command on. + // /// The transaction to use for the command. + // /// The command string to create the command with. + // /// A new command with the given transaction. + // public static IDbCommand CreateCommand(this IDbConnection self, IDbTransaction? transaction, string? cmdtext = null) + // { + // var cmd = self.CreateCommand(); + // cmd.Transaction = transaction; + // if (!string.IsNullOrEmpty(cmdtext)) + // cmd.SetCommandAndParameters(cmdtext); + // return cmd; + // } + + // /// + // /// Sets the command text and adds parameters to the command. + // /// + // /// The command to set the command text and add parameters to. + // /// The transaction to use for the command. + // /// The command text to set. + // /// The command with the command text set and parameters added. + // public static IDbCommand SetCommandAndParameters(this IDbCommand cmd, IDbTransaction transaction, string cmdtext) + // { + // cmd.Transaction = transaction; + // return cmd.SetCommandAndParameters(cmdtext); + // } + + // /// + // /// Sets the command text and adds parameters to the command. + // /// + // /// The command to set the command text and add parameters to. + // /// The command text to set. + // /// The command with the command text set and parameters added. + // public static IDbCommand SetCommandAndParameters(this IDbCommand cmd, string cmdtext) + // { + // cmd.CommandText = cmdtext; + // cmd.Parameters.Clear(); + + // #if DEBUG + // if (cmd.CommandText.Contains("?", StringComparison.OrdinalIgnoreCase)) + // throw new ArgumentException("Command text cannot contain '?' as a parameter placeholder, use '@' instead.", nameof(cmdtext)); + // #endif + + // var parameters = Regex.Matches(cmdtext, @"@\w+"); + // var found = new HashSet(StringComparer.OrdinalIgnoreCase); + // foreach (Match match in parameters) + // { + // if (found.Contains(match.Value)) + // continue; + // found.Add(match.Value); + // cmd.AddNamedParameter(match.Value); + // } + + // return cmd; + // } + + // /// + // /// Creates the transaction for the given connection and executes a "BEGIN IMMEDIATE" command to ensure that the transaction can be comitted or rolled back. + // /// This works around a quirk in SQLite where the transaction is not initialized until the first command is executed. + // /// This means that if the transaction is rolled back or committed with no commands executed, it will fail with an exception. + // /// + // /// The connection to create the transaction on + // /// The transaction + // //public static IDbTransaction BeginTransactionSafe(this IDbConnection self) + // //{ + // // var transaction = self.BeginTransaction(); + // // // using (var cmd = self.CreateCommand(transaction, "BEGIN IMMEDIATE;")) + // // // try { cmd.ExecuteNonQuery(); } + // // // catch { + // // // ignore + // // } + // // + // // return transaction; + // //} + + + // /// + // /// Creates a command with the given command string, and adds parameters to fit the input + // /// + // /// The connection to create the command on + // /// The command string to create the command with + // /// The command with the parameters added + // public static IDbCommand CreateCommand(this IDbConnection self, string cmdtext) + // => CreateCommand(self, null, cmdtext); + + // /// + // /// Expands the given parameter name to a list of parameters for an IN clause. + // /// + // /// The type of the command + // /// The type of the values + // /// The command to expand the parameter for + // /// The original parameter name to expand + // /// The values to expand the parameter for + // public static TCommand ExpandInClauseParameter(this TCommand cmd, string originalParamName, IEnumerable values) + // where TCommand : IDbCommand + // { + // if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) + // throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); + + // foreach (var p in cmd.Parameters) + // if (p is IDataParameter parameter && parameter.ParameterName.Equals(originalParamName, StringComparison.OrdinalIgnoreCase)) + // { + // cmd.Parameters.Remove(parameter); + // break; + // } + + // foreach ((var value, var index) in values.Select((value, index) => (value, index))) + // cmd.AddNamedParameter($"{originalParamName}{index}", value); + + // var inClause = string.Join(", ", values.Select((_, index) => $"{originalParamName}{index}")); + // if (string.IsNullOrWhiteSpace(inClause) && values.Any()) + // throw new ArgumentException("IN clause cannot be empty", nameof(values)); + + // #if DEBUG + // if (!cmd.CommandText.Contains(originalParamName, StringComparison.OrdinalIgnoreCase)) + // throw new ArgumentException($"Command text does not contain parameter '{originalParamName}'", nameof(originalParamName)); + // #endif + // cmd.CommandText = cmd.CommandText.Replace(originalParamName, inClause, StringComparison.OrdinalIgnoreCase); + + // return cmd; + // } + + // /// + // /// Expands the given parameter name to a list of parameters for an IN clause. + // /// + // /// The type of the command + // /// The command to expand the parameter for + // /// The original parameter name to expand + // /// The values to expand the parameter for + // public static T ExpandInClauseParameter(this T cmd, string originalParamName, TemporaryDbValueList values) + // where T : IDbCommand + // { + // if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) + // throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); + + // if (!values.IsTableCreated) + // return ExpandInClauseParameter(cmd, originalParamName, values.Values); + + // // We have a temporary table, so we need to replace the parameter with the table name + // cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(), StringComparison.OrdinalIgnoreCase); + // return cmd; + // } + // // TODO comment to here } \ No newline at end of file From 6782597791aeb50faca646a03e871374b259516b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 15 May 2025 12:39:38 +0200 Subject: [PATCH 035/362] Moved FormatInvariant to inner function and restructured chain of calls when executing commands. --- .../Library/Main/Database/ExtensionMethods.cs | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index c81fc4772..2ba6e4860 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -72,16 +72,15 @@ public static partial class ExtensionMethods public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext) { var cmd = self.CreateCommand(); - cmd.SetCommandAndParameters(Library.Utility.Utility.FormatInvariant(cmdtext)); + cmd.SetCommandAndParameters(cmdtext); await cmd.PrepareAsync(); return cmd; } - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd) + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmdtext) { - self.CommandText = cmd; - self.Parameters.Clear(); + self.SetCommandAndParameters(cmdtext); // TODO "late format string-ing" using (new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", $"ExecuteNonQueryAsync: {self.CommandText}")) @@ -118,10 +117,9 @@ public static partial class ExtensionMethods // return self.ExecuteReader(); //} - public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmd) + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext) { - self.CommandText = cmd; - self.Parameters.Clear(); + self.SetCommandAndParameters(cmdtext); // TODO "late format string-ing" using (new Logging.Timer(LOGTAG, "ExecuteReaderAsync", $"ExecuteReaderAsync: {self.CommandText}")) @@ -152,9 +150,10 @@ public static partial class ExtensionMethods yield return rd; } - public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmd) + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext) { - self.CommandText = cmd; + self.SetCommandAndParameters(cmdtext); + return ExecuteReaderEnumerableAsync(self); } @@ -163,10 +162,9 @@ public static partial class ExtensionMethods return await self.ExecuteScalarAsync().ConfigureAwait(false); } - public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmd) + public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext) { - self.CommandText = cmd; - self.Parameters.Clear(); + self.SetCommandAndParameters(cmdtext); return await ExecuteScalarAsync(self).ConfigureAwait(false); } @@ -198,10 +196,9 @@ public static partial class ExtensionMethods return defaultvalue; } - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string cmd, long defaultvalue = -1) + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string cmdtext, long defaultvalue = -1) { - self.CommandText = cmd; - self.Parameters.Clear(); + self.SetCommandAndParameters(cmdtext); return await ExecuteScalarInt64Async(self, defaultvalue).ConfigureAwait(false); } @@ -249,7 +246,7 @@ public static partial class ExtensionMethods public static SqliteCommand SetCommandAndParameters(this SqliteCommand cmd, string cmdtext) { - cmd.CommandText = cmdtext; + cmd.CommandText = Library.Utility.Utility.FormatInvariant(cmdtext); cmd.Parameters.Clear(); #if DEBUG From 1cb7c882f9c3b7aa046ea98f10581145169e209f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 15 May 2025 12:54:03 +0200 Subject: [PATCH 036/362] Removed calls to a helper function that's been moved to extension methods --- .../Library/Main/Database/LocalDatabase.cs | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 31d225d91..a19825eac 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -301,6 +301,8 @@ namespace Duplicati.Library.Main.Database return db; } + // TODO to here is fine. + public static bool Exists(string path) { return File.Exists(path); @@ -579,51 +581,51 @@ namespace Duplicati.Library.Main.Database var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); deletecmd.Parameters.Clear(); - var bsIdsSubQuery = FormatInvariant(@$" + var bsIdsSubQuery = @" SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) UNION ALL SELECT DISTINCT ""BlocksetID"" FROM ""BlocklistHash"" -WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))"); +WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))"; // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMP TABLE ""{blocksetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery}")); - bsIdsSubQuery = FormatInvariant($@"SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"CREATE TEMP TABLE ""{blocksetidstable}"" (""ID"" INTEGER PRIMARY KEY)"); + await deletecmd.ExecuteNonQueryAsync($@"INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery}"); + bsIdsSubQuery = $@"SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" "; deletecmd.Parameters.Clear(); // Create a temp table to associate metadata that is being deleted to a fileset - var metadataFilesetQuery = FormatInvariant($@"SELECT Metadataset.ID, FilesetEntry.FilesetID + var metadataFilesetQuery = $@"SELECT Metadataset.ID, FilesetEntry.FilesetD FROM Metadataset INNER JOIN FileLookup ON FileLookup.MetadataID = Metadataset.ID INNER JOIN FilesetEntry ON FilesetEntry.FileID = FileLookup.ID WHERE Metadataset.BlocksetID IN ({bsIdsSubQuery}) -OR Metadataset.ID IN (SELECT MetadataID FROM FileLookup WHERE BlocksetID IN ({bsIdsSubQuery}))"); +OR Metadataset.ID IN (SELECT MetadataID FROM FileLookup WHERE BlocksetID IN ({bsIdsSubQuery}))"; var metadataFilesetTable = @"DelMetadataFilesetIds-" + temptransguid; - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMP TABLE ""{metadataFilesetTable}"" (MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER)")); - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (MetadataID, FilesetID) {metadataFilesetQuery}")); + await deletecmd.ExecuteNonQueryAsync($@"CREATE TEMP TABLE ""{metadataFilesetTable}"" (MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER)"); + await deletecmd.ExecuteNonQueryAsync($@"INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (MetadataID, FilesetID) {metadataFilesetQuery}"); // Delete FilesetEntry rows that had their metadata deleted - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM FilesetEntry + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FilesetID IN (SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"") AND FilesetEntry.FileID IN ( SELECT FilesetEntry.FileID FROM FilesetEntry INNER JOIN FileLookup ON FileLookup.ID = FilesetEntry.FileID - WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}""))")); + WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}""))"); // Delete FilesetEntry rows that had their blocks deleted - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FileID IN ( + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FileID IN ( SELECT ID FROM FileLookup -WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}))")); - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")")); +WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}))"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")"); - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery})")); - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - await deletecmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery})")); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery})"); // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); @@ -1000,7 +1002,7 @@ ON if (real_count != unique_count) throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); - var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")"), 0); + var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async($@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")", 0); if (itemswithnoblocklisthash != 0) throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); @@ -1095,7 +1097,7 @@ ON cmd2.Transaction = transaction; await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) { - var expandedCmd = FormatInvariant($@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FILESETS}) UNION SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FOLDERS_AND_SYMLINKS}))"); + var expandedCmd = $@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FILESETS}) UNION SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FOLDERS_AND_SYMLINKS}))"; cmd2.SetCommandAndParameters(expandedCmd); cmd2.SetParameterValue("@FilesetId", filesetid); cmd2.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); @@ -1488,7 +1490,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER { using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ")).Await(); + cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ").Await(); return; } @@ -1497,8 +1499,8 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")).Await(); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{Tablename}"" (""Path"") VALUES (@Path)")); + cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)").Await(); + cmd.SetCommandAndParameters($@"INSERT INTO ""{Tablename}"" (""Path"") VALUES (@Path)"); using (var c2 = m_connection.CreateCommand()) using (var rd = c2.ExecuteReaderAsync(@"SELECT DISTINCT ""Path"" FROM ""File"" ").Await()) while (rd.ReadAsync().Await()) @@ -1525,20 +1527,20 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER var argName = $"@Arg{args.Count}"; if (type == FilterType.Wildcard) { - sb.Append(FormatInvariant(@$"""Path"" LIKE {argName}")); + sb.Append(@$"""Path"" LIKE {argName}"); args.Add(argName, f.Replace('*', '%').Replace('?', '_')); } else { - sb.Append(FormatInvariant(@$"""Path"" = {argName}")); + sb.Append(@$"""Path"" = {argName}"); args.Add(argName, f); } } using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")).Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}"), args).Await(); + cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)").Await(); + cmd.ExecuteNonQueryAsync($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}", args).Await(); transaction.CommitAsync().Await(); } } @@ -1549,7 +1551,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER try { using var cmd = m_connection.CreateCommand(); - cmd.ExecuteNonQueryAsync(FormatInvariant(@$"DROP TABLE IF EXISTS ""{Tablename}"" ")).Await(); + cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{Tablename}"" ").Await(); } catch { } finally { Tablename = null!; } @@ -1637,11 +1639,11 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; // Group subquery by hash to ensure that each blocklist hash appears only once in the result - var sql = FormatInvariant($@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM + var sql = $@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM (SELECT ""BlocklistHash"".""BlocksetID"", ""Block"".""Hash"", ""BlocklistHash"".""Index"" FROM ""BlocklistHash"",""Block"" WHERE ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId GROUP BY ""Block"".""Hash"", ""Block"".""Size"") A, ""BlocksetEntry"" B, ""Block"" C WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" AND ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize}) AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize}) AND ""C"".""ID"" = ""B"".""BlockID"" - ORDER BY ""A"".""BlocksetID"", ""B"".""Index"""); + ORDER BY ""A"".""BlocksetID"", ""B"".""Index"""; string? curHash = null; var count = 0; @@ -1725,7 +1727,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER { using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; - using var rd = await cmd.ExecuteReaderAsync(FormatInvariant(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')")); + using var rd = await cmd.ExecuteReaderAsync(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')"); while (await rd.ReadAsync()) { yield return new KeyValuePair( From 029f560f6424198e57b90e2e4a9ab422dd4d9ef3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 06:43:07 +0200 Subject: [PATCH 037/362] Changed FilteredFilenameTable to a factory-like construction. --- .../Library/Main/Database/LocalDatabase.cs | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index a19825eac..6a260d162 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1474,10 +1474,23 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER public string Tablename { get; private set; } private readonly SqliteConnection m_connection; + private FilteredFilenameTable(SqliteConnection connection) + { + Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + m_connection = connection; + } + + [Obsolete("Calling this constructor will throw an exception. Use the Create method instead.")] public FilteredFilenameTable(SqliteConnection connection, IFilter filter, SqliteTransaction transaction) { - m_connection = connection; - Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + throw new NotImplementedException("Use the Create method instead."); + } + + public async Task CreateFilteredFilenameTableAsync(SqliteConnection connection, IFilter filter, SqliteTransaction transaction) + { + var ftt = new FilteredFilenameTable(connection); + + // TODO factory var type = FilterType.Regexp; if (filter is FilterExpression expression) type = expression.Type; @@ -1490,8 +1503,8 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER { using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; - cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ").Await(); - return; + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" "); + return ftt; } if (type == FilterType.Regexp || type == FilterType.Group) @@ -1499,11 +1512,11 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full - cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)").Await(); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)"); cmd.SetCommandAndParameters($@"INSERT INTO ""{Tablename}"" (""Path"") VALUES (@Path)"); using (var c2 = m_connection.CreateCommand()) - using (var rd = c2.ExecuteReaderAsync(@"SELECT DISTINCT ""Path"" FROM ""File"" ").Await()) - while (rd.ReadAsync().Await()) + using (var rd = await c2.ExecuteReaderAsync(@"SELECT DISTINCT ""Path"" FROM ""File"" ")) + while (await rd.ReadAsync()) { var p = rd.ConvertValueToString(0); if (FilterExpression.Matches(filter, p)) @@ -1513,7 +1526,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } } - transaction.CommitAsync().Await(); + await transaction.CommitAsync(); } else { @@ -1539,10 +1552,12 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; - cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)").Await(); - cmd.ExecuteNonQueryAsync($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}", args).Await(); - transaction.CommitAsync().Await(); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)"); + await cmd.ExecuteNonQueryAsync($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}", args); + await transaction.CommitAsync(); } + + return ftt; } public void Dispose() From 82308292660cb100a88ac89a266d8f55141b2555 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 07:12:47 +0200 Subject: [PATCH 038/362] Restructered all of the internal SQL queries for better readability. Also changed most executions of the queries to a chain of method calls instead of a series of statements. --- .../Library/Main/Database/LocalDatabase.cs | 1405 +++++++++++------ 1 file changed, 962 insertions(+), 443 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 6a260d162..ab582c84b 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -33,6 +33,7 @@ using Duplicati.Library.Modules.Builtin.ResultSerialization; using Duplicati.Library.Utility; using Duplicati.Library.Interface; using Microsoft.Data.Sqlite; +using CommunityToolkit.HighPerformance; // Expose internal classes to UnitTests, so that Database classes can be tested [assembly: InternalsVisibleTo("Duplicati.UnitTest")] @@ -57,6 +58,9 @@ namespace Duplicati.Library.Main.Database protected long m_operationid = -1; protected long m_pagecachesize; private bool m_hasExecutedVacuum; + // TODO use this rather than passing down transactions. + // TODO dispose should check this, and report if it was not disposed prior to calling dispose. Maybe the rtr itself should check this during dispose? And just throw a warning during runtime. + protected ReusableTransaction m_rtr; private SqliteCommand m_updateremotevolumeCommand = null!; private SqliteCommand m_selectremotevolumesCommand = null!; @@ -180,8 +184,14 @@ namespace Duplicati.Library.Main.Database if (operation != null) { db.m_operationid = await cmd.SetCommandAndParameters(@" - INSERT INTO ""Operation"" (""Description"", ""Timestamp"") - VALUES (@Description, @Timestamp); + INSERT INTO ""Operation"" ( + ""Description"", + ""Timestamp"" + ) + VALUES ( + @Description, + @Timestamp + ); SELECT last_insert_rowid(); ") .SetParameterValue("@Description", operation) @@ -192,7 +202,9 @@ namespace Duplicati.Library.Main.Database { // Get last operation using var rd = await cmd.ExecuteReaderAsync(@" - SELECT ""ID"", ""Timestamp"" + SELECT + ""ID"", + ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1 @@ -211,7 +223,15 @@ namespace Duplicati.Library.Main.Database private static async Task CreateLocalDatabaseAsync(SqliteConnection connection) { var selectremotevolumes_sql = @" - SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" + SELECT + ""ID"", + ""Name"", + ""Type"", + ""Size"", + ""Hash"", + ""State"", + ""DeleteGraceTime"", + ""ArchiveTime"" FROM ""Remotevolume"" "; @@ -219,18 +239,46 @@ namespace Duplicati.Library.Main.Database { m_connection = connection, m_insertlogCommand = await connection.CreateCommandAsync(@" - INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") - VALUES (@OperationID, @Timestamp, @Type, @Message, @Exception) + INSERT INTO ""LogData"" ( + ""OperationID"", + ""Timestamp"", + ""Type"", + ""Message"", + ""Exception"" + ) + VALUES ( + @OperationID, + @Timestamp, + @Type, + @Message, + @Exception + ) "), m_insertremotelogCommand = await connection.CreateCommandAsync(@" - INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") - VALUES (@OperationID, @Timestamp, @Operation, @Path, @Data) + INSERT INTO ""RemoteOperation"" ( + ""OperationID"", + ""Timestamp"", + ""Operation"", + ""Path"", + ""Data"" + ) + VALUES ( + @OperationID, + @Timestamp, + @Operation, + @Path, + @Data + ) "), m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" UPDATE ""Remotevolume"" - SET ""OperationID"" = @OperationID, ""State"" = @State, ""Hash"" = @Hash, ""Size"" = @Size + SET + ""OperationID"" = @OperationID, + ""State"" = @State, + ""Hash"" = @Hash, + ""Size"" = @Size WHERE ""Name"" = @Name "), @@ -256,7 +304,8 @@ namespace Duplicati.Library.Main.Database DELETE FROM ""Remotevolume"" WHERE ""Name"" = @Name AND ( - ""DeleteGraceTime"" < @Now OR ""State"" != @State + ""DeleteGraceTime"" < @Now + OR ""State"" != @State ) "), @@ -264,7 +313,8 @@ namespace Duplicati.Library.Main.Database DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted}' AND ( - ""DeleteGraceTime"" < @Now OR LENGTH(""DeleteGraceTime"") > 12 + ""DeleteGraceTime"" < @Now + OR LENGTH(""DeleteGraceTime"") > 12 ) "), // >12 is to handle removal of old records that were in ticks @@ -275,14 +325,38 @@ namespace Duplicati.Library.Main.Database "), m_createremotevolumeCommand = await connection.CreateCommandAsync(@" - INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"", ""ArchiveTime"") - VALUES (@OperationID, @Name, @Type, @State, @Size, @VerificationCount, @DeleteGraceTime, @ArchiveTime); + INSERT INTO ""Remotevolume"" ( + ""OperationID"", + ""Name"", + ""Type"", + ""State"", + ""Size"", + ""VerificationCount"", + ""DeleteGraceTime"", + ""ArchiveTime"" + ) + VALUES ( + @OperationID, + @Name, + @Type, + @State, + @Size, + @VerificationCount, + @DeleteGraceTime, + @ArchiveTime + ); SELECT last_insert_rowid(); "), m_insertIndexBlockLink = await connection.CreateCommandAsync(@" - INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") - VALUES (@IndexVolumeId, @BlockVolumeId) + INSERT INTO ""IndexBlockLink"" ( + ""IndexVolumeID"", + ""BlockVolumeID"" + ) + VALUES ( + @IndexVolumeId, + @BlockVolumeId + ) "), m_findpathprefixCommand = await connection.CreateCommandAsync(@" @@ -301,14 +375,8 @@ namespace Duplicati.Library.Main.Database return db; } - // TODO to here is fine. - - public static bool Exists(string path) - { - return File.Exists(path); - } - /// + /// TODO Remove from here and call the utility function directly - or maybe rename this so that the utility function makes more sense? /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value /// public static DateTime ParseFromEpochSeconds(long seconds) @@ -328,13 +396,13 @@ namespace Duplicati.Library.Main.Database public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, SqliteTransaction transaction) { - m_updateremotevolumeCommand.Transaction = transaction; - m_updateremotevolumeCommand.SetParameterValue("@OperationID", m_operationid); - m_updateremotevolumeCommand.SetParameterValue("@State", state.ToString()); - m_updateremotevolumeCommand.SetParameterValue("@Hash", hash); - m_updateremotevolumeCommand.SetParameterValue("@Size", size); - m_updateremotevolumeCommand.SetParameterValue("@Name", name); - var c = await m_updateremotevolumeCommand.ExecuteNonQueryAsync(); + var c = await m_updateremotevolumeCommand.SetTransaction(transaction) + .SetParameterValue("@OperationID", m_operationid) + .SetParameterValue("@State", state.ToString()) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync(); if (c != 1) { @@ -343,12 +411,15 @@ namespace Duplicati.Library.Main.Database if (deleteGraceTime.Ticks > 0) { - using var cmd = m_connection.CreateCommand(); - cmd.CommandText = @"UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = @DeleteGraceTime WHERE ""Name"" = @Name "; - cmd.Transaction = transaction; - cmd.SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)); - cmd.SetParameterValue("@Name", name); - c = await cmd.ExecuteNonQueryAsync(); + using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""RemoteVolume"" + SET ""DeleteGraceTime"" = @DeleteGraceTime + WHERE ""Name"" = @Name + "); + c = await cmd.SetTransaction(transaction) + .SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)) + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Unexpected number of updates when recording remote volume updates: {c}!"); @@ -356,12 +427,15 @@ namespace Duplicati.Library.Main.Database if (setArchived.HasValue) { - using var cmd = m_connection.CreateCommand(); - cmd.CommandText = @"UPDATE ""RemoteVolume"" SET ""ArchiveTime"" = @ArchiveTime WHERE ""Name"" = @Name "; - cmd.Transaction = transaction; - cmd.SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0); - cmd.SetParameterValue("@Name", name); - c = await cmd.ExecuteNonQueryAsync(); + using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""RemoteVolume"" + SET ""ArchiveTime"" = @ArchiveTime + WHERE ""Name"" = @Name + "); + c = await cmd.SetTransaction(transaction) + .SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0) + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Unexpected number of updates when recording remote volume archive-time updates: {c}!"); @@ -375,10 +449,13 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable> FilesetTimes() { - using var cmd = m_connection.CreateCommand(); - cmd.CommandText = @"SELECT ""ID"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"; + using var cmd = await m_connection.CreateCommandAsync(@" + SELECT ""ID"", ""Timestamp"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); using var rd = await cmd.ExecuteReaderAsync(); - while (rd.Read()) + while (await rd.ReadAsync()) yield return new KeyValuePair(rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime()); } @@ -439,17 +516,24 @@ namespace Duplicati.Library.Main.Database return (query.ToString(), args); } - public async Task GetRemoteVolumeID(string file, SqliteTransaction? transaction = null) + public async Task GetRemoteVolumeID(string file, SqliteTransaction transaction) { - m_selectremotevolumeIdCommand.Transaction = transaction; - m_selectremotevolumeIdCommand.SetParameterValue("@Name", file); - return await m_selectremotevolumeIdCommand.ExecuteScalarInt64Async(-1); + return await m_selectremotevolumeIdCommand + .SetTransaction(transaction) + .SetParameterValue("@Name", file) + .ExecuteScalarInt64Async(-1); } - public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files, SqliteTransaction? transaction = null) + public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files, SqliteTransaction transaction) { - using var cmd = await m_connection.CreateCommandAsync(@"SELECT ""Name"", ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name)"); - cmd.Transaction = transaction; + using var cmd = await m_connection.CreateCommandAsync(@" + SELECT + ""Name"", + ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Name) + "); + cmd.SetTransaction(transaction); using var tmptable = new TemporaryDbValueList(m_connection, transaction, files); cmd.ExpandInClauseParameter("@Name", tmptable); @@ -458,10 +542,12 @@ namespace Duplicati.Library.Main.Database yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } - public async Task GetRemoteVolume(string file, SqliteTransaction? transaction = null) + public async Task GetRemoteVolume(string file, SqliteTransaction transaction) { - m_selectremotevolumeCommand.Transaction = transaction; - m_selectremotevolumeCommand.SetParameterValue("@Name", file); + m_selectremotevolumeCommand + .SetTransaction(transaction) + .SetParameterValue("@Name", file); + using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync()) if (await rd.ReadAsync()) return new RemoteVolumeEntry( @@ -478,9 +564,10 @@ namespace Duplicati.Library.Main.Database return RemoteVolumeEntry.Empty; } - public async IAsyncEnumerable> DuplicateRemoteVolumes(SqliteTransaction? transaction) + public async IAsyncEnumerable> DuplicateRemoteVolumes(SqliteTransaction transaction) { - m_selectduplicateRemoteVolumesCommand.Transaction = transaction; + m_selectduplicateRemoteVolumesCommand.SetTransaction(transaction); + await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync()) { yield return new KeyValuePair( @@ -490,9 +577,9 @@ namespace Duplicati.Library.Main.Database } } - public async IAsyncEnumerable GetRemoteVolumes(SqliteTransaction? transaction = null) + public async IAsyncEnumerable GetRemoteVolumes(SqliteTransaction transaction) { - m_selectremotevolumesCommand.Transaction = transaction; + m_selectremotevolumesCommand.SetTransaction(transaction); using var rd = await m_selectremotevolumesCommand.ExecuteReaderAsync(); while (await rd.ReadAsync()) { @@ -515,15 +602,16 @@ namespace Duplicati.Library.Main.Database /// The operation performed /// The path involved /// Any data relating to the operation - public async Task LogRemoteOperation(string operation, string path, string? data, SqliteTransaction? transaction) + public async Task LogRemoteOperation(string operation, string path, string? data, SqliteTransaction transaction) { - m_insertremotelogCommand.Transaction = transaction; - m_insertremotelogCommand.SetParameterValue("@OperationID", m_operationid); - m_insertremotelogCommand.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); - m_insertremotelogCommand.SetParameterValue("@Operation", operation); - m_insertremotelogCommand.SetParameterValue("@Path", path); - m_insertremotelogCommand.SetParameterValue("@Data", data); - await m_insertremotelogCommand.ExecuteNonQueryAsync(); + await m_insertremotelogCommand + .SetTransaction(transaction) + .SetParameterValue("@OperationID", m_operationid) + .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) + .SetParameterValue("@Operation", operation) + .SetParameterValue("@Path", path) + .SetParameterValue("@Data", data) + .ExecuteNonQueryAsync(); } /// @@ -532,24 +620,29 @@ namespace Duplicati.Library.Main.Database /// The message type /// The message /// An optional exception - public async Task LogMessage(string type, string message, Exception? exception, SqliteTransaction? transaction) + public async Task LogMessage(string type, string message, Exception? exception, SqliteTransaction transaction) { - m_insertlogCommand.Transaction = transaction; - m_insertlogCommand.SetParameterValue("@OperationID", m_operationid); - m_insertlogCommand.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); - m_insertlogCommand.SetParameterValue("@Type", type); - m_insertlogCommand.SetParameterValue("@Message", message); - m_insertlogCommand.SetParameterValue("@Exception", exception?.ToString()); - await m_insertlogCommand.ExecuteNonQueryAsync(); + await m_insertlogCommand + .SetTransaction(transaction) + .SetParameterValue("@OperationID", m_operationid) + .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) + .SetParameterValue("@Type", type) + .SetParameterValue("@Message", message) + .SetParameterValue("@Exception", exception?.ToString()) + .ExecuteNonQueryAsync(); } public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state, SqliteTransaction transaction) { - using var cmd = await m_connection.CreateCommandAsync(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = @Name AND ""State"" = @State "); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Name", name); - cmd.SetParameterValue("@State", state.ToString()); - var c = await cmd.ExecuteNonQueryAsync(); + using var cmd = await m_connection.CreateCommandAsync(@" + DELETE FROM ""RemoteVolume"" + WHERE ""Name"" = @Name + AND ""State"" = @State + "); + var c = await cmd.SetTransaction(transaction) + .SetParameterValue("@Name", name) + .SetParameterValue("@State", state.ToString()) + .ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); @@ -581,51 +674,122 @@ namespace Duplicati.Library.Main.Database var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); deletecmd.Parameters.Clear(); - var bsIdsSubQuery = @" -SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" -WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) -UNION ALL -SELECT DISTINCT ""BlocksetID"" FROM ""BlocklistHash"" -WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))"; + var bsIdsSubQuery = @$" + SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" + FROM ""BlocksetEntry"", ""Block"" + WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) + UNION ALL + SELECT DISTINCT ""BlocksetID"" + FROM ""BlocklistHash"" + WHERE ""Hash"" IN ( + SELECT ""Hash"" + FROM ""Block"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ) + "; // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). - await deletecmd.ExecuteNonQueryAsync($@"CREATE TEMP TABLE ""{blocksetidstable}"" (""ID"" INTEGER PRIMARY KEY)"); - await deletecmd.ExecuteNonQueryAsync($@"INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery}"); - bsIdsSubQuery = $@"SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" "; + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TEMP TABLE ""{blocksetidstable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + "); + await deletecmd.ExecuteNonQueryAsync($@" + INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") + {bsIdsSubQuery} + "); + bsIdsSubQuery = $@" + SELECT DISTINCT ""ID"" + FROM ""{blocksetidstable}"" + "; deletecmd.Parameters.Clear(); // Create a temp table to associate metadata that is being deleted to a fileset - var metadataFilesetQuery = $@"SELECT Metadataset.ID, FilesetEntry.FilesetD -FROM Metadataset -INNER JOIN FileLookup ON FileLookup.MetadataID = Metadataset.ID -INNER JOIN FilesetEntry ON FilesetEntry.FileID = FileLookup.ID -WHERE Metadataset.BlocksetID IN ({bsIdsSubQuery}) -OR Metadataset.ID IN (SELECT MetadataID FROM FileLookup WHERE BlocksetID IN ({bsIdsSubQuery}))"; + var metadataFilesetQuery = $@" + SELECT Metadataset.ID, FilesetEntry.FilesetD + FROM Metadataset + INNER JOIN FileLookup + ON FileLookup.MetadataID = Metadataset.ID + INNER JOIN FilesetEntry + ON FilesetEntry.FileID = FileLookup.ID + WHERE Metadataset.BlocksetID IN ({bsIdsSubQuery}) + OR Metadataset.ID IN ( + SELECT MetadataID + FROM FileLookup + WHERE BlocksetID IN ({bsIdsSubQuery}) + ) + "; - var metadataFilesetTable = @"DelMetadataFilesetIds-" + temptransguid; - await deletecmd.ExecuteNonQueryAsync($@"CREATE TEMP TABLE ""{metadataFilesetTable}"" (MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER)"); - await deletecmd.ExecuteNonQueryAsync($@"INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (MetadataID, FilesetID) {metadataFilesetQuery}"); + var metadataFilesetTable = $"DelMetadataFilesetIds-{temptransguid}"; + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TEMP TABLE ""{metadataFilesetTable}"" ( + MetadataID INTEGER PRIMARY KEY, + FilesetID INTEGER + ) + "); + await deletecmd.ExecuteNonQueryAsync($@" + INSERT OR IGNORE INTO ""{metadataFilesetTable}"" ( + MetadataID, + FilesetID + ) + {metadataFilesetQuery} + "); // Delete FilesetEntry rows that had their metadata deleted - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM FilesetEntry -WHERE FilesetEntry.FilesetID IN (SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"") -AND FilesetEntry.FileID IN ( - SELECT FilesetEntry.FileID - FROM FilesetEntry - INNER JOIN FileLookup ON FileLookup.ID = FilesetEntry.FileID - WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}""))"); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM FilesetEntry + WHERE FilesetEntry.FilesetID IN ( + SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"" + ) + AND FilesetEntry.FileID IN ( + SELECT FilesetEntry.FileID + FROM FilesetEntry + INNER JOIN FileLookup + ON FileLookup.ID = FilesetEntry.FileID + WHERE FileLookup.MetadataID IN ( + SELECT MetadataID FROM ""{metadataFilesetTable}"" + ) + ) + "); // Delete FilesetEntry rows that had their blocks deleted - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FileID IN ( -SELECT ID FROM FileLookup -WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}))"); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")"); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM FilesetEntry + WHERE FilesetEntry.FileID IN ( + SELECT ID + FROM FileLookup + WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}) + ) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM FileLookup + WHERE FileLookup.MetadataID IN ( + SELECT MetadataID + FROM ""{metadataFilesetTable}"" + ) + "); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})"); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})"); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery})"); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})"); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Metadataset"" + WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""FileLookup"" + WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Blockset"" + WHERE ""ID"" IN ({bsIdsSubQuery}) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""BlocksetEntry"" + WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""BlocklistHash"" + WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery}) + "); // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); @@ -668,17 +832,25 @@ AND ""Fileset"".""ID"" NOT IN } catch { /* Ignore, will be deleted on close anyway. */ } - m_removeremotevolumeCommand.Transaction = transaction; - m_removeremotevolumeCommand.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); - m_removeremotevolumeCommand.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + m_removeremotevolumeCommand + .SetTransaction(transaction) + .SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) + .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); foreach (var name in names) { - m_removeremotevolumeCommand.SetParameterValue("@Name", name); - await m_removeremotevolumeCommand.ExecuteNonQueryAsync(); + await m_removeremotevolumeCommand + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync(); } // Validate before commiting changes - var nonAttachedFiles = await deletecmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); + var nonAttachedFiles = await deletecmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FileID"" NOT IN ( + SELECT ""ID"" FROM ""FileLookup"" + ) + "); if (nonAttachedFiles > 0) throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); @@ -689,6 +861,7 @@ AND ""Fileset"".""ID"" NOT IN { m_hasExecutedVacuum = true; using var cmd = m_connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync("VACUUM"); } @@ -710,16 +883,17 @@ AND ""Fileset"".""ID"" NOT IN public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, SqliteTransaction transaction) { - m_createremotevolumeCommand.Transaction = transaction; - m_createremotevolumeCommand.SetParameterValue("@OperationId", m_operationid); - m_createremotevolumeCommand.SetParameterValue("@Name", name); - m_createremotevolumeCommand.SetParameterValue("@Type", type.ToString()); - m_createremotevolumeCommand.SetParameterValue("@State", state.ToString()); - m_createremotevolumeCommand.SetParameterValue("@Size", size); - m_createremotevolumeCommand.SetParameterValue("@VerificationCount", 0); - m_createremotevolumeCommand.SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks); - m_createremotevolumeCommand.SetParameterValue("@ArchiveTime", 0); - var r = await m_createremotevolumeCommand.ExecuteScalarInt64Async(); + var r = await m_createremotevolumeCommand + .SetTransaction(transaction) + .SetParameterValue("@OperationId", m_operationid) + .SetParameterValue("@Name", name) + .SetParameterValue("@Type", type.ToString()) + .SetParameterValue("@State", state.ToString()) + .SetParameterValue("@Size", size) + .SetParameterValue("@VerificationCount", 0) + .SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks) + .SetParameterValue("@ArchiveTime", 0) + .ExecuteScalarInt64Async(); await transaction.CommitAsync(); @@ -731,17 +905,28 @@ AND ""Fileset"".""ID"" NOT IN if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); - (var query, var values) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch); + (var wherequery, var values) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch); var res = new List(); using var cmd = m_connection.CreateCommand(); - using (var rd = await cmd.ExecuteReaderAsync($@"SELECT ""ID"" FROM ""Fileset"" {query} ORDER BY ""Timestamp"" DESC", values)) + cmd.SetCommandAndParameters($@" + SELECT ""ID"" + FROM ""Fileset"" + {wherequery} + ORDER BY ""Timestamp"" DESC + ") + .SetParameterValues(values); + using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) res.Add(rd.ConvertValueToInt64(0)); if (res.Count == 0) { - cmd.Parameters.Clear(); - using (var rd = await cmd.ExecuteReaderAsync(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) + cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); + using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) res.Add(rd.ConvertValueToInt64(0)); @@ -759,11 +944,18 @@ AND ""Fileset"".""ID"" NOT IN if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); - var (query, args) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true); + var (wherequery, args) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true); var res = new List(); - using (var cmd = m_connection.CreateCommand()) - using (var rd = await cmd.ExecuteReaderAsync(@"SELECT ""ID"" FROM ""Fileset"" " + query + @" ORDER BY ""Timestamp"" DESC", args)) + using var cmd = m_connection.CreateCommand(); + cmd.SetCommandAndParameters(@$" + SELECT ""ID"" + FROM ""Fileset"" + {wherequery} + ORDER BY ""Timestamp"" DESC + ") + .SetParameterValues(args); + using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) res.Add(rd.ConvertValueToInt64(0)); @@ -773,9 +965,14 @@ AND ""Fileset"".""ID"" NOT IN public async Task IsFilesetFullBackup(DateTime filesetTime, SqliteTransaction transaction) { using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = @Timestamp"); - cmd.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)); + cmd.SetCommandAndParameters($@" + SELECT ""IsFullBackup"" + FROM ""Fileset"" + WHERE ""Timestamp"" = @Timestamp + ") + .SetTransaction(transaction) + .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)); + using var rd = await cmd.ExecuteReaderAsync(); if (!await rd.ReadAsync()) return false; @@ -826,18 +1023,25 @@ AND ""Fileset"".""ID"" NOT IN private IEnumerable> GetDbOptionList(IDbTransaction? transaction = null) { using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - using var rd = await cmd.ExecuteReaderAsync(@"SELECT ""Key"", ""Value"" FROM ""Configuration"" "); + cmd.SetCommandAndParameters(@" + SELECT + ""Key"", + ""Value"" + FROM ""Configuration"" + "); + cmd.SetTransaction(transaction); + + using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? ""); } - public async Task> GetDbOptions(SqliteTransaction? transaction) + public async Task> GetDbOptions(SqliteTransaction transaction) { - var t = transaction ?? m_connection.BeginTransaction(); - var res = await GetDbOptionList(t).ToDictionaryAsync(x => x.Key, x => x.Value); - if (transaction == null) - await t.CommitAsync(); + var res = await GetDbOptionList(transaction) + .ToDictionaryAsync(x => x.Key, x => x.Value); + + await transaction.CommitAsync(); return res; } @@ -863,28 +1067,46 @@ AND ""Fileset"".""ID"" NOT IN /// /// Flag indicating if a repair is in progress /// - public bool RepairInProgress + public async Task RepairInProgress(SqliteTransaction transaction, bool? value = null) { - get => GetDbOptions(null).Await().ContainsKey("repair-in-progress"); - set => UpdateDbOption("repair-in-progress", value).Await(); + if (value is bool v) + { + await UpdateDbOption("repair-in-progress", v); + return v; + } + + var opts = await GetDbOptions(transaction); + return opts.ContainsKey("repair-in-progress"); } /// /// Flag indicating if a repair is in progress /// - public bool PartiallyRecreated + public async Task PartiallyRecreated(SqliteTransaction transaction, bool? value = null) { - get => GetDbOptions(null).Await().ContainsKey("partially-recreated"); - set => UpdateDbOption("partially-recreated", value).Await(); + if (value is bool v) + { + await UpdateDbOption("partially-recreated", v); + return v; + } + + var opts = await GetDbOptions(transaction); + return opts.ContainsKey("partially-recreated"); } /// /// Flag indicating if the database can contain partial uploads /// - public bool TerminatedWithActiveUploads + public async Task TerminatedWithActiveUploads(SqliteTransaction transaction, bool? value = null) { - get => GetDbOptions(null).Await().ContainsKey("terminated-with-active-uploads"); - set => UpdateDbOption("terminated-with-active-uploads", value).Await(); + if (value is bool v) + { + await UpdateDbOption("terminated-with-active-uploads", v); + return v; + } + + var opts = await GetDbOptions(transaction); + return opts.ContainsKey("terminated-with-active-uploads"); } /// @@ -895,14 +1117,25 @@ AND ""Fileset"".""ID"" NOT IN public async Task SetDbOptions(IDictionary options, SqliteTransaction transaction) { using var cmd = m_connection.CreateCommand(); - await cmd.ExecuteNonQueryAsync(@"DELETE FROM ""Configuration"" "); + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""Configuration"" + "); foreach (var kp in options) { - cmd.SetCommandAndParameters(@"INSERT INTO ""Configuration"" (""Key"", ""Value"") VALUES (@Key, @Value) "); - cmd.SetParameterValue("@Key", kp.Key); - cmd.SetParameterValue("@Value", kp.Value); - await cmd.ExecuteNonQueryAsync(); + await cmd.SetCommandAndParameters(@" + INSERT INTO ""Configuration"" ( + ""Key"", + ""Value"" + ) + VALUES ( + @Key, + @Value + ) + ") + .SetParameterValue("@Key", kp.Key) + .SetParameterValue("@Value", kp.Value) + .ExecuteNonQueryAsync(); } await transaction.CommitAsync(); @@ -910,9 +1143,14 @@ AND ""Fileset"".""ID"" NOT IN public async Task GetBlocksLargerThan(long fhblocksize) { - using var cmd = await m_connection.CreateCommandAsync(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size"); - cmd.SetParameterValue("@Size", fhblocksize); - return await cmd.ExecuteScalarInt64Async(-1); + using var cmd = await m_connection.CreateCommandAsync(@" + SELECT COUNT(*) + FROM ""Block"" + WHERE ""Size"" > @Size + "); + + return await cmd.SetParameterValue("@Size", fhblocksize) + .ExecuteScalarInt64Async(-1); } /// @@ -949,31 +1187,33 @@ AND ""Fileset"".""ID"" NOT IN cmd.Transaction = transaction; // Calculate the lengths for each blockset var combinedLengths = @" -SELECT - ""A"".""ID"" AS ""BlocksetID"", - IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"", - ""A"".""Length"" -FROM - ""Blockset"" A -LEFT OUTER JOIN - ( - SELECT - ""BlocksetEntry"".""BlocksetID"", - SUM(""Block"".""Size"") AS ""CalcLen"" - FROM - ""BlocksetEntry"" - LEFT OUTER JOIN - ""Block"" - ON - ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" - GROUP BY ""BlocksetEntry"".""BlocksetID"" - ) B -ON - ""A"".""ID"" = ""B"".""BlocksetID"" + SELECT + ""A"".""ID"" AS ""BlocksetID"", + IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"", + ""A"".""Length"" + FROM ""Blockset"" A + LEFT OUTER JOIN ( + SELECT + ""BlocksetEntry"".""BlocksetID"", + SUM(""Block"".""Size"") AS ""CalcLen"" + FROM ""BlocksetEntry"" + LEFT OUTER JOIN ""Block"" + ON ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" + GROUP BY ""BlocksetEntry"".""BlocksetID"" + ) B + ON ""A"".""ID"" = ""B"".""BlocksetID"" + "; -"; // For each blockset with wrong lengths, fetch the file path - var reportDetails = @"SELECT ""CalcLen"", ""Length"", ""A"".""BlocksetID"", ""File"".""Path"" FROM (" + combinedLengths + @") A, ""File"" WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" AND ""A"".""CalcLen"" != ""A"".""Length"" "; + var reportDetails = @$" + SELECT + ""CalcLen"", + ""Length"", ""A"".""BlocksetID"", + ""File"".""Path"" + FROM ({combinedLengths}) A, ""File"" + WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" + AND ""A"".""CalcLen"" != ""A"".""Length"" + "; using (var rd = await cmd.ExecuteReaderAsync(reportDetails)) if (await rd.ReadAsync()) @@ -996,20 +1236,76 @@ ON throw new DatabaseInconsistencyException(sb.ToString()); } - var real_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM ""BlocklistHash""", 0); - var unique_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); + var real_count = await cmd.ExecuteScalarInt64Async(@" + SELECT Count(*) + FROM ""BlocklistHash"" + ", 0); + var unique_count = await cmd.ExecuteScalarInt64Async(@" + SELECT Count(*) + FROM ( + SELECT DISTINCT + ""BlocksetID"", + ""Index"" + FROM ""BlocklistHash"" + )", 0); if (real_count != unique_count) throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); - var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async($@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")", 0); + var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ( + SELECT * + FROM ( + SELECT + ""N"".""BlocksetID"", + ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", + CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" + FROM ( + SELECT + ""BlocksetID"", + COUNT(*) AS ""BlockCount"" + FROM ""BlocksetEntry"" + GROUP BY ""BlocksetID"" + ) ""N"" + LEFT OUTER JOIN ( + SELECT + ""BlocksetID"", + COUNT(*) AS ""BlocklistHashCount"" + FROM ""BlocklistHash"" + GROUP BY ""BlocksetID"" + ) ""G"" + ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" + WHERE ""N"".""BlockCount"" > 1 + ) + WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"" + ) + ", 0); if (itemswithnoblocklisthash != 0) throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); - if (await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM ""Blockset"" WHERE ""Length"" > 0 AND ""ID"" NOT IN (SELECT ""BlocksetId"" FROM ""BlocksetEntry"")") != 0) + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""Blockset"" + WHERE ""Length"" > 0 + AND ""ID"" NOT IN ( + SELECT ""BlocksetId"" + FROM ""BlocksetEntry"" + ) + "); + if (await cmd.ExecuteScalarInt64Async() != 0) throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); - cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""BlocksetID"" != @FolderBlocksetId AND ""BlocksetID"" != @SymlinkBlocksetId AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")"); + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""FileLookup"" + WHERE ""BlocksetID"" != @FolderBlocksetId + AND ""BlocksetID"" != @SymlinkBlocksetId + AND NOT ""BlocksetID"" IN ( + SELECT ""ID"" + FROM ""Blockset"" + ) + "); cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); if (await cmd.ExecuteScalarInt64Async(0) != 0) @@ -1017,7 +1313,16 @@ ON if (!laxVerifyForRepair) { - cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)"); + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""Fileset"" + WHERE ""VolumeID"" NOT IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Type"" = @Type + AND ""State"" != @State + ) + "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); var filesetsMissingVolumes = await cmd.ExecuteScalarInt64Async(0); @@ -1026,7 +1331,19 @@ ON { if (filesetsMissingVolumes == 1) { - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)"); + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Timestamp"", + ""VolumeID"" + FROM ""Fileset"" + WHERE ""VolumeID"" NOT IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Type"" = @Type + AND ""State"" != @State + ) + "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); using var reader = await cmd.ExecuteReaderAsync(); @@ -1037,7 +1354,16 @@ ON throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes"); } - cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")"); + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""RemoteVolume"" + WHERE ""Type"" = @Type + AND ""State"" != @State + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) + "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); var volumesMissingFilests = await cmd.ExecuteScalarInt64Async(0); @@ -1045,7 +1371,19 @@ ON { if (volumesMissingFilests == 1) { - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")"); + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""State"" + FROM ""RemoteVolume"" + WHERE ""Type"" = @Type + AND ""State"" != @State + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) + "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); using var reader = await cmd.ExecuteReaderAsync(); @@ -1057,11 +1395,26 @@ ON } } - var nonAttachedFiles = await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); + var nonAttachedFiles = await cmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FileID"" NOT IN ( + SELECT ""ID"" + FROM ""FileLookup"" + ) + "); if (nonAttachedFiles != 0) { // Attempt to create a better error message by finding the first 10 fileset ids with the issue - using var filesetIdReader = await cmd.ExecuteReaderAsync(@"SELECT DISTINCT(FilesetID) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"") LIMIT 11"); + using var filesetIdReader = await cmd.ExecuteReaderAsync(@" + SELECT DISTINCT(FilesetID) + FROM ""FilesetEntry"" + WHERE ""FileID"" NOT IN ( + SELECT ""ID"" + FROM ""FileLookup"" + ) + LIMIT 11 + "); var filesetIds = new HashSet(); var overflow = false; while (await filesetIdReader.ReadAsync()) @@ -1094,19 +1447,37 @@ ON var anyError = new List(); using (var cmd2 = m_connection.CreateCommand()) { - cmd2.Transaction = transaction; - await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) + cmd2.SetTransaction(transaction); + cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + "); + await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync().Select(x => x.ConvertValueToInt64(0, -1))) { - var expandedCmd = $@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FILESETS}) UNION SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FOLDERS_AND_SYMLINKS}))"; - cmd2.SetCommandAndParameters(expandedCmd); - cmd2.SetParameterValue("@FilesetId", filesetid); - cmd2.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); - cmd2.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); - var expandedlist = await cmd.ExecuteScalarInt64Async(0); + var expandedlist = await cmd2.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT ""Path"" + FROM ({LIST_FILESETS}) + UNION + SELECT DISTINCT ""Path"" + FROM ({LIST_FOLDERS_AND_SYMLINKS}) + ) + ") + .SetParameterValue("@FilesetId", filesetid) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64Async(0); + //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != @FolderBlocksetId AND ""FileLookup"".""BlocksetID"" != @SymlinkBlocksetId"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); - cmd2.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId"); - cmd2.SetParameterValue("@FilesetId", filesetid); - var storedlist = await cmd2.ExecuteScalarInt64Async(0); + + var storedlist = await cmd2.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetid) + .ExecuteScalarInt64Async(0); if (expandedlist != storedlist) { @@ -1119,7 +1490,7 @@ ON } } - if (anyError.Any()) + if (anyError.Count != 0) { throw new DatabaseInconsistencyException(string.Join("\n\r", anyError), "FilesetDifferences"); } @@ -1140,9 +1511,15 @@ ON public async IAsyncEnumerable GetBlocks(long volumeid, SqliteTransaction transaction) { - using var cmd = await m_connection.CreateCommandAsync(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@VolumeId", volumeid); + using var cmd = await m_connection.CreateCommandAsync(@" + SELECT DISTINCT + ""Hash"", + ""Size"" + FROM ""Block"" + WHERE ""VolumeID"" = @VolumeId + "); + cmd.SetTransaction(transaction) + .SetParameterValue("@VolumeId", volumeid); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return new Block(rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), rd.ConvertValueToInt64(1)); @@ -1238,141 +1615,140 @@ ON } public const string LIST_FILESETS = @" -SELECT - ""L"".""Path"", - ""L"".""Lastmodified"", - ""L"".""Filelength"", - ""L"".""Filehash"", - ""L"".""Metahash"", - ""L"".""Metalength"", - ""L"".""BlocklistHash"", - ""L"".""FirstBlockHash"", - ""L"".""FirstBlockSize"", - ""L"".""FirstMetaBlockHash"", - ""L"".""FirstMetaBlockSize"", - ""M"".""Hash"" AS ""MetaBlocklistHash"" -FROM - ( - SELECT - ""J"".""Path"", - ""J"".""Lastmodified"", - ""J"".""Filelength"", - ""J"".""Filehash"", - ""J"".""Metahash"", - ""J"".""Metalength"", - ""K"".""Hash"" AS ""BlocklistHash"", - ""J"".""FirstBlockHash"", - ""J"".""FirstBlockSize"", - ""J"".""FirstMetaBlockHash"", - ""J"".""FirstMetaBlockSize"", - ""J"".""MetablocksetID"" - FROM - ( - SELECT - ""A"".""Path"" AS ""Path"", - ""D"".""Lastmodified"" AS ""Lastmodified"", - ""B"".""Length"" AS ""Filelength"", - ""B"".""FullHash"" AS ""Filehash"", - ""E"".""FullHash"" AS ""Metahash"", - ""E"".""Length"" AS ""Metalength"", - ""A"".""BlocksetID"" AS ""BlocksetID"", - ""F"".""Hash"" AS ""FirstBlockHash"", - ""F"".""Size"" AS ""FirstBlockSize"", - ""H"".""Hash"" AS ""FirstMetaBlockHash"", - ""H"".""Size"" AS ""FirstMetaBlockSize"", - ""C"".""BlocksetID"" AS ""MetablocksetID"" - FROM - ""File"" A - LEFT JOIN ""Blockset"" B - ON ""A"".""BlocksetID"" = ""B"".""ID"" - LEFT JOIN ""Metadataset"" C - ON ""A"".""MetadataID"" = ""C"".""ID"" - LEFT JOIN ""FilesetEntry"" D - ON ""A"".""ID"" = ""D"".""FileID"" - LEFT JOIN ""Blockset"" E - ON ""E"".""ID"" = ""C"".""BlocksetID"" - LEFT JOIN ""BlocksetEntry"" G - ON ""B"".""ID"" = ""G"".""BlocksetID"" - LEFT JOIN ""Block"" F - ON ""G"".""BlockID"" = ""F"".""ID"" - LEFT JOIN ""BlocksetEntry"" I - ON ""E"".""ID"" = ""I"".""BlocksetID"" - LEFT JOIN ""Block"" H - ON ""I"".""BlockID"" = ""H"".""ID"" - WHERE - ""A"".""BlocksetId"" >= 0 AND - ""D"".""FilesetID"" = @FilesetId AND - (""I"".""Index"" = 0 OR ""I"".""Index"" IS NULL) AND - (""G"".""Index"" = 0 OR ""G"".""Index"" IS NULL) - ) J - LEFT OUTER JOIN - ""BlocklistHash"" K - ON - ""K"".""BlocksetID"" = ""J"".""BlocksetID"" - ORDER BY ""J"".""Path"", ""K"".""Index"" - ) L - -LEFT OUTER JOIN - ""BlocklistHash"" M -ON - ""M"".""BlocksetID"" = ""L"".""MetablocksetID"" + SELECT + ""L"".""Path"", + ""L"".""Lastmodified"", + ""L"".""Filelength"", + ""L"".""Filehash"", + ""L"".""Metahash"", + ""L"".""Metalength"", + ""L"".""BlocklistHash"", + ""L"".""FirstBlockHash"", + ""L"".""FirstBlockSize"", + ""L"".""FirstMetaBlockHash"", + ""L"".""FirstMetaBlockSize"", + ""M"".""Hash"" AS ""MetaBlocklistHash"" + FROM ( + SELECT + ""J"".""Path"", + ""J"".""Lastmodified"", + ""J"".""Filelength"", + ""J"".""Filehash"", + ""J"".""Metahash"", + ""J"".""Metalength"", + ""K"".""Hash"" AS ""BlocklistHash"", + ""J"".""FirstBlockHash"", + ""J"".""FirstBlockSize"", + ""J"".""FirstMetaBlockHash"", + ""J"".""FirstMetaBlockSize"", + ""J"".""MetablocksetID"" + FROM ( + SELECT + ""A"".""Path"" AS ""Path"", + ""D"".""Lastmodified"" AS ""Lastmodified"", + ""B"".""Length"" AS ""Filelength"", + ""B"".""FullHash"" AS ""Filehash"", + ""E"".""FullHash"" AS ""Metahash"", + ""E"".""Length"" AS ""Metalength"", + ""A"".""BlocksetID"" AS ""BlocksetID"", + ""F"".""Hash"" AS ""FirstBlockHash"", + ""F"".""Size"" AS ""FirstBlockSize"", + ""H"".""Hash"" AS ""FirstMetaBlockHash"", + ""H"".""Size"" AS ""FirstMetaBlockSize"", + ""C"".""BlocksetID"" AS ""MetablocksetID"" + FROM ""File"" A + LEFT JOIN ""Blockset"" B + ON ""A"".""BlocksetID"" = ""B"".""ID"" + LEFT JOIN ""Metadataset"" C + ON ""A"".""MetadataID"" = ""C"".""ID"" + LEFT JOIN ""FilesetEntry"" D + ON ""A"".""ID"" = ""D"".""FileID"" + LEFT JOIN ""Blockset"" E + ON ""E"".""ID"" = ""C"".""BlocksetID"" + LEFT JOIN ""BlocksetEntry"" G + ON ""B"".""ID"" = ""G"".""BlocksetID"" + LEFT JOIN ""Block"" F + ON ""G"".""BlockID"" = ""F"".""ID"" + LEFT JOIN ""BlocksetEntry"" I + ON ""E"".""ID"" = ""I"".""BlocksetID"" + LEFT JOIN ""Block"" H + ON ""I"".""BlockID"" = ""H"".""ID"" + WHERE ""A"".""BlocksetId"" >= 0 + AND ""D"".""FilesetID"" = @FilesetId + AND ( + ""I"".""Index"" = 0 + OR ""I"".""Index"" IS NULL + ) + AND ( + ""G"".""Index"" = 0 + OR ""G"".""Index"" IS NULL + ) + ) J + LEFT OUTER JOIN ""BlocklistHash"" K + ON ""K"".""BlocksetID"" = ""J"".""BlocksetID"" + ORDER BY + ""J"".""Path"", + ""K"".""Index"" + ) L + LEFT OUTER JOIN ""BlocklistHash"" M + ON ""M"".""BlocksetID"" = ""L"".""MetablocksetID"" "; public const string LIST_FOLDERS_AND_SYMLINKS = @" -SELECT - ""G"".""BlocksetID"", - ""G"".""ID"", - ""G"".""Path"", - ""G"".""Length"", - ""G"".""FullHash"", - ""G"".""Lastmodified"", - ""G"".""FirstMetaBlockHash"", - ""H"".""Hash"" AS ""MetablocklistHash"" -FROM - ( - SELECT - ""B"".""BlocksetID"", - ""B"".""ID"", - ""B"".""Path"", - ""D"".""Length"", - ""D"".""FullHash"", - ""A"".""Lastmodified"", - ""F"".""Hash"" AS ""FirstMetaBlockHash"", - ""C"".""BlocksetID"" AS ""MetaBlocksetID"" - FROM - ""FilesetEntry"" A, - ""File"" B, - ""Metadataset"" C, - ""Blockset"" D, - ""BlocksetEntry"" E, - ""Block"" F - WHERE - ""A"".""FileID"" = ""B"".""ID"" - AND ""B"".""MetadataID"" = ""C"".""ID"" - AND ""C"".""BlocksetID"" = ""D"".""ID"" - AND ""E"".""BlocksetID"" = ""C"".""BlocksetID"" - AND ""E"".""BlockID"" = ""F"".""ID"" - AND ""E"".""Index"" = 0 - AND (""B"".""BlocksetID"" = @FolderBlocksetId OR ""B"".""BlocksetID"" = @SymlinkBlocksetId) - AND ""A"".""FilesetID"" = @FilesetId - ) G -LEFT OUTER JOIN - ""BlocklistHash"" H -ON - ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID"" -ORDER BY - ""G"".""Path"", ""H"".""Index"" - -"; + SELECT + ""G"".""BlocksetID"", + ""G"".""ID"", + ""G"".""Path"", + ""G"".""Length"", + ""G"".""FullHash"", + ""G"".""Lastmodified"", + ""G"".""FirstMetaBlockHash"", + ""H"".""Hash"" AS ""MetablocklistHash"" + FROM ( + SELECT + ""B"".""BlocksetID"", + ""B"".""ID"", + ""B"".""Path"", + ""D"".""Length"", + ""D"".""FullHash"", + ""A"".""Lastmodified"", + ""F"".""Hash"" AS ""FirstMetaBlockHash"", + ""C"".""BlocksetID"" AS ""MetaBlocksetID"" + FROM + ""FilesetEntry"" A, + ""File"" B, + ""Metadataset"" C, + ""Blockset"" D, + ""BlocksetEntry"" E, + ""Block"" F + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""MetadataID"" = ""C"".""ID"" + AND ""C"".""BlocksetID"" = ""D"".""ID"" + AND ""E"".""BlocksetID"" = ""C"".""BlocksetID"" + AND ""E"".""BlockID"" = ""F"".""ID"" + AND ""E"".""Index"" = 0 + AND ( + ""B"".""BlocksetID"" = @FolderBlocksetId + OR ""B"".""BlocksetID"" = @SymlinkBlocksetId + ) + AND ""A"".""FilesetID"" = @FilesetId + ) G + LEFT OUTER JOIN ""BlocklistHash"" H + ON ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID"" + ORDER BY + ""G"".""Path"", + ""H"".""Index"" + "; public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS); - cmd.SetParameterValue("@FilesetId", filesetId); - cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); - cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); + using var cmd = m_connection.CreateCommand() + .SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS) + .SetTransaction(transaction) + .SetParameterValue("@FilesetId", filesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); string? lastpath = null; using (var rd = await cmd.ExecuteReaderAsync()) @@ -1439,12 +1815,15 @@ ORDER BY public async Task LinkFilesetToVolume(long filesetid, long volumeid, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"UPDATE ""Fileset"" SET ""VolumeID"" = @VolumeId WHERE ""ID"" = @FilesetId"); - cmd.SetParameterValue("@VolumeId", volumeid); - cmd.SetParameterValue("@FilesetId", filesetid); - var c = await cmd.ExecuteNonQueryAsync(); + using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""Fileset"" + SET ""VolumeID"" = @VolumeId + WHERE ""ID"" = @FilesetId + "); + var c = await cmd.SetTransaction(transaction) + .SetParameterValue("@VolumeId", volumeid) + .SetParameterValue("@FilesetId", filesetid) + .ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); @@ -1453,17 +1832,24 @@ ORDER BY public async Task PushTimestampChangesToPreviousVersion(long filesetId, SqliteTransaction transaction) { var query = @" -UPDATE FilesetEntry AS oldVersion -SET Lastmodified = tempVersion.Lastmodified -FROM FilesetEntry AS tempVersion -WHERE oldVersion.FileID = tempVersion.FileID -AND tempVersion.FilesetID = @FilesetId -AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER BY Timestamp DESC LIMIT 1)"; + UPDATE FilesetEntry AS oldVersion + SET Lastmodified = tempVersion.Lastmodified + FROM FilesetEntry AS tempVersion + WHERE oldVersion.FileID = tempVersion.FileID + AND tempVersion.FilesetID = @FilesetId + AND oldVersion.FilesetID = ( + SELECT ID + FROM Fileset + WHERE ID != @FilesetId + ORDER BY Timestamp DESC + LIMIT 1 + ) + "; using var cmd = await m_connection.CreateCommandAsync(query); - cmd.Transaction = transaction; - cmd.SetParameterValue("@FilesetId", filesetId); - await cmd.ExecuteNonQueryAsync(); + await cmd.SetTransaction(transaction) + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(); } /// @@ -1501,28 +1887,44 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER if (filter.Empty) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" "); + using var cmd = await m_connection.CreateCommandAsync($@" + CREATE TEMPORARY TABLE ""{Tablename}"" AS + SELECT DISTINCT ""Path"" + FROM ""File"" + "); + await cmd.SetTransaction(transaction) + .ExecuteNonQueryAsync(); return ftt; } if (type == FilterType.Regexp || type == FilterType.Group) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)"); - cmd.SetCommandAndParameters($@"INSERT INTO ""{Tablename}"" (""Path"") VALUES (@Path)"); - using (var c2 = m_connection.CreateCommand()) - using (var rd = await c2.ExecuteReaderAsync(@"SELECT DISTINCT ""Path"" FROM ""File"" ")) + using var cmd = await m_connection.CreateCommandAsync($@" + CREATE TEMPORARY TABLE ""{Tablename}"" ( + ""Path"" TEXT NOT NULL + ) + "); + await cmd.SetTransaction(transaction) + .ExecuteNonQueryAsync(); + + cmd.SetCommandAndParameters($@" + INSERT INTO ""{Tablename}"" (""Path"") + VALUES (@Path) + "); + using var c2 = await m_connection.CreateCommandAsync(@" + SELECT DISTINCT ""Path"" + FROM ""File"" + "); + c2.SetTransaction(transaction); + using (var rd = await c2.ExecuteReaderAsync()) while (await rd.ReadAsync()) { var p = rd.ConvertValueToString(0); if (FilterExpression.Matches(filter, p)) { - cmd.SetParameterValue("@Path", p); - cmd.ExecuteNonQueryAsync().Await(); + await cmd.SetParameterValue("@Path", p) + .ExecuteNonQueryAsync(); } } @@ -1551,9 +1953,18 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)"); - await cmd.ExecuteNonQueryAsync($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}", args); + cmd.SetTransaction(transaction); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{Tablename}"" ( + ""Path"" TEXT NOT NULL + ) + "); + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{Tablename}"" + SELECT DISTINCT ""Path"" + FROM ""File"" + WHERE {sb} + ", args); await transaction.CommitAsync(); } @@ -1561,12 +1972,19 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } public void Dispose() + { + DisposeAsync().Await(); + } + + public async Task DisposeAsync() { if (Tablename != null) try { using var cmd = m_connection.CreateCommand(); - cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{Tablename}"" ").Await(); + await cmd.ExecuteNonQueryAsync(@$" + DROP TABLE IF EXISTS ""{Tablename}"" + "); } catch { } finally { Tablename = null!; } @@ -1575,20 +1993,27 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER public async Task RenameRemoteFile(string oldname, string newname, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; //Rename the old entry, to preserve ID links - cmd.SetCommandAndParameters(@"UPDATE ""Remotevolume"" SET ""Name"" = @Newname WHERE ""Name"" = @Oldname"); - cmd.SetParameterValue("@Newname", newname); - cmd.SetParameterValue("@Oldname", oldname); - var c = await cmd.ExecuteNonQueryAsync(); + using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""Remotevolume"" + SET ""Name"" = @Newname + WHERE ""Name"" = @Oldname + "); + var c = await cmd.SetTransaction(transaction) + .SetParameterValue("@Newname", newname) + .SetParameterValue("@Oldname", oldname) + .ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}"); // Grab the type of entry - cmd.SetCommandAndParameters(@"SELECT ""Type"" FROM ""Remotevolume"" WHERE ""Name"" = @Name"); - cmd.SetParameterValue("@Name", newname); + cmd.SetCommandAndParameters(@" + SELECT ""Type"" + FROM ""Remotevolume"" + WHERE ""Name"" = @Name + ") + .SetParameterValue("@Name", newname); var type = (RemoteVolumeType)Enum.Parse( typeof(RemoteVolumeType), (await cmd.ExecuteScalarAsync())?.ToString() ?? "", true); @@ -1608,14 +2033,27 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// An optional external transaction public virtual async Task CreateFileset(long volumeid, DateTime timestamp, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"", ""IsFullBackup"") VALUES (@OperationId, @Timestamp, @VolumeId, @IsFullBackup); SELECT last_insert_rowid();"); - cmd.SetParameterValue("@OperationId", m_operationid); - cmd.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)); - cmd.SetParameterValue("@VolumeId", volumeid); - cmd.SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP); - var id = await cmd.ExecuteScalarInt64Async(-1); + using var cmd = await m_connection.CreateCommandAsync(@" + INSERT INTO ""Fileset"" ( + ""OperationID"", + ""Timestamp"", + ""VolumeID"", + ""IsFullBackup"" + ) + VALUES ( + @OperationId, + @Timestamp, + @VolumeId, + @IsFullBackup + ); + SELECT last_insert_rowid(); + "); + var id = await cmd.SetTransaction(transaction) + .SetParameterValue("@OperationId", m_operationid) + .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) + .SetParameterValue("@VolumeId", volumeid) + .SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP) + .ExecuteScalarInt64Async(-1); await transaction.CommitAsync(); @@ -1635,10 +2073,10 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER if (blockVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0."); - m_insertIndexBlockLink.Transaction = transaction; - m_insertIndexBlockLink.SetParameterValue("@IndexVolumeId", indexVolumeID); - m_insertIndexBlockLink.SetParameterValue("@BlockVolumeId", blockVolumeID); - await m_insertIndexBlockLink.ExecuteNonQueryAsync(); + await m_insertIndexBlockLink.SetTransaction(transaction) + .SetParameterValue("@IndexVolumeId", indexVolumeID) + .SetParameterValue("@BlockVolumeId", blockVolumeID) + .ExecuteNonQueryAsync(); } /// @@ -1651,21 +2089,42 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data public async IAsyncEnumerable> GetBlocklists(long volumeid, long blocksize, int hashsize, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; // Group subquery by hash to ensure that each blocklist hash appears only once in the result - var sql = $@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM -(SELECT ""BlocklistHash"".""BlocksetID"", ""Block"".""Hash"", ""BlocklistHash"".""Index"" FROM ""BlocklistHash"",""Block"" WHERE ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId GROUP BY ""Block"".""Hash"", ""Block"".""Size"") A, - ""BlocksetEntry"" B, ""Block"" C WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" AND - ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize}) AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize}) AND ""C"".""ID"" = ""B"".""BlockID"" - ORDER BY ""A"".""BlocksetID"", ""B"".""Index"""; + using var cmd = await m_connection.CreateCommandAsync($@" + SELECT + ""A"".""Hash"", + ""C"".""Hash"" + FROM ( + SELECT + ""BlocklistHash"".""BlocksetID"", + ""Block"".""Hash"", + ""BlocklistHash"".""Index"" + FROM + ""BlocklistHash"", + ""Block"" + WHERE ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" + AND ""Block"".""VolumeID"" = @VolumeId + GROUP BY + ""Block"".""Hash"", + ""Block"".""Size"" + ) A, + ""BlocksetEntry"" B, + ""Block"" C + WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" + AND ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize}) + AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize}) + AND ""C"".""ID"" = ""B"".""BlockID"" + ORDER BY + ""A"".""BlocksetID"", + ""B"".""Index"" + "); string? curHash = null; var count = 0; var buffer = new byte[blocksize]; - cmd.SetCommandAndParameters(sql); - cmd.SetParameterValue("@VolumeId", volumeid); + cmd.SetTransaction(transaction) + .SetParameterValue("@VolumeId", volumeid); using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) { @@ -1695,12 +2154,15 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// An optional external transaction public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"UPDATE ""Fileset"" SET ""IsFullBackup"" = @IsFullBackup WHERE ""ID"" = @FilesetId;"); - cmd.SetParameterValue("@FilesetId", fileSetId); - cmd.SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP); - await cmd.ExecuteNonQueryAsync(); + using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""Fileset"" + SET ""IsFullBackup"" = @IsFullBackup + WHERE ""ID"" = @FilesetId + "); + await cmd.SetTransaction(transaction) + .SetParameterValue("@FilesetId", fileSetId) + .SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP) + .ExecuteNonQueryAsync(); await transaction.CommitAsync(); } @@ -1712,11 +2174,13 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// The transaction to use public async Task ClearFilesetEntries(long filesetId, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId"); - cmd.SetParameterValue("@FilesetId", filesetId); - await cmd.ExecuteNonQueryAsync(); + using var cmd = await m_connection.CreateCommandAsync(@" + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + "); + await cmd.SetTransaction(transaction) + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(); } /// @@ -1726,7 +2190,9 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// The last incomplete fileset or default public async Task GetLastIncompleteFilesetVolume(SqliteTransaction transaction) { - var candidates = GetIncompleteFilesets(transaction).OrderBy(x => x.Value); + var candidates = GetIncompleteFilesets(transaction) + .OrderBy(x => x.Value); + if (await candidates.AnyAsync()) return await GetRemoteVolumeFromFilesetID((await candidates.LastAsync()).Key, transaction); @@ -1740,14 +2206,32 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// A list of fileset IDs and timestamps public async IAsyncEnumerable> GetIncompleteFilesets(SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - using var rd = await cmd.ExecuteReaderAsync(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')"); + using var cmd = await m_connection.CreateCommandAsync(@$" + SELECT DISTINCT + ""Fileset"".""ID"", + ""Fileset"".""Timestamp"" + FROM + ""Fileset"", + ""RemoteVolume"" + WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" + AND ""Fileset"".""ID"" IN ( + SELECT ""FilesetID"" + FROM ""FilesetEntry"" + ) + AND ( + ""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' + OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}' + ) + "); + + using var rd = await cmd.SetTransaction(transaction) + .ExecuteReaderAsync(); while (await rd.ReadAsync()) { yield return new KeyValuePair( rd.ConvertValueToInt64(0), - ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)) + .ToLocalTime() ); } } @@ -1760,11 +2244,26 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// The remote volume entry or default public async Task GetRemoteVolumeFromFilesetID(long filesetID, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"SELECT ""RemoteVolume"".""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" FROM ""RemoteVolume"", ""Fileset"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Fileset"".""ID"" = @FilesetId"); - cmd.SetParameterValue("@FilesetId", filesetID); - using var rd = await cmd.ExecuteReaderAsync(); + using var cmd = await m_connection.CreateCommandAsync(@" + SELECT + ""RemoteVolume"".""ID"", + ""Name"", + ""Type"", + ""Size"", + ""Hash"", + ""State"", + ""DeleteGraceTime"", + ""ArchiveTime"" + FROM + ""RemoteVolume"", + ""Fileset"" + WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" + AND ""Fileset"".""ID"" = @FilesetId + "); + + using var rd = await cmd.SetTransaction(transaction) + .SetParameterValue("@FilesetId", filesetID) + .ExecuteReaderAsync(); if (await rd.ReadAsync()) return new RemoteVolumeEntry( rd.ConvertValueToInt64(0, -1), @@ -1786,12 +2285,20 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold); - cmd.SetCommandAndParameters(@"DELETE FROM ""LogData"" WHERE ""Timestamp"" < @Timestamp"); - cmd.SetParameterValue("@Timestamp", t); - await cmd.ExecuteNonQueryAsync(); - cmd.SetCommandAndParameters(@"DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < @Timestamp"); - cmd.SetParameterValue("@Timestamp", t); - await cmd.ExecuteNonQueryAsync(); + + await cmd.SetCommandAndParameters(@" + DELETE FROM ""LogData"" + WHERE ""Timestamp"" < @Timestamp + ") + .SetParameterValue("@Timestamp", t) + .ExecuteNonQueryAsync(); + + await cmd.SetCommandAndParameters(@" + DELETE FROM ""RemoteOperation"" + WHERE ""Timestamp"" < @Timestamp + ") + .SetParameterValue("@Timestamp", t) + .ExecuteNonQueryAsync(); await transaction.CommitAsync(); } @@ -1799,15 +2306,21 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER public async Task PurgeDeletedVolumes(DateTime threshold) { using var transaction = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - m_removedeletedremotevolumeCommand.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)); - await m_removedeletedremotevolumeCommand.ExecuteNonQueryAsync(); + + await m_removedeletedremotevolumeCommand + .SetTransaction(transaction) + .SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)) + .ExecuteNonQueryAsync(); await transaction.CommitAsync(); } public virtual void Dispose() + { + DisposeAsync().Await(); + } + + public virtual async Task DisposeAsync() { if (IsDisposed) return; @@ -1823,11 +2336,11 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER { command.Transaction = transaction; // SQLite recommends that PRAGMA optimize is run just before closing each database connection. - command.ExecuteNonQueryAsync("PRAGMA optimize").Await(); + await command.ExecuteNonQueryAsync("PRAGMA optimize"); try { - transaction.CommitAsync().Await(); + await transaction.CommitAsync(); } catch (Exception ex) { @@ -1835,10 +2348,10 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } } - m_connection.CloseAsync().Await(); + await m_connection.CloseAsync(); } - m_connection.Dispose(); + await m_connection.DisposeAsync(); } IsDisposed = true; @@ -1865,8 +2378,14 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER var fields = typechain.SelectMany(x => - x.GetFields(System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.FlattenHierarchy) - ).Distinct().Where(x => x.FieldType.IsAssignableFrom(typeof(T))); + x.GetFields( + System.Reflection.BindingFlags.NonPublic | + System.Reflection.BindingFlags.Public | + System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.FlattenHierarchy + ) + ).Distinct() + .Where(x => x.FieldType.IsAssignableFrom(typeof(T))); foreach (var p in fields) try @@ -1933,15 +2452,15 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER return m_pathPrefixLookup[ix].Value; } - m_findpathprefixCommand.Transaction = transaction; - m_findpathprefixCommand.SetParameterValue("@Prefix", prefix); - var id = await m_findpathprefixCommand.ExecuteScalarInt64Async(); + var id = await m_findpathprefixCommand.SetTransaction(transaction) + .SetParameterValue("@Prefix", prefix) + .ExecuteScalarInt64Async(); if (id < 0) { - m_insertpathprefixCommand.SetParameterValue("@Prefix", prefix); - m_insertpathprefixCommand.Transaction = transaction; - id = await m_insertpathprefixCommand.ExecuteScalarInt64Async(); + id = await m_insertpathprefixCommand.SetTransaction(transaction) + .SetParameterValue("@Prefix", prefix) + .ExecuteScalarInt64Async(); } m_pathPrefixIndex = (m_pathPrefixIndex + 1) % m_pathPrefixLookup.Length; From 84b50e5db8feb11d884b5c5eba342381c7ea0ecd Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 09:05:38 +0200 Subject: [PATCH 039/362] Added the SetTransaction extension method. The main use is to ensure that the given transaction parameter is not null. Maybe it will be performing more checks in the future. --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 2ba6e4860..1fea1d30f 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -301,6 +301,13 @@ public static partial class ExtensionMethods return self; } + public static SqliteCommand SetTransaction(this SqliteCommand self, SqliteTransaction transaction) + { + self.Transaction = transaction; + + return self; + } + /// /// The tag used for logging /// From 2de3f1d5b55c9b319e33d4d258f69313a3f5c084 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 12:38:02 +0200 Subject: [PATCH 040/362] Reintroduced the synchronous CreateCommand extension method --- .../Library/Main/Database/ExtensionMethods.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 1fea1d30f..31fb07ebc 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -60,14 +60,14 @@ public static partial class ExtensionMethods return v.ToString(); } - // public static SqliteCommand CreateCommand(this SqliteConnection self, string cmdtext) - // { - // var cmd = self.CreateCommand(); - // cmd.SetCommandAndParameters(cmdtext); - // cmd.PrepareAsync().Wait(); + public static SqliteCommand CreateCommand(this SqliteConnection self, string cmdtext) + { + var cmd = self.CreateCommand(); + cmd.SetCommandAndParameters(cmdtext); + cmd.PrepareAsync().Wait(); - // return cmd; - // } + return cmd; + } public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext) { From 7b3163b851d78c60ed73acfbb08cc9593ebe07b8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 12:38:13 +0200 Subject: [PATCH 041/362] Post-rebase fixes to LocalDatabase --- .../Library/Main/Database/LocalDatabase.cs | 180 +++++++++++------- 1 file changed, 106 insertions(+), 74 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index ab582c84b..64f797cdd 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -666,12 +666,21 @@ namespace Duplicati.Library.Main.Database var filesetidstable = "DelFilesetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames)")) - .ExpandInClauseParameter("@VolumeNames", names.ToArray()) - .ExecuteNonQuery(); + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TEMP TABLE ""{volidstable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + "); + await deletecmd.SetCommandAndParameters($@" + INSERT OR IGNORE INTO ""{volidstable}"" + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@VolumeNames) + ") + .ExpandInClauseParameter("@VolumeNames", [.. names]) + .ExecuteNonQueryAsync(); - var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); + var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" "; deletecmd.Parameters.Clear(); var bsIdsSubQuery = @$" @@ -792,43 +801,107 @@ namespace Duplicati.Library.Main.Database "); // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetID IN (SELECT ID FROM Fileset WHERE VolumeID IN ({volIdsSubQuery}))")); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""IndexBlockLink"" + WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) + OR ""IndexVolumeID"" IN ({volIdsSubQuery}) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Block"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""DeletedBlock"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""ChangeJournalData"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ) + "); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM FilesetEntry + WHERE FilesetID IN ( + SELECT ID + FROM Fileset + WHERE VolumeID IN ({volIdsSubQuery}) + ) + "); - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{filesetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{filesetidstable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + "); + await deletecmd.ExecuteNonQueryAsync($@" + INSERT OR IGNORE INTO ""{filesetidstable}"" + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + "); // Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""Fileset"".""ID"" IN -(SELECT DISTINCT ""FilesetID"" FROM ""{metadataFilesetTable}"") -AND ""Fileset"".""ID"" NOT IN - (SELECT DISTINCT ""FilesetID"" FROM FilesetEntry)")); + await deletecmd.ExecuteNonQueryAsync($@" + INSERT OR IGNORE INTO ""{filesetidstable}"" + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""Fileset"".""ID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{metadataFilesetTable}"" + ) + AND ""Fileset"".""ID"" NOT IN ( + SELECT DISTINCT ""FilesetID"" + FROM FilesetEntry + ) + "); // Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later - deletecmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""RemoteVolume"" SET ""State"" = @NewState WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Fileset"" WHERE ""Fileset"".""ID"" IN (SELECT ""ID"" FROM ""{filesetidstable}"")) AND ""State"" IN (@AllowedStates)")) + await deletecmd.SetCommandAndParameters($@" + UPDATE ""RemoteVolume"" + SET ""State"" = @NewState + WHERE ""ID"" IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Fileset"" + WHERE ""Fileset"".""ID"" IN ( + SELECT ""ID"" + FROM ""{filesetidstable}"" + ) + ) + AND ""State"" IN (@AllowedStates) + ") .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) - .ExpandInClauseParameter("@AllowedStates", [RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString()]) - .ExecuteNonQuery(); - - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Fileset"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{filesetidstable}"")")); + .ExpandInClauseParameter("@AllowedStates", [ + RemoteVolumeState.Uploading.ToString(), + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString(), + RemoteVolumeState.Temporary.ToString() + ]) + .ExecuteNonQueryAsync(); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Fileset"" + WHERE ""ID"" IN ( + SELECT ""ID"" + FROM ""{filesetidstable}"" + ) + "); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{blocksetidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{metadataFilesetTable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{filesetidstable}"" ")); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{blocksetidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{metadataFilesetTable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{filesetidstable}"" "); try { deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ")); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filesetidstable}"" "); } catch { /* Ignore, will be deleted on close anyway. */ } @@ -980,56 +1053,15 @@ AND ""Fileset"".""ID"" NOT IN return isFullBackup == BackupType.FULL_BACKUP; } - // TODO: Remove this - public IDbTransaction BeginTransaction() + private async IAsyncEnumerable> GetDbOptionList(SqliteTransaction transaction) { - return m_connection.BeginTransactionSafe(); - } - - protected class TemporaryTransactionWrapper : IDisposable - { - private readonly IDbTransaction m_parent; - private readonly bool m_isTemporary; - - public TemporaryTransactionWrapper(IDbConnection connection, IDbTransaction? transaction) - { - if (transaction != null) - { - m_parent = transaction; - m_isTemporary = false; - } - else - { - m_parent = connection.BeginTransactionSafe(); - m_isTemporary = true; - } - } - - public void Commit() - { - if (m_isTemporary) - m_parent.Commit(); - } - - public void Dispose() - { - if (m_isTemporary) - m_parent.Dispose(); - } - - public IDbTransaction Parent { get { return m_parent; } } - } - - private IEnumerable> GetDbOptionList(IDbTransaction? transaction = null) - { - using var cmd = m_connection.CreateCommand(); - cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(@" SELECT ""Key"", ""Value"" FROM ""Configuration"" - "); - cmd.SetTransaction(transaction); + ") + .SetTransaction(transaction); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) From 44c037e86401d9c34ddec2d9fd601c566dbaa74e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 13:44:57 +0200 Subject: [PATCH 042/362] Implemented factory construction of LocalRestoreDatabase --- .../Library/Main/Database/LocalDatabase.cs | 307 +++++++++--------- .../Main/Database/LocalRestoreDatabase.cs | 18 +- 2 files changed, 170 insertions(+), 155 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 64f797cdd..3d0e4d397 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -126,30 +126,38 @@ namespace Duplicati.Library.Main.Database return c; } - public static LocalDatabase CreateLocalDatabase(string path, string operation, bool shouldclose, long pagecachesize) - { - return CreateLocalDatabaseAsync(path, operation, shouldclose, pagecachesize).Await(); - } + // TODO Create these once the signature is stable. + // public static LocalDatabase CreateLocalDatabase(string path, string operation, bool shouldclose, long pagecachesize) + // { + // return CreateLocalDatabaseAsync(path, operation, shouldclose, pagecachesize).Await(); + // } - public static LocalDatabase CreateLocalDatabase(LocalDatabase db) - { - return CreateLocalDatabaseAsync(db).Await(); - } + // public static LocalDatabase CreateLocalDatabase(LocalDatabase db) + // { + // return CreateLocalDatabaseAsync(db).Await(); + // } - public static LocalDatabase CreateLocalDatabase(SqliteConnection connection, string operation) - { - return CreateLocalDatabaseAsync(connection, operation).Await(); - } + // public static LocalDatabase CreateLocalDatabase(SqliteConnection connection, string operation) + // { + // return CreateLocalDatabaseAsync(connection, operation).Await(); + // } - private static LocalDatabase CreateLocalDatabase(SqliteConnection connection) - { - return CreateLocalDatabaseAsync(connection).Await(); - } + // private static LocalDatabase CreateLocalDatabase(SqliteConnection connection) + // { + // return CreateLocalDatabaseAsync(connection).Await(); + // } public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize) + { + var db = new LocalDatabase(); + + return await CreateLocalDatabaseAsync(db, path, operation, shouldclose, pagecachesize); + } + + public static async Task CreateLocalDatabaseAsync(LocalDatabase db, string path, string operation, bool shouldclose, long pagecachesize) { var connection = await CreateConnectionAsync(path, pagecachesize); - var db = await CreateLocalDatabaseAsync(connection, operation); + db = await CreateLocalDatabaseAsync(db, connection, operation); db.ShouldCloseConnection = shouldclose; db.m_pagecachesize = pagecachesize; @@ -157,21 +165,21 @@ namespace Duplicati.Library.Main.Database return db; } - public static async Task CreateLocalDatabaseAsync(LocalDatabase db) + public static async Task CreateLocalDatabaseAsync(LocalDatabase dbparent, LocalDatabase dbnew) { - var new_db = await CreateLocalDatabaseAsync(db.m_connection); + dbnew = await CreateLocalDatabaseAsync(dbnew, dbparent.m_connection); - new_db.OperationTimestamp = db.OperationTimestamp; - new_db.m_connection = db.m_connection; - new_db.m_operationid = db.m_operationid; - new_db.m_pagecachesize = db.m_pagecachesize; + dbnew.OperationTimestamp = dbparent.OperationTimestamp; + dbnew.m_connection = dbparent.m_connection; + dbnew.m_operationid = dbparent.m_operationid; + dbnew.m_pagecachesize = dbparent.m_pagecachesize; - return new_db; + return dbnew; } - public static async Task CreateLocalDatabaseAsync(SqliteConnection connection, string operation) + public static async Task CreateLocalDatabaseAsync(LocalDatabase db, SqliteConnection connection, string operation) { - var db = await CreateLocalDatabaseAsync(connection); + db = await CreateLocalDatabaseAsync(db, connection); db.OperationTimestamp = DateTime.UtcNow; @@ -220,8 +228,10 @@ namespace Duplicati.Library.Main.Database return db; } - private static async Task CreateLocalDatabaseAsync(SqliteConnection connection) + private static async Task CreateLocalDatabaseAsync(LocalDatabase db, SqliteConnection connection) { + db.m_connection = connection; + var selectremotevolumes_sql = @" SELECT ""ID"", @@ -235,142 +245,139 @@ namespace Duplicati.Library.Main.Database FROM ""Remotevolume"" "; - var db = new LocalDatabase - { - m_connection = connection, - m_insertlogCommand = await connection.CreateCommandAsync(@" - INSERT INTO ""LogData"" ( - ""OperationID"", - ""Timestamp"", - ""Type"", - ""Message"", - ""Exception"" - ) - VALUES ( - @OperationID, - @Timestamp, - @Type, - @Message, - @Exception - ) - "), + db.m_insertlogCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""LogData"" ( + ""OperationID"", + ""Timestamp"", + ""Type"", + ""Message"", + ""Exception"" + ) + VALUES ( + @OperationID, + @Timestamp, + @Type, + @Message, + @Exception + ) + "); - m_insertremotelogCommand = await connection.CreateCommandAsync(@" - INSERT INTO ""RemoteOperation"" ( - ""OperationID"", - ""Timestamp"", - ""Operation"", - ""Path"", - ""Data"" - ) - VALUES ( - @OperationID, - @Timestamp, - @Operation, - @Path, - @Data - ) - "), + db.m_insertremotelogCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""RemoteOperation"" ( + ""OperationID"", + ""Timestamp"", + ""Operation"", + ""Path"", + ""Data"" + ) + VALUES ( + @OperationID, + @Timestamp, + @Operation, + @Path, + @Data + ) + "); - m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" - UPDATE ""Remotevolume"" - SET - ""OperationID"" = @OperationID, - ""State"" = @State, - ""Hash"" = @Hash, - ""Size"" = @Size - WHERE ""Name"" = @Name - "), + db.m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" + UPDATE ""Remotevolume"" + SET + ""OperationID"" = @OperationID, + ""State"" = @State, + ""Hash"" = @Hash, + ""Size"" = @Size + WHERE ""Name"" = @Name + "); - m_selectremotevolumesCommand = await connection.CreateCommandAsync(selectremotevolumes_sql), + db.m_selectremotevolumesCommand = await connection.CreateCommandAsync(selectremotevolumes_sql); - m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$" - {selectremotevolumes_sql} - WHERE ""Name"" = @Name - "), + db.m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$" + {selectremotevolumes_sql} + WHERE ""Name"" = @Name + "); - m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" - SELECT DISTINCT ""Name"", ""State"" + db.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" + SELECT DISTINCT ""Name"", ""State"" + FROM ""Remotevolume"" + WHERE ""Name"" IN ( + SELECT ""Name"" FROM ""Remotevolume"" - WHERE ""Name"" IN ( - SELECT ""Name"" - FROM ""Remotevolume"" - WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') - ) - AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') - "), + WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') + ) + AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') + "); - m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" - DELETE FROM ""Remotevolume"" - WHERE ""Name"" = @Name - AND ( - ""DeleteGraceTime"" < @Now - OR ""State"" != @State - ) - "), + db.m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" + DELETE FROM ""Remotevolume"" + WHERE ""Name"" = @Name + AND ( + ""DeleteGraceTime"" < @Now + OR ""State"" != @State + ) + "); - m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@" - DELETE FROM ""Remotevolume"" - WHERE ""State"" == '{RemoteVolumeState.Deleted}' - AND ( - ""DeleteGraceTime"" < @Now - OR LENGTH(""DeleteGraceTime"") > 12 - ) - "), // >12 is to handle removal of old records that were in ticks + // >12 is to handle removal of old records that were in ticks + db.m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@" + DELETE FROM ""Remotevolume"" + WHERE ""State"" == '{RemoteVolumeState.Deleted}' + AND ( + ""DeleteGraceTime"" < @Now + OR LENGTH(""DeleteGraceTime"") > 12 + ) + "); - m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" - SELECT ""ID"" - FROM ""Remotevolume"" - WHERE ""Name"" = @Name - "), + db.m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Remotevolume"" + WHERE ""Name"" = @Name + "); - m_createremotevolumeCommand = await connection.CreateCommandAsync(@" - INSERT INTO ""Remotevolume"" ( - ""OperationID"", - ""Name"", - ""Type"", - ""State"", - ""Size"", - ""VerificationCount"", - ""DeleteGraceTime"", - ""ArchiveTime"" - ) - VALUES ( - @OperationID, - @Name, - @Type, - @State, - @Size, - @VerificationCount, - @DeleteGraceTime, - @ArchiveTime - ); - SELECT last_insert_rowid(); - "), + db.m_createremotevolumeCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""Remotevolume"" ( + ""OperationID"", + ""Name"", + ""Type"", + ""State"", + ""Size"", + ""VerificationCount"", + ""DeleteGraceTime"", + ""ArchiveTime"" + ) + VALUES ( + @OperationID, + @Name, + @Type, + @State, + @Size, + @VerificationCount, + @DeleteGraceTime, + @ArchiveTime + ); + SELECT last_insert_rowid(); + "); - m_insertIndexBlockLink = await connection.CreateCommandAsync(@" - INSERT INTO ""IndexBlockLink"" ( - ""IndexVolumeID"", - ""BlockVolumeID"" - ) - VALUES ( - @IndexVolumeId, - @BlockVolumeId - ) - "), + db.m_insertIndexBlockLink = await connection.CreateCommandAsync(@" + INSERT INTO ""IndexBlockLink"" ( + ""IndexVolumeID"", + ""BlockVolumeID"" + ) + VALUES ( + @IndexVolumeId, + @BlockVolumeId + ) + "); - m_findpathprefixCommand = await connection.CreateCommandAsync(@" - SELECT ""ID"" - FROM ""PathPrefix"" - WHERE ""Prefix"" = @Prefix - "), + db.m_findpathprefixCommand = await connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""PathPrefix"" + WHERE ""Prefix"" = @Prefix + "); - m_insertpathprefixCommand = await connection.CreateCommandAsync(@" - INSERT INTO ""PathPrefix"" (""Prefix"") - VALUES (@Prefix); - SELECT last_insert_rowid(); - ") - }; + db.m_insertpathprefixCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""PathPrefix"" (""Prefix"") + VALUES (@Prefix); + SELECT last_insert_rowid(); + "); return db; } diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index f790a4b1d..0473cccc0 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -58,15 +58,23 @@ namespace Duplicati.Library.Main.Database public DateTime RestoreTime { get { return m_restoreTime; } } - public LocalRestoreDatabase(string path, long pagecachesize) - : this(new LocalDatabase(path, "Restore", false, pagecachesize)) + private LocalRestoreDatabase() { } + + public static async Task CreateAsync(string path, long pagecachesize) { - ShouldCloseConnection = true; + var db = new LocalRestoreDatabase(); + + db = (LocalRestoreDatabase)await CreateLocalDatabaseAsync(db, path, "Restore", false, pagecachesize); + db.ShouldCloseConnection = true; + + return db; } - public LocalRestoreDatabase(LocalDatabase dbparent) - : base(dbparent) + public static async Task CreateAsync(LocalDatabase dbparent) { + var dbnew = new LocalRestoreDatabase(); + + return (LocalRestoreDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); } /// From d9584c5acb5a5d277dcc0e3dec919706ad93b854 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 13:46:05 +0200 Subject: [PATCH 043/362] Removed using introduced by IDE --- Duplicati/Library/Main/Database/LocalDatabase.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 3d0e4d397..9ddc76d76 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -33,7 +33,6 @@ using Duplicati.Library.Modules.Builtin.ResultSerialization; using Duplicati.Library.Utility; using Duplicati.Library.Interface; using Microsoft.Data.Sqlite; -using CommunityToolkit.HighPerformance; // Expose internal classes to UnitTests, so that Database classes can be tested [assembly: InternalsVisibleTo("Duplicati.UnitTest")] From 43dc194acd41e4ad38c2b28658e318d00d9ae127 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 14:46:08 +0200 Subject: [PATCH 044/362] Changed heavy properties of LocalRestoreDatabase to methods instead --- .../Main/Database/LocalRestoreDatabase.cs | 179 ++++++++---------- 1 file changed, 80 insertions(+), 99 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 0473cccc0..19a1c575b 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -525,7 +525,7 @@ END "); string TargetHash { get; } long TargetFileID { get; } long Length { get; } - IEnumerable Blocks { get; } + IAsyncEnumerable Blocks(); } public interface IBlockSource @@ -542,14 +542,14 @@ END "); long Offset { get; } long Index { get; } bool IsMetadata { get; } - IEnumerable Blocksources { get; } + IAsyncEnumerable BlockSources(); } public interface ILocalBlockSource { string TargetPath { get; } long TargetFileID { get; } - IEnumerable Blocks { get; } + IAsyncEnumerable Blocks(); } /// @@ -573,7 +573,7 @@ END "); { string Path { get; } long FileID { get; } - IEnumerable Blocks { get; } + IAsyncEnumerable Blocks(); } private class ExistingFile : IExistingFile @@ -600,20 +600,17 @@ END "); public long Size { get { return m_reader.ConvertValueToInt64(6); } } } - public IEnumerable Blocks + public async IAsyncEnumerable Blocks() { - get + string p = TargetPath; + while (HasMore && p == TargetPath) { - string p = TargetPath; - while (HasMore && p == TargetPath) - { - yield return new ExistingFileBlock(m_reader); - HasMore = m_reader.ReadAsync().Await(); - } + yield return new ExistingFileBlock(m_reader); + HasMore = await m_reader.ReadAsync(); } } - public static async IAsyncEnumerable GetExistingFilesWithBlocks(SqliteConnection connection, string tablename) + public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename) { using var cmd = connection.CreateCommand(FormatInvariant($@"SELECT ""{tablename}"".""TargetPath"", ""Blockset"".""FullHash"", ""{tablename}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{tablename}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index""")); cmd.Transaction = connection.BeginTransaction(deferred: true); @@ -632,7 +629,7 @@ END "); more = await rd.ReadAsync(); } } - await cmd.Transaction.CommitAsync(); + await db.Transaction.CommitAsync(); } } @@ -670,21 +667,19 @@ END "); public bool HasMore { get; private set; } - public IEnumerable Blocksources + public async IAsyncEnumerable BlockSources() { - get - { - var p = TargetPath; - var h = Hash; - var s = Size; + var p = TargetPath; + var h = Hash; + var s = Size; - while (HasMore && p == TargetPath && h == Hash && s == Size) - { - yield return new BlockSource(m_reader); - HasMore = m_reader.ReadAsync().Await(); - } + while (HasMore && p == TargetPath && h == Hash && s == Size) + { + yield return new BlockSource(m_reader); + HasMore = await m_reader.ReadAsync(); } } + } private readonly SqliteDataReader m_reader; @@ -695,23 +690,20 @@ END "); public bool HasMore { get; private set; } - public IEnumerable Blocks + public async IAsyncEnumerable Blocks() { - get + var p = TargetPath; + while (HasMore && p == TargetPath) { - var p = TargetPath; - while (HasMore && p == TargetPath) - { - var c = new BlockDescriptor(m_reader); - var h = c.Hash; - var s = c.Size; + var c = new BlockDescriptor(m_reader); + var h = c.Hash; + var s = c.Size; - yield return c; + yield return c; - HasMore = c.HasMore; - while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) - HasMore = m_reader.ReadAsync().Await(); - } + HasMore = c.HasMore; + while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) + HasMore = await m_reader.ReadAsync(); } } @@ -784,8 +776,8 @@ ORDER BY ""BB"".""MaxIndex"" ")); public interface IFilesAndMetadata : IDisposable { - IEnumerable FilesWithMissingBlocks { get; } - IEnumerable MetadataWithMissingBlocks { get; } + IAsyncEnumerable FilesWithMissingBlocks(); + IAsyncEnumerable MetadataWithMissingBlocks(); } private class FilesAndMetadata : IFilesAndMetadata @@ -796,9 +788,11 @@ ORDER BY ""BB"".""MaxIndex"" ")); private readonly long m_blocksize; private readonly SqliteConnection m_connection; + private readonly LocalDatabase m_db; - public FilesAndMetadata(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + public FilesAndMetadata(LocalDatabase db, SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) { + m_db = db; m_filetablename = filetablename; m_blocktablename = blocktablename; m_blocksize = blocksize; @@ -852,77 +846,68 @@ ORDER BY ""BB"".""MaxIndex"" ")); public long FileID { get { return m_reader.ConvertValueToInt64(1); } } public bool HasMore { get; private set; } - public IEnumerable Blocks + public async IAsyncEnumerable Blocks() { - get + string p = Path; + while (HasMore && p == Path) { - string p = Path; - while (HasMore && p == Path) - { - yield return new PatchBlock(m_reader); - HasMore = m_reader.ReadAsync().Await(); - } + yield return new PatchBlock(m_reader); + HasMore = await m_reader.ReadAsync(); } } } - public IEnumerable FilesWithMissingBlocks + public async IAsyncEnumerable FilesWithMissingBlocks() { - get - { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! + cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"0"} AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); - using var rd = cmd.ExecuteReaderAsync().Await(); - if (rd.ReadAsync().Await()) + using var rd = await cmd.ExecuteReaderAsync(); + if (await rd.ReadAsync()) + { + var more = true; + while (more) { - var more = true; - while (more) - { - var f = new VolumePatch(rd); - var current = f.Path; - yield return f; + var f = new VolumePatch(rd); + var current = f.Path; + yield return f; - more = f.HasMore; - while (more && current == f.Path) - more = rd.ReadAsync().Await(); - } + more = f.HasMore; + while (more && current == f.Path) + more = await rd.ReadAsync(); } - cmd.Transaction.CommitAsync().Await(); } + await m_db.Transaction.Transaction.CommitAsync(); } - public IEnumerable MetadataWithMissingBlocks + public async IAsyncEnumerable MetadataWithMissingBlocks() { - get - { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_connection.BeginTransaction(deferred: true); + // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! + cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"1"} AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); - using var rd = cmd.ExecuteReaderAsync().Await(); - if (rd.ReadAsync().Await()) + using var rd = await cmd.ExecuteReaderAsync(); + if (await rd.ReadAsync()) + { + var more = true; + while (more) { - var more = true; - while (more) - { - var f = new VolumePatch(rd); - string current = f.Path; - yield return f; + var f = new VolumePatch(rd); + string current = f.Path; + yield return f; - more = f.HasMore; - while (more && current == f.Path) - more = rd.ReadAsync().Await(); - } + more = f.HasMore; + while (more && current == f.Path) + more = await rd.ReadAsync(); } } } @@ -1351,7 +1336,7 @@ WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND string TargetPath { get; } long TargetFileID { get; } string SourcePath { get; } - IEnumerable Blocks { get; } + IAsyncEnumerable Blocks(); } public interface IBlockEntry @@ -1383,18 +1368,14 @@ WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } public string SourcePath { get { return m_rd.ConvertValueToString(1) ?? ""; } } - public IEnumerable Blocks + public async IAsyncEnumerable Blocks() { - get + var tid = TargetFileID; + + do { - var tid = TargetFileID; - - do - { - yield return new BlockEntry(m_rd, m_blocksize); - } while ((MoreData = m_rd.ReadAsync().Await()) && tid == TargetFileID); - - } + yield return new BlockEntry(m_rd, m_blocksize); + } while ((MoreData = await m_rd.ReadAsync()) && tid == TargetFileID); } } From 88f5fbb96320ae4e4dbc61885b1471132c621149 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 15:48:27 +0200 Subject: [PATCH 045/362] Moved async code from constructors to factory functions --- .../Main/Database/LocalRestoreDatabase.cs | 163 ++++++++++++------ 1 file changed, 110 insertions(+), 53 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 19a1c575b..6a0a0e962 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -782,38 +782,62 @@ ORDER BY ""BB"".""MaxIndex"" ")); private class FilesAndMetadata : IFilesAndMetadata { - private readonly string m_tmptable; - private readonly string m_filetablename; - private readonly string m_blocktablename; - private readonly long m_blocksize; + private string m_tmptable = null!; + private string m_filetablename = null!; + private string m_blocktablename = null!; + private long m_blocksize; - private readonly SqliteConnection m_connection; - private readonly LocalDatabase m_db; + private LocalDatabase m_db = null!; - public FilesAndMetadata(LocalDatabase db, SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] + public FilesAndMetadata(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) { - m_db = db; - m_filetablename = filetablename; - m_blocktablename = blocktablename; - m_blocksize = blocksize; - m_connection = connection; + throw new NotImplementedException("Use CreateAsync instead of the constructor"); + } - using var c = m_connection.CreateCommand(); - c.Transaction = m_connection.BeginTransaction(deferred: true); - m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - c.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tmptable}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )")).Await(); + private FilesAndMetadata() { } - c.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tmptable}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); + public static async Task CreateAsync(LocalDatabase db, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + { + var fam = new FilesAndMetadata() + { + m_db = db, + m_filetablename = filetablename, + m_blocktablename = blocktablename, + m_blocksize = blocksize, + }; + + using var c = db.Connection.CreateCommand() + .SetTransaction(db.Transaction.Transaction); + fam.m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await c.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{fam.m_tmptable}"" ( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + ) + "); + + c.SetCommandAndParameters($@" + INSERT INTO ""{fam.m_tmptable}"" ( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size) + "); foreach (var s in curvolume.Blocks) { - c.SetParameterValue("@Hash", s.Key); - c.SetParameterValue("@Size", s.Value); - c.ExecuteNonQueryAsync().Await(); + await c.SetParameterValue("@Hash", s.Key) + .SetParameterValue("@Size", s.Value) + .ExecuteNonQueryAsync(); } // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks - c.Transaction.CommitAsync().Await(); + await db.Transaction.CommitAsync(); + + return fam; } public void Dispose() @@ -1178,46 +1202,78 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// private class DirectBlockMarker : IBlockMarker { - private readonly SqliteCommand m_insertblockCommand; - private readonly SqliteCommand m_resetfileCommand; - private readonly SqliteCommand m_updateAsRestoredCommand; - private readonly SqliteCommand m_updateFileAsDataVerifiedCommand; - private readonly SqliteCommand m_statUpdateCommand; - private SqliteTransaction m_transaction; + private SqliteCommand m_insertblockCommand = null!; + private SqliteCommand m_resetfileCommand = null!; + private SqliteCommand m_updateAsRestoredCommand = null!; + private SqliteCommand m_updateFileAsDataVerifiedCommand = null!; + private SqliteCommand m_statUpdateCommand = null!; + private ReusableTransaction m_rtr = null!; private bool m_hasUpdates = false; - private readonly string m_blocktablename; - private readonly string m_filetablename; + private string m_blocktablename = null!; + private string m_filetablename = null!; + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public DirectBlockMarker(SqliteConnection connection, string blocktablename, string filetablename, string statstablename) { - m_transaction = connection.BeginTransaction(deferred: true); - m_blocktablename = blocktablename; - m_filetablename = filetablename; + throw new NotImplementedException("Use CreateAsync instead of the constructor"); + } - m_insertblockCommand = connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 -WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND ""Size"" = @Size AND ""Metadata"" = @Metadata AND ""Restored"" = 0 ")); - m_insertblockCommand.Transaction = m_transaction; + private DirectBlockMarker() { } - m_resetfileCommand = connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = @TargetFileId ")); - m_resetfileCommand.Transaction = m_transaction; - - m_updateAsRestoredCommand = connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 WHERE ""FileID"" = @TargetFileId AND ""Metadata"" <= @Metadata ")); - m_updateAsRestoredCommand.Transaction = m_transaction; - - m_updateFileAsDataVerifiedCommand = connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = @TargetFileId")); - m_updateFileAsDataVerifiedCommand.Transaction = m_transaction; - - if (statstablename != null) + public static async Task CreateAsync(LocalDatabase db, string blocktablename, string filetablename, string statstablename) + { + var dbm = new DirectBlockMarker() { - // Fields in Stats: TotalFiles, TotalBlocks, TotalSize - // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored - m_statUpdateCommand = connection.CreateCommand(FormatInvariant($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" ")); - } - else // very slow fallback if stats tables were not created - m_statUpdateCommand = connection.CreateCommand(FormatInvariant($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{m_blocktablename}"" WHERE ""Restored"" = 1 ")); - m_statUpdateCommand.Transaction = m_transaction; + m_rtr = db.Transaction, + m_blocktablename = blocktablename, + m_filetablename = filetablename, + m_insertblockCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" SET ""Restored"" = 1 + WHERE ""FileID"" = @TargetFileId + AND ""Index"" = @Index + AND ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""Metadata"" = @Metadata + AND ""Restored"" = 0 + "), + + m_resetfileCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" + SET ""Restored"" = 0 + WHERE ""FileID"" = @TargetFileId + "), + + m_updateAsRestoredCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" + SET ""Restored"" = 1 + WHERE ""FileID"" = @TargetFileId + AND ""Metadata"" <= @Metadata + "), + + m_updateFileAsDataVerifiedCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{filetablename}"" + SET ""DataVerified"" = 1 + WHERE ""ID"" = @TargetFileId + "), + + m_statUpdateCommand = statstablename == null ? + // very slow fallback if stats tables were not created + await db.Connection.CreateCommandAsync($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{blocktablename}"" WHERE ""Restored"" = 1 ") + : + // Fields in Stats: TotalFiles, TotalBlocks, TotalSize + // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored + await db.Connection.CreateCommandAsync($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" ") + }; + + dbm.m_insertblockCommand.SetTransaction(db.Transaction.Transaction); + dbm.m_resetfileCommand.SetTransaction(db.Transaction.Transaction); + dbm.m_updateAsRestoredCommand.SetTransaction(db.Transaction.Transaction); + dbm.m_updateFileAsDataVerifiedCommand.SetTransaction(db.Transaction.Transaction); + dbm.m_statUpdateCommand.SetTransaction(db.Transaction.Transaction); + + return dbm; } public async Task UpdateProcessed(IOperationProgressUpdater updater) @@ -1300,12 +1356,13 @@ WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND } public IBlockMarker CreateBlockMarker() + public async Task CreateBlockMarkerAsync() { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); if (string.IsNullOrWhiteSpace(m_totalprogtable)) throw new InvalidOperationException("No progress table set up for this restore."); - return new DirectBlockMarker(m_connection, m_tempblocktable, m_tempfiletable, m_totalprogtable); + return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable); } public override void Dispose() From 1a0c6dcb379959ae6d90387b43df25f5d0d190ea Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 15:49:40 +0200 Subject: [PATCH 046/362] Restructered SQL statements, ensured all calls to the database are async, and that commands are created using a chan of method calls rather than a sequence of statements --- .../Main/Database/LocalRestoreDatabase.cs | 1186 ++++++++++++----- 1 file changed, 859 insertions(+), 327 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 6a0a0e962..1b305b91b 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -30,6 +30,7 @@ using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Main.Operation.Restore; using Duplicati.Library.Main.Volumes; +using Duplicati.Library.SQLiteHelper; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -98,30 +99,42 @@ namespace Duplicati.Library.Main.Database m_totalprogtable = "TotalProgress-" + m_temptabsetguid; m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr.Transaction); // How to handle METADATA? - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( -""FileId"" INTEGER PRIMARY KEY, -""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL, -""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL -)")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( + ""FileId"" INTEGER PRIMARY KEY, + ""TotalBlocks"" INTEGER NOT NULL, + ""TotalSize"" INTEGER NOT NULL, + ""BlocksRestored"" INTEGER NOT NULL, + ""SizeRestored"" INTEGER NOT NULL + ) + "); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( -""TotalFiles"" INTEGER NOT NULL, ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL, -""FilesFullyRestored"" INTEGER NOT NULL, ""FilesPartiallyRestored"" INTEGER NOT NULL, -""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL -)")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( + ""TotalFiles"" INTEGER NOT NULL, + ""TotalBlocks"" INTEGER NOT NULL, + ""TotalSize"" INTEGER NOT NULL, + ""FilesFullyRestored"" INTEGER NOT NULL, + ""FilesPartiallyRestored"" INTEGER NOT NULL, + ""BlocksRestored"" INTEGER NOT NULL, + ""SizeRestored"" INTEGER NOT NULL + ) + "); if (createFilesNewlyDoneTracker) { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( -""ID"" INTEGER PRIMARY KEY -)")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + "); } try @@ -129,33 +142,51 @@ namespace Duplicati.Library.Main.Database // Initialize statistics with File- and Block-Data (it is valid to already have restored blocks in files) // A rebuild with this function should be valid anytime. // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. - string sql; // We use a LEFT JOIN to allow for empty files (no data Blocks) - sql = FormatInvariant($@" INSERT INTO ""{m_fileprogtable}"" (""FileId"", ""TotalBlocks"", ""TotalSize"", ""BlocksRestored"", ""SizeRestored"") -SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size""), 0) - , IFNULL(COUNT(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""ID"" ELSE NULL END), 0) - , IFNULL(SUM(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""Size"" ELSE 0 END), 0) - FROM ""{m_tempfiletable}"" ""F"" LEFT JOIN ""{m_tempblocktable}"" ""B"" - ON ""B"".""FileID"" = ""F"".""ID"" - WHERE ""B"".""Metadata"" IS NOT 1 - GROUP BY ""F"".""ID"" "); // Will be one row per file. - await cmd.ExecuteNonQueryAsync(sql); - - sql = FormatInvariant($@"INSERT INTO ""{m_totalprogtable}"" ( - ""TotalFiles"", ""TotalBlocks"", ""TotalSize"" -, ""FilesFullyRestored"", ""FilesPartiallyRestored"", ""BlocksRestored"", ""SizeRestored"" - ) - SELECT IFNULL(COUNT(""P"".""FileId""), 0), IFNULL(SUM(""P"".""TotalBlocks""), 0), IFNULL(SUM(""P"".""TotalSize""), 0) - , IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" THEN 1 ELSE NULL END), 0) - , IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" BETWEEN 1 AND ""P"".""TotalBlocks"" - 1 THEN 1 ELSE NULL END), 0) - , IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) - FROM ""{m_fileprogtable}"" ""P"" "); + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_fileprogtable}"" ( + ""FileId"", + ""TotalBlocks"", + ""TotalSize"", + ""BlocksRestored"", + ""SizeRestored"" + ) + SELECT + ""F"".""ID"", + IFNULL(COUNT(""B"".""ID""), 0), + IFNULL(SUM(""B"".""Size""), 0), + IFNULL(COUNT(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""ID"" ELSE NULL END), 0), + IFNULL(SUM(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""Size"" ELSE 0 END), 0) + FROM ""{m_tempfiletable}"" ""F"" + LEFT JOIN ""{m_tempblocktable}"" ""B"" + ON ""B"".""FileID"" = ""F"".""ID"" + WHERE ""B"".""Metadata"" IS NOT 1 + GROUP BY ""F"".""ID"" + "); // Will result in a single line (no support to also track metadata) - await cmd.ExecuteNonQueryAsync(sql); + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_totalprogtable}"" ( + ""TotalFiles"", + ""TotalBlocks"", + ""TotalSize"", + ""FilesFullyRestored"", + ""FilesPartiallyRestored"", + ""BlocksRestored"", + ""SizeRestored"" + ) + SELECT + IFNULL(COUNT(""P"".""FileId""), 0), + IFNULL(SUM(""P"".""TotalBlocks""), 0), + IFNULL(SUM(""P"".""TotalSize""), 0), + IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" THEN 1 ELSE NULL END), 0), + IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" BETWEEN 1 AND ""P"".""TotalBlocks"" - 1 THEN 1 ELSE NULL END), 0), + IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) + FROM ""{m_fileprogtable}"" ""P"" + "); // Finally we create TRIGGERs to keep all our statistics up to date. // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic. @@ -164,39 +195,53 @@ SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size"" // with additional INSERT and DELETE triggers. // A trigger to update the file-stat entry each time a block changes restoration state. - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" AFTER UPDATE OF ""Restored"" ON ""{m_tempblocktable}"" -WHEN OLD.""Restored"" != NEW.""Restored"" AND NEW.""Metadata"" = 0 -BEGIN UPDATE ""{m_fileprogtable}"" - SET ""BlocksRestored"" = ""{m_fileprogtable}"".""BlocksRestored"" + (NEW.""Restored"" - OLD.""Restored"") - , ""SizeRestored"" = ""{m_fileprogtable}"".""SizeRestored"" + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) - WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" -; END "); - await cmd.ExecuteNonQueryAsync(sql); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" + AFTER UPDATE OF ""Restored"" + ON ""{m_tempblocktable}"" + WHEN OLD.""Restored"" != NEW.""Restored"" + AND NEW.""Metadata"" = 0 + BEGIN UPDATE ""{m_fileprogtable}"" + SET + ""BlocksRestored"" = ""{m_fileprogtable}"".""BlocksRestored"" + (NEW.""Restored"" - OLD.""Restored""), + ""SizeRestored"" = ""{m_fileprogtable}"".""SizeRestored"" + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) + WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" + ; END + "); // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" AFTER UPDATE ON ""{m_fileprogtable}"" -BEGIN UPDATE ""{m_totalprogtable}"" - SET ""FilesFullyRestored"" = ""{m_totalprogtable}"".""FilesFullyRestored"" - + (CASE WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" THEN 1 ELSE 0 END) - - (CASE WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" THEN 1 ELSE 0 END) - , ""FilesPartiallyRestored"" = ""{m_totalprogtable}"".""FilesPartiallyRestored"" - + (CASE WHEN NEW.""BlocksRestored"" BETWEEN 1 AND NEW.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) - - (CASE WHEN OLD.""BlocksRestored"" BETWEEN 1 AND OLD.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) - , ""BlocksRestored"" = ""{m_totalprogtable}"".""BlocksRestored"" + NEW.""BlocksRestored"" - OLD.""BlocksRestored"" - , ""SizeRestored"" = ""{m_totalprogtable}"".""SizeRestored"" + NEW.""SizeRestored"" - OLD.""SizeRestored"" -; END"); - await cmd.ExecuteNonQueryAsync(sql); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" + AFTER UPDATE ON ""{m_fileprogtable}"" + BEGIN UPDATE ""{m_totalprogtable}"" + SET + ""FilesFullyRestored"" = ""{m_totalprogtable}"".""FilesFullyRestored"" + + (CASE WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" THEN 1 ELSE 0 END) + - (CASE WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" THEN 1 ELSE 0 END), + ""FilesPartiallyRestored"" = ""{m_totalprogtable}"".""FilesPartiallyRestored"" + + (CASE WHEN NEW.""BlocksRestored"" BETWEEN 1 AND NEW.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) + - (CASE WHEN OLD.""BlocksRestored"" BETWEEN 1 AND OLD.""TotalBlocks"" - 1 THEN 1 ELSE 0 END), + ""BlocksRestored"" = ""{m_totalprogtable}"".""BlocksRestored"" + NEW.""BlocksRestored"" - OLD.""BlocksRestored"", + ""SizeRestored"" = ""{m_totalprogtable}"".""SizeRestored"" + NEW.""SizeRestored"" - OLD.""SizeRestored"" + ; END + "); if (createFilesNewlyDoneTracker) { // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" AFTER UPDATE OF ""BlocksRestored"", ""TotalBlocks"" ON ""{m_fileprogtable}"" -WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" -BEGIN - INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") VALUES (NEW.""FileId""); -END "); - await cmd.ExecuteNonQueryAsync(sql); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" + AFTER UPDATE OF + ""BlocksRestored"", + ""TotalBlocks"" + ON ""{m_fileprogtable}"" + WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" + BEGIN + INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") + VALUES (NEW.""FileId""); + END + "); } } @@ -209,7 +254,7 @@ END "); } finally { - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } } @@ -220,18 +265,23 @@ END "); using (var cmd = m_connection.CreateCommand()) { - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + cmd.SetTransaction(m_rtr.Transaction); var filesetIds = (await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions)).ToList(); while (filesetIds.Count > 0) { var filesetId = filesetIds[0]; filesetIds.RemoveAt(0); - cmd.SetCommandAndParameters(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = @FilesetId"); - cmd.SetParameterValue("@FilesetId", filesetId); + cmd.SetCommandAndParameters(@" + SELECT ""Timestamp"" + FROM ""Fileset"" + WHERE ""ID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); m_restoreTime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); - var ix = await FilesetTimes().Select((value, index) => new { value.Key, index }) + var ix = await FilesetTimes() + .Select((value, index) => new { value.Key, index }) .Where(n => n.Key == filesetId) .Select(pair => pair.index + 1) .FirstOrDefaultAsync() - 1; @@ -240,98 +290,224 @@ END "); cmd.Parameters.Clear(); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempfiletable}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""TargetPath"" TEXT NULL, ""DataVerified"" BOOLEAN NOT NULL, ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN) ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocktable}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL, ""Metadata"" BOOLEAN NOT NULL, ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL)")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_tempfiletable}"" ( + ""ID"" INTEGER PRIMARY KEY, + ""Path"" TEXT NOT NULL, + ""BlocksetID"" INTEGER NOT NULL, + ""MetadataID"" INTEGER NOT NULL, + ""TargetPath"" TEXT NULL, + ""DataVerified"" BOOLEAN NOT NULL, + ""LatestBlocksetId"" INTEGER, + ""LocalSourceExists"" BOOLEAN + ) + "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_tempblocktable}"" ( + ""ID"" INTEGER PRIMARY KEY, + ""FileID"" INTEGER NOT NULL, + ""Index"" INTEGER NOT NULL, + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Restored"" BOOLEAN NOT NULL, + ""Metadata"" BOOLEAN NOT NULL, + ""VolumeID"" INTEGER NOT NULL, + ""BlockID"" INTEGER NOT NULL + ) + "); // TODO: Optimize to use the path prefix if (filter == null || filter.Empty) { // Simple case, restore everything - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId")); - cmd.SetParameterValue("@FilesetId", filesetId); - await cmd.ExecuteNonQueryAsync(); + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"", + 0 + FROM + ""File"", + ""FilesetEntry"" + WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple) { using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset")) - await cmd.Transaction.CommitAsync(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + await m_rtr.CommitAsync(); + cmd.SetTransaction(m_rtr.Transaction); // If we get a list of filenames, the lookup table is faster // unfortunately we cannot do this if the filesystem is case sensitive as // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var m_filenamestable = "Filenames-" + m_temptabsetguid; - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filenamestable}"" (""Path"" TEXT NOT NULL) ")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_filenamestable}"" (""Path"") VALUES (@Path)")); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_filenamestable}"" ( + ""Path"" TEXT NOT NULL + ) + "); + cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_filenamestable}"" (""Path"") + VALUES (@Path) + "); foreach (var s in p) { - cmd.SetParameterValue("@Path", s); - await cmd.ExecuteNonQueryAsync(); + await cmd.SetParameterValue("@Path", s) + .ExecuteNonQueryAsync(); } - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{m_filenamestable}"") ")); - cmd.SetParameterValue("@FilesetId", filesetId); - var c = await cmd.ExecuteNonQueryAsync(); - - cmd.Parameters.Clear(); + var c = await cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"", + 0 + FROM + ""File"", + ""FilesetEntry"" + WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId + AND ""Path"" IN ( + SELECT DISTINCT ""Path"" + FROM ""{m_filenamestable}"" + ) + ") + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(); if (c != p.Length && c != 0) { var sb = new StringBuilder(); sb.AppendLine(); - using (var rd = await cmd.ExecuteReaderAsync(FormatInvariant($@"SELECT ""Path"" FROM ""{m_filenamestable}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tempfiletable}"")"))) + cmd.SetCommandAndParameters($@" + SELECT ""Path"" + FROM ""{m_filenamestable}"" + WHERE ""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ) + "); + using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) sb.AppendLine(rd.ConvertValueToString(0)); - cmd.SetCommandAndParameters(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = @FilesetId"); - cmd.SetParameterValue("@FilesetId", filesetId); + cmd.SetCommandAndParameters(@" + SELECT ""Timestamp"" + FROM ""Fileset"" + WHERE ""ID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); cmd.Parameters.Clear(); } - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" "); using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset")) - await cmd.Transaction.CommitAsync(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + await m_rtr.CommitAsync(); + cmd.SetTransaction(m_rtr.Transaction); } else { // Restore but filter elements based on the filter expression // If this is too slow, we could add a special handler for wildcard searches too - cmd.SetCommandAndParameters(@"SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetID"" = @FilesetId"); - cmd.SetParameterValue("@FilesetId", filesetId); + cmd.SetCommandAndParameters(@" + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"" + FROM + ""File"", + ""FilesetEntry"" + WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); object[] values = new object[4]; - using var cmd2 = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (@ID, @Path, @BlocksetID, @MetadataID, 0)")); + using var cmd2 = m_connection.CreateCommand($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + VALUES ( + @ID, + @Path, + @BlocksetID, + @MetadataID, + 0 + ) + "); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) { rd.GetValues(values); if (values[1] != null && values[1] != DBNull.Value && FilterExpression.Matches(filter, values[1].ToString())) { - cmd2.SetParameterValue("@ID", values[0]); - cmd2.SetParameterValue("@Path", values[1]); - cmd2.SetParameterValue("@BlocksetID", values[2]); - cmd2.SetParameterValue("@MetadataID", values[3]); - await cmd2.ExecuteNonQueryAsync(); + await cmd2.SetParameterValue("@ID", values[0]) + .SetParameterValue("@Path", values[1]) + .SetParameterValue("@BlocksetID", values[2]) + .SetParameterValue("@MetadataID", values[3]) + .ExecuteNonQueryAsync(); } } } //creating indexes after insertion is much faster - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_ID"" ON ""{m_tempfiletable}"" (""ID"")")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_TargetPath"" ON ""{m_tempfiletable}"" (""TargetPath"")")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_Path"" ON ""{m_tempfiletable}"" (""Path"")")); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_ID"" + ON ""{m_tempfiletable}"" (""ID"") + "); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_TargetPath"" + ON ""{m_tempfiletable}"" (""TargetPath"") + "); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_Path"" + ON ""{m_tempfiletable}"" (""Path"") + "); - using (var rd = await cmd.ExecuteReaderAsync(FormatInvariant($@"SELECT COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), SUM(""Blockset"".""Length"") FROM ""{m_tempfiletable}"", ""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" "))) + cmd.SetCommandAndParameters($@" + SELECT + COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), + SUM(""Blockset"".""Length"") + FROM + ""{m_tempfiletable}"", + ""Blockset"" + WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + "); + using (var rd = await cmd.ExecuteReaderAsync()) { var filecount = 0L; var filesize = 0L; @@ -350,7 +526,7 @@ END "); } } - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } return new Tuple(0, 0); @@ -358,39 +534,59 @@ END "); public async Task GetFirstPath() { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - var v0 = await cmd.ExecuteScalarAsync(FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); + using var cmd = m_connection.CreateCommand($@" + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + ") + .SetTransaction(m_rtr.Transaction); + + var v0 = await cmd.ExecuteScalarAsync(); if (v0 == null || v0 == DBNull.Value) return null; - await cmd.Transaction.CommitAsync(); + + await m_rtr.CommitAsync(); return v0.ToString(); } public async Task GetLargestPrefix() { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - var v0 = await cmd.ExecuteScalarAsync(FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); + using var cmd = m_connection.CreateCommand($@" + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + ") + .SetTransaction(m_rtr.Transaction); + + var v0 = await cmd.ExecuteScalarAsync(); var maxpath = ""; if (v0 != null && v0 != DBNull.Value) maxpath = v0.ToString()!; var dirsep = Util.GuessDirSeparator(maxpath); - var filecount = await cmd.ExecuteScalarInt64Async(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"""), -1); + var filecount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{m_tempfiletable}"" + ", -1); var foundfiles = -1L; //TODO: Handle FS case-sensitive? - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix")); + cmd.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ""{m_tempfiletable}"" + WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix + "); while (filecount != foundfiles && maxpath.Length > 0) { var mp = Util.AppendDirSeparator(maxpath, dirsep); - cmd.SetParameterValue("@PrefixLength", mp.Length); - cmd.SetParameterValue("@Prefix", mp); - foundfiles = await cmd.ExecuteScalarInt64Async(-1); + foundfiles = await cmd.SetParameterValue("@PrefixLength", mp.Length) + .SetParameterValue("@Prefix", mp) + .ExecuteScalarInt64Async(-1); if (filecount != foundfiles) { @@ -403,7 +599,7 @@ END "); } } - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); } @@ -412,8 +608,8 @@ END "); { var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? await GetFirstPath() : largest_prefix); - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr.Transaction); if (string.IsNullOrEmpty(destination)) { //The string fixing here is meant to provide some non-random @@ -423,21 +619,47 @@ END "); if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") { // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END")); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 2, 1) == ':' + THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) + ELSE ""Path"" + END + "); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 1, 2) == '\\' + THEN '\\' || SUBSTR(""Path"", 2) + ELSE ""Path"" + END + "); } else if (OperatingSystem.IsWindows() && dirsep == "/") { // For Linux -> Win, we use the temporary folder's drive as the root path - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 1) == '/' THEN @Path || SUBSTR(""Path"", 2) ELSE ""Path"" END")); - cmd.SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")); - await cmd.ExecuteNonQueryAsync(); + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 1, 1) == '/' + THEN @Path || SUBSTR(""Path"", 2) + ELSE ""Path"" + END") + .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) + .ExecuteNonQueryAsync(); } else { // Same OS, just use the path directly - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = ""Path"" ")); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = ""Path"" + "); } } else @@ -447,69 +669,165 @@ END "); //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) // We use the format / / - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = + CASE + WHEN SUBSTR(""Path"", 2, 1) == ':' + THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) + ELSE ""Path"" + END + "); // For UNC paths, we use \\server\folder -> / / - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END")); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = + CASE + WHEN SUBSTR(""Path"", 1, 2) == '\\' + THEN SUBSTR(""Path"", 2) + ELSE ""TargetPath"" + END + "); } else { largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength)")); - cmd.SetParameterValue("@PrefixLength", largest_prefix.Length + 1); - await cmd.ExecuteNonQueryAsync(); + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength) + ") + .SetParameterValue("@PrefixLength", largest_prefix.Length + 1) + .ExecuteNonQueryAsync(); } } // Cross-os path remapping support if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") // For Win paths on Linux - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')")); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/') + "); else if (OperatingSystem.IsWindows() && dirsep == "/") // For Linux paths on Windows - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')")); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\') + "); if (!string.IsNullOrEmpty(destination)) { // Paths are now relative with target-os naming system // so we prefix them with the target path - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @Destination || ""TargetPath"" ")); - cmd.SetParameterValue("@Destination", Util.AppendDirSeparator(destination)); - await cmd.ExecuteNonQueryAsync(); + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = @Destination || ""TargetPath"" + ") + .SetParameterValue("@Destination", Util.AppendDirSeparator(destination)) + .ExecuteNonQueryAsync(); } - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public async Task FindMissingBlocks(bool skipMetadata) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - var p1 = await cmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ")); + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr.Transaction); + var p1 = await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_tempblocktable}"" ( + ""FileID"", + ""Index"", + ""Hash"", + ""Size"", + ""Restored"", + ""Metadata"", + ""VolumeId"", + ""BlockId"" + ) + SELECT DISTINCT + ""{m_tempfiletable}"".""ID"", + ""BlocksetEntry"".""Index"", + ""Block"".""Hash"", + ""Block"".""Size"", + 0, + 0, + ""Block"".""VolumeID"", + ""Block"".""ID"" + FROM + ""{m_tempfiletable}"", + ""BlocksetEntry"", + ""Block"" + WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + "); var p2 = 0; if (!skipMetadata) - p2 = await cmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ")); + p2 = await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_tempblocktable}"" ( + ""FileID"", + ""Index"", + ""Hash"", + ""Size"", + ""Restored"", + ""Metadata"", + ""VolumeId"", + ""BlockId"" + ) + SELECT DISTINCT + ""{m_tempfiletable}"".""ID"", + ""BlocksetEntry"".""Index"", + ""Block"".""Hash"", + ""Block"".""Size"", + 0, + 1, + ""Block"".""VolumeID"", + ""Block"".""ID"" + FROM + ""{m_tempfiletable}"", + ""BlocksetEntry"", + ""Block"", + ""Metadataset"" + WHERE ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" + AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + "); //creating indexes after insertion is much faster - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" ON ""{m_tempblocktable}"" (""Hash"", ""Size"")")); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" + ON ""{m_tempblocktable}"" (""Hash"", ""Size"") + "); // better suited to speed up commit on UpdateBlocks - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" ON ""{m_tempblocktable}"" (""FileId"", ""Index"")")); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" + ON ""{m_tempblocktable}"" (""FileId"", ""Index"") + "); - var size = await cmd.ExecuteScalarInt64Async(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_tempblocktable}"" "), 0); + var size = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_tempblocktable}"" + ", 0); Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public async Task UpdateTargetPath(long ID, string newname) { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @TargetPath WHERE ""ID"" = @ID")); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - cmd.SetParameterValue("@TargetPath", newname); - cmd.SetParameterValue("@ID", ID); - await cmd.ExecuteNonQueryAsync(); - await cmd.Transaction.CommitAsync(); + using var cmd = m_connection.CreateCommand($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = @TargetPath + WHERE ""ID"" = @ID + "); + await cmd.SetTransaction(m_rtr.Transaction) + .SetParameterValue("@TargetPath", newname) + .SetParameterValue("@ID", ID) + .ExecuteNonQueryAsync(); + + await m_rtr.CommitAsync(); } public interface IExistingFileBlock @@ -612,8 +930,26 @@ END "); public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename) { - using var cmd = connection.CreateCommand(FormatInvariant($@"SELECT ""{tablename}"".""TargetPath"", ""Blockset"".""FullHash"", ""{tablename}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{tablename}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index""")); - cmd.Transaction = connection.BeginTransaction(deferred: true); + using var cmd = db.Connection.CreateCommand($@" + SELECT + ""{tablename}"".""TargetPath"", + ""Blockset"".""FullHash"", + ""{tablename}"".""ID"", + ""Blockset"".""Length"", + ""Block"".""Hash"", + ""BlocksetEntry"".""Index"", + ""Block"".""Size"" + FROM + ""{tablename}"", + ""Blockset"", + ""BlocksetEntry"", + ""Block"" + WHERE ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + ORDER BY ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index"" + ") + .SetTransaction(db.Transaction.Transaction); using var rd = await cmd.ExecuteReaderAsync(); if (await rd.ReadAsync()) { @@ -637,7 +973,7 @@ END "); { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); - return ExistingFile.GetExistingFilesWithBlocks(m_connection, m_tempfiletable); + return ExistingFile.GetExistingFilesWithBlocks(this, m_tempfiletable); } private class LocalBlockSource : ILocalBlockSource @@ -707,14 +1043,42 @@ END "); } } - public static async IAsyncEnumerable GetFilesAndSourceBlocks(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, bool skipMetadata) + public static async IAsyncEnumerable GetFilesAndSourceBlocks(LocalDatabase db, string filetablename, string blocktablename, long blocksize, bool skipMetadata) { // TODO: Skip metadata as required // Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows - using var cmd = connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {blocksize}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {blocksize}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{filetablename}"" ""A"", ""{blocktablename}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0 ORDER BY ""A"".""TargetPath"", ""B"".""Index""")); - cmd.Transaction = connection.BeginTransaction(deferred: true); + using var cmd = db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""A"".""ID"", + ""B"".""Hash"", + (""B"".""Index"" * {blocksize}), + ""B"".""Index"", + ""B"".""Size"", + ""C"".""Path"", + (""D"".""Index"" * {blocksize}), + ""E"".""Size"", + ""B"".""Metadata"" + FROM + ""{filetablename}"" ""A"", + ""{blocktablename}"" ""B"", + ""File"" ""C"", + ""BlocksetEntry"" ""D"", + ""Block"" E + WHERE ""A"".""ID"" = ""B"".""FileID"" + AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" + AND ""D"".""BlockID"" = ""E"".""ID"" + AND ""B"".""Hash"" = ""E"".""Hash"" + AND ""B"".""Size"" = ""E"".""Size"" + AND ""B"".""Restored"" = 0 + ORDER BY + ""A"".""TargetPath"", + ""B"".""Index"" + ") + .SetTransaction(db.Transaction.Transaction); using var rd = await cmd.ExecuteReaderAsync(); var more = await rd.ReadAsync(); + while (more) { var f = new LocalBlockSource(rd); @@ -725,7 +1089,8 @@ END "); while (more && current == f.TargetPath) more = await rd.ReadAsync(); } - await cmd.Transaction.CommitAsync(); + + await db.Transaction.CommitAsync(); } } @@ -733,13 +1098,31 @@ END "); { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); - return LocalBlockSource.GetFilesAndSourceBlocks(m_connection, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata); + return LocalBlockSource.GetFilesAndSourceBlocks(this, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata); } public async IAsyncEnumerable GetMissingVolumes() { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + using var cmd = m_connection.CreateCommand($@" + SELECT + ""RV"".""Name"", + ""RV"".""Hash"", + ""RV"".""Size"", + ""BB"".""MaxIndex"" + FROM ""RemoteVolume"" ""RV"" + INNER JOIN ( + SELECT + ""TB"".""VolumeID"", + MAX(""TB"".""Index"") as ""MaxIndex"" + FROM ""{m_tempblocktable}"" ""TB"" + WHERE ""TB"".""Restored"" = 0 + GROUP BY ""TB"".""VolumeID"" + ) as ""BB"" + ON ""RV"".""ID"" = ""BB"".""VolumeID"" + ORDER BY ""BB"".""MaxIndex"" + ") + .SetTransaction(m_rtr.Transaction); + // Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore. // If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file. // If we manage to get the volumes in an order restoring front blocks first, this can save time. @@ -750,15 +1133,6 @@ END "); // Now it is likely to restore all files from front to back. Large files will always be done last. // One could also use like the average block number in a volume, that needs to be measured. - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT ""RV"".""Name"", ""RV"".""Hash"", ""RV"".""Size"", ""BB"".""MaxIndex"" -FROM ""RemoteVolume"" ""RV"" INNER JOIN - (SELECT ""TB"".""VolumeID"", MAX(""TB"".""Index"") as ""MaxIndex"" - FROM ""{m_tempblocktable}"" ""TB"" - WHERE ""TB"".""Restored"" = 0 - GROUP BY ""TB"".""VolumeID"" - ) as ""BB"" ON ""RV"".""ID"" = ""BB"".""VolumeID"" -ORDER BY ""BB"".""MaxIndex"" ")); - using var rd = await cmd.ExecuteReaderAsync(); object[] r = new object[3]; while (await rd.ReadAsync()) @@ -771,7 +1145,7 @@ ORDER BY ""BB"".""MaxIndex"" ")); ); } - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public interface IFilesAndMetadata : IDisposable @@ -841,13 +1215,18 @@ ORDER BY ""BB"".""MaxIndex"" ")); } public void Dispose() + { + DisposeAsync().Await(); + } + + public async Task DisposeAsync() { if (m_tmptable != null) { - using var c = m_connection.CreateCommand(); - c.Transaction = m_connection.BeginTransaction(deferred: true); - c.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tmptable}""")); - c.Transaction.CommitAsync().Await(); + using var c = m_db.Connection.CreateCommand(@"DROP TABLE IF EXISTS ""{m_tmptable}""") + .SetTransaction(m_db.Transaction.Transaction); + await c.ExecuteNonQueryAsync(); + await m_db.Transaction.CommitAsync(); } } @@ -883,14 +1262,34 @@ ORDER BY ""BB"".""MaxIndex"" ")); public async IAsyncEnumerable FilesWithMissingBlocks() { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" -FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" -WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"0"} -AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") -ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); + using var cmd = m_db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""BB"".""FileID"", + (""BB"".""Index"" * {m_blocksize}), + ""BB"".""Size"", + ""BB"".""Hash"" + FROM + ""{m_filetablename}"" ""A"", + ""{m_blocktablename}"" ""BB"" + WHERE ""A"".""ID"" = ""BB"".""FileID"" + AND ""BB"".""Restored"" = 0 + AND ""BB"".""Metadata"" = {"0"} + AND ""BB"".""ID"" IN ( + SELECT ""B"".""ID"" + FROM + ""{m_blocktablename}"" ""B"", + ""{m_tmptable}"" ""C"" + WHERE ""B"".""Hash"" = ""C"".""Hash"" + AND ""B"".""Size"" = ""C"".""Size"" + ) + ORDER BY + ""A"".""TargetPath"", + ""BB"".""Index"" + ") + .SetTransaction(m_db.Transaction.Transaction); + using var rd = await cmd.ExecuteReaderAsync(); if (await rd.ReadAsync()) { @@ -911,14 +1310,34 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async IAsyncEnumerable MetadataWithMissingBlocks() { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" - FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" -WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"1"} - AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") -ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); + using var cmd = m_db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""BB"".""FileID"", + (""BB"".""Index"" * {m_blocksize}), + ""BB"".""Size"", + ""BB"".""Hash"" + FROM + ""{m_filetablename}"" ""A"", + ""{m_blocktablename}"" ""BB"" + WHERE ""A"".""ID"" = ""BB"".""FileID"" + AND ""BB"".""Restored"" = 0 + AND ""BB"".""Metadata"" = {"1"} + AND ""BB"".""ID"" IN ( + SELECT ""B"".""ID"" + FROM + ""{m_blocktablename}"" ""B"", + ""{m_tmptable}"" ""C"" + WHERE ""B"".""Hash"" = ""C"".""Hash"" + AND ""B"".""Size"" = ""C"".""Size"" + ) + ORDER BY + ""A"".""TargetPath"", + ""BB"".""Index"" + ") + .SetTransaction(m_db.Transaction.Transaction); + using var rd = await cmd.ExecuteReaderAsync(); if (await rd.ReadAsync()) { @@ -937,11 +1356,11 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); } } - public IFilesAndMetadata GetMissingBlockData(BlockVolumeReader curvolume, long blocksize) + public async Task GetMissingBlockData(BlockVolumeReader curvolume, long blocksize) { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); - return new FilesAndMetadata(m_connection, m_tempfiletable, m_tempblocktable, blocksize, curvolume); + return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume); } /// @@ -952,11 +1371,11 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); { if (!m_connection_pool.TryTake(out var connection)) { - connection = await SQLiteHelper.SQLiteLoader.LoadConnection(); - connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; + connection = await SQLiteLoader.LoadConnectionAsync(); + connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared"; connection.Open(); - await SQLiteHelper.SQLiteLoader.ApplyCustomPragmas(connection, m_pagecachesize); + await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize); } return connection; @@ -978,16 +1397,27 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified) { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT ""{m_tempfiletable}"".""ID"", ""{m_tempfiletable}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{m_tempfiletable}"",""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified")); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - cmd.SetParameterValue("@Verified", !onlyNonVerified); + using var cmd = m_connection.CreateCommand($@" + SELECT + ""{m_tempfiletable}"".""ID"", + ""{m_tempfiletable}"".""TargetPath"", + ""Blockset"".""FullHash"", + ""Blockset"".""Length"" + FROM + ""{m_tempfiletable}"", + ""Blockset"" + WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@Verified", !onlyNonVerified); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return new FileToRestore( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } /// @@ -996,17 +1426,25 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// A list of files and symlinks to restore. public async IAsyncEnumerable GetFilesAndSymlinksToRestore() { - using var cmd = m_connection.CreateCommand(FormatInvariant($@" - SELECT F.ID, F.Path, F.TargetPath, IFNULL(B.FullHash, ''), IFNULL(B.Length, 0), F.BlocksetID + using var cmd = m_connection.CreateCommand($@" + SELECT + F.ID, + F.Path, + F.TargetPath, + IFNULL(B.FullHash, ''), + IFNULL(B.Length, 0), + F.BlocksetID FROM ""{m_tempfiletable}"" F - LEFT JOIN Blockset B ON F.BlocksetID = B.ID - WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID}")); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + LEFT JOIN Blockset B + ON F.BlocksetID = B.ID + WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID} + ") + .SetTransaction(m_rtr.Transaction); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } /// @@ -1017,15 +1455,31 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); { using var cmd = m_connection.CreateCommand(); cmd.Transaction = m_connection.BeginTransaction(deferred: true); - using var rd = await cmd.ExecuteReaderAsync(FormatInvariant($@" - SELECT F.ID, '', F.TargetPath, '', 0, {FOLDER_BLOCKSET_ID} + using var rd = await cmd.ExecuteReaderAsync($@" + SELECT + F.ID, + '', + F.TargetPath, + '', + 0, + {FOLDER_BLOCKSET_ID} FROM ""{m_tempfiletable}"" F - WHERE F.BlocksetID = {FOLDER_BLOCKSET_ID} AND F.MetadataID IS NOT NULL AND F.MetadataID >= 0")); + WHERE F.BlocksetID = {FOLDER_BLOCKSET_ID} + AND F.MetadataID IS NOT NULL + AND F.MetadataID >= 0 + "); while (await rd.ReadAsync()) - yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); + yield return new FileRequest( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToString(3), + rd.ConvertValueToInt64(4), + rd.ConvertValueToInt64(5) + ); - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } /// @@ -1035,26 +1489,39 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// A list of tuples containing the block ID and the volume ID of the block. public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) { - using var cmd = Connection.CreateCommand(); - cmd.Transaction = Connection.BeginTransaction(deferred: true); - using var reader = await cmd.ExecuteReaderAsync(FormatInvariant($@" - SELECT ""Block"".""ID"", ""Block"".""VolumeID"" - FROM ""BlocksetEntry"" - INNER JOIN ""{m_tempfiletable}"" ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" - INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - ") - + (skipMetadata ? "" : FormatInvariant($@" + var metadata_query = skipMetadata ? "" : $@" UNION ALL - SELECT ""Block"".""ID"", ""Block"".""VolumeID"" + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" FROM ""{m_tempfiletable}"" - INNER JOIN ""Metadataset"" ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" - INNER JOIN ""BlocksetEntry"" ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - "))); + INNER JOIN ""Metadataset"" + ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""BlocksetEntry"" + ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + "; + using var cmd = Connection.CreateCommand($@" + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""BlocksetEntry"" + INNER JOIN ""{m_tempfiletable}"" + ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + {metadata_query} + ") + .SetTransaction(m_rtr.Transaction); + using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) - yield return (reader.ConvertValueToInt64(0), reader.ConvertValueToInt64(1)); + yield return ( + reader.ConvertValueToInt64(0), + reader.ConvertValueToInt64(1) + ); - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } /// @@ -1065,19 +1532,32 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async IAsyncEnumerable GetBlocksFromFile(long blocksetID) { var connection = await GetConnectionFromPool(); - using var cmd = connection.CreateCommand(FormatInvariant(@$" - SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" - FROM ""BlocksetEntry"" INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID")); - cmd.SetParameterValue("@BlocksetID", blocksetID); - cmd.Transaction = connection.BeginTransaction(deferred: true); + using var cmd = connection.CreateCommand(@$" + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""BlocksetEntry"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@BlocksetID", blocksetID); using var reader = await cmd.ExecuteReaderAsync(); for (long i = 0; await reader.ReadAsync(); i++) - yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); + yield return new BlockRequest( + reader.ConvertValueToInt64(0), + i, + reader.ConvertValueToString(1), + reader.ConvertValueToInt64(2), + reader.ConvertValueToInt64(3), + false + ); - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); // Return the connection to the pool m_connection_pool.Add(connection); @@ -1091,16 +1571,23 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID) { var connection = await GetConnectionFromPool(); - using var cmd = connection.CreateCommand(FormatInvariant($@" - SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" + using var cmd = connection.CreateCommand($@" + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" FROM ""File"" - INNER JOIN ""Metadataset"" ON ""File"".""MetadataID"" = ""Metadataset"".""ID"" - INNER JOIN ""BlocksetEntry"" ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + INNER JOIN ""Metadataset"" + ON ""File"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""BlocksetEntry"" + ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" WHERE ""File"".""ID"" = @FileID - ")); - cmd.Transaction = connection.BeginTransaction(deferred: true); - cmd.SetParameterValue("@FileID", fileID); + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@FileID", fileID); using var reader = await cmd.ExecuteReaderAsync(); for (long i = 0; await reader.ReadAsync(); i++) @@ -1108,7 +1595,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); } - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); // Return the connection to the pool m_connection_pool.Add(connection); @@ -1121,23 +1608,33 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); /// A tuple containing the name, size, and hash of the volume. public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) { - using var cmd = m_connection.CreateCommand("SELECT Name, Size, Hash FROM RemoteVolume WHERE ID = @VolumeID"); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - cmd.SetParameterValue("@VolumeID", VolumeID); + using var cmd = m_connection.CreateCommand(@" + SELECT + Name, + Size, + Hash + FROM RemoteVolume + WHERE ID = @VolumeID + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@VolumeID", VolumeID); + using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) yield return (reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), reader.ConvertValueToString(2) ?? ""); - await cmd.Transaction.CommitAsync(); + + await m_rtr.CommitAsync(); } public async Task DropRestoreTable() { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr.Transaction); + if (m_tempfiletable != null) try { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"""); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_tempfiletable = null; } @@ -1145,7 +1642,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); if (m_tempblocktable != null) try { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"""); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_tempblocktable = null; } @@ -1153,7 +1650,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); if (m_latestblocktable != null) try { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"""); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_latestblocktable = null; } @@ -1161,7 +1658,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); if (m_fileprogtable != null) try { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"""); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_fileprogtable = null; } @@ -1169,7 +1666,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); if (m_totalprogtable != null) try { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"""); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_totalprogtable = null; } @@ -1177,12 +1674,12 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); if (m_filesnewlydonetable != null) try { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"""); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_filesnewlydonetable = null; } - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public interface IBlockMarker : IDisposable @@ -1299,6 +1796,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); { m_hasUpdates = true; m_resetfileCommand.SetParameterValue("@TargetFileId", targetfileid); + var r = await m_resetfileCommand.ExecuteNonQueryAsync(); if (r <= 0) throw new Exception("Unexpected reset result"); @@ -1307,8 +1805,9 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata) { m_hasUpdates = true; - m_updateAsRestoredCommand.SetParameterValue("@TargetFileId", targetfileid); - m_updateAsRestoredCommand.SetParameterValue("@Metadata", includeMetadata ? 1 : 0); + m_updateAsRestoredCommand.SetParameterValue("@TargetFileId", targetfileid) + .SetParameterValue("@Metadata", includeMetadata ? 1 : 0); + var r = await m_updateAsRestoredCommand.ExecuteNonQueryAsync(); if (r <= 0) throw new Exception("Unexpected reset result"); @@ -1318,6 +1817,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); { m_hasUpdates = true; m_updateFileAsDataVerifiedCommand.SetParameterValue("@TargetFileId", targetfileid); + var r = await m_updateFileAsDataVerifiedCommand.ExecuteNonQueryAsync(); if (r != 1) throw new Exception("Unexpected result when marking file as verified."); @@ -1326,11 +1826,12 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata) { m_hasUpdates = true; - m_insertblockCommand.SetParameterValue("@TargetFileId", targetfileid); - m_insertblockCommand.SetParameterValue("@Index", index); - m_insertblockCommand.SetParameterValue("@Hash", hash); - m_insertblockCommand.SetParameterValue("@Size", size); - m_insertblockCommand.SetParameterValue("@Metadata", metadata); + m_insertblockCommand.SetParameterValue("@TargetFileId", targetfileid) + .SetParameterValue("@Index", index) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .SetParameterValue("@Metadata", metadata); + var r = await m_insertblockCommand.ExecuteNonQueryAsync(); if (r != 1) throw new Exception("Unexpected result when marking block."); @@ -1339,9 +1840,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async Task CommitAsync() { using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) - await m_transaction.CommitAsync(); - await m_transaction.DisposeAsync(); - m_transaction = null!; + await m_rtr.CommitAsync(); } public void Dispose() @@ -1351,11 +1850,15 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); m_updateAsRestoredCommand?.Dispose(); m_updateFileAsDataVerifiedCommand?.Dispose(); m_statUpdateCommand?.Dispose(); - m_transaction?.Dispose(); } } + [Obsolete("Calling this constructor will throw an exception. Use CreateBlockMarkerAsync instead.")] public IBlockMarker CreateBlockMarker() + { + throw new NotImplementedException("Use CreateBlockMarkerAsync instead of the constructor"); + } + public async Task CreateBlockMarkerAsync() { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) @@ -1379,13 +1882,19 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async IAsyncEnumerable GetTargetFolders() { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT ""TargetPath"" FROM ""{m_tempfiletable}"" WHERE ""BlocksetID"" == @BlocksetID")); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); - cmd.SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); + using var cmd = m_connection.CreateCommand($@" + SELECT ""TargetPath"" + FROM ""{m_tempfiletable}"" + WHERE ""BlocksetID"" == @BlocksetID + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); + using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return rd.ConvertValueToString(0) ?? ""; - await cmd.Transaction.CommitAsync(); + + await m_rtr.CommitAsync(); } public interface IFastSource @@ -1438,22 +1947,29 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize) { - using (var transaction = m_connection.BeginTransaction(deferred: true)) using (var cmdReader = m_connection.CreateCommand()) using (var cmd = m_connection.CreateCommand()) { - cmdReader.Transaction = transaction; - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""LocalSourceExists"" = 1 WHERE Path = @Path")); - using (var rd = await cmdReader.ExecuteReaderAsync(FormatInvariant($@"SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"""))) + cmdReader.SetTransaction(m_rtr.Transaction); + cmd.SetTransaction(m_rtr.Transaction); + cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""LocalSourceExists"" = 1 + WHERE Path = @Path + "); + cmdReader.SetCommandAndParameters($@" + SELECT DISTINCT ""{m_tempfiletable}"".""Path"" + FROM ""{m_tempfiletable}"" + "); + using (var rd = await cmdReader.ExecuteReaderAsync()) { while (await rd.ReadAsync()) { var sourcepath = rd.ConvertValueToString(0); if (SystemIO.IO_OS.FileExists(sourcepath)) { - cmd.SetParameterValue("@Path", sourcepath); - await cmd.ExecuteNonQueryAsync(); + await cmd.SetParameterValue("@Path", sourcepath) + .ExecuteNonQueryAsync(); } else { @@ -1463,18 +1979,24 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); } //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"")")); - await transaction.CommitAsync(); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" + ON ""{m_tempfiletable}"" (""LocalSourceExists"") + "); + + await m_rtr.CommitAsync(); } m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; - var whereclause = FormatInvariant($@" - ""{m_tempfiletable}"".""LocalSourceExists"" = 1 AND - ""{m_tempblocktable}"".""Restored"" = 0 AND ""{m_tempblocktable}"".""Metadata"" = 0 AND - ""{m_tempfiletable}"".""TargetPath"" != ""{m_tempfiletable}"".""Path"""); + var whereclause = $@" + ""{m_tempfiletable}"".""LocalSourceExists"" = 1 + AND ""{m_tempblocktable}"".""Restored"" = 0 + AND ""{m_tempblocktable}"".""Metadata"" = 0 + AND ""{m_tempfiletable}"".""TargetPath"" != ""{m_tempfiletable}"".""Path"" + "; - var latestBlocksetIds = FormatInvariant($@" + var latestBlocksetIds = $@" SELECT ""File"".""Path"" AS ""PATH"", ""File"".""BlocksetID"" AS ""BlocksetID"", @@ -1483,54 +2005,64 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); ""File"", ""FilesetEntry"", ""Fileset"" - WHERE - ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND - ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" AND - ""File"".""Path"" IN - (SELECT DISTINCT - ""{m_tempfiletable}"".""Path"" - FROM - ""{m_tempfiletable}"", - ""{m_tempblocktable}"" - WHERE - ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" AND - {whereclause}) - GROUP BY ""File"".""Path"""); + WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" + AND ""File"".""Path"" IN ( + SELECT DISTINCT ""{m_tempfiletable}"".""Path"" + FROM + ""{m_tempfiletable}"", + ""{m_tempblocktable}"" + WHERE ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" + AND {whereclause} + ) + GROUP BY ""File"".""Path"" + "; - using var transaction2 = m_connection.BeginTransaction(deferred: true); using (var cmd = m_connection.CreateCommand()) { - cmd.Transaction = transaction2; - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{m_latestblocktable}_path"" ON ""{m_latestblocktable}"" (""Path"")")); + cmd.SetTransaction(m_rtr.Transaction); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" "); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}"); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_latestblocktable}_path"" + ON ""{m_latestblocktable}"" (""Path"") + "); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET LatestBlocksetId = (SELECT BlocksetId FROM ""{m_latestblocktable}"" WHERE Path = ""{m_tempfiletable}"".Path)")); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET LatestBlocksetId = ( + SELECT BlocksetId + FROM ""{m_latestblocktable}"" + WHERE Path = ""{m_tempfiletable}"".Path + ) + "); } - var sources = FormatInvariant($@" -SELECT DISTINCT - ""{m_tempfiletable}"".""TargetPath"", - ""{m_tempfiletable}"".""Path"", - ""{m_tempfiletable}"".""ID"", - ""{m_tempblocktable}"".""Index"", - ""{m_tempblocktable}"".""Hash"", - ""{m_tempblocktable}"".""Size"" -FROM - ""{m_tempfiletable}"", - ""{m_tempblocktable}"", - ""BlocksetEntry"" -WHERE - ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" AND - ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" AND - ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" AND - ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" AND - {whereclause} -ORDER BY ""{m_tempfiletable}"".""ID"", ""{m_tempblocktable}"".""Index"" "); + var sources = $@" + SELECT DISTINCT + ""{m_tempfiletable}"".""TargetPath"", + ""{m_tempfiletable}"".""Path"", + ""{m_tempfiletable}"".""ID"", + ""{m_tempblocktable}"".""Index"", + ""{m_tempblocktable}"".""Hash"", + ""{m_tempblocktable}"".""Size"" + FROM + ""{m_tempfiletable}"", + ""{m_tempblocktable}"", + ""BlocksetEntry"" + WHERE ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" + AND ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" + AND {whereclause} + ORDER BY + ""{m_tempfiletable}"".""ID"", + ""{m_tempblocktable}"".""Index"" + "; using (var cmd = m_connection.CreateCommand()) { - cmd.Transaction = transaction2; + cmd.SetTransaction(m_rtr.Transaction); using var rd = await cmd.ExecuteReaderAsync(sources); if (await rd.ReadAsync()) { @@ -1549,7 +2081,7 @@ ORDER BY ""{m_tempfiletable}"".""ID"", ""{m_tempblocktable}"".""Index"" "); } } - await transaction2.CommitAsync(); + await m_rtr.CommitAsync(); } } From c380989aad70ca8cd7b38b9f1e2c8a0e0deac740 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 15:50:11 +0200 Subject: [PATCH 047/362] Added a readonly property to LocalDatabase's transaction, so derived classes can access it --- Duplicati/Library/Main/Database/LocalDatabase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 9ddc76d76..c1428552c 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -60,6 +60,7 @@ namespace Duplicati.Library.Main.Database // TODO use this rather than passing down transactions. // TODO dispose should check this, and report if it was not disposed prior to calling dispose. Maybe the rtr itself should check this during dispose? And just throw a warning during runtime. protected ReusableTransaction m_rtr; + public ReusableTransaction Transaction { get { return m_rtr; } } private SqliteCommand m_updateremotevolumeCommand = null!; private SqliteCommand m_selectremotevolumesCommand = null!; From 32b14908ddc4bff14cfdd4f23a5cfbaafbbebb0b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 16 May 2025 15:50:36 +0200 Subject: [PATCH 048/362] Default message to reusable transaction should be null - i.e. empty. --- Duplicati/Library/Main/Database/ReusableTransaction.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 7052b1a6f..5d2179e6e 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -82,7 +82,7 @@ internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transact /// True if the transaction should be restarted. /// An awaitable task. /// If the transaction is already Disposed. - public async Task CommitAsync(string? message, bool restart = true) + public async Task CommitAsync(string? message = null, bool restart = true) { if (m_disposed) throw new InvalidOperationException("Transaction is already disposed"); From cb4e9eb657083d9d4a25f5bf6f67a4f54d16c1fc Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 06:40:18 +0200 Subject: [PATCH 049/362] Made the LocalDatabase constructor protected to ensure that derived classes won't call the obsolete warning throwing constructor. --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index c1428552c..ef6ad45f6 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -90,7 +90,7 @@ namespace Duplicati.Library.Main.Database public bool ShouldCloseConnection { get; set; } // Constructor is private to force use of CreateLocalDatabaseAsync - private LocalDatabase() { } + protected LocalDatabase() { } // Arguments are not used, but are required to match the constructor signatures [Obsolete("Calling the constructor will throw an exception. Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead")] diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 1b305b91b..f2fcb795b 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -59,8 +59,6 @@ namespace Duplicati.Library.Main.Database public DateTime RestoreTime { get { return m_restoreTime; } } - private LocalRestoreDatabase() { } - public static async Task CreateAsync(string path, long pagecachesize) { var db = new LocalRestoreDatabase(); From 8bc81a45db5518a6044051ad67a0cf7ae5f3321a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 06:41:49 +0200 Subject: [PATCH 050/362] Changed LocalRepairDatabase to use factory for construction --- .../Main/Database/LocalRepairDatabase.cs | 2123 +++++++++-------- 1 file changed, 1099 insertions(+), 1024 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 18c7155dd..b7ce0fdea 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1,1024 +1,1099 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Duplicati.Library.Utility; -using Microsoft.Data.Sqlite; - -namespace Duplicati.Library.Main.Database -{ - internal class LocalRepairDatabase : LocalDatabase - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRepairDatabase)); - - /// - /// Creates a new local repair database - /// - /// The path to the database - /// The page cache size - public LocalRepairDatabase(string path, long pagecachesize) - : base(path, "Repair", true, pagecachesize) - { - - } - - /// - /// Gets the fileset ID from the remote name - /// - /// The remote name of the fileset - /// An optional transaction - /// >The fileset ID - public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist, SqliteTransaction? transaction) - { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name"); - cmd.SetParameterValue("@Name", filelist); - var rd = await cmd.ExecuteReaderAsync(); - - if (!await rd.ReadAsync()) - throw new Exception($"No such remote file: {filelist}"); - - return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); - } - - /// - /// Moves entries in the FilesetEntry table from previous fileset to current fileset - /// - /// Current fileset ID - /// Source fileset ID - /// The transaction - public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId, SqliteTransaction transaction) - { - if (filesetid <= 0) - throw new ArgumentException("filesetid must be > 0"); - - if (prevFilesetId <= 0) - throw new ArgumentException("prevId must be > 0"); - - using var cmd = m_connection.CreateCommand(); - cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId "); - cmd.Transaction = transaction; - cmd.SetParameterValue("@CurrentFilesetId", filesetid); - cmd.SetParameterValue("@PreviousFilesetId", prevFilesetId); - await cmd.ExecuteNonQueryAsync(); - } - - /// - /// Gets the list of index files that reference a given block file - /// - /// The block file ID - /// >A list of index file IDs - public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(@"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@BlockFileId", blockfileid); - cmd.SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); - } - - /// - /// Gets a list of filesets that are missing files - /// - /// An optional transaction - /// A list of fileset IDs and timestamps - public async IAsyncEnumerable> GetFilesetsWithMissingFiles(SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - using var rd = await cmd.ExecuteReaderAsync(@"SELECT ID, Timestamp FROM Fileset WHERE ID IN (SELECT FilesetID FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup""))"); - while (await rd.ReadAsync()) - { - yield return new KeyValuePair( - rd.ConvertValueToInt64(0), - ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() - ); - } - } - - /// - /// Deletes all fileset entries for a given fileset - /// - /// The fileset ID - /// An optional transaction - /// The number of deleted entries - public async Task DeleteFilesetEntries(long filesetid, SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId"); - cmd.SetParameterValue("@FilesetId", filesetid); - return await cmd.ExecuteNonQueryAsync(); - } - - private class RemoteVolume : IRemoteVolume - { - public string Name { get; private set; } - public string Hash { get; private set; } - public long Size { get; private set; } - - public RemoteVolume(string name, string hash, long size) - { - this.Name = name; - this.Hash = hash; - this.Size = size; - } - } - - public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName, SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) "); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Name", indexName); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - - /// - /// A single block with source data for repair - /// - /// The block hash - /// The block size - /// The file that contains the block - /// The offset of the block in the file - public sealed record BlockWithSourceData(string Hash, long Size, string File, long Offset); - - /// - /// A single block with metadata source data for repair - /// - /// The block hash - /// The block size - /// The path of the file or directory that contains the block - public sealed record BlockWithMetadataSourceData(string Hash, long Size, string Path); - - /// - /// A single blocklist hash entry - /// - /// The blockset id - /// The hash of the blocklist entry (when done) - /// The total length of the blockset - /// The index of the blocklist hash - /// The index of the block in the blockset - /// The hash of the entry - public sealed record BlocklistHashesEntry( - long BlocksetId, - string BlocklistHash, - long BlocklistHashLength, - long BlocklistHashIndex, - int Index, - string Hash); - - /// - /// Helper interface for the missing block list - /// - public interface IMissingBlockList : IDisposable - { - /// - /// Registers a block as restored - /// - /// The block hash - /// The block size - /// The volume ID of the new target volume - /// The restored blocks - Task SetBlockRestored(string hash, long size, long volumeId); - /// - /// Gets the list of files that contains missing blocks - /// - /// The blocksize setting - /// A list of files with missing blocks - IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize); - /// - /// Gets a list for filesystem entries that contain missing blocks in metadata - /// - /// A list of filesystem entries with missing blocks - IAsyncEnumerable GetSourceItemsWithMetadataBlocks(); - /// - /// Gets missing blocklist hashes - /// - /// The number of hashes for each block - /// >A list of blocklist hashes - IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock); - /// - /// Gets the number of missing blocks - /// - /// >The number of missing blocks - Task GetMissingBlockCount(); - /// - /// Gets all the filesets that are affected by missing blocks - /// - /// >A list of filesets - IAsyncEnumerable GetFilesetsUsingMissingBlocks(); - /// - /// Gets a list of remote files that may contain missing blocks - /// - /// >A list of remote files - IAsyncEnumerable GetMissingBlockSources(); - } - - /// - /// Implementation of the missing block list - /// - private class MissingBlockList : IMissingBlockList - { - /// - /// The connection to the database - /// - private readonly SqliteConnection m_connection; - /// - /// The transaction to use - /// - private readonly ReusableTransaction m_transaction; - /// - /// The insert command to use for restoring blocks - /// - private readonly SqliteCommand m_insertCommand; - /// - /// The command to copy blocks into the duplicate block table - /// - private readonly SqliteCommand m_copyIntoDuplicatedBlocks; - /// - /// The command to assign blocks to a new volume - /// - private readonly SqliteCommand m_assignBlocksToNewVolume; - /// - /// The name of the temporary table - /// - private readonly string m_tablename; - /// - /// The name of the volume where blocks are missing - /// - private readonly string m_volumename; - /// - /// The query to use for getting missing blocks - /// - private readonly string m_missingBlocksQuery; - - /// - /// Whether the object has been disposed - /// - private bool m_isDisposed = false; - - /// - /// Creates a new missing block list - /// - /// The name of the volume with missing blocks - /// The connection to the database - /// The transaction to use - public MissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) - { - m_connection = connection; - m_transaction = transaction; - m_volumename = volumename; - var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = transaction.Transaction; - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ")).Await(); - m_tablename = tablename; - - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = @Name ")); - cmd.SetParameterValue("@Name", volumename); - var blockCount = cmd.ExecuteNonQueryAsync().Await(); - - if (blockCount == 0) - throw new Exception($"Unexpected empty block volume: {0}"); - - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")")).Await(); - } - - m_insertCommand = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue ")); - m_copyIntoDuplicatedBlocks = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""Block"".""ID"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""Hash"" = @Hash AND ""Block"".""Size"" = @Size"); - m_assignBlocksToNewVolume = m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @TargetVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - m_missingBlocksQuery = FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored "); - } - - /// - public async Task SetBlockRestored(string hash, long size, long targetVolumeId) - { - //m_insertCommand.SetTransaction(m_transaction.Transaction); - m_insertCommand.Transaction = m_transaction.Transaction; - m_insertCommand.SetParameterValue("@NewRestoredValue", 1); - m_insertCommand.SetParameterValue("@Hash", hash); - m_insertCommand.SetParameterValue("@Size", size); - m_insertCommand.SetParameterValue("@PreviousRestoredValue", 0); - var restored = await m_insertCommand.ExecuteNonQueryAsync() == 1; - - if (restored) - { - m_copyIntoDuplicatedBlocks.Transaction = m_transaction.Transaction; - m_copyIntoDuplicatedBlocks.SetParameterValue("@Hash", hash); - m_copyIntoDuplicatedBlocks.SetParameterValue("@Size", size); - await m_copyIntoDuplicatedBlocks.ExecuteNonQueryAsync(); - - m_assignBlocksToNewVolume.Transaction = m_transaction.Transaction; - m_assignBlocksToNewVolume.SetParameterValue("@TargetVolumeId", targetVolumeId); - m_assignBlocksToNewVolume.SetParameterValue("@Hash", hash); - m_assignBlocksToNewVolume.SetParameterValue("@Size", size); - var c = await m_assignBlocksToNewVolume.ExecuteNonQueryAsync(); - - if (c != 1) - throw new Exception($"Unexpected number of updated blocks: {c} != 1"); - } - - return restored; - } - - /// - public async IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize) - { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}")); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - { - var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); - var size = rd.ConvertValueToInt64(1); - var file = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); - var offset = rd.ConvertValueToInt64(3); - - yield return new BlockWithSourceData(hash, size, file, offset); - } - } - - /// - public async IAsyncEnumerable GetSourceItemsWithMetadataBlocks() - { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path""")); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - { - var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); - var size = rd.ConvertValueToInt64(1); - var path = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); - - yield return new BlockWithMetadataSourceData(hash, size, path); - } - } - - /// - public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock) - { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_transaction.Transaction; - var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; - try - { - // We need to create a snapshot as we will be updating the m_tablename table during enumeration - cmd.SetCommandAndParameters(FormatInvariant($@" - CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS - SELECT - b.""Hash"" AS ""BlockHash"", - bs.""Id"" AS ""BlocksetId"", - bs.""Length"" AS ""BlocksetLength"", - bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", - ( - SELECT blh.""Hash"" - FROM ""BlocklistHash"" blh - WHERE blh.""BlocksetID"" = bs.""ID"" - AND blh.""Index"" == bse.""Index"" / @HashesPerBlock - LIMIT 1 - ) AS ""BlocklistHashHash"", - bse.""Index"" AS ""BlocksetEntryIndex"" - FROM - ""BlocksetEntry"" bse - JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" - JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" - WHERE - EXISTS ( - SELECT 1 - FROM ""BlocklistHash"" blh - JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash"" - WHERE blh.""BlocksetID"" = bs.""ID"" - AND mt.""Restored"" = @Restored - ) - ")); - cmd.SetParameterValue("@HashesPerBlock", hashesPerBlock); - cmd.SetParameterValue("@Restored", 0); - await cmd.ExecuteNonQueryAsync(); - - cmd.SetCommandAndParameters(FormatInvariant($@" - SELECT - ""BlockHash"", - ""BlocksetId"", - ""BlocksetLength"", - ""BlocklistHashIndex"", - ""BlocklistHashHash"", - ""BlocksetEntryIndex"" - FROM ""{blocklistTableName}"" - ORDER BY ""BlocksetId"", ""BlocklistHashIndex"", ""BlocksetEntryIndex""")); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - { - var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); - var blocksetId = rd.ConvertValueToInt64(1); - var length = rd.ConvertValueToInt64(2); - var blocklistHashIndex = rd.ConvertValueToInt64(3); - var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); - var index = rd.ConvertValueToInt64(5); - - yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash); - } - } - finally - { - try { await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); } - catch { } - } - } - - /// - public async Task GetMissingBlockCount() - { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})")); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); - - return await cmd.ExecuteScalarInt64Async(0); - } - - /// - /// Gets the list of missing blocks - /// - /// >A list of missing blocks - public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks() - { - using var cmd = m_connection.CreateCommand(m_missingBlocksQuery); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); - } - - /// - public async Task MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId) - { - if (targetVolumeId <= 0) - throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); - - // Move the source blocks into the DuplicateBlock table - using var cmd = m_connection.CreateCommand(FormatInvariant($@" - INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") - SELECT b.""ID"", b.""VolumeID"" - FROM ""Block"" b - WHERE b.""VolumeID"" = @SourceVolumeId - AND (b.""Hash"", b.""Size"") IN ( - SELECT ""Hash"", ""Size"" - FROM ""{m_tablename}"" - WHERE ""Restored"" = @Restored - ) - ")); - cmd.SetParameterValue("@SourceVolumeId", sourceVolumeId); - cmd.SetParameterValue("@Restored", 1); - - var moved = await cmd.ExecuteNonQueryAsync(); - - // Then update the blocks table to point to the new volume - cmd.SetCommandAndParameters(FormatInvariant($@" - UPDATE ""Block"" - SET ""VolumeID"" = @TargetVolumeId - WHERE ""VolumeID"" = @SourceVolumeId - AND (""Hash"", ""Size"") IN ( - SELECT ""Hash"", ""Size"" - FROM ""{m_tablename}"" - WHERE ""Restored"" = @Restored - ) - ")); - cmd.SetParameterValue("@TargetVolumeId", targetVolumeId); - cmd.SetParameterValue("@SourceVolumeId", sourceVolumeId); - cmd.SetParameterValue("@Restored", 1); - - var updated = await cmd.ExecuteNonQueryAsync(); - if (updated != moved) - throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); - - return updated; - } - - public async IAsyncEnumerable GetFilesetsUsingMissingBlocks() - { - var blocks = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); - var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); - var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = @Type AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))"); - - using var cmd = m_connection.CreateCommand(cmdtxt); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - - public async IAsyncEnumerable GetMissingBlockSources() - { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); - cmd.SetParameterValue("@Name", m_volumename); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - - public void Dispose() - { - if (m_isDisposed) - return; - - m_isDisposed = true; - try - { - if (m_tablename != null) - { - - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_transaction.Transaction; - cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")).Await(); - } - } - catch { } - - try { m_insertCommand?.Dispose(); } - catch { } - } - } - - public IMissingBlockList CreateBlockList(string volumename, ReusableTransaction transaction) - { - return new MissingBlockList(volumename, m_connection, transaction); - } - - public async Task FixDuplicateMetahash() - { - using var tr = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); - - cmd.Transaction = tr; - var sql_count = - @"SELECT COUNT(*) FROM (" + - @" SELECT DISTINCT c1 FROM (" + - @"SELECT COUNT(*) AS ""C1"" FROM (SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") UNION SELECT COUNT(*) AS ""C1"" FROM ""Metadataset"" " + - @")" + - @")"; - - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 1) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashes", "Found duplicate metadatahashes, repairing"); - - var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); - - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); - - var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; - - using (var c2 = m_connection.CreateCommand()) - { - c2.Transaction = tr; - c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""MetadataID"" = @MetadataId WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId)") - + @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId AND ""ID"" != @MetadataId"); - using (var rd = await cmd.ExecuteReaderAsync(sql)) - while (await rd.ReadAsync()) - { - c2.SetParameterValue("@MetadataId", rd.GetValue(0)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(1)); - await c2.ExecuteNonQueryAsync(); - } - } - - sql = FormatInvariant($@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") - WHERE ""Entries"" > 1 ORDER BY ""ID"""); - - using (var c2 = m_connection.CreateCommand()) - { - c2.Transaction = tr; - c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId)") - + FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId")); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql)) - { - c2.SetParameterValue("@FileId", rd.GetValue(0)); - c2.SetParameterValue("@Path", rd.GetValue(1)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(2)); - c2.SetParameterValue("@MetadataId", rd.GetValue(3)); - await c2.ExecuteNonQueryAsync(); - } - } - - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); - - x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 1) - throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); - await tr.CommitAsync(); - } - } - - public async Task FixDuplicateFileentries() - { - using var tr = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); - - cmd.Transaction = tr; - var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; - - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); - - var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") - WHERE ""Entries"" > 1 ORDER BY ""ID"""; - - using (var c2 = m_connection.CreateCommand()) - { - c2.Transaction = tr; - c2.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetatadataId)" - + @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId"); - cmd.SetCommandAndParameters(sql); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - { - c2.SetParameterValue("@FileId", rd.GetValue(0)); - c2.SetParameterValue("@PrefixId", rd.GetValue(1)); - c2.SetParameterValue("@Path", rd.GetValue(2)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(3)); - c2.SetParameterValue("@MetadataId", rd.GetValue(4)); - await c2.ExecuteNonQueryAsync(); - } - } - - x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 1) - throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); - await tr.CommitAsync(); - } - - } - - public async Task FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize) - { - var blocklistbuffer = new byte[blocksize]; - - using var tr = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); - using var blockhasher = HashFactory.CreateHasher(blockhashalgorithm); - - cmd.Transaction = tr; - var hashsize = blockhasher.HashSize / 8; - - var sql = FormatInvariant($@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"""); - var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; - - var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); - if (itemswithnoblocklisthash != 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); - using (var c2 = m_connection.CreateCommand()) - using (var c3 = m_connection.CreateCommand()) - using (var c4 = m_connection.CreateCommand()) - using (var c5 = m_connection.CreateCommand()) - using (var c6 = m_connection.CreateCommand()) - { - c2.Transaction = tr; - c3.Transaction = tr; - c4.Transaction = tr; - c5.Transaction = tr; - c6.Transaction = tr; - c3.SetCommandAndParameters(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash) "); - c4.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - c5.SetCommandAndParameters(@"SELECT ""ID"" FROM ""DeletedBlock"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND (""State"" IN (@State1, @State2)))"); - c6.SetCommandAndParameters(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1; DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId;"); - - await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql)) - { - var blocksetid = e.ConvertValueToInt64(0); - var ix = 0L; - int blocklistoffset = 0; - - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId"); - c2.SetParameterValue("@BlocksetId", blocksetid); - await c2.ExecuteNonQueryAsync(); - - c2.SetCommandAndParameters(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = @BlocksetId ORDER BY ""B"".""Index"""); - c2.SetParameterValue("@BlocksetId", blocksetid); - - await foreach (var h in c2.ExecuteReaderEnumerableAsync()) - { - var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); - if (blocklistbuffer.Length - blocklistoffset < tmp.Length) - { - var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); - - // Check if the block exists in "blocks" - c4.SetParameterValue("@Hash", blkey); - c4.SetParameterValue("@Size", blocklistoffset); - var existingBlockId = await c4.ExecuteScalarInt64Async(-1); - - if (existingBlockId <= 0) - { - c5.SetParameterValue("@Hash", blkey); - c5.SetParameterValue("@Size", blocklistoffset); - c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); - c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); - var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); - - if (deletedBlockId <= 0) - throw new Exception($"Missing block for blocklisthash: {blkey}"); - else - { - c6.SetParameterValue("@DeletedBlockId", deletedBlockId); - var rc = await c6.ExecuteNonQueryAsync(); - - if (rc != 2) - throw new Exception($"Unexpected update count: {rc}"); - } - } - - // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid); - c3.SetParameterValue("@Index", ix); - c3.SetParameterValue("@Hash", blkey); - await c3.ExecuteNonQueryAsync(); - - ix++; - blocklistoffset = 0; - } - - Array.Copy(tmp, 0, blocklistbuffer, blocklistoffset, tmp.Length); - blocklistoffset += tmp.Length; - - } - - if (blocklistoffset != 0) - { - var blkeyfinal = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); - - // Ensure that the block exists in "blocks" - c4.SetParameterValue("@Hash", blkeyfinal); - c4.SetParameterValue("@Size", blocklistoffset); - var existingBlockId = await c4.ExecuteScalarInt64Async(-1); - - if (existingBlockId <= 0) - { - c5.SetParameterValue("@Hash", blkeyfinal); - c5.SetParameterValue("@Size", blocklistoffset); - c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); - c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); - var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); - - if (deletedBlockId == 0) - throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); - else - { - c6.SetParameterValue("@DeletedBlockId", deletedBlockId); - var rc = await c6.ExecuteNonQueryAsync(); - if (rc != 2) - throw new Exception($"Unexpected update count: {rc}"); - } - } - - // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid); - c3.SetParameterValue("@Index", ix); - c3.SetParameterValue("@Hash", blkeyfinal); - await c3.ExecuteNonQueryAsync(); - } - } - } - - itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); - if (itemswithnoblocklisthash != 0) - throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); - tr.Commit(); - } - - } - - public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize) - { - using var tr = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); - - cmd.Transaction = tr; - var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1"; - - var sql_count = @"SELECT COUNT(*) FROM (" + dup_sql + ")"; - - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); - if (x > 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashes", "Found duplicate blocklisthash entries, repairing"); - - var unique_count = await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); - - using (var c2 = m_connection.CreateCommand()) - { - c2.Transaction = tr; - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId AND ""Index"" = @Index LIMIT @Limit)"); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql)) - { - var expected = rd.GetInt32(2) - 1; - c2.SetParameterValue("@BlocksetId", rd.GetValue(0)); - c2.SetParameterValue("@Index", rd.GetValue(1)); - c2.SetParameterValue("@Limit", expected); - var actual = await c2.ExecuteNonQueryAsync(); - - if (actual != expected) - throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); - } - } - - x = await cmd.ExecuteScalarInt64Async(sql_count); - if (x > 1) - throw new Exception("Repair failed, there are still duplicate file entries!"); - - var real_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM ""BlocklistHash""", 0); - - if (real_count != unique_count) - throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); - - try - { - await VerifyConsistency(blocksize, hashsize, true, tr); - } - catch (Exception ex) - { - throw new Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); - } - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); - await tr.CommitAsync(); - } - } - - public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - try - { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); - - foreach (var kp in blocks) - { - cmd.SetParameterValue("@Hash", kp.Key); - cmd.SetParameterValue("@Size", kp.Value); - await cmd.ExecuteNonQueryAsync(); - } - - cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name"); - cmd.SetParameterValue("@Name", filename); - var id = await cmd.ExecuteScalarInt64Async(-1); - - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{tablename}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != @VolumeId ")); - cmd.SetParameterValue("@VolumeId", id); - var aliens = await cmd.ExecuteScalarInt64Async(0); - - if (aliens != 0) - throw new Exception($"Not all blocks were found in {filename}"); - } - finally - { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); - } - } - - public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(); - - cmd.Transaction = transaction; - var query = FormatInvariant($@" -SELECT - ""C"".""Hash"", - ""C"".""Size"" -FROM - ""BlocksetEntry"" A, - ( - SELECT - ""Y"".""BlocksetID"", - ""Y"".""Hash"" AS ""BlocklistHash"", - ""Y"".""Index"" AS ""BlocklistHashIndex"", - ""Z"".""Size"" AS ""BlocklistSize"", - ""Z"".""ID"" AS ""BlocklistHashBlockID"" - FROM - ""BlocklistHash"" Y, - ""Block"" Z - WHERE - ""Y"".""Hash"" = ""Z"".""Hash"" AND ""Y"".""Hash"" = @Hash AND ""Z"".""Size"" = @Size - LIMIT 1 - ) B, - ""Block"" C -WHERE - ""A"".""BlocksetID"" = ""B"".""BlocksetID"" - AND - ""A"".""BlockID"" = ""C"".""ID"" - AND - ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({blocksize} / {blockhashlength}) - AND - ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({blocksize} / {blockhashlength}) -ORDER BY - ""A"".""Index"" - -"); - - using var en = blocklist.GetEnumerator(); - - cmd.SetCommandAndParameters(query); - cmd.SetParameterValue("@Hash", hash); - cmd.SetParameterValue("@Size", length); - await foreach (var r in cmd.ExecuteReaderEnumerableAsync()) - { - if (!en.MoveNext()) - throw new Exception($"Too few entries in source blocklist with hash {hash}"); - if (en.Current != r.ConvertValueToString(0)) - throw new Exception($"Mismatch in blocklist with hash {hash}"); - } - - if (en.MoveNext()) - throw new Exception($"Too many source blocklist entries in {hash}"); - } - - public async IAsyncEnumerable MissingLocalFilesets(SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return rd.ConvertValueToString(0) ?? ""; - } - - public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets(SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(@"SELECT ""ID"", ""Timestamp"", ""IsFullBackup"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return (rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP); - } - - public async IAsyncEnumerable EmptyIndexFiles(SqliteTransaction transaction) - { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ID"" NOT IN (SELECT ""IndexVolumeId"" FROM ""IndexBlockLink"")"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); - cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - } -} - +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; + +namespace Duplicati.Library.Main.Database +{ + internal class LocalRepairDatabase : LocalDatabase + { + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRepairDatabase)); + + /// + /// Creates a new local repair database + /// + /// The path to the database + /// The page cache size + public static async Task CreateRepairDatabase(string path, long pagecachesize) + { + var db = new LocalRepairDatabase(); + + db = (LocalRepairDatabase)await CreateLocalDatabaseAsync(db, path, "Repair", false, pagecachesize); + + return db; + } + + /// + /// Gets the fileset ID from the remote name + /// + /// The remote name of the fileset + /// An optional transaction + /// >The fileset ID + public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist, SqliteTransaction? transaction) + { + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name"); + cmd.SetParameterValue("@Name", filelist); + var rd = await cmd.ExecuteReaderAsync(); + + if (!await rd.ReadAsync()) + throw new Exception($"No such remote file: {filelist}"); + + return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); + } + + /// + /// Moves entries in the FilesetEntry table from previous fileset to current fileset + /// + /// Current fileset ID + /// Source fileset ID + /// The transaction + public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId, SqliteTransaction transaction) + { + if (filesetid <= 0) + throw new ArgumentException("filesetid must be > 0"); + + if (prevFilesetId <= 0) + throw new ArgumentException("prevId must be > 0"); + + using var cmd = m_connection.CreateCommand(); + cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId "); + cmd.Transaction = transaction; + cmd.SetParameterValue("@CurrentFilesetId", filesetid); + cmd.SetParameterValue("@PreviousFilesetId", prevFilesetId); + await cmd.ExecuteNonQueryAsync(); + } + + /// + /// Gets the list of index files that reference a given block file + /// + /// The block file ID + /// >A list of index file IDs + public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(@"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@BlockFileId", blockfileid); + cmd.SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); + } + + /// + /// Gets a list of filesets that are missing files + /// + /// An optional transaction + /// A list of fileset IDs and timestamps + public async IAsyncEnumerable> GetFilesetsWithMissingFiles(SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + using var rd = await cmd.ExecuteReaderAsync(@"SELECT ID, Timestamp FROM Fileset WHERE ID IN (SELECT FilesetID FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup""))"); + while (await rd.ReadAsync()) + { + yield return new KeyValuePair( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() + ); + } + } + + /// + /// Deletes all fileset entries for a given fileset + /// + /// The fileset ID + /// An optional transaction + /// The number of deleted entries + public async Task DeleteFilesetEntries(long filesetid, SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId"); + cmd.SetParameterValue("@FilesetId", filesetid); + return await cmd.ExecuteNonQueryAsync(); + } + + private class RemoteVolume : IRemoteVolume + { + public string Name { get; private set; } + public string Hash { get; private set; } + public long Size { get; private set; } + + public RemoteVolume(string name, string hash, long size) + { + this.Name = name; + this.Hash = hash; + this.Size = size; + } + } + + public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName, SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) "); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Name", indexName); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); + } + + /// + /// A single block with source data for repair + /// + /// The block hash + /// The block size + /// The file that contains the block + /// The offset of the block in the file + public sealed record BlockWithSourceData(string Hash, long Size, string File, long Offset); + + /// + /// A single block with metadata source data for repair + /// + /// The block hash + /// The block size + /// The path of the file or directory that contains the block + public sealed record BlockWithMetadataSourceData(string Hash, long Size, string Path); + + /// + /// A single blocklist hash entry + /// + /// The blockset id + /// The hash of the blocklist entry (when done) + /// The total length of the blockset + /// The index of the blocklist hash + /// The index of the block in the blockset + /// The hash of the entry + public sealed record BlocklistHashesEntry( + long BlocksetId, + string BlocklistHash, + long BlocklistHashLength, + long BlocklistHashIndex, + int Index, + string Hash); + + /// + /// Helper interface for the missing block list + /// + public interface IMissingBlockList : IDisposable + { + /// + /// Registers a block as restored + /// + /// The block hash + /// The block size + /// The volume ID of the new target volume + /// The restored blocks + Task SetBlockRestored(string hash, long size, long volumeId); + /// + /// Gets the list of files that contains missing blocks + /// + /// The blocksize setting + /// A list of files with missing blocks + IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize); + /// + /// Gets a list for filesystem entries that contain missing blocks in metadata + /// + /// A list of filesystem entries with missing blocks + IAsyncEnumerable GetSourceItemsWithMetadataBlocks(); + /// + /// Gets missing blocklist hashes + /// + /// The number of hashes for each block + /// >A list of blocklist hashes + IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock); + /// + /// Gets the number of missing blocks + /// + /// >The number of missing blocks + Task GetMissingBlockCount(); + /// + /// Gets all the filesets that are affected by missing blocks + /// + /// >A list of filesets + IAsyncEnumerable GetFilesetsUsingMissingBlocks(); + /// + /// Gets a list of remote files that may contain missing blocks + /// + /// >A list of remote files + IAsyncEnumerable GetMissingBlockSources(); + } + + /// + /// Implementation of the missing block list + /// + private class MissingBlockList : IMissingBlockList + { + /// + /// The connection to the database + /// + private readonly SqliteConnection m_connection; + /// + /// The transaction to use + /// + private readonly ReusableTransaction m_transaction; + /// + /// The insert command to use for restoring blocks + /// + private SqliteCommand m_insertCommand = null!; + /// + /// The command to copy blocks into the duplicate block table + /// + private SqliteCommand m_copyIntoDuplicatedBlocks = null!; + /// + /// The command to assign blocks to a new volume + /// + private SqliteCommand m_assignBlocksToNewVolume = null!; + private SqliteCommand m_missingBlocksCommand = null!; + private SqliteCommand m_missingBlocksCountCommand = null!; + /// + /// The name of the temporary table + /// + private readonly string m_tablename; + /// + /// The name of the volume where blocks are missing + /// + private readonly string m_volumename; + // TODO remove + /// + /// The query to use for getting missing blocks + /// + //private readonly string m_missingBlocksQuery; + + /// + /// Whether the object has been disposed + /// + private bool m_isDisposed = false; + + /// + /// Creates a new missing block list + /// + /// The name of the volume with missing blocks + /// The connection to the database + /// The transaction to use + private MissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) + { + m_connection = connection; + m_transaction = transaction; + m_volumename = volumename; + var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + m_tablename = tablename; + } + + public static async Task CreateMissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) + { + var blocklist = new MissingBlockList(volumename, connection, transaction); + using (var cmd = connection.CreateCommand()) + { + cmd.SetTransaction(transaction.Transaction); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{blocklist.m_tablename}"" ( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Restored"" INTEGER NOT NULL + ) + "); + + cmd.SetCommandAndParameters($@" + INSERT INTO ""{blocklist.m_tablename}"" ( + ""Hash"", + ""Size"", + ""Restored"" + ) + SELECT DISTINCT + ""Block"".""Hash"", + ""Block"".""Size"", + 0 AS ""Restored"" FROM ""Block"", + ""Remotevolume"" + WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Remotevolume"".""Name"" = @Name + ") + .SetParameterValue("@Name", volumename); + var blockCount = await cmd.ExecuteNonQueryAsync(); + + if (blockCount == 0) + throw new Exception($"Unexpected empty block volume: {0}"); + + await cmd.ExecuteNonQueryAsync($@" + CREATE UNIQUE INDEX ""{blocklist.m_tablename}-Ix"" + ON ""{blocklist.m_tablename}"" ( + ""Hash"", + ""Size"", + ""Restored"" + ) + "); + } + + blocklist.m_insertCommand = await connection.CreateCommandAsync($@" + UPDATE ""{blocklist.m_tablename}"" + SET ""Restored"" = @NewRestoredValue + WHERE ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""Restored"" = @PreviousRestoredValue + "); + + blocklist.m_copyIntoDuplicatedBlocks = await connection.CreateCommandAsync(@" + INSERT OR IGNORE INTO ""DuplicateBlock"" ( + ""BlockID"", + ""VolumeID"" + ) + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""Block"" + WHERE ""Block"".""Hash"" = @Hash + AND ""Block"".""Size"" = @Size + "); + + blocklist.m_assignBlocksToNewVolume = await connection.CreateCommandAsync(@" + UPDATE ""Block"" + SET ""VolumeID"" = @TargetVolumeId + WHERE ""Hash"" = @Hash + AND ""Size"" = @Size + "); + + var m_missingBlocksQuery = $@" + SELECT + ""{blocklist.m_tablename}"".""Hash"", + ""{blocklist.m_tablename}"".""Size"" + FROM ""{blocklist.m_tablename}"" + WHERE ""{blocklist.m_tablename}"".""Restored"" = @Restored "; + blocklist.m_missingBlocksCommand = await connection.CreateCommandAsync(m_missingBlocksQuery); + + blocklist.m_missingBlocksCountCommand = await connection.CreateCommandAsync($@" + SELECT COUNT(*) + FROM ({m_missingBlocksQuery}) + "); + + return blocklist; + } + + /// + public async Task SetBlockRestored(string hash, long size, long targetVolumeId) + { + //m_insertCommand.SetTransaction(m_transaction.Transaction); + m_insertCommand.Transaction = m_transaction.Transaction; + m_insertCommand.SetParameterValue("@NewRestoredValue", 1); + m_insertCommand.SetParameterValue("@Hash", hash); + m_insertCommand.SetParameterValue("@Size", size); + m_insertCommand.SetParameterValue("@PreviousRestoredValue", 0); + var restored = await m_insertCommand.ExecuteNonQueryAsync() == 1; + + if (restored) + { + m_copyIntoDuplicatedBlocks.Transaction = m_transaction.Transaction; + m_copyIntoDuplicatedBlocks.SetParameterValue("@Hash", hash); + m_copyIntoDuplicatedBlocks.SetParameterValue("@Size", size); + await m_copyIntoDuplicatedBlocks.ExecuteNonQueryAsync(); + + m_assignBlocksToNewVolume.Transaction = m_transaction.Transaction; + m_assignBlocksToNewVolume.SetParameterValue("@TargetVolumeId", targetVolumeId); + m_assignBlocksToNewVolume.SetParameterValue("@Hash", hash); + m_assignBlocksToNewVolume.SetParameterValue("@Size", size); + var c = await m_assignBlocksToNewVolume.ExecuteNonQueryAsync(); + + if (c != 1) + throw new Exception($"Unexpected number of updated blocks: {c} != 1"); + } + + return restored; + } + + /// + public async IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize) + { + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}")); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); + var size = rd.ConvertValueToInt64(1); + var file = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); + var offset = rd.ConvertValueToInt64(3); + + yield return new BlockWithSourceData(hash, size, file, offset); + } + } + + /// + public async IAsyncEnumerable GetSourceItemsWithMetadataBlocks() + { + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path""")); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); + var size = rd.ConvertValueToInt64(1); + var path = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); + + yield return new BlockWithMetadataSourceData(hash, size, path); + } + } + + /// + public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock) + { + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_transaction.Transaction; + var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; + try + { + // We need to create a snapshot as we will be updating the m_tablename table during enumeration + cmd.SetCommandAndParameters(FormatInvariant($@" + CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS + SELECT + b.""Hash"" AS ""BlockHash"", + bs.""Id"" AS ""BlocksetId"", + bs.""Length"" AS ""BlocksetLength"", + bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", + ( + SELECT blh.""Hash"" + FROM ""BlocklistHash"" blh + WHERE blh.""BlocksetID"" = bs.""ID"" + AND blh.""Index"" == bse.""Index"" / @HashesPerBlock + LIMIT 1 + ) AS ""BlocklistHashHash"", + bse.""Index"" AS ""BlocksetEntryIndex"" + FROM + ""BlocksetEntry"" bse + JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" + JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" + WHERE + EXISTS ( + SELECT 1 + FROM ""BlocklistHash"" blh + JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash"" + WHERE blh.""BlocksetID"" = bs.""ID"" + AND mt.""Restored"" = @Restored + ) + ")); + cmd.SetParameterValue("@HashesPerBlock", hashesPerBlock); + cmd.SetParameterValue("@Restored", 0); + await cmd.ExecuteNonQueryAsync(); + + cmd.SetCommandAndParameters(FormatInvariant($@" + SELECT + ""BlockHash"", + ""BlocksetId"", + ""BlocksetLength"", + ""BlocklistHashIndex"", + ""BlocklistHashHash"", + ""BlocksetEntryIndex"" + FROM ""{blocklistTableName}"" + ORDER BY ""BlocksetId"", ""BlocklistHashIndex"", ""BlocksetEntryIndex""")); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); + var blocksetId = rd.ConvertValueToInt64(1); + var length = rd.ConvertValueToInt64(2); + var blocklistHashIndex = rd.ConvertValueToInt64(3); + var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); + var index = rd.ConvertValueToInt64(5); + + yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash); + } + } + finally + { + try { await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); } + catch { } + } + } + + /// + public async Task GetMissingBlockCount() + { + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})")); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); + + return await cmd.ExecuteScalarInt64Async(0); + } + + /// + /// Gets the list of missing blocks + /// + /// >A list of missing blocks + public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks() + { + using var cmd = m_connection.CreateCommand(m_missingBlocksQuery); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + } + + /// + public async Task MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId) + { + if (targetVolumeId <= 0) + throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); + + // Move the source blocks into the DuplicateBlock table + using var cmd = m_connection.CreateCommand(FormatInvariant($@" + INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") + SELECT b.""ID"", b.""VolumeID"" + FROM ""Block"" b + WHERE b.""VolumeID"" = @SourceVolumeId + AND (b.""Hash"", b.""Size"") IN ( + SELECT ""Hash"", ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ")); + cmd.SetParameterValue("@SourceVolumeId", sourceVolumeId); + cmd.SetParameterValue("@Restored", 1); + + var moved = await cmd.ExecuteNonQueryAsync(); + + // Then update the blocks table to point to the new volume + cmd.SetCommandAndParameters(FormatInvariant($@" + UPDATE ""Block"" + SET ""VolumeID"" = @TargetVolumeId + WHERE ""VolumeID"" = @SourceVolumeId + AND (""Hash"", ""Size"") IN ( + SELECT ""Hash"", ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ")); + cmd.SetParameterValue("@TargetVolumeId", targetVolumeId); + cmd.SetParameterValue("@SourceVolumeId", sourceVolumeId); + cmd.SetParameterValue("@Restored", 1); + + var updated = await cmd.ExecuteNonQueryAsync(); + if (updated != moved) + throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); + + return updated; + } + + public async IAsyncEnumerable GetFilesetsUsingMissingBlocks() + { + var blocks = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); + var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); + var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = @Type AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))"); + + using var cmd = m_connection.CreateCommand(cmdtxt); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); + } + + public async IAsyncEnumerable GetMissingBlockSources() + { + using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")); + cmd.Transaction = m_transaction.Transaction; + cmd.SetParameterValue("@Restored", 0); + cmd.SetParameterValue("@Name", m_volumename); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); + } + + public void Dispose() + { + if (m_isDisposed) + return; + + m_isDisposed = true; + try + { + if (m_tablename != null) + { + + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_transaction.Transaction; + cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")).Await(); + } + } + catch { } + + try { m_insertCommand?.Dispose(); } + catch { } + } + } + + public IMissingBlockList CreateBlockList(string volumename, ReusableTransaction transaction) + { + return new MissingBlockList(volumename, m_connection, transaction); + } + + public async Task FixDuplicateMetahash() + { + using var tr = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + + cmd.Transaction = tr; + var sql_count = + @"SELECT COUNT(*) FROM (" + + @" SELECT DISTINCT c1 FROM (" + + @"SELECT COUNT(*) AS ""C1"" FROM (SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") UNION SELECT COUNT(*) AS ""C1"" FROM ""Metadataset"" " + + @")" + + @")"; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 1) + { + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashes", "Found duplicate metadatahashes, repairing"); + + var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); + + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); + + var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; + + using (var c2 = m_connection.CreateCommand()) + { + c2.Transaction = tr; + c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""MetadataID"" = @MetadataId WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId)") + + @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId AND ""ID"" != @MetadataId"); + using (var rd = await cmd.ExecuteReaderAsync(sql)) + while (await rd.ReadAsync()) + { + c2.SetParameterValue("@MetadataId", rd.GetValue(0)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(1)); + await c2.ExecuteNonQueryAsync(); + } + } + + sql = FormatInvariant($@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( + SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") + WHERE ""Entries"" > 1 ORDER BY ""ID"""); + + using (var c2 = m_connection.CreateCommand()) + { + c2.Transaction = tr; + c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId)") + + FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId")); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql)) + { + c2.SetParameterValue("@FileId", rd.GetValue(0)); + c2.SetParameterValue("@Path", rd.GetValue(1)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(2)); + c2.SetParameterValue("@MetadataId", rd.GetValue(3)); + await c2.ExecuteNonQueryAsync(); + } + } + + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); + + x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 1) + throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); + await tr.CommitAsync(); + } + } + + public async Task FixDuplicateFileentries() + { + using var tr = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + + cmd.Transaction = tr; + var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 0) + { + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); + + var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( + SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") + WHERE ""Entries"" > 1 ORDER BY ""ID"""; + + using (var c2 = m_connection.CreateCommand()) + { + c2.Transaction = tr; + c2.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetatadataId)" + + @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId"); + cmd.SetCommandAndParameters(sql); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + { + c2.SetParameterValue("@FileId", rd.GetValue(0)); + c2.SetParameterValue("@PrefixId", rd.GetValue(1)); + c2.SetParameterValue("@Path", rd.GetValue(2)); + c2.SetParameterValue("@BlocksetId", rd.GetValue(3)); + c2.SetParameterValue("@MetadataId", rd.GetValue(4)); + await c2.ExecuteNonQueryAsync(); + } + } + + x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 1) + throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); + await tr.CommitAsync(); + } + + } + + public async Task FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize) + { + var blocklistbuffer = new byte[blocksize]; + + using var tr = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + using var blockhasher = HashFactory.CreateHasher(blockhashalgorithm); + + cmd.Transaction = tr; + var hashsize = blockhasher.HashSize / 8; + + var sql = FormatInvariant($@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"""); + var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; + + var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); + if (itemswithnoblocklisthash != 0) + { + Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); + using (var c2 = m_connection.CreateCommand()) + using (var c3 = m_connection.CreateCommand()) + using (var c4 = m_connection.CreateCommand()) + using (var c5 = m_connection.CreateCommand()) + using (var c6 = m_connection.CreateCommand()) + { + c2.Transaction = tr; + c3.Transaction = tr; + c4.Transaction = tr; + c5.Transaction = tr; + c6.Transaction = tr; + c3.SetCommandAndParameters(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash) "); + c4.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + c5.SetCommandAndParameters(@"SELECT ""ID"" FROM ""DeletedBlock"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND (""State"" IN (@State1, @State2)))"); + c6.SetCommandAndParameters(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1; DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId;"); + + await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql)) + { + var blocksetid = e.ConvertValueToInt64(0); + var ix = 0L; + int blocklistoffset = 0; + + c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId"); + c2.SetParameterValue("@BlocksetId", blocksetid); + await c2.ExecuteNonQueryAsync(); + + c2.SetCommandAndParameters(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = @BlocksetId ORDER BY ""B"".""Index"""); + c2.SetParameterValue("@BlocksetId", blocksetid); + + await foreach (var h in c2.ExecuteReaderEnumerableAsync()) + { + var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); + if (blocklistbuffer.Length - blocklistoffset < tmp.Length) + { + var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); + + // Check if the block exists in "blocks" + c4.SetParameterValue("@Hash", blkey); + c4.SetParameterValue("@Size", blocklistoffset); + var existingBlockId = await c4.ExecuteScalarInt64Async(-1); + + if (existingBlockId <= 0) + { + c5.SetParameterValue("@Hash", blkey); + c5.SetParameterValue("@Size", blocklistoffset); + c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); + c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); + var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); + + if (deletedBlockId <= 0) + throw new Exception($"Missing block for blocklisthash: {blkey}"); + else + { + c6.SetParameterValue("@DeletedBlockId", deletedBlockId); + var rc = await c6.ExecuteNonQueryAsync(); + + if (rc != 2) + throw new Exception($"Unexpected update count: {rc}"); + } + } + + // Add to table + c3.SetParameterValue("@BlocksetId", blocksetid); + c3.SetParameterValue("@Index", ix); + c3.SetParameterValue("@Hash", blkey); + await c3.ExecuteNonQueryAsync(); + + ix++; + blocklistoffset = 0; + } + + Array.Copy(tmp, 0, blocklistbuffer, blocklistoffset, tmp.Length); + blocklistoffset += tmp.Length; + + } + + if (blocklistoffset != 0) + { + var blkeyfinal = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); + + // Ensure that the block exists in "blocks" + c4.SetParameterValue("@Hash", blkeyfinal); + c4.SetParameterValue("@Size", blocklistoffset); + var existingBlockId = await c4.ExecuteScalarInt64Async(-1); + + if (existingBlockId <= 0) + { + c5.SetParameterValue("@Hash", blkeyfinal); + c5.SetParameterValue("@Size", blocklistoffset); + c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); + c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); + var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); + + if (deletedBlockId == 0) + throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); + else + { + c6.SetParameterValue("@DeletedBlockId", deletedBlockId); + var rc = await c6.ExecuteNonQueryAsync(); + if (rc != 2) + throw new Exception($"Unexpected update count: {rc}"); + } + } + + // Add to table + c3.SetParameterValue("@BlocksetId", blocksetid); + c3.SetParameterValue("@Index", ix); + c3.SetParameterValue("@Hash", blkeyfinal); + await c3.ExecuteNonQueryAsync(); + } + } + } + + itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); + if (itemswithnoblocklisthash != 0) + throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); + tr.Commit(); + } + + } + + public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize) + { + using var tr = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(); + + cmd.Transaction = tr; + var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1"; + + var sql_count = @"SELECT COUNT(*) FROM (" + dup_sql + ")"; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 0) + { + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashes", "Found duplicate blocklisthash entries, repairing"); + + var unique_count = await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); + + using (var c2 = m_connection.CreateCommand()) + { + c2.Transaction = tr; + c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId AND ""Index"" = @Index LIMIT @Limit)"); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql)) + { + var expected = rd.GetInt32(2) - 1; + c2.SetParameterValue("@BlocksetId", rd.GetValue(0)); + c2.SetParameterValue("@Index", rd.GetValue(1)); + c2.SetParameterValue("@Limit", expected); + var actual = await c2.ExecuteNonQueryAsync(); + + if (actual != expected) + throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); + } + } + + x = await cmd.ExecuteScalarInt64Async(sql_count); + if (x > 1) + throw new Exception("Repair failed, there are still duplicate file entries!"); + + var real_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM ""BlocklistHash""", 0); + + if (real_count != unique_count) + throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); + + try + { + await VerifyConsistency(blocksize, hashsize, true, tr); + } + catch (Exception ex) + { + throw new Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); + } + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); + await tr.CommitAsync(); + } + } + + public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(); + cmd.Transaction = transaction; + var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + try + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); + cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); + + foreach (var kp in blocks) + { + cmd.SetParameterValue("@Hash", kp.Key); + cmd.SetParameterValue("@Size", kp.Value); + await cmd.ExecuteNonQueryAsync(); + } + + cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name"); + cmd.SetParameterValue("@Name", filename); + var id = await cmd.ExecuteScalarInt64Async(-1); + + cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{tablename}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != @VolumeId ")); + cmd.SetParameterValue("@VolumeId", id); + var aliens = await cmd.ExecuteScalarInt64Async(0); + + if (aliens != 0) + throw new Exception($"Not all blocks were found in {filename}"); + } + finally + { + await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); + } + } + + public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(); + + cmd.Transaction = transaction; + var query = FormatInvariant($@" +SELECT + ""C"".""Hash"", + ""C"".""Size"" +FROM + ""BlocksetEntry"" A, + ( + SELECT + ""Y"".""BlocksetID"", + ""Y"".""Hash"" AS ""BlocklistHash"", + ""Y"".""Index"" AS ""BlocklistHashIndex"", + ""Z"".""Size"" AS ""BlocklistSize"", + ""Z"".""ID"" AS ""BlocklistHashBlockID"" + FROM + ""BlocklistHash"" Y, + ""Block"" Z + WHERE + ""Y"".""Hash"" = ""Z"".""Hash"" AND ""Y"".""Hash"" = @Hash AND ""Z"".""Size"" = @Size + LIMIT 1 + ) B, + ""Block"" C +WHERE + ""A"".""BlocksetID"" = ""B"".""BlocksetID"" + AND + ""A"".""BlockID"" = ""C"".""ID"" + AND + ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({blocksize} / {blockhashlength}) + AND + ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({blocksize} / {blockhashlength}) +ORDER BY + ""A"".""Index"" + +"); + + using var en = blocklist.GetEnumerator(); + + cmd.SetCommandAndParameters(query); + cmd.SetParameterValue("@Hash", hash); + cmd.SetParameterValue("@Size", length); + await foreach (var r in cmd.ExecuteReaderEnumerableAsync()) + { + if (!en.MoveNext()) + throw new Exception($"Too few entries in source blocklist with hash {hash}"); + if (en.Current != r.ConvertValueToString(0)) + throw new Exception($"Mismatch in blocklist with hash {hash}"); + } + + if (en.MoveNext()) + throw new Exception($"Too many source blocklist entries in {hash}"); + } + + public async IAsyncEnumerable MissingLocalFilesets(SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + yield return rd.ConvertValueToString(0) ?? ""; + } + + public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets(SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(@"SELECT ""ID"", ""Timestamp"", ""IsFullBackup"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + yield return (rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP); + } + + public async IAsyncEnumerable EmptyIndexFiles(SqliteTransaction transaction) + { + using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ID"" NOT IN (SELECT ""IndexVolumeId"" FROM ""IndexBlockLink"")"); + cmd.Transaction = transaction; + cmd.SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); + cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); + } + } +} + From ba5f9e8a3d537c1e48acb15d4362f03b8e4f852a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 10:37:13 +0200 Subject: [PATCH 051/362] Restructured SQL statements in LocalRepairDatabase as well as changing calls to SqliteCommands to use a chain of method calls rather than a sequence of statements --- .../Main/Database/LocalRepairDatabase.cs | 1274 ++++++++++++----- 1 file changed, 931 insertions(+), 343 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index b7ce0fdea..308e9b066 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -56,18 +56,32 @@ namespace Duplicati.Library.Main.Database /// The remote name of the fileset /// An optional transaction /// >The fileset ID - public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist, SqliteTransaction? transaction) + public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name"); - cmd.SetParameterValue("@Name", filelist); + using var cmd = m_connection.CreateCommand(@" + SELECT + ""Fileset"".""ID"", + ""Fileset"".""Timestamp"", + ""Fileset"".""IsFullBackup"" + FROM + ""Fileset"", + ""RemoteVolume"" + WHERE + ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" + AND ""RemoteVolume"".""Name"" = @Name + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@Name", filelist); var rd = await cmd.ExecuteReaderAsync(); if (!await rd.ReadAsync()) throw new Exception($"No such remote file: {filelist}"); - return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); + return ( + rd.ConvertValueToInt64(0, -1), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), + rd.GetInt32(2) == BackupType.FULL_BACKUP + ); } /// @@ -76,7 +90,7 @@ namespace Duplicati.Library.Main.Database /// Current fileset ID /// Source fileset ID /// The transaction - public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId, SqliteTransaction transaction) + public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId) { if (filesetid <= 0) throw new ArgumentException("filesetid must be > 0"); @@ -84,11 +98,15 @@ namespace Duplicati.Library.Main.Database if (prevFilesetId <= 0) throw new ArgumentException("prevId must be > 0"); - using var cmd = m_connection.CreateCommand(); - cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId "); - cmd.Transaction = transaction; - cmd.SetParameterValue("@CurrentFilesetId", filesetid); - cmd.SetParameterValue("@PreviousFilesetId", prevFilesetId); + using var cmd = m_connection.CreateCommand(@" + UPDATE ""FilesetEntry"" + SET ""FilesetID"" = @CurrentFilesetId + WHERE ""FilesetID"" = @PreviousFilesetId + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@CurrentFilesetId", filesetid) + .SetParameterValue("@PreviousFilesetId", prevFilesetId); + await cmd.ExecuteNonQueryAsync(); } @@ -97,12 +115,21 @@ namespace Duplicati.Library.Main.Database /// /// The block file ID /// >A list of index file IDs - public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, SqliteTransaction transaction) + public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid) { - using var cmd = m_connection.CreateCommand(@"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@BlockFileId", blockfileid); - cmd.SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); + using var cmd = m_connection.CreateCommand(@" + SELECT ""RemoteVolume"".""Name"" + FROM + ""RemoteVolume"", + ""IndexBlockLink"" + WHERE + ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId + AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" + AND ""RemoteVolume"".""Type"" = @Type + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@BlockFileId", blockfileid) + .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); @@ -113,11 +140,25 @@ namespace Duplicati.Library.Main.Database /// /// An optional transaction /// A list of fileset IDs and timestamps - public async IAsyncEnumerable> GetFilesetsWithMissingFiles(SqliteTransaction transaction) + public async IAsyncEnumerable> GetFilesetsWithMissingFiles() { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - using var rd = await cmd.ExecuteReaderAsync(@"SELECT ID, Timestamp FROM Fileset WHERE ID IN (SELECT FilesetID FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup""))"); + using var cmd = m_connection.CreateCommand(@" + SELECT + ID, + Timestamp + FROM Fileset + WHERE ID IN ( + SELECT FilesetID + FROM ""FilesetEntry"" + WHERE ""FileID"" NOT IN ( + SELECT ""ID"" + FROM ""FileLookup"" + ) + ) + ") + .SetTransaction(m_rtr.Transaction); + + using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) { yield return new KeyValuePair( @@ -133,12 +174,15 @@ namespace Duplicati.Library.Main.Database /// The fileset ID /// An optional transaction /// The number of deleted entries - public async Task DeleteFilesetEntries(long filesetid, SqliteTransaction transaction) + public async Task DeleteFilesetEntries(long filesetid) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; - cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId"); - cmd.SetParameterValue("@FilesetId", filesetid); + using var cmd = m_connection.CreateCommand(@" + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@FilesetId", filesetid); + return await cmd.ExecuteNonQueryAsync(); } @@ -156,14 +200,33 @@ namespace Duplicati.Library.Main.Database } } - public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName, SqliteTransaction transaction) + public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName) { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) "); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Name", indexName); + using var cmd = m_connection.CreateCommand(@" + SELECT + ""Name"", + ""Hash"", + ""Size"" + FROM ""RemoteVolume"" + WHERE ""ID"" IN ( + SELECT ""BlockVolumeID"" + FROM ""IndexBlockLink"" + WHERE ""IndexVolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" = @Name + ) + ) + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@Name", indexName); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "", + rd.ConvertValueToInt64(2) + ); } /// @@ -311,9 +374,8 @@ namespace Duplicati.Library.Main.Database public static async Task CreateMissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) { var blocklist = new MissingBlockList(volumename, connection, transaction); - using (var cmd = connection.CreateCommand()) + using (var cmd = connection.CreateCommand(transaction.Transaction)) { - cmd.SetTransaction(transaction.Transaction); await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{blocklist.m_tablename}"" ( ""Hash"" TEXT NOT NULL, @@ -333,8 +395,9 @@ namespace Duplicati.Library.Main.Database ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"", ""Remotevolume"" - WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" - AND ""Remotevolume"".""Name"" = @Name + WHERE + ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Remotevolume"".""Name"" = @Name ") .SetParameterValue("@Name", volumename); var blockCount = await cmd.ExecuteNonQueryAsync(); @@ -355,9 +418,10 @@ namespace Duplicati.Library.Main.Database blocklist.m_insertCommand = await connection.CreateCommandAsync($@" UPDATE ""{blocklist.m_tablename}"" SET ""Restored"" = @NewRestoredValue - WHERE ""Hash"" = @Hash - AND ""Size"" = @Size - AND ""Restored"" = @PreviousRestoredValue + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""Restored"" = @PreviousRestoredValue "); blocklist.m_copyIntoDuplicatedBlocks = await connection.CreateCommandAsync(@" @@ -369,15 +433,17 @@ namespace Duplicati.Library.Main.Database ""Block"".""ID"", ""Block"".""VolumeID"" FROM ""Block"" - WHERE ""Block"".""Hash"" = @Hash - AND ""Block"".""Size"" = @Size + WHERE + ""Block"".""Hash"" = @Hash + AND ""Block"".""Size"" = @Size "); blocklist.m_assignBlocksToNewVolume = await connection.CreateCommandAsync(@" UPDATE ""Block"" SET ""VolumeID"" = @TargetVolumeId - WHERE ""Hash"" = @Hash - AND ""Size"" = @Size + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size "); var m_missingBlocksQuery = $@" @@ -399,26 +465,25 @@ namespace Duplicati.Library.Main.Database /// public async Task SetBlockRestored(string hash, long size, long targetVolumeId) { - //m_insertCommand.SetTransaction(m_transaction.Transaction); - m_insertCommand.Transaction = m_transaction.Transaction; - m_insertCommand.SetParameterValue("@NewRestoredValue", 1); - m_insertCommand.SetParameterValue("@Hash", hash); - m_insertCommand.SetParameterValue("@Size", size); - m_insertCommand.SetParameterValue("@PreviousRestoredValue", 0); - var restored = await m_insertCommand.ExecuteNonQueryAsync() == 1; + var restored = await m_insertCommand.SetTransaction(m_transaction.Transaction) + .SetParameterValue("@NewRestoredValue", 1) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .SetParameterValue("@PreviousRestoredValue", 0) + .ExecuteNonQueryAsync() == 1; if (restored) { - m_copyIntoDuplicatedBlocks.Transaction = m_transaction.Transaction; - m_copyIntoDuplicatedBlocks.SetParameterValue("@Hash", hash); - m_copyIntoDuplicatedBlocks.SetParameterValue("@Size", size); - await m_copyIntoDuplicatedBlocks.ExecuteNonQueryAsync(); + await m_copyIntoDuplicatedBlocks.SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(); - m_assignBlocksToNewVolume.Transaction = m_transaction.Transaction; - m_assignBlocksToNewVolume.SetParameterValue("@TargetVolumeId", targetVolumeId); - m_assignBlocksToNewVolume.SetParameterValue("@Hash", hash); - m_assignBlocksToNewVolume.SetParameterValue("@Size", size); - var c = await m_assignBlocksToNewVolume.ExecuteNonQueryAsync(); + var c = await m_assignBlocksToNewVolume.SetTransaction(m_transaction.Transaction) + .SetParameterValue("@TargetVolumeId", targetVolumeId) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Unexpected number of updated blocks: {c} != 1"); @@ -430,9 +495,31 @@ namespace Duplicati.Library.Main.Database /// public async IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize) { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}")); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); + using var cmd = m_connection.CreateCommand($@" + SELECT DISTINCT + ""{m_tablename}"".""Hash"", + ""{m_tablename}"".""Size"", + ""File"".""Path"", + ""BlocksetEntry"".""Index"" * {blocksize} + FROM + ""{m_tablename}"", + ""Block"", + ""BlocksetEntry"", + ""File"" + WHERE + ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" + AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" + AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" + AND ""{m_tablename}"".""Restored"" = @Restored + ORDER BY + ""{m_tablename}"".""Hash"", + ""{m_tablename}"".""Size"", + ""File"".""Path"", + ""BlocksetEntry"".""Index"" * {blocksize} + ") + .SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Restored", 0); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { @@ -448,9 +535,31 @@ namespace Duplicati.Library.Main.Database /// public async IAsyncEnumerable GetSourceItemsWithMetadataBlocks() { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path""")); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); + using var cmd = m_connection.CreateCommand($@" + SELECT DISTINCT + ""{m_tablename}"".""Hash"", + ""{m_tablename}"".""Size"", + ""File"".""Path"" + FROM + ""{m_tablename}"", + ""Block"", + ""BlocksetEntry"", + ""Metadataset"", + ""File"" + WHERE + ""File"".""MetadataID"" == ""Metadataset"".""ID"" + AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" + AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" + AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" + AND ""{m_tablename}"".""Restored"" = @Restored + ORDER BY + ""{m_tablename}"".""Hash"", + ""{m_tablename}"".""Size"", + ""File"".""Path"" + ") + .SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Restored", 0); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { @@ -465,13 +574,14 @@ namespace Duplicati.Library.Main.Database /// public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_transaction.Transaction; + using var cmd = m_connection.CreateCommand(m_transaction.Transaction); + var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; + try { // We need to create a snapshot as we will be updating the m_tablename table during enumeration - cmd.SetCommandAndParameters(FormatInvariant($@" + await cmd.SetCommandAndParameters($@" CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS SELECT b.""Hash"" AS ""BlockHash"", @@ -481,29 +591,32 @@ namespace Duplicati.Library.Main.Database ( SELECT blh.""Hash"" FROM ""BlocklistHash"" blh - WHERE blh.""BlocksetID"" = bs.""ID"" - AND blh.""Index"" == bse.""Index"" / @HashesPerBlock + WHERE + blh.""BlocksetID"" = bs.""ID"" + AND blh.""Index"" == bse.""Index"" / @HashesPerBlock LIMIT 1 ) AS ""BlocklistHashHash"", bse.""Index"" AS ""BlocksetEntryIndex"" - FROM - ""BlocksetEntry"" bse - JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" - JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" - WHERE - EXISTS ( - SELECT 1 - FROM ""BlocklistHash"" blh - JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash"" - WHERE blh.""BlocksetID"" = bs.""ID"" + FROM ""BlocksetEntry"" bse + JOIN ""Block"" b + ON b.""ID"" = bse.""BlockID"" + JOIN ""Blockset"" bs + ON bs.""ID"" = bse.""BlocksetID"" + WHERE EXISTS ( + SELECT 1 + FROM ""BlocklistHash"" blh + JOIN ""{m_tablename}"" mt + ON mt.""Hash"" = blh.""Hash"" + WHERE + blh.""BlocksetID"" = bs.""ID"" AND mt.""Restored"" = @Restored - ) - ")); - cmd.SetParameterValue("@HashesPerBlock", hashesPerBlock); - cmd.SetParameterValue("@Restored", 0); - await cmd.ExecuteNonQueryAsync(); + ) + ") + .SetParameterValue("@HashesPerBlock", hashesPerBlock) + .SetParameterValue("@Restored", 0) + .ExecuteNonQueryAsync(); - cmd.SetCommandAndParameters(FormatInvariant($@" + cmd.SetCommandAndParameters($@" SELECT ""BlockHash"", ""BlocksetId"", @@ -512,7 +625,11 @@ namespace Duplicati.Library.Main.Database ""BlocklistHashHash"", ""BlocksetEntryIndex"" FROM ""{blocklistTableName}"" - ORDER BY ""BlocksetId"", ""BlocklistHashIndex"", ""BlocksetEntryIndex""")); + ORDER BY + ""BlocksetId"", + ""BlocklistHashIndex"", + ""BlocksetEntryIndex"" + "); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { @@ -523,12 +640,24 @@ namespace Duplicati.Library.Main.Database var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); var index = rd.ConvertValueToInt64(5); - yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash); + yield return new BlocklistHashesEntry( + blocksetId, + blocklistHash, + length, + blocklistHashIndex, + (int)index, + hash + ); } } finally { - try { await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); } + try + { + await cmd.ExecuteNonQueryAsync($@" + DROP TABLE IF EXISTS ""{blocklistTableName}"" + "); + } catch { } } } @@ -536,11 +665,10 @@ namespace Duplicati.Library.Main.Database /// public async Task GetMissingBlockCount() { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})")); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); - - return await cmd.ExecuteScalarInt64Async(0); + return await m_missingBlocksCountCommand + .SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Restored", 0) + .ExecuteScalarInt64Async(0); } /// @@ -549,11 +677,11 @@ namespace Duplicati.Library.Main.Database /// >A list of missing blocks public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks() { - using var cmd = m_connection.CreateCommand(m_missingBlocksQuery); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); + m_missingBlocksCommand + .SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Restored", 0); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in m_missingBlocksCommand.ExecuteReaderEnumerableAsync()) yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } @@ -564,38 +692,49 @@ namespace Duplicati.Library.Main.Database throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); // Move the source blocks into the DuplicateBlock table - using var cmd = m_connection.CreateCommand(FormatInvariant($@" - INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") - SELECT b.""ID"", b.""VolumeID"" - FROM ""Block"" b - WHERE b.""VolumeID"" = @SourceVolumeId - AND (b.""Hash"", b.""Size"") IN ( - SELECT ""Hash"", ""Size"" - FROM ""{m_tablename}"" - WHERE ""Restored"" = @Restored + using var cmd = m_connection.CreateCommand($@" + INSERT OR IGNORE INTO ""DuplicateBlock"" ( + ""BlockID"", + ""VolumeID"" ) - ")); - cmd.SetParameterValue("@SourceVolumeId", sourceVolumeId); - cmd.SetParameterValue("@Restored", 1); + SELECT + b.""ID"", + b.""VolumeID"" + FROM ""Block"" b + WHERE + b.""VolumeID"" = @SourceVolumeId + AND (b.""Hash"", b.""Size"") IN ( + SELECT + ""Hash"", + ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ") + .SetParameterValue("@SourceVolumeId", sourceVolumeId) + .SetParameterValue("@Restored", 1); var moved = await cmd.ExecuteNonQueryAsync(); // Then update the blocks table to point to the new volume - cmd.SetCommandAndParameters(FormatInvariant($@" + var updated = await cmd.SetCommandAndParameters($@" UPDATE ""Block"" SET ""VolumeID"" = @TargetVolumeId - WHERE ""VolumeID"" = @SourceVolumeId - AND (""Hash"", ""Size"") IN ( - SELECT ""Hash"", ""Size"" - FROM ""{m_tablename}"" - WHERE ""Restored"" = @Restored - ) - ")); - cmd.SetParameterValue("@TargetVolumeId", targetVolumeId); - cmd.SetParameterValue("@SourceVolumeId", sourceVolumeId); - cmd.SetParameterValue("@Restored", 1); + WHERE + ""VolumeID"" = @SourceVolumeId + AND (""Hash"", ""Size"") IN ( + SELECT + ""Hash"", + ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ") + .SetParameterValue("@TargetVolumeId", targetVolumeId) + .SetParameterValue("@SourceVolumeId", sourceVolumeId) + .SetParameterValue("@Restored", 1) + .ExecuteNonQueryAsync(); - var updated = await cmd.ExecuteNonQueryAsync(); if (updated != moved) throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); @@ -604,30 +743,108 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable GetFilesetsUsingMissingBlocks() { - var blocks = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); - var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); - var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = @Type AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))"); + var blocks = $@" + SELECT DISTINCT ""FileLookup"".""ID"" AS ID + FROM + ""{m_tablename}"", + ""Block"", + ""Blockset"", + ""BlocksetEntry"", + ""FileLookup"" + WHERE + ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" + AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" + "; + var blocklists = $@" + SELECT DISTINCT ""FileLookup"".""ID"" AS ID + FROM + ""{m_tablename}"", + ""Block"", + ""Blockset"", + ""BlocklistHash"", + ""FileLookup"" + WHERE + ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" + AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" + AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" + AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" + "; + var cmdtxt = $@" + SELECT DISTINCT + ""RemoteVolume"".""Name"", + ""RemoteVolume"".""Hash"", + ""RemoteVolume"".""Size"" + FROM + ""RemoteVolume"", + ""FilesetEntry"", + ""Fileset"" + WHERE + ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + AND ""RemoteVolume"".""Type"" = @Type + AND ""FilesetEntry"".""FileID"" IN ( + SELECT DISTINCT ""ID"" + FROM ( + {blocks} UNION {blocklists} + ) + ) + "; - using var cmd = m_connection.CreateCommand(cmdtxt); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + using var cmd = m_connection.CreateCommand(cmdtxt) + .SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "", + rd.ConvertValueToInt64(2) + ); } public async IAsyncEnumerable GetMissingBlockSources() { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")); - cmd.Transaction = m_transaction.Transaction; - cmd.SetParameterValue("@Restored", 0); - cmd.SetParameterValue("@Name", m_volumename); + using var cmd = m_connection.CreateCommand($@" + SELECT DISTINCT + ""RemoteVolume"".""Name"", + ""RemoteVolume"".""Hash"", + ""RemoteVolume"".""Size"" + FROM + ""RemoteVolume"", + ""Block"", + ""{m_tablename}"" + WHERE + ""{m_tablename}"".""Restored"" = @Restored + AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" + AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" + AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" + AND ""Remotevolume"".""Name"" != @Name + ") + .SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Restored", 0) + .SetParameterValue("@Name", m_volumename); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "", + rd.ConvertValueToInt64(2) + ); } public void Dispose() + { + if (m_isDisposed) + return; + + DisposeAsync().Await(); + } + + public async Task DisposeAsync() { if (m_isDisposed) return; @@ -638,9 +855,9 @@ namespace Duplicati.Library.Main.Database if (m_tablename != null) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_transaction.Transaction; - cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")).Await(); + using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}"""); + await cmd.SetTransaction(m_transaction.Transaction) + .ExecuteNonQueryAsync(); } } catch { } @@ -650,124 +867,279 @@ namespace Duplicati.Library.Main.Database } } - public IMissingBlockList CreateBlockList(string volumename, ReusableTransaction transaction) + public async Task CreateBlockList(string volumename) { - return new MissingBlockList(volumename, m_connection, transaction); + return await MissingBlockList.CreateMissingBlockList(volumename, m_connection, m_rtr); } public async Task FixDuplicateMetahash() { - using var tr = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); + using var cmd = m_connection.CreateCommand(m_rtr.Transaction); - cmd.Transaction = tr; - var sql_count = - @"SELECT COUNT(*) FROM (" + - @" SELECT DISTINCT c1 FROM (" + - @"SELECT COUNT(*) AS ""C1"" FROM (SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") UNION SELECT COUNT(*) AS ""C1"" FROM ""Metadataset"" " + - @")" + - @")"; + var sql_count = @" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT c1 + FROM ( + SELECT COUNT(*) AS ""C1"" + FROM ( + SELECT DISTINCT ""BlocksetID"" + FROM ""Metadataset"" + ) + UNION SELECT COUNT(*) AS ""C1"" + FROM ""Metadataset"" + ) + ) + "; var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 1) { Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashes", "Found duplicate metadatahashes, repairing"); var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tablename}"" + AS SELECT * FROM ""File"" + "); - var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; + var sql = @" + SELECT + ""A"".""ID"", + ""B"".""BlocksetID"" + FROM ( + SELECT + MIN(""ID"") AS ""ID"", + COUNT(""ID"") AS ""Duplicates"" + FROM ""Metadataset"" + GROUP BY ""BlocksetID"" + ) ""A"", + ""Metadataset"" ""B"" + WHERE + ""A"".""Duplicates"" > 1 + AND ""A"".""ID"" = ""B"".""ID"" + "; - using (var c2 = m_connection.CreateCommand()) + using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) { - c2.Transaction = tr; - c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""MetadataID"" = @MetadataId WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId)") - + @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId AND ""ID"" != @MetadataId"); - using (var rd = await cmd.ExecuteReaderAsync(sql)) - while (await rd.ReadAsync()) - { - c2.SetParameterValue("@MetadataId", rd.GetValue(0)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(1)); - await c2.ExecuteNonQueryAsync(); - } - } + c2.SetCommandAndParameters($@" + UPDATE ""{tablename}"" + SET ""MetadataID"" = @MetadataId + WHERE ""MetadataID"" IN ( + SELECT ""ID"" + FROM ""Metadataset"" + WHERE ""BlocksetID"" = @BlocksetId + ); + DELETE FROM ""Metadataset"" + WHERE + ""BlocksetID"" = @BlocksetId + AND ""ID"" != @MetadataId + "); - sql = FormatInvariant($@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") - WHERE ""Entries"" > 1 ORDER BY ""ID"""); + using var rd = await cmd.ExecuteReaderAsync(sql); - using (var c2 = m_connection.CreateCommand()) - { - c2.Transaction = tr; - c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId)") - + FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId")); - - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql)) + while (await rd.ReadAsync()) { - c2.SetParameterValue("@FileId", rd.GetValue(0)); - c2.SetParameterValue("@Path", rd.GetValue(1)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(2)); - c2.SetParameterValue("@MetadataId", rd.GetValue(3)); - await c2.ExecuteNonQueryAsync(); + await c2.SetParameterValue("@MetadataId", rd.GetValue(0)) + .SetParameterValue("@BlocksetId", rd.GetValue(1)) + .ExecuteNonQueryAsync(); } } - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); + sql = $@" + SELECT + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""Entries"" + FROM ( + SELECT + MIN(""ID"") AS ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + COUNT(*) as ""Entries"" + FROM ""{tablename}"" + GROUP BY + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + WHERE ""Entries"" > 1 + ORDER BY ""ID"" + "; + + using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + { + c2.SetCommandAndParameters($@" + UPDATE ""FilesetEntry"" + SET ""FileID"" = @FileId + WHERE ""FileID"" IN ( + SELECT ""ID"" + FROM ""{tablename}"" + WHERE + ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + ); + DELETE FROM ""{tablename}"" + WHERE + ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + AND ""ID"" != @FileId + "); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql)) + { + await c2.SetParameterValue("@FileId", rd.GetValue(0)) + .SetParameterValue("@Path", rd.GetValue(1)) + .SetParameterValue("@BlocksetId", rd.GetValue(2)) + .SetParameterValue("@MetadataId", rd.GetValue(3)) + .ExecuteNonQueryAsync(); + } + } + + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""FileLookup"" + WHERE ""ID"" NOT IN ( + SELECT ""ID"" + FROM ""{tablename}"" + ) + "); + + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{tablename}-Ix"" + ON ""{tablename}"" ( + ""ID"", + ""MetadataID"" + ) + "); + + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""FileLookup"" + SET ""MetadataID"" = ( + SELECT ""MetadataID"" + FROM ""{tablename}"" A + WHERE ""A"".""ID"" = ""FileLookup"".""ID"" + ) + "); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{tablename}"" "); x = await cmd.ExecuteScalarInt64Async(sql_count, 0); if (x > 1) throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); - await tr.CommitAsync(); + + await m_rtr.CommitAsync(); } } public async Task FixDuplicateFileentries() { - using var tr = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); + using var cmd = m_connection.CreateCommand(m_rtr.Transaction); - cmd.Transaction = tr; - var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; + var sql_count = @" + SELECT COUNT(*) + FROM ( + SELECT + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + COUNT(*) as ""Duplicates"" + FROM ""FileLookup"" + GROUP BY + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + WHERE ""Duplicates"" > 1 + "; var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 0) { Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); - var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") - WHERE ""Entries"" > 1 ORDER BY ""ID"""; + var sql = @" + SELECT + ""ID"", + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""Entries"" + FROM ( + SELECT + MIN(""ID"") AS ""ID"", + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + COUNT(*) as ""Entries"" + FROM ""FileLookup"" + GROUP BY + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + WHERE ""Entries"" > 1 + ORDER BY ""ID"" + "; - using (var c2 = m_connection.CreateCommand()) + using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) { - c2.Transaction = tr; - c2.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetatadataId)" - + @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId"); + c2.SetCommandAndParameters(@" + UPDATE ""FilesetEntry"" + SET ""FileID"" = @FileId + WHERE ""FileID"" IN ( + SELECT ""ID"" + FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetatadataId + ); + DELETE FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + AND ""ID"" != @FileId + "); + cmd.SetCommandAndParameters(sql); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { - c2.SetParameterValue("@FileId", rd.GetValue(0)); - c2.SetParameterValue("@PrefixId", rd.GetValue(1)); - c2.SetParameterValue("@Path", rd.GetValue(2)); - c2.SetParameterValue("@BlocksetId", rd.GetValue(3)); - c2.SetParameterValue("@MetadataId", rd.GetValue(4)); - await c2.ExecuteNonQueryAsync(); + await c2.SetParameterValue("@FileId", rd.GetValue(0)) + .SetParameterValue("@PrefixId", rd.GetValue(1)) + .SetParameterValue("@Path", rd.GetValue(2)) + .SetParameterValue("@BlocksetId", rd.GetValue(3)) + .SetParameterValue("@MetadataId", rd.GetValue(4)) + .ExecuteNonQueryAsync(); } } x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 1) throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); - await tr.CommitAsync(); + + await m_rtr.CommitAsync(); } } @@ -776,35 +1148,111 @@ namespace Duplicati.Library.Main.Database { var blocklistbuffer = new byte[blocksize]; - using var tr = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); + using var cmd = m_connection.CreateCommand(m_rtr.Transaction); using var blockhasher = HashFactory.CreateHasher(blockhashalgorithm); - cmd.Transaction = tr; var hashsize = blockhasher.HashSize / 8; - var sql = FormatInvariant($@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"""); - var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; + var sql = $@" + SELECT * + FROM ( + SELECT + ""N"".""BlocksetID"", + ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", + CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" + FROM ( + SELECT + ""BlocksetID"", + COUNT(*) AS ""BlockCount"" + FROM ""BlocksetEntry"" + GROUP BY ""BlocksetID"" + ) ""N"" + LEFT OUTER JOIN ( + SELECT + ""BlocksetID"", + COUNT(*) AS ""BlocklistHashCount"" + FROM ""BlocklistHash"" + GROUP BY ""BlocksetID"" + ) ""G"" + ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" + WHERE ""N"".""BlockCount"" > 1 + ) + WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"" + "; + + var countsql = @$" + SELECT COUNT(*) + FROM ({sql}) + "; var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); + if (itemswithnoblocklisthash != 0) { Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); - using (var c2 = m_connection.CreateCommand()) - using (var c3 = m_connection.CreateCommand()) - using (var c4 = m_connection.CreateCommand()) - using (var c5 = m_connection.CreateCommand()) - using (var c6 = m_connection.CreateCommand()) + using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + using (var c3 = m_connection.CreateCommand(m_rtr.Transaction)) + using (var c4 = m_connection.CreateCommand(m_rtr.Transaction)) + using (var c5 = m_connection.CreateCommand(m_rtr.Transaction)) + using (var c6 = m_connection.CreateCommand(m_rtr.Transaction)) { - c2.Transaction = tr; - c3.Transaction = tr; - c4.Transaction = tr; - c5.Transaction = tr; - c6.Transaction = tr; - c3.SetCommandAndParameters(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash) "); - c4.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - c5.SetCommandAndParameters(@"SELECT ""ID"" FROM ""DeletedBlock"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND (""State"" IN (@State1, @State2)))"); - c6.SetCommandAndParameters(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1; DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId;"); + c3.SetCommandAndParameters(@" + INSERT INTO ""BlocklistHash"" ( + ""BlocksetID"", + ""Index"", + ""Hash"" + ) + VALUES ( + @BlocksetId, + @Index, + @Hash + ) + "); + + c4.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + "); + + c5.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""DeletedBlock"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ( + ""State"" IN ( + @State1, + @State2 + ) + ) + ) + "); + + c6.SetCommandAndParameters(@" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId + LIMIT 1; + DELETE FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId; + "); await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql)) { @@ -812,12 +1260,24 @@ namespace Duplicati.Library.Main.Database var ix = 0L; int blocklistoffset = 0; - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId"); - c2.SetParameterValue("@BlocksetId", blocksetid); - await c2.ExecuteNonQueryAsync(); + await c2.SetCommandAndParameters(@" + DELETE FROM ""BlocklistHash"" + WHERE ""BlocksetID"" = @BlocksetId + ") + .SetParameterValue("@BlocksetId", blocksetid) + .ExecuteNonQueryAsync(); - c2.SetCommandAndParameters(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = @BlocksetId ORDER BY ""B"".""Index"""); - c2.SetParameterValue("@BlocksetId", blocksetid); + c2.SetCommandAndParameters(@" + SELECT ""A"".""Hash"" + FROM + ""Block"" ""A"", + ""BlocksetEntry"" ""B"" + WHERE + ""A"".""ID"" = ""B"".""BlockID"" + AND ""B"".""BlocksetID"" = @BlocksetId + ORDER BY ""B"".""Index"" + ") + .SetParameterValue("@BlocksetId", blocksetid); await foreach (var h in c2.ExecuteReaderEnumerableAsync()) { @@ -827,25 +1287,28 @@ namespace Duplicati.Library.Main.Database var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); // Check if the block exists in "blocks" - c4.SetParameterValue("@Hash", blkey); - c4.SetParameterValue("@Size", blocklistoffset); - var existingBlockId = await c4.ExecuteScalarInt64Async(-1); + var existingBlockId = await c4 + .SetParameterValue("@Hash", blkey) + .SetParameterValue("@Size", blocklistoffset) + .ExecuteScalarInt64Async(-1); if (existingBlockId <= 0) { - c5.SetParameterValue("@Hash", blkey); - c5.SetParameterValue("@Size", blocklistoffset); - c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); - c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); - var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); + var deletedBlockId = await c5 + .SetParameterValue("@Hash", blkey) + .SetParameterValue("@Size", blocklistoffset) + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) + .ExecuteScalarInt64Async(-1); if (deletedBlockId <= 0) throw new Exception($"Missing block for blocklisthash: {blkey}"); else { - c6.SetParameterValue("@DeletedBlockId", deletedBlockId); - var rc = await c6.ExecuteNonQueryAsync(); + var rc = await c6 + .SetParameterValue("@DeletedBlockId", deletedBlockId) + .ExecuteNonQueryAsync(); if (rc != 2) throw new Exception($"Unexpected update count: {rc}"); @@ -853,10 +1316,10 @@ namespace Duplicati.Library.Main.Database } // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid); - c3.SetParameterValue("@Index", ix); - c3.SetParameterValue("@Hash", blkey); - await c3.ExecuteNonQueryAsync(); + await c3.SetParameterValue("@BlocksetId", blocksetid) + .SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", blkey) + .ExecuteNonQueryAsync(); ix++; blocklistoffset = 0; @@ -872,77 +1335,117 @@ namespace Duplicati.Library.Main.Database var blkeyfinal = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); // Ensure that the block exists in "blocks" - c4.SetParameterValue("@Hash", blkeyfinal); - c4.SetParameterValue("@Size", blocklistoffset); - var existingBlockId = await c4.ExecuteScalarInt64Async(-1); + var existingBlockId = await c4 + .SetParameterValue("@Hash", blkeyfinal) + .SetParameterValue("@Size", blocklistoffset) + .ExecuteScalarInt64Async(-1); if (existingBlockId <= 0) { - c5.SetParameterValue("@Hash", blkeyfinal); - c5.SetParameterValue("@Size", blocklistoffset); - c5.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - c5.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()); - c5.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()); - var deletedBlockId = await c5.ExecuteScalarInt64Async(-1); + var deletedBlockId = await c5 + .SetParameterValue("@Hash", blkeyfinal) + .SetParameterValue("@Size", blocklistoffset) + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) + .ExecuteScalarInt64Async(-1); if (deletedBlockId == 0) throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); else { - c6.SetParameterValue("@DeletedBlockId", deletedBlockId); - var rc = await c6.ExecuteNonQueryAsync(); + var rc = await c6 + .SetParameterValue("@DeletedBlockId", deletedBlockId) + .ExecuteNonQueryAsync(); if (rc != 2) throw new Exception($"Unexpected update count: {rc}"); } } // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid); - c3.SetParameterValue("@Index", ix); - c3.SetParameterValue("@Hash", blkeyfinal); - await c3.ExecuteNonQueryAsync(); + await c3 + .SetParameterValue("@BlocksetId", blocksetid) + .SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", blkeyfinal) + .ExecuteNonQueryAsync(); } } } - itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); + itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); + if (itemswithnoblocklisthash != 0) throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); - tr.Commit(); + + await m_rtr.CommitAsync(); } } public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize) { - using var tr = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); + using var cmd = m_connection.CreateCommand(m_rtr.Transaction); - cmd.Transaction = tr; - var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1"; + var dup_sql = @" + SELECT * + FROM ( + SELECT + ""BlocksetID"", + ""Index"", + COUNT(*) AS ""EC"" + FROM ""BlocklistHash"" + GROUP BY + ""BlocksetID"", + ""Index"" + ) + WHERE ""EC"" > 1 + "; - var sql_count = @"SELECT COUNT(*) FROM (" + dup_sql + ")"; + var sql_count = @$" + SELECT COUNT(*) + FROM ({dup_sql}) + "; var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + if (x > 0) { Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashes", "Found duplicate blocklisthash entries, repairing"); - var unique_count = await cmd.ExecuteScalarInt64Async(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); + var unique_count = await cmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT + ""BlocksetID"", + ""Index"" + FROM ""BlocklistHash"" + ) + ", 0); - using (var c2 = m_connection.CreateCommand()) + using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) { - c2.Transaction = tr; - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId AND ""Index"" = @Index LIMIT @Limit)"); + c2.SetCommandAndParameters(@" + DELETE FROM ""BlocklistHash"" + WHERE rowid IN ( + SELECT rowid + FROM ""BlocklistHash"" + WHERE + ""BlocksetID"" = @BlocksetId + AND ""Index"" = @Index + LIMIT @Limit + ) + "); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql)) { var expected = rd.GetInt32(2) - 1; - c2.SetParameterValue("@BlocksetId", rd.GetValue(0)); - c2.SetParameterValue("@Index", rd.GetValue(1)); - c2.SetParameterValue("@Limit", expected); - var actual = await c2.ExecuteNonQueryAsync(); + var actual = await c2 + .SetParameterValue("@BlocksetId", rd.GetValue(0)) + .SetParameterValue("@Index", rd.GetValue(1)) + .SetParameterValue("@Limit", expected) + .ExecuteNonQueryAsync(); if (actual != expected) throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); @@ -950,17 +1453,21 @@ namespace Duplicati.Library.Main.Database } x = await cmd.ExecuteScalarInt64Async(sql_count); + if (x > 1) throw new Exception("Repair failed, there are still duplicate file entries!"); - var real_count = await cmd.ExecuteScalarInt64Async(@"SELECT Count(*) FROM ""BlocklistHash""", 0); + var real_count = await cmd.ExecuteScalarInt64Async(@" + SELECT Count(*) + FROM ""BlocklistHash"" + ", 0); if (real_count != unique_count) throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); try { - await VerifyConsistency(blocksize, hashsize, true, tr); + await VerifyConsistency(blocksize, hashsize, true, m_rtr.Transaction); } catch (Exception ex) { @@ -968,88 +1475,116 @@ namespace Duplicati.Library.Main.Database } Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); - await tr.CommitAsync(); + + await m_rtr.CommitAsync(); } } - public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, SqliteTransaction transaction) + public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; + using var cmd = m_connection.CreateCommand(m_rtr.Transaction); var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + try { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tablename}"" ( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + ) + "); + + cmd.SetCommandAndParameters($@" + INSERT INTO ""{tablename}"" ( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size + ) + "); foreach (var kp in blocks) { - cmd.SetParameterValue("@Hash", kp.Key); - cmd.SetParameterValue("@Size", kp.Value); - await cmd.ExecuteNonQueryAsync(); + await cmd + .SetParameterValue("@Hash", kp.Key) + .SetParameterValue("@Size", kp.Value) + .ExecuteNonQueryAsync(); } - cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name"); - cmd.SetParameterValue("@Name", filename); - var id = await cmd.ExecuteScalarInt64Async(-1); + var id = await cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" = @Name + ") + .SetParameterValue("@Name", filename) + .ExecuteScalarInt64Async(-1); - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{tablename}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != @VolumeId ")); - cmd.SetParameterValue("@VolumeId", id); - var aliens = await cmd.ExecuteScalarInt64Async(0); + var aliens = await cmd.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ( + SELECT ""A"".""VolumeID"" + FROM ""{tablename}"" B + LEFT OUTER JOIN ""Block"" A + ON ""A"".""Hash"" = ""B"".""Hash"" + AND ""A"".""Size"" = ""B"".""Size"" + ) + WHERE ""VolumeID"" != @VolumeId + ") + .SetParameterValue("@VolumeId", id) + .ExecuteScalarInt64Async(0); if (aliens != 0) throw new Exception($"Not all blocks were found in {filename}"); } finally { - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" "); } } - public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, SqliteTransaction transaction) + public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength) { - using var cmd = m_connection.CreateCommand(); + using var cmd = m_connection.CreateCommand(m_rtr.Transaction); - cmd.Transaction = transaction; - var query = FormatInvariant($@" -SELECT - ""C"".""Hash"", - ""C"".""Size"" -FROM - ""BlocksetEntry"" A, - ( - SELECT - ""Y"".""BlocksetID"", - ""Y"".""Hash"" AS ""BlocklistHash"", - ""Y"".""Index"" AS ""BlocklistHashIndex"", - ""Z"".""Size"" AS ""BlocklistSize"", - ""Z"".""ID"" AS ""BlocklistHashBlockID"" - FROM - ""BlocklistHash"" Y, - ""Block"" Z - WHERE - ""Y"".""Hash"" = ""Z"".""Hash"" AND ""Y"".""Hash"" = @Hash AND ""Z"".""Size"" = @Size - LIMIT 1 - ) B, - ""Block"" C -WHERE - ""A"".""BlocksetID"" = ""B"".""BlocksetID"" - AND - ""A"".""BlockID"" = ""C"".""ID"" - AND - ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({blocksize} / {blockhashlength}) - AND - ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({blocksize} / {blockhashlength}) -ORDER BY - ""A"".""Index"" - -"); + var query = $@" + SELECT + ""C"".""Hash"", + ""C"".""Size"" + FROM + ""BlocksetEntry"" A, + ( + SELECT + ""Y"".""BlocksetID"", + ""Y"".""Hash"" AS ""BlocklistHash"", + ""Y"".""Index"" AS ""BlocklistHashIndex"", + ""Z"".""Size"" AS ""BlocklistSize"", + ""Z"".""ID"" AS ""BlocklistHashBlockID"" + FROM + ""BlocklistHash"" Y, + ""Block"" Z + WHERE + ""Y"".""Hash"" = ""Z"".""Hash"" + AND ""Y"".""Hash"" = @Hash + AND ""Z"".""Size"" = @Size + LIMIT 1 + ) B, + ""Block"" C + WHERE + ""A"".""BlocksetID"" = ""B"".""BlocksetID"" + AND ""A"".""BlockID"" = ""C"".""ID"" + AND ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({blocksize} / {blockhashlength}) + AND ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({blocksize} / {blockhashlength}) + ORDER BY ""A"".""Index"" + "; using var en = blocklist.GetEnumerator(); - cmd.SetCommandAndParameters(query); - cmd.SetParameterValue("@Hash", hash); - cmd.SetParameterValue("@Size", length); + cmd.SetCommandAndParameters(query) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", length); + await foreach (var r in cmd.ExecuteReaderEnumerableAsync()) { if (!en.MoveNext()) @@ -1062,38 +1597,91 @@ ORDER BY throw new Exception($"Too many source blocklist entries in {hash}"); } - public async IAsyncEnumerable MissingLocalFilesets(SqliteTransaction transaction) + public async IAsyncEnumerable MissingLocalFilesets() { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); + using var cmd = m_connection.CreateCommand(@" + SELECT ""Name"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" NOT IN (@States) + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) + .ExpandInClauseParameter("@States", [ + RemoteVolumeState.Deleting.ToString(), + RemoteVolumeState.Deleted.ToString() + ]); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return rd.ConvertValueToString(0) ?? ""; } - public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets(SqliteTransaction transaction) + public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets() { - using var cmd = m_connection.CreateCommand(@"SELECT ""ID"", ""Timestamp"", ""IsFullBackup"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); + using var cmd = m_connection.CreateCommand(@" + SELECT + ""ID"", + ""Timestamp"", + ""IsFullBackup"" + FROM ""Fileset"" + WHERE ""VolumeID"" NOT IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" NOT IN (@States) + ) + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) + .ExpandInClauseParameter("@States", [ + RemoteVolumeState.Deleting.ToString(), + RemoteVolumeState.Deleted.ToString() + ]); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return (rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP); + yield return ( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), + rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP + ); } - public async IAsyncEnumerable EmptyIndexFiles(SqliteTransaction transaction) + public async IAsyncEnumerable EmptyIndexFiles() { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ID"" NOT IN (SELECT ""IndexVolumeId"" FROM ""IndexBlockLink"")"); - cmd.Transaction = transaction; - cmd.SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); - cmd.ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + using var cmd = m_connection.CreateCommand(@" + SELECT + ""Name"", + ""Hash"", + ""Size"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" IN (@States) + AND ""ID"" NOT IN ( + SELECT ""IndexVolumeId"" + FROM ""IndexBlockLink"" + ) + ") + .SetTransaction(m_rtr.Transaction) + .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) + .ExpandInClauseParameter("@States", [ + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString() + ]); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "", + rd.ConvertValueToInt64(2) + ); } } -} +} From ab6b2a64c93a4154373c1060e4ff053e7a755601 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 10:37:37 +0200 Subject: [PATCH 052/362] Added the extension method to pass ReusableTransactions directly on SqliteCommands --- .../Library/Main/Database/ExtensionMethods.cs | 27 +++++++++-- .../Main/Database/LocalRepairDatabase.cs | 48 +++++++++---------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 31fb07ebc..6edc22fa9 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -24,6 +24,7 @@ using System.Data; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; #nullable enable @@ -62,11 +63,7 @@ public static partial class ExtensionMethods public static SqliteCommand CreateCommand(this SqliteConnection self, string cmdtext) { - var cmd = self.CreateCommand(); - cmd.SetCommandAndParameters(cmdtext); - cmd.PrepareAsync().Wait(); - - return cmd; + return CreateCommandAsync(self, cmdtext).Await(); } public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext) @@ -78,6 +75,19 @@ public static partial class ExtensionMethods return cmd; } + public static SqliteCommand CreateCommand(this SqliteConnection self, SqliteTransaction transaction) + { + var cmd = self.CreateCommand(); + cmd.SetTransaction(transaction); + + return cmd; + } + + internal static SqliteCommand CreateCommand(this SqliteConnection self, ReusableTransaction rtr) + { + return self.CreateCommand(rtr.Transaction); + } + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmdtext) { self.SetCommandAndParameters(cmdtext); @@ -308,6 +318,13 @@ public static partial class ExtensionMethods return self; } + internal static SqliteCommand SetTransaction(this SqliteCommand self, ReusableTransaction rtr) + { + self.Transaction = rtr.Transaction; + + return self; + } + /// /// The tag used for logging /// diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 308e9b066..a2d9bc5f2 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -70,7 +70,7 @@ namespace Duplicati.Library.Main.Database ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Name", filelist); var rd = await cmd.ExecuteReaderAsync(); @@ -103,7 +103,7 @@ namespace Duplicati.Library.Main.Database SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@CurrentFilesetId", filesetid) .SetParameterValue("@PreviousFilesetId", prevFilesetId); @@ -127,7 +127,7 @@ namespace Duplicati.Library.Main.Database AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@BlockFileId", blockfileid) .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); @@ -156,7 +156,7 @@ namespace Duplicati.Library.Main.Database ) ) ") - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) @@ -180,7 +180,7 @@ namespace Duplicati.Library.Main.Database DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetid); return await cmd.ExecuteNonQueryAsync(); @@ -218,7 +218,7 @@ namespace Duplicati.Library.Main.Database ) ) ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Name", indexName); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -322,7 +322,7 @@ namespace Duplicati.Library.Main.Database /// /// The transaction to use /// - private readonly ReusableTransaction m_transaction; + private readonly ReusableTransaction m_rtr; /// /// The insert command to use for restoring blocks /// @@ -361,11 +361,11 @@ namespace Duplicati.Library.Main.Database /// /// The name of the volume with missing blocks /// The connection to the database - /// The transaction to use - private MissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) + /// The transaction to use + private MissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction rtr) { m_connection = connection; - m_transaction = transaction; + m_rtr = rtr; m_volumename = volumename; var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); m_tablename = tablename; @@ -465,7 +465,7 @@ namespace Duplicati.Library.Main.Database /// public async Task SetBlockRestored(string hash, long size, long targetVolumeId) { - var restored = await m_insertCommand.SetTransaction(m_transaction.Transaction) + var restored = await m_insertCommand.SetTransaction(m_rtr) .SetParameterValue("@NewRestoredValue", 1) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) @@ -474,12 +474,12 @@ namespace Duplicati.Library.Main.Database if (restored) { - await m_copyIntoDuplicatedBlocks.SetTransaction(m_transaction.Transaction) + await m_copyIntoDuplicatedBlocks.SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .ExecuteNonQueryAsync(); - var c = await m_assignBlocksToNewVolume.SetTransaction(m_transaction.Transaction) + var c = await m_assignBlocksToNewVolume.SetTransaction(m_rtr) .SetParameterValue("@TargetVolumeId", targetVolumeId) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) @@ -518,7 +518,7 @@ namespace Duplicati.Library.Main.Database ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} ") - .SetTransaction(m_transaction.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -558,7 +558,7 @@ namespace Duplicati.Library.Main.Database ""{m_tablename}"".""Size"", ""File"".""Path"" ") - .SetTransaction(m_transaction.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -574,7 +574,7 @@ namespace Duplicati.Library.Main.Database /// public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock) { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction); + using var cmd = m_connection.CreateCommand(m_rtr); var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; @@ -666,7 +666,7 @@ namespace Duplicati.Library.Main.Database public async Task GetMissingBlockCount() { return await m_missingBlocksCountCommand - .SetTransaction(m_transaction.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0) .ExecuteScalarInt64Async(0); } @@ -678,7 +678,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks() { m_missingBlocksCommand - .SetTransaction(m_transaction.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); await foreach (var rd in m_missingBlocksCommand.ExecuteReaderEnumerableAsync()) @@ -795,7 +795,7 @@ namespace Duplicati.Library.Main.Database "; using var cmd = m_connection.CreateCommand(cmdtxt) - .SetTransaction(m_transaction.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -824,7 +824,7 @@ namespace Duplicati.Library.Main.Database AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ") - .SetTransaction(m_transaction.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0) .SetParameterValue("@Name", m_volumename); @@ -856,7 +856,7 @@ namespace Duplicati.Library.Main.Database { using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}"""); - await cmd.SetTransaction(m_transaction.Transaction) + await cmd.SetTransaction(m_rtr) .ExecuteNonQueryAsync(); } } @@ -1610,7 +1610,7 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" ) ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) .ExpandInClauseParameter("@States", [ RemoteVolumeState.Deleting.ToString(), @@ -1637,7 +1637,7 @@ namespace Duplicati.Library.Main.Database AND ""State"" NOT IN (@States) ) ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) .ExpandInClauseParameter("@States", [ RemoteVolumeState.Deleting.ToString(), @@ -1668,7 +1668,7 @@ namespace Duplicati.Library.Main.Database FROM ""IndexBlockLink"" ) ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) .ExpandInClauseParameter("@States", [ RemoteVolumeState.Uploaded.ToString(), From 2dabf56c82e7d427c3df15a1d8420caf6cdc6c8f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 10:46:50 +0200 Subject: [PATCH 053/362] Additional SQL query whitespaces --- .../Library/Main/Database/LocalDatabase.cs | 245 ++++++++++-------- .../Main/Database/LocalRestoreDatabase.cs | 185 +++++++------ 2 files changed, 241 insertions(+), 189 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index ef6ad45f6..15fba1402 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -299,31 +299,34 @@ namespace Duplicati.Library.Main.Database db.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" - WHERE ""Name"" IN ( - SELECT ""Name"" - FROM ""Remotevolume"" - WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') - ) - AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') + WHERE + ""Name"" IN ( + SELECT ""Name"" + FROM ""Remotevolume"" + WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') + ) + AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') "); db.m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" DELETE FROM ""Remotevolume"" - WHERE ""Name"" = @Name - AND ( - ""DeleteGraceTime"" < @Now - OR ""State"" != @State - ) + WHERE + ""Name"" = @Name + AND ( + ""DeleteGraceTime"" < @Now + OR ""State"" != @State + ) "); // >12 is to handle removal of old records that were in ticks db.m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@" DELETE FROM ""Remotevolume"" - WHERE ""State"" == '{RemoteVolumeState.Deleted}' - AND ( - ""DeleteGraceTime"" < @Now - OR LENGTH(""DeleteGraceTime"") > 12 - ) + WHERE + ""State"" == '{RemoteVolumeState.Deleted}' + AND ( + ""DeleteGraceTime"" < @Now + OR LENGTH(""DeleteGraceTime"") > 12 + ) "); db.m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" @@ -643,8 +646,9 @@ namespace Duplicati.Library.Main.Database { using var cmd = await m_connection.CreateCommandAsync(@" DELETE FROM ""RemoteVolume"" - WHERE ""Name"" = @Name - AND ""State"" = @State + WHERE + ""Name"" = @Name + AND ""State"" = @State "); var c = await cmd.SetTransaction(transaction) .SetParameterValue("@Name", name) @@ -693,8 +697,9 @@ namespace Duplicati.Library.Main.Database var bsIdsSubQuery = @$" SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" - WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) + WHERE + ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) UNION ALL SELECT DISTINCT ""BlocksetID"" FROM ""BlocklistHash"" @@ -755,18 +760,19 @@ namespace Duplicati.Library.Main.Database // Delete FilesetEntry rows that had their metadata deleted await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM FilesetEntry - WHERE FilesetEntry.FilesetID IN ( - SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"" - ) - AND FilesetEntry.FileID IN ( - SELECT FilesetEntry.FileID - FROM FilesetEntry - INNER JOIN FileLookup - ON FileLookup.ID = FilesetEntry.FileID - WHERE FileLookup.MetadataID IN ( - SELECT MetadataID FROM ""{metadataFilesetTable}"" + WHERE + FilesetEntry.FilesetID IN ( + SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"" + ) + AND FilesetEntry.FileID IN ( + SELECT FilesetEntry.FileID + FROM FilesetEntry + INNER JOIN FileLookup + ON FileLookup.ID = FilesetEntry.FileID + WHERE FileLookup.MetadataID IN ( + SELECT MetadataID FROM ""{metadataFilesetTable}"" + ) ) - ) "); // Delete FilesetEntry rows that had their blocks deleted @@ -854,29 +860,31 @@ namespace Duplicati.Library.Main.Database INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" - WHERE ""Fileset"".""ID"" IN ( - SELECT DISTINCT ""FilesetID"" - FROM ""{metadataFilesetTable}"" - ) - AND ""Fileset"".""ID"" NOT IN ( - SELECT DISTINCT ""FilesetID"" - FROM FilesetEntry - ) + WHERE + ""Fileset"".""ID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{metadataFilesetTable}"" + ) + AND ""Fileset"".""ID"" NOT IN ( + SELECT DISTINCT ""FilesetID"" + FROM FilesetEntry + ) "); // Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later await deletecmd.SetCommandAndParameters($@" UPDATE ""RemoteVolume"" SET ""State"" = @NewState - WHERE ""ID"" IN ( - SELECT DISTINCT ""VolumeID"" - FROM ""Fileset"" - WHERE ""Fileset"".""ID"" IN ( - SELECT ""ID"" - FROM ""{filesetidstable}"" + WHERE + ""ID"" IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Fileset"" + WHERE ""Fileset"".""ID"" IN ( + SELECT ""ID"" + FROM ""{filesetidstable}"" + ) ) - ) - AND ""State"" IN (@AllowedStates) + AND ""State"" IN (@AllowedStates) ") .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) .ExpandInClauseParameter("@AllowedStates", [ @@ -1250,8 +1258,9 @@ namespace Duplicati.Library.Main.Database ""Length"", ""A"".""BlocksetID"", ""File"".""Path"" FROM ({combinedLengths}) A, ""File"" - WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" - AND ""A"".""CalcLen"" != ""A"".""Length"" + WHERE + ""A"".""BlocksetID"" = ""File"".""BlocksetID"" + AND ""A"".""CalcLen"" != ""A"".""Length"" "; using (var rd = await cmd.ExecuteReaderAsync(reportDetails)) @@ -1326,11 +1335,12 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""Blockset"" - WHERE ""Length"" > 0 - AND ""ID"" NOT IN ( - SELECT ""BlocksetId"" - FROM ""BlocksetEntry"" - ) + WHERE + ""Length"" > 0 + AND ""ID"" NOT IN ( + SELECT ""BlocksetId"" + FROM ""BlocksetEntry"" + ) "); if (await cmd.ExecuteScalarInt64Async() != 0) throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); @@ -1338,12 +1348,13 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""FileLookup"" - WHERE ""BlocksetID"" != @FolderBlocksetId - AND ""BlocksetID"" != @SymlinkBlocksetId - AND NOT ""BlocksetID"" IN ( - SELECT ""ID"" - FROM ""Blockset"" - ) + WHERE + ""BlocksetID"" != @FolderBlocksetId + AND ""BlocksetID"" != @SymlinkBlocksetId + AND NOT ""BlocksetID"" IN ( + SELECT ""ID"" + FROM ""Blockset"" + ) "); cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); @@ -1358,8 +1369,9 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" NOT IN ( SELECT ""ID"" FROM ""RemoteVolume"" - WHERE ""Type"" = @Type - AND ""State"" != @State + WHERE + ""Type"" = @Type + AND ""State"" != @State ) "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); @@ -1379,8 +1391,9 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" NOT IN ( SELECT ""ID"" FROM ""RemoteVolume"" - WHERE ""Type"" = @Type - AND ""State"" != @State + WHERE + ""Type"" = @Type + AND ""State"" != @State ) "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); @@ -1396,12 +1409,13 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""RemoteVolume"" - WHERE ""Type"" = @Type - AND ""State"" != @State - AND ""ID"" NOT IN ( - SELECT ""VolumeID"" - FROM ""Fileset"" - ) + WHERE + ""Type"" = @Type + AND ""State"" != @State + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); @@ -1416,12 +1430,13 @@ namespace Duplicati.Library.Main.Database ""Name"", ""State"" FROM ""RemoteVolume"" - WHERE ""Type"" = @Type - AND ""State"" != @State - AND ""ID"" NOT IN ( - SELECT ""VolumeID"" - FROM ""Fileset"" - ) + WHERE + ""Type"" = @Type + AND ""State"" != @State + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); @@ -1712,16 +1727,17 @@ namespace Duplicati.Library.Main.Database ON ""E"".""ID"" = ""I"".""BlocksetID"" LEFT JOIN ""Block"" H ON ""I"".""BlockID"" = ""H"".""ID"" - WHERE ""A"".""BlocksetId"" >= 0 - AND ""D"".""FilesetID"" = @FilesetId - AND ( - ""I"".""Index"" = 0 - OR ""I"".""Index"" IS NULL - ) - AND ( - ""G"".""Index"" = 0 - OR ""G"".""Index"" IS NULL - ) + WHERE + ""A"".""BlocksetId"" >= 0 + AND ""D"".""FilesetID"" = @FilesetId + AND ( + ""I"".""Index"" = 0 + OR ""I"".""Index"" IS NULL + ) + AND ( + ""G"".""Index"" = 0 + OR ""G"".""Index"" IS NULL + ) ) J LEFT OUTER JOIN ""BlocklistHash"" K ON ""K"".""BlocksetID"" = ""J"".""BlocksetID"" @@ -1874,15 +1890,16 @@ namespace Duplicati.Library.Main.Database UPDATE FilesetEntry AS oldVersion SET Lastmodified = tempVersion.Lastmodified FROM FilesetEntry AS tempVersion - WHERE oldVersion.FileID = tempVersion.FileID - AND tempVersion.FilesetID = @FilesetId - AND oldVersion.FilesetID = ( - SELECT ID - FROM Fileset - WHERE ID != @FilesetId - ORDER BY Timestamp DESC - LIMIT 1 - ) + WHERE + oldVersion.FileID = tempVersion.FileID + AND tempVersion.FilesetID = @FilesetId + AND oldVersion.FilesetID = ( + SELECT ID + FROM Fileset + WHERE ID != @FilesetId + ORDER BY Timestamp DESC + LIMIT 1 + ) "; using var cmd = await m_connection.CreateCommandAsync(query); @@ -2141,18 +2158,20 @@ namespace Duplicati.Library.Main.Database FROM ""BlocklistHash"", ""Block"" - WHERE ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" - AND ""Block"".""VolumeID"" = @VolumeId + WHERE + ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" + AND ""Block"".""VolumeID"" = @VolumeId GROUP BY ""Block"".""Hash"", ""Block"".""Size"" ) A, ""BlocksetEntry"" B, ""Block"" C - WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" - AND ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize}) - AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize}) - AND ""C"".""ID"" = ""B"".""BlockID"" + WHERE + ""B"".""BlocksetID"" = ""A"".""BlocksetID"" + AND ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize}) + AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize}) + AND ""C"".""ID"" = ""B"".""BlockID"" ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" @@ -2252,15 +2271,16 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"", ""RemoteVolume"" - WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" - AND ""Fileset"".""ID"" IN ( - SELECT ""FilesetID"" - FROM ""FilesetEntry"" - ) - AND ( - ""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' - OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}' - ) + WHERE + ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" + AND ""Fileset"".""ID"" IN ( + SELECT ""FilesetID"" + FROM ""FilesetEntry"" + ) + AND ( + ""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' + OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}' + ) "); using var rd = await cmd.SetTransaction(transaction) @@ -2296,8 +2316,9 @@ namespace Duplicati.Library.Main.Database FROM ""RemoteVolume"", ""Fileset"" - WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" - AND ""Fileset"".""ID"" = @FilesetId + WHERE + ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" + AND ""Fileset"".""ID"" = @FilesetId "); using var rd = await cmd.SetTransaction(transaction) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index f2fcb795b..89bc1c238 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -338,8 +338,9 @@ namespace Duplicati.Library.Main.Database FROM ""File"", ""FilesetEntry"" - WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" - AND ""FilesetEntry"".""FilesetID"" = @FilesetId + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQueryAsync(); @@ -387,12 +388,13 @@ namespace Duplicati.Library.Main.Database FROM ""File"", ""FilesetEntry"" - WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" - AND ""FilesetEntry"".""FilesetID"" = @FilesetId - AND ""Path"" IN ( - SELECT DISTINCT ""Path"" - FROM ""{m_filenamestable}"" - ) + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId + AND ""Path"" IN ( + SELECT DISTINCT ""Path"" + FROM ""{m_filenamestable}"" + ) ") .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQueryAsync(); @@ -445,8 +447,9 @@ namespace Duplicati.Library.Main.Database FROM ""File"", ""FilesetEntry"" - WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" - AND ""FilesetID"" = @FilesetId + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId); @@ -757,8 +760,9 @@ namespace Duplicati.Library.Main.Database ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"" - WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE + ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "); var p2 = 0; @@ -788,9 +792,10 @@ namespace Duplicati.Library.Main.Database ""BlocksetEntry"", ""Block"", ""Metadataset"" - WHERE ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" - AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE + ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" + AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "); //creating indexes after insertion is much faster @@ -942,10 +947,13 @@ namespace Duplicati.Library.Main.Database ""Blockset"", ""BlocksetEntry"", ""Block"" - WHERE ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" - AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" - AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - ORDER BY ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index"" + WHERE + ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + ORDER BY + ""{tablename}"".""TargetPath"", + ""BlocksetEntry"".""Index"" ") .SetTransaction(db.Transaction.Transaction); using var rd = await cmd.ExecuteReaderAsync(); @@ -1063,12 +1071,13 @@ namespace Duplicati.Library.Main.Database ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E - WHERE ""A"".""ID"" = ""B"".""FileID"" - AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" - AND ""D"".""BlockID"" = ""E"".""ID"" - AND ""B"".""Hash"" = ""E"".""Hash"" - AND ""B"".""Size"" = ""E"".""Size"" - AND ""B"".""Restored"" = 0 + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" + AND ""D"".""BlockID"" = ""E"".""ID"" + AND ""B"".""Hash"" = ""E"".""Hash"" + AND ""B"".""Size"" = ""E"".""Size"" + AND ""B"".""Restored"" = 0 ORDER BY ""A"".""TargetPath"", ""B"".""Index"" @@ -1271,17 +1280,19 @@ namespace Duplicati.Library.Main.Database FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" - WHERE ""A"".""ID"" = ""BB"".""FileID"" - AND ""BB"".""Restored"" = 0 - AND ""BB"".""Metadata"" = {"0"} - AND ""BB"".""ID"" IN ( - SELECT ""B"".""ID"" - FROM - ""{m_blocktablename}"" ""B"", - ""{m_tmptable}"" ""C"" - WHERE ""B"".""Hash"" = ""C"".""Hash"" - AND ""B"".""Size"" = ""C"".""Size"" - ) + WHERE + ""A"".""ID"" = ""BB"".""FileID"" + AND ""BB"".""Restored"" = 0 + AND ""BB"".""Metadata"" = {"0"} + AND ""BB"".""ID"" IN ( + SELECT ""B"".""ID"" + FROM + ""{m_blocktablename}"" ""B"", + ""{m_tmptable}"" ""C"" + WHERE + ""B"".""Hash"" = ""C"".""Hash"" + AND ""B"".""Size"" = ""C"".""Size"" + ) ORDER BY ""A"".""TargetPath"", ""BB"".""Index"" @@ -1319,17 +1330,19 @@ namespace Duplicati.Library.Main.Database FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" - WHERE ""A"".""ID"" = ""BB"".""FileID"" - AND ""BB"".""Restored"" = 0 - AND ""BB"".""Metadata"" = {"1"} - AND ""BB"".""ID"" IN ( - SELECT ""B"".""ID"" - FROM - ""{m_blocktablename}"" ""B"", - ""{m_tmptable}"" ""C"" - WHERE ""B"".""Hash"" = ""C"".""Hash"" - AND ""B"".""Size"" = ""C"".""Size"" - ) + WHERE + ""A"".""ID"" = ""BB"".""FileID"" + AND ""BB"".""Restored"" = 0 + AND ""BB"".""Metadata"" = {"1"} + AND ""BB"".""ID"" IN ( + SELECT ""B"".""ID"" + FROM + ""{m_blocktablename}"" ""B"", + ""{m_tmptable}"" ""C"" + WHERE + ""B"".""Hash"" = ""C"".""Hash"" + AND ""B"".""Size"" = ""C"".""Size"" + ) ORDER BY ""A"".""TargetPath"", ""BB"".""Index"" @@ -1404,8 +1417,9 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tempfiletable}"", ""Blockset"" - WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" - AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified + WHERE + ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified ") .SetTransaction(m_rtr.Transaction) .SetParameterValue("@Verified", !onlyNonVerified); @@ -1462,9 +1476,10 @@ namespace Duplicati.Library.Main.Database 0, {FOLDER_BLOCKSET_ID} FROM ""{m_tempfiletable}"" F - WHERE F.BlocksetID = {FOLDER_BLOCKSET_ID} - AND F.MetadataID IS NOT NULL - AND F.MetadataID >= 0 + WHERE + F.BlocksetID = {FOLDER_BLOCKSET_ID} + AND F.MetadataID IS NOT NULL + AND F.MetadataID >= 0 "); while (await rd.ReadAsync()) @@ -1726,12 +1741,13 @@ namespace Duplicati.Library.Main.Database m_insertblockCommand = await db.Connection.CreateCommandAsync($@" UPDATE ""{blocktablename}"" SET ""Restored"" = 1 - WHERE ""FileID"" = @TargetFileId - AND ""Index"" = @Index - AND ""Hash"" = @Hash - AND ""Size"" = @Size - AND ""Metadata"" = @Metadata - AND ""Restored"" = 0 + WHERE + ""FileID"" = @TargetFileId + AND ""Index"" = @Index + AND ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""Metadata"" = @Metadata + AND ""Restored"" = 0 "), m_resetfileCommand = await db.Connection.CreateCommandAsync($@" @@ -1743,8 +1759,9 @@ namespace Duplicati.Library.Main.Database m_updateAsRestoredCommand = await db.Connection.CreateCommandAsync($@" UPDATE ""{blocktablename}"" SET ""Restored"" = 1 - WHERE ""FileID"" = @TargetFileId - AND ""Metadata"" <= @Metadata + WHERE + ""FileID"" = @TargetFileId + AND ""Metadata"" <= @Metadata "), m_updateFileAsDataVerifiedCommand = await db.Connection.CreateCommandAsync($@" @@ -1755,11 +1772,22 @@ namespace Duplicati.Library.Main.Database m_statUpdateCommand = statstablename == null ? // very slow fallback if stats tables were not created - await db.Connection.CreateCommandAsync($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{blocktablename}"" WHERE ""Restored"" = 1 ") + await db.Connection.CreateCommandAsync($@" + SELECT + COUNT(DISTINCT ""FileID""), + SUM(""Size"") + FROM ""{blocktablename}"" + WHERE ""Restored"" = 1 + ") : // Fields in Stats: TotalFiles, TotalBlocks, TotalSize // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored - await db.Connection.CreateCommandAsync($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" ") + await db.Connection.CreateCommandAsync($@" + SELECT + SUM(""FilesFullyRestored""), + SUM(""SizeRestored"") + FROM ""{statstablename}"" + ") }; dbm.m_insertblockCommand.SetTransaction(db.Transaction.Transaction); @@ -2003,16 +2031,18 @@ namespace Duplicati.Library.Main.Database ""File"", ""FilesetEntry"", ""Fileset"" - WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" - AND ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" - AND ""File"".""Path"" IN ( - SELECT DISTINCT ""{m_tempfiletable}"".""Path"" - FROM - ""{m_tempfiletable}"", - ""{m_tempblocktable}"" - WHERE ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" - AND {whereclause} - ) + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" + AND ""File"".""Path"" IN ( + SELECT DISTINCT ""{m_tempfiletable}"".""Path"" + FROM + ""{m_tempfiletable}"", + ""{m_tempblocktable}"" + WHERE + ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" + AND {whereclause} + ) GROUP BY ""File"".""Path"" "; @@ -2048,11 +2078,12 @@ namespace Duplicati.Library.Main.Database ""{m_tempfiletable}"", ""{m_tempblocktable}"", ""BlocksetEntry"" - WHERE ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" - AND ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" - AND ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" - AND ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" - AND {whereclause} + WHERE + ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" + AND ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" + AND {whereclause} ORDER BY ""{m_tempfiletable}"".""ID"", ""{m_tempblocktable}"".""Index"" From bdaa976f6b9aa4114d782c1ad51ec1c16db1ec7b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 10:49:22 +0200 Subject: [PATCH 054/362] Changed LocalRestoreDatabase to pass the ReusableTransaction instead of its inner transaction --- .../Main/Database/LocalRestoreDatabase.cs | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 89bc1c238..eabe64069 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -98,7 +98,7 @@ namespace Duplicati.Library.Main.Database m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; using var cmd = m_connection.CreateCommand() - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); // How to handle METADATA? await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" "); @@ -263,7 +263,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - cmd.SetTransaction(m_rtr.Transaction); + cmd.SetTransaction(m_rtr); var filesetIds = (await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions)).ToList(); while (filesetIds.Count > 0) { @@ -349,7 +349,7 @@ namespace Duplicati.Library.Main.Database { using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset")) await m_rtr.CommitAsync(); - cmd.SetTransaction(m_rtr.Transaction); + cmd.SetTransaction(m_rtr); // If we get a list of filenames, the lookup table is faster // unfortunately we cannot do this if the filesystem is case sensitive as // SQLite only supports ASCII compares @@ -432,7 +432,7 @@ namespace Duplicati.Library.Main.Database using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset")) await m_rtr.CommitAsync(); - cmd.SetTransaction(m_rtr.Transaction); + cmd.SetTransaction(m_rtr); } else { @@ -541,7 +541,7 @@ namespace Duplicati.Library.Main.Database ORDER BY LENGTH(""Path"") DESC LIMIT 1 ") - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); var v0 = await cmd.ExecuteScalarAsync(); if (v0 == null || v0 == DBNull.Value) @@ -560,7 +560,7 @@ namespace Duplicati.Library.Main.Database ORDER BY LENGTH(""Path"") DESC LIMIT 1 ") - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); var v0 = await cmd.ExecuteScalarAsync(); var maxpath = ""; @@ -610,7 +610,7 @@ namespace Duplicati.Library.Main.Database var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? await GetFirstPath() : largest_prefix); using var cmd = m_connection.CreateCommand() - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); if (string.IsNullOrEmpty(destination)) { //The string fixing here is meant to provide some non-random @@ -735,7 +735,7 @@ namespace Duplicati.Library.Main.Database public async Task FindMissingBlocks(bool skipMetadata) { using var cmd = m_connection.CreateCommand() - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); var p1 = await cmd.ExecuteNonQueryAsync($@" INSERT INTO ""{m_tempblocktable}"" ( ""FileID"", @@ -825,7 +825,7 @@ namespace Duplicati.Library.Main.Database SET ""TargetPath"" = @TargetPath WHERE ""ID"" = @ID "); - await cmd.SetTransaction(m_rtr.Transaction) + await cmd.SetTransaction(m_rtr) .SetParameterValue("@TargetPath", newname) .SetParameterValue("@ID", ID) .ExecuteNonQueryAsync(); @@ -955,7 +955,7 @@ namespace Duplicati.Library.Main.Database ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index"" ") - .SetTransaction(db.Transaction.Transaction); + .SetTransaction(db.Transaction); using var rd = await cmd.ExecuteReaderAsync(); if (await rd.ReadAsync()) { @@ -1082,7 +1082,7 @@ namespace Duplicati.Library.Main.Database ""A"".""TargetPath"", ""B"".""Index"" ") - .SetTransaction(db.Transaction.Transaction); + .SetTransaction(db.Transaction); using var rd = await cmd.ExecuteReaderAsync(); var more = await rd.ReadAsync(); @@ -1128,7 +1128,7 @@ namespace Duplicati.Library.Main.Database ON ""RV"".""ID"" = ""BB"".""VolumeID"" ORDER BY ""BB"".""MaxIndex"" ") - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); // Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore. // If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file. @@ -1189,7 +1189,7 @@ namespace Duplicati.Library.Main.Database }; using var c = db.Connection.CreateCommand() - .SetTransaction(db.Transaction.Transaction); + .SetTransaction(db.Transaction); fam.m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); await c.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{fam.m_tmptable}"" ( @@ -1231,7 +1231,7 @@ namespace Duplicati.Library.Main.Database if (m_tmptable != null) { using var c = m_db.Connection.CreateCommand(@"DROP TABLE IF EXISTS ""{m_tmptable}""") - .SetTransaction(m_db.Transaction.Transaction); + .SetTransaction(m_db.Transaction); await c.ExecuteNonQueryAsync(); await m_db.Transaction.CommitAsync(); } @@ -1297,7 +1297,7 @@ namespace Duplicati.Library.Main.Database ""A"".""TargetPath"", ""BB"".""Index"" ") - .SetTransaction(m_db.Transaction.Transaction); + .SetTransaction(m_db.Transaction); using var rd = await cmd.ExecuteReaderAsync(); if (await rd.ReadAsync()) @@ -1347,7 +1347,7 @@ namespace Duplicati.Library.Main.Database ""A"".""TargetPath"", ""BB"".""Index"" ") - .SetTransaction(m_db.Transaction.Transaction); + .SetTransaction(m_db.Transaction); using var rd = await cmd.ExecuteReaderAsync(); if (await rd.ReadAsync()) @@ -1421,7 +1421,7 @@ namespace Duplicati.Library.Main.Database ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Verified", !onlyNonVerified); using var rd = await cmd.ExecuteReaderAsync(); @@ -1451,7 +1451,7 @@ namespace Duplicati.Library.Main.Database ON F.BlocksetID = B.ID WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID} ") - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); @@ -1526,7 +1526,7 @@ namespace Duplicati.Library.Main.Database ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" {metadata_query} ") - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) yield return ( @@ -1556,7 +1556,7 @@ namespace Duplicati.Library.Main.Database ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetID", blocksetID); using var reader = await cmd.ExecuteReaderAsync(); @@ -1599,7 +1599,7 @@ namespace Duplicati.Library.Main.Database ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" WHERE ""File"".""ID"" = @FileID ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@FileID", fileID); using var reader = await cmd.ExecuteReaderAsync(); @@ -1629,7 +1629,7 @@ namespace Duplicati.Library.Main.Database FROM RemoteVolume WHERE ID = @VolumeID ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@VolumeID", VolumeID); using var reader = await cmd.ExecuteReaderAsync(); @@ -1642,7 +1642,7 @@ namespace Duplicati.Library.Main.Database public async Task DropRestoreTable() { using var cmd = m_connection.CreateCommand() - .SetTransaction(m_rtr.Transaction); + .SetTransaction(m_rtr); if (m_tempfiletable != null) try @@ -1790,11 +1790,11 @@ namespace Duplicati.Library.Main.Database ") }; - dbm.m_insertblockCommand.SetTransaction(db.Transaction.Transaction); - dbm.m_resetfileCommand.SetTransaction(db.Transaction.Transaction); - dbm.m_updateAsRestoredCommand.SetTransaction(db.Transaction.Transaction); - dbm.m_updateFileAsDataVerifiedCommand.SetTransaction(db.Transaction.Transaction); - dbm.m_statUpdateCommand.SetTransaction(db.Transaction.Transaction); + dbm.m_insertblockCommand.SetTransaction(db.Transaction); + dbm.m_resetfileCommand.SetTransaction(db.Transaction); + dbm.m_updateAsRestoredCommand.SetTransaction(db.Transaction); + dbm.m_updateFileAsDataVerifiedCommand.SetTransaction(db.Transaction); + dbm.m_statUpdateCommand.SetTransaction(db.Transaction); return dbm; } @@ -1913,7 +1913,7 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tempfiletable}"" WHERE ""BlocksetID"" == @BlocksetID ") - .SetTransaction(m_rtr.Transaction) + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); using var rd = await cmd.ExecuteReaderAsync(); @@ -1976,8 +1976,8 @@ namespace Duplicati.Library.Main.Database using (var cmdReader = m_connection.CreateCommand()) using (var cmd = m_connection.CreateCommand()) { - cmdReader.SetTransaction(m_rtr.Transaction); - cmd.SetTransaction(m_rtr.Transaction); + cmdReader.SetTransaction(m_rtr); + cmd.SetTransaction(m_rtr); cmd.SetCommandAndParameters($@" UPDATE ""{m_tempfiletable}"" SET ""LocalSourceExists"" = 1 @@ -2048,7 +2048,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - cmd.SetTransaction(m_rtr.Transaction); + cmd.SetTransaction(m_rtr); await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" "); await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}"); await cmd.ExecuteNonQueryAsync($@" @@ -2091,7 +2091,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - cmd.SetTransaction(m_rtr.Transaction); + cmd.SetTransaction(m_rtr); using var rd = await cmd.ExecuteReaderAsync(sources); if (await rd.ReadAsync()) { From 4ab7f290a7415475bb42129e35781b87e1f22c17 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 13:19:24 +0200 Subject: [PATCH 055/362] Updated LocalBackupDatabase to use factory construction --- .../Main/Database/LocalBackupDatabase.cs | 195 +++++++++--------- 1 file changed, 102 insertions(+), 93 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 8e1918bcc..817778f3c 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -1,30 +1,30 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; using System.Collections.Generic; -using System.Data; -using System.Globalization; using System.Linq; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -36,29 +36,29 @@ namespace Duplicati.Library.Main.Database /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private readonly IDbCommand m_findblockCommand; - private readonly IDbCommand m_findblocksetCommand; - private readonly IDbCommand m_findfilesetCommand; - private readonly IDbCommand m_findmetadatasetCommand; + private SqliteCommand m_findblockCommand = null!; + private SqliteCommand m_findblocksetCommand = null!; + private SqliteCommand m_findfilesetCommand = null!; + private SqliteCommand m_findmetadatasetCommand = null!; - private readonly IDbCommand m_insertblockCommand; + private SqliteCommand m_insertblockCommand = null!; - private readonly IDbCommand m_insertfileCommand; + private SqliteCommand m_insertfileCommand = null!; - private readonly IDbCommand m_insertblocksetCommand; - private readonly IDbCommand m_insertblocksetentryCommand; - private readonly IDbCommand m_insertblocklistHashesCommand; + private SqliteCommand m_insertblocksetCommand = null!; + private SqliteCommand m_insertblocksetentryCommand = null!; + private SqliteCommand m_insertblocklistHashesCommand = null!; - private readonly IDbCommand m_insertmetadatasetCommand; + private SqliteCommand m_insertmetadatasetCommand = null!; - private readonly IDbCommand m_findfileCommand; - private readonly IDbCommand m_selectfilelastmodifiedCommand; - private readonly IDbCommand m_selectfilelastmodifiedWithSizeCommand; - private readonly IDbCommand m_selectfileHashCommand; + private SqliteCommand m_findfileCommand = null!; + private SqliteCommand m_selectfilelastmodifiedCommand = null!; + private SqliteCommand m_selectfilelastmodifiedWithSizeCommand = null!; + private SqliteCommand m_selectfileHashCommand = null!; - private readonly IDbCommand m_insertfileOperationCommand; - private readonly IDbCommand m_selectfilemetadatahashandsizeCommand; - private readonly IDbCommand m_getfirstfilesetwithblockinblockset; + private SqliteCommand m_insertfileOperationCommand = null!; + private SqliteCommand m_selectfilemetadatahashandsizeCommand = null!; + private SqliteCommand m_getfirstfilesetwithblockinblockset = null!; private HashSet m_blocklistHashes; @@ -73,41 +73,48 @@ namespace Duplicati.Library.Main.Database /// /// The command used to move deleted blocks to the main block table; null if not used /// - private readonly IDbCommand? m_moveblockfromdeletedCommand; + private SqliteCommand? m_moveblockfromdeletedCommand; /// /// The command used to find blocks in the deleted blocks table; null if not used /// - private readonly IDbCommand? m_findindeletedCommand; + private SqliteCommand? m_findindeletedCommand; private long m_filesetId; - private readonly bool m_logQueries; + private bool m_logQueries; - public LocalBackupDatabase(string path, Options options) - : this(new LocalDatabase(path, "Backup", false, options.SqlitePageCache), options) + public static async Task CreateAsync(string path, Options options) { - this.ShouldCloseConnection = true; + var db = new LocalBackupDatabase(); + + db = (LocalBackupDatabase)await CreateLocalDatabaseAsync(db, path, "Backup", false, options.SqlitePageCache); + db.ShouldCloseConnection = true; + + return await CreateAsync(db, options); } - public LocalBackupDatabase(LocalDatabase db, Options options) - : base(db) + public static async Task CreateAsync(LocalBackupDatabase dbparent, Options options) { - m_logQueries = options.ProfileAllDatabaseQueries; + var dbnew = new LocalBackupDatabase(); - m_findblockCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @hash AND ""Size"" = @Size"); - m_findblocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = @Fullhash AND ""Length"" = @Length"); - m_findmetadatasetCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = @Hash AND ""C"".""Size"" = @Size"); - m_findfilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""Path"" = @Path AND ""PrefixID"" = @PrefixId"); - m_insertblockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (@Hash, @VolumeId, @Size); SELECT last_insert_rowid();"); - m_insertfileOperationCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId, @FileId, @LastModified)"); - m_insertfileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (@PrefixId, @Path, @BlocksetId, @MetadataId); SELECT last_insert_rowid();"); - m_insertblocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length, @Fullhash); SELECT last_insert_rowid();"); - m_insertblocksetentryCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT @BlocksetId AS A, @Index AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - m_insertblocklistHashesCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash)"); - m_insertmetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); - m_selectfilelastmodifiedCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId"); - m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand(@"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" "); - m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = @FileId AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" "); + dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + + dbnew.m_logQueries = options.ProfileAllDatabaseQueries; + + dbnew.m_findblockCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @hash AND ""Size"" = @Size"); + dbnew.m_findblocksetCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = @Fullhash AND ""Length"" = @Length"); + dbnew.m_findmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = @Hash AND ""C"".""Size"" = @Size"); + dbnew.m_findfilesetCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""Path"" = @Path AND ""PrefixID"" = @PrefixId"); + dbnew.m_insertblockCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (@Hash, @VolumeId, @Size); SELECT last_insert_rowid();"); + dbnew.m_insertfileOperationCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId, @FileId, @LastModified)"); + dbnew.m_insertfileCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (@PrefixId, @Path, @BlocksetId, @MetadataId); SELECT last_insert_rowid();"); + dbnew.m_insertblocksetCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length, @Fullhash); SELECT last_insert_rowid();"); + dbnew.m_insertblocksetentryCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT @BlocksetId AS A, @Index AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + dbnew.m_insertblocklistHashesCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash)"); + dbnew.m_insertmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); + dbnew.m_selectfilelastmodifiedCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId"); + dbnew.m_selectfilelastmodifiedWithSizeCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" "); + dbnew.m_selectfilemetadatahashandsizeCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = @FileId AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" "); // Experimental toggling of the deleted block cache // If the value is less than zero, the lookup is disabled @@ -121,58 +128,58 @@ namespace Duplicati.Library.Main.Database if (deletedBlockCacheSizeLong >= 0) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = dbnew.Connection.CreateCommand()) { - m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempDeletedBlockTable}"" AS ") + dbnew.m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + cmd.SetCommandAndParameters($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS " + @$"SELECT MAX(""ID"") AS ""ID"", ""Hash"", ""Size"" FROM ""DeletedBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""State"" NOT IN (@States)) GROUP BY ""Hash"", ""Size""") .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted, RemoteVolumeState.Deleting]) .ExecuteNonQuery(); - var deletedBlocks = cmd.ExecuteScalarInt64(FormatInvariant(@$"SELECT COUNT(*) FROM ""{m_tempDeletedBlockTable}"""), 0); + var deletedBlocks = cmd.ExecuteScalarInt64(@$"SELECT COUNT(*) FROM ""{dbnew.m_tempDeletedBlockTable}""", 0); // There are no deleted blocks, so we can drop the table if (deletedBlocks == 0) { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{m_tempDeletedBlockTable}""")); - m_tempDeletedBlockTable = null; + cmd.ExecuteNonQuery($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}"""); + dbnew.m_tempDeletedBlockTable = null; } // The deleted blocks are small enough to fit in memory else if (deletedBlocks <= deletedBlockCacheSizeLong) { - m_deletedBlockLookup = new Dictionary>(); - using (var reader = cmd.ExecuteReader(FormatInvariant(@$"SELECT ""ID"", ""Hash"", ""Size"" FROM ""{m_tempDeletedBlockTable}"""))) + dbnew.m_deletedBlockLookup = new Dictionary>(); + using (var reader = cmd.ExecuteReader(@$"SELECT ""ID"", ""Hash"", ""Size"" FROM ""{dbnew.m_tempDeletedBlockTable}""")) while (reader.Read()) { var id = reader.ConvertValueToInt64(0); var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null"); var size = reader.ConvertValueToInt64(2); - if (!m_deletedBlockLookup.TryGetValue(hash, out var sizes)) - m_deletedBlockLookup[hash] = sizes = new Dictionary(); + if (!dbnew.m_deletedBlockLookup.TryGetValue(hash, out var sizes)) + dbnew.m_deletedBlockLookup[hash] = sizes = new Dictionary(); sizes[size] = id; } - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{m_tempDeletedBlockTable}""")); - m_tempDeletedBlockTable = null; + cmd.ExecuteNonQuery($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}"""); + dbnew.m_tempDeletedBlockTable = null; } // The deleted blocks are too large to fit in memory, so we use a temporary table else { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""unique_{m_tempDeletedBlockTable}"" ON ""{m_tempDeletedBlockTable}"" (""Hash"", ""Size"")")); - m_findindeletedCommand = m_connection.CreateCommand(FormatInvariant($@"SELECT ""ID"" FROM ""{m_tempDeletedBlockTable}"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size")); - m_moveblockfromdeletedCommand = m_connection.CreateCommand(string.Join(";", + cmd.ExecuteNonQuery($@"CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" ON ""{dbnew.m_tempDeletedBlockTable}"" (""Hash"", ""Size"")"); + dbnew.m_findindeletedCommand = dbnew.m_connection.CreateCommand($@"SELECT ""ID"" FROM ""{dbnew.m_tempDeletedBlockTable}"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + dbnew.m_moveblockfromdeletedCommand = dbnew.m_connection.CreateCommand(string.Join(";", @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", - FormatInvariant(@$"DELETE FROM {m_tempDeletedBlockTable} WHERE ""ID"" = @DeletedBlockId"), - "SELECT last_insert_rowid()")); + @$"DELETE FROM {dbnew.m_tempDeletedBlockTable} WHERE ""ID"" = @DeletedBlockId"), + "SELECT last_insert_rowid()"); } if (deletedBlocks > 0) { - m_moveblockfromdeletedCommand = m_connection.CreateCommand(string.Join(";", + dbnew.m_moveblockfromdeletedCommand = dbnew.m_connection.CreateCommand(string.Join(";", @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", "SELECT last_insert_rowid()")); @@ -188,13 +195,13 @@ namespace Duplicati.Library.Main.Database if (testqueryversion != 0) Logging.Log.WriteWarningMessage(LOGTAG, "TestFileQuery", null, "Using performance test query version {0} as the TEST_QUERY_VERSION environment variable is set", testqueryversion); - // The original query (v==1) finds the most recent entry of the file in question, + // The original query (v==1) finds the most recent entry of the file in question, // but it requires some large joins to extract the required information. // To speed it up, we use a slightly simpler approach that only looks at the // previous fileset, and uses information here. // If there is a case where a file is sometimes there and sometimes not // (i.e. filter file, remove filter) we will not find the file. - // We currently use this faster version, + // We currently use this faster version, // but allow users to switch back via an environment variable // such that we can get performance feedback @@ -225,10 +232,10 @@ namespace Duplicati.Library.Main.Database @" WHERE ""A"".""ID"" = ""B"".""FileID"" " + @" AND ""B"".""FilesetID"" = @FilesetId "; - findQuery = FormatInvariant($@"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" -FROM ({getLastFileEntryForPath}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" -WHERE ""C"".""BlocksetID"" == ""D"".""ID"" AND ""C"".""MetadataID"" == ""F"".""ID"" AND ""F"".""BlocksetID"" = ""E"".""ID"" -LIMIT 1"); + findQuery = $@"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" +FROM ({getLastFileEntryForPath}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" +WHERE ""C"".""BlocksetID"" == ""D"".""ID"" AND ""C"".""MetadataID"" == ""F"".""ID"" AND ""F"".""BlocksetID"" = ""E"".""ID"" +LIMIT 1"; break; // Potentially faster query: https://forum.duplicati.com/t/release-2-0-3-10-canary-2018-08-30/4497/25 @@ -261,14 +268,16 @@ LIMIT 1"); } - m_findfileCommand = m_connection.CreateCommand(findQuery); + dbnew.m_findfileCommand = dbnew.m_connection.CreateCommand(findQuery); - m_selectfileHashCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = @FileId "); - m_getfirstfilesetwithblockinblockset = m_connection.CreateCommand(@"SELECT MIN(""FilesetEntry"".""FilesetID"") FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" IN ( + dbnew.m_selectfileHashCommand = dbnew.m_connection.CreateCommand(@"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = @FileId "); + dbnew.m_getfirstfilesetwithblockinblockset = dbnew.m_connection.CreateCommand(@"SELECT MIN(""FilesetEntry"".""FilesetID"") FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" IN ( SELECT ""File"".""ID"" FROM ""File"" WHERE ""File"".""BlocksetID"" IN( SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""Hash"" = @Hash))"); - m_blocklistHashes = new HashSet(); + dbnew.m_blocklistHashes = new HashSet(); + + return dbnew; } /// @@ -714,7 +723,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't + /// Populates FilesetEntry table with files from previous fileset, which aren't /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. /// /// Transaction @@ -725,7 +734,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't + /// Populates FilesetEntry table with files from previous fileset, which aren't /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. /// /// Transaction @@ -762,8 +771,8 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// /// Transaction @@ -774,8 +783,8 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// /// Transaction @@ -806,7 +815,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist .SetParameterValue("@CurrentFilesetId", fileSetId) .ExecuteNonQuery(); - // now we need to remove, from the above, any entries that were enumerated by the + // now we need to remove, from the above, any entries that were enumerated by the // UNC-driven backup cmdDelete.SetTransaction(tr.Parent) .SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = @FileId")); @@ -828,7 +837,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } // now copy the temporary table into the FileSetEntry table - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") + cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @FilesetId, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}""")) .SetParameterValue("@FilesetId", fileSetId) .ExecuteNonQuery(); From d4c56ec9ef60cd5688eca74499f5465e26a89a44 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 13:48:07 +0200 Subject: [PATCH 056/362] Changed TemporaryDbValueList from IDb to Microsoft.Data.Sqlite --- .../Library/Main/Database/ExtensionMethods.cs | 2 +- .../Main/Database/TemporaryDbValueList.cs | 56 ++++++++----------- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 6edc22fa9..7cbd9e953 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -241,7 +241,7 @@ public static partial class ExtensionMethods return cmd; } - public static SqliteCommand ExpandInClauseParameter(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) + internal static SqliteCommand ExpandInClauseParameter(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); diff --git a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs index 91f1aa834..36c8dac05 100644 --- a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs +++ b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs @@ -1,27 +1,28 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Data; using System.Linq; +using Microsoft.Data.Sqlite; #nullable enable @@ -30,7 +31,7 @@ namespace Duplicati.Library.Main.Database; /// /// A list of values that can be used in a query /// -public class TemporaryDbValueList : IDisposable +internal class TemporaryDbValueList : IDisposable { /// /// The tag used for logging @@ -40,15 +41,8 @@ public class TemporaryDbValueList : IDisposable /// /// The command to use /// - private IDbCommand? _cmd; - /// - /// The connection to use - /// - private readonly IDbConnection _con; - /// - /// The transaction to use, if any - /// - private readonly IDbTransaction? _transaction; + private SqliteCommand? _cmd; + private LocalDatabase _db; /// /// The table to use /// @@ -77,10 +71,9 @@ public class TemporaryDbValueList : IDisposable /// The connection to use /// The transaction to use /// The values to use - public TemporaryDbValueList(IDbConnection con, IDbTransaction? transaction, IEnumerable values) + public TemporaryDbValueList(LocalDatabase db, IEnumerable values) { - _con = con; - _transaction = transaction; + _db = db; _valuesType = "INTEGER"; if (values == null) throw new ArgumentNullException(nameof(values)); @@ -95,10 +88,9 @@ public class TemporaryDbValueList : IDisposable /// The command to use /// The connection to use /// The values to use - public TemporaryDbValueList(IDbConnection con, IDbTransaction? transaction, IEnumerable values) + public TemporaryDbValueList(LocalDatabase db, IEnumerable values) { - _con = con; - _transaction = transaction; + _db = db; _valuesType = "TEXT"; if (values == null) throw new ArgumentNullException(nameof(values)); @@ -142,7 +134,7 @@ public class TemporaryDbValueList : IDisposable if (_isTable) return; - _cmd = _con.CreateCommand(_transaction); + _cmd = _db.Connection.CreateCommand(_db.Transaction); _cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType})"); _isTable = true; foreach (var slice in _values.Chunk(LocalDatabase.CHUNK_SIZE)) From ae12d63468607aa30346e7e66ff07a88ce81f07d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 14:20:10 +0200 Subject: [PATCH 057/362] Changed TemporaryDbValueList to use factory construction and to be proper async. --- .../Library/Main/Database/ExtensionMethods.cs | 4 +- .../Library/Main/Database/LocalDatabase.cs | 4 +- .../Main/Database/TemporaryDbValueList.cs | 90 +++++++++++-------- 3 files changed, 57 insertions(+), 41 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 7cbd9e953..247912a58 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -241,7 +241,7 @@ public static partial class ExtensionMethods return cmd; } - internal static SqliteCommand ExpandInClauseParameter(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) + internal static async Task ExpandInClauseParameter(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); @@ -250,7 +250,7 @@ public static partial class ExtensionMethods return ExpandInClauseParameter(cmd, originalParamName, values.Values); // We have a temporary table, so we need to replace the parameter with the table name - cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(), StringComparison.OrdinalIgnoreCase); + cmd.CommandText = cmd.CommandText.Replace(originalParamName, await values.GetInClause(), StringComparison.OrdinalIgnoreCase); return cmd; } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 15fba1402..dd3c3ccea 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -544,8 +544,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" IN (@Name) "); cmd.SetTransaction(transaction); - using var tmptable = new TemporaryDbValueList(m_connection, transaction, files); - cmd.ExpandInClauseParameter("@Name", tmptable); + using var tmptable = await TemporaryDbValueList.CreateAsync(this, files); + await cmd.ExpandInClauseParameter("@Name", tmptable); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) diff --git a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs index 36c8dac05..a442b1773 100644 --- a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs +++ b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs @@ -22,6 +22,8 @@ using System; using System.Collections.Generic; using System.Data; using System.Linq; +using System.Threading.Tasks; +using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; #nullable enable @@ -42,7 +44,7 @@ internal class TemporaryDbValueList : IDisposable /// The command to use /// private SqliteCommand? _cmd; - private LocalDatabase _db; + private readonly LocalDatabase _db; /// /// The table to use /// @@ -59,44 +61,43 @@ internal class TemporaryDbValueList : IDisposable /// /// The values to use /// - private IEnumerable _values; + private readonly IEnumerable _values; /// /// The type of the values /// private readonly string _valuesType; - /// - /// Creates a new reusable transaction - /// - /// The connection to use - /// The transaction to use - /// The values to use - public TemporaryDbValueList(LocalDatabase db, IEnumerable values) - { - _db = db; - _valuesType = "INTEGER"; - if (values == null) - throw new ArgumentNullException(nameof(values)); - _values = values.Cast(); - if (values.Count() > LocalDatabase.CHUNK_SIZE) - ForceToTable(); - } - /// /// Creates a new reusable transaction /// /// The command to use /// The connection to use /// The values to use - public TemporaryDbValueList(LocalDatabase db, IEnumerable values) + private TemporaryDbValueList(LocalDatabase db, IEnumerable values, string valuesType) { _db = db; - _valuesType = "TEXT"; - if (values == null) - throw new ArgumentNullException(nameof(values)); - _values = values.Cast(); - if (values.Count() > LocalDatabase.CHUNK_SIZE) - ForceToTable(); + _valuesType = valuesType; + _values = values; + + ArgumentNullException.ThrowIfNull(values); + } + + internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) + { + return await DoCreateAsync(new TemporaryDbValueList(db, values.Cast(), "INTEGER")); + } + + internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) + { + return await DoCreateAsync(new TemporaryDbValueList(db, values.Cast(), "TEXT")); + } + + private static async Task DoCreateAsync(TemporaryDbValueList valueList) + { + if (valueList._values.Count() > LocalDatabase.CHUNK_SIZE) + await valueList.ForceToTable(); + + return valueList; } /// @@ -113,20 +114,23 @@ internal class TemporaryDbValueList : IDisposable /// Get the in clause for the values, creating the table if needed /// /// The in clause for the values - public string GetInClause() + public async Task GetInClause() { if (_disposed) throw new ObjectDisposedException(nameof(TemporaryDbValueList)); - ForceToTable(); + await ForceToTable(); - return $@"SELECT ""Value"" FROM ""{_tableName}"""; + return $@" + SELECT ""Value"" + FROM ""{_tableName}"" + "; } /// /// Force the values to be written to a table, if not already done /// - public void ForceToTable() + public async Task ForceToTable() { if (_disposed) throw new ObjectDisposedException(nameof(TemporaryDbValueList)); @@ -135,18 +139,23 @@ internal class TemporaryDbValueList : IDisposable return; _cmd = _db.Connection.CreateCommand(_db.Transaction); - _cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType})"); + await _cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType}) + "); _isTable = true; foreach (var slice in _values.Chunk(LocalDatabase.CHUNK_SIZE)) { var parameterNames = slice.Select((_, i) => $"@p{i}").ToArray(); - var sql = $@"INSERT INTO ""{_tableName}"" (""Value"") VALUES {string.Join(", ", parameterNames.Select(p => $"({p})"))}"; + var sql = $@" + INSERT INTO ""{_tableName}"" (""Value"") + VALUES {string.Join(", ", parameterNames.Select(p => $"({p})"))} + "; - _cmd.CommandText = sql; + _cmd.SetCommandAndParameters(sql); for (int i = 0; i < slice.Length; i++) - _cmd.AddNamedParameter(parameterNames[i], slice[i]); + _cmd.SetParameterValue(parameterNames[i], slice[i]); - _cmd.ExecuteNonQuery(); + await _cmd.ExecuteNonQueryAsync(); } } @@ -157,6 +166,11 @@ internal class TemporaryDbValueList : IDisposable /// public void Dispose() + { + DisposeAsync().Await(); + } + + public async Task DisposeAsync() { if (_disposed) return; @@ -167,8 +181,10 @@ internal class TemporaryDbValueList : IDisposable try { if (_cmd != null) - _cmd.ExecuteNonQuery($@"DROP TABLE ""{_tableName}"""); - _cmd?.Dispose(); + { + await _cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{_tableName}"""); + await _cmd.DisposeAsync(); + } _cmd = null; } catch (Exception ex) From 48e6b619772e3e172f11ea457fc748ed55aeef39 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 14:40:00 +0200 Subject: [PATCH 058/362] Purged IDb from LocalBackupDatabase --- .../Main/Database/LocalBackupDatabase.cs | 340 +++++++++--------- 1 file changed, 161 insertions(+), 179 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 817778f3c..ec0a1eb0d 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -172,8 +172,8 @@ namespace Duplicati.Library.Main.Database dbnew.m_moveblockfromdeletedCommand = dbnew.m_connection.CreateCommand(string.Join(";", @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", - @$"DELETE FROM {dbnew.m_tempDeletedBlockTable} WHERE ""ID"" = @DeletedBlockId"), - "SELECT last_insert_rowid()"); + @$"DELETE FROM {dbnew.m_tempDeletedBlockTable} WHERE ""ID"" = @DeletedBlockId", + "SELECT last_insert_rowid()")); } @@ -286,9 +286,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The block key /// The size of the block /// True if the block should be added to the current output - public long FindBlockID(string key, long size, IDbTransaction? transaction = null) + public long FindBlockID(string key, long size) { - return m_findblockCommand.SetTransaction(transaction) + return m_findblockCommand.SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) .ExecuteScalarInt64(m_logQueries, -1); @@ -300,9 +300,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The block key /// The size of the block /// True if the block should be added to the current output - public bool AddBlock(string key, long size, long volumeid, IDbTransaction? transaction = null) + public bool AddBlock(string key, long size, long volumeid) { - var r = FindBlockID(key, size, transaction); + var r = FindBlockID(key, size); if (r == -1L) { if (m_moveblockfromdeletedCommand != null) @@ -312,7 +312,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist if (m_deletedBlockLookup.TryGetValue(key, out var sizes)) if (sizes.TryGetValue(size, out var id)) { - m_moveblockfromdeletedCommand.SetTransaction(transaction) + m_moveblockfromdeletedCommand.SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) .ExecuteNonQuery(m_logQueries); @@ -332,7 +332,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist if (id != -1) { - var c = m_moveblockfromdeletedCommand.SetTransaction(transaction) + var c = m_moveblockfromdeletedCommand.SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) .ExecuteNonQuery(m_logQueries); @@ -345,7 +345,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - var ins = m_insertblockCommand.SetTransaction(transaction) + var ins = m_insertblockCommand.SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@Size", size) @@ -371,77 +371,74 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The list of hashes /// The id of the blockset, new or old /// True if the blockset was created, false otherwise - public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid, IDbTransaction? transaction = null) + public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid) { - blocksetid = m_findblocksetCommand.SetTransaction(transaction) + blocksetid = m_findblocksetCommand.SetTransaction(m_rtr) .SetParameterValue("@Fullhash", filehash) .SetParameterValue("@Length", size) .ExecuteScalarInt64(m_logQueries, null, -1); if (blocksetid != -1) return false; //Found it - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + blocksetid = m_insertblocksetCommand.SetTransaction(m_rtr) + .SetParameterValue("@Length", size) + .SetParameterValue("@Fullhash", filehash) + .ExecuteScalarInt64(m_logQueries); + + long ix = 0; + if (blocklistHashes != null) { - blocksetid = m_insertblocksetCommand.SetTransaction(tr.Parent) - .SetParameterValue("@Length", size) - .SetParameterValue("@Fullhash", filehash) - .ExecuteScalarInt64(m_logQueries); - - long ix = 0; - if (blocklistHashes != null) - { - m_insertblocklistHashesCommand.SetTransaction(tr.Parent) - .SetParameterValue("@BlocksetId", blocksetid); - - foreach (var bh in blocklistHashes) - { - var c = m_insertblocklistHashesCommand.SetParameterValue("@Index", ix) - .SetParameterValue("@Hash", bh) - .ExecuteNonQuery(m_logQueries); - if (c != 1) - throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}"); - ix++; - } - } - - m_insertblocksetentryCommand.SetTransaction(tr.Parent) + m_insertblocklistHashesCommand.SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid); - ix = 0; - long remainsize = size; - foreach (var h in hashes) + foreach (var bh in blocklistHashes) { - var exsize = remainsize < blocksize ? remainsize : blocksize; - var c = m_insertblocksetentryCommand.SetParameterValue("@Index", ix) - .SetParameterValue("@Hash", h) - .SetParameterValue("@Size", exsize) + var c = m_insertblocklistHashesCommand.SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", bh) .ExecuteNonQuery(m_logQueries); if (c != 1) - { - Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size); - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - var bid = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash") - .SetParameterValue("@Hash", h) - .ExecuteScalarInt64(-1); - if (bid == -1) - throw new Exception($"Could not find any blocks with the given hash: {h}"); - cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = @Hash") - .SetParameterValue("@Hash", h); - foreach (var rd in cmd.ExecuteReaderEnumerable()) - Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); - } + throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}"); + ix++; + } + } - throw new Exception($"Unexpected result count: {c}, expected {1}, check log for more messages"); + m_insertblocksetentryCommand.SetTransaction(m_rtr) + .SetParameterValue("@BlocksetId", blocksetid); + + ix = 0; + long remainsize = size; + foreach (var h in hashes) + { + var exsize = remainsize < blocksize ? remainsize : blocksize; + var c = m_insertblocksetentryCommand.SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", h) + .SetParameterValue("@Size", exsize) + .ExecuteNonQuery(m_logQueries); + if (c != 1) + { + Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size); + using (var cmd = m_connection.CreateCommand(m_rtr)) + { + var bid = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash") + .SetParameterValue("@Hash", h) + .ExecuteScalarInt64(-1); + if (bid == -1) + throw new Exception($"Could not find any blocks with the given hash: {h}"); + cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = @Hash") + .SetParameterValue("@Hash", h); + foreach (var rd in cmd.ExecuteReaderEnumerable()) + Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); } - ix++; - remainsize -= blocksize; + throw new Exception($"Unexpected result count: {c}, expected {1}, check log for more messages"); } - tr.Commit(); + ix++; + remainsize -= blocksize; } + m_rtr.Commit(); + return true; } @@ -453,11 +450,11 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The size of the metadata. /// The ID of the metadataset. /// An optional transaction. - public bool GetMetadatasetID(string filehash, long size, out long metadataid, IDbTransaction? transaction = null) + public bool GetMetadatasetID(string filehash, long size, out long metadataid) { if (size > 0) { - metadataid = m_findmetadatasetCommand.SetTransaction(transaction) + metadataid = m_findmetadatasetCommand.SetTransaction(m_rtr) .SetParameterValue("@Hash", filehash) .SetParameterValue("@Size", size) .ExecuteScalarInt64(m_logQueries, null, -1); @@ -477,19 +474,18 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The id of the blockset to add /// The id of the metadata set /// True if the set was added to the database, false otherwise - public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, IDbTransaction? transaction = null) + public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid) { - if (GetMetadatasetID(filehash, size, out metadataid, transaction)) + if (GetMetadatasetID(filehash, size, out metadataid)) return false; - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - metadataid = m_insertmetadatasetCommand.SetTransaction(tr.Parent) - .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64(m_logQueries); - tr.Commit(); - return true; - } + metadataid = m_insertmetadatasetCommand.SetTransaction(m_rtr) + .SetParameterValue("@BlocksetId", blocksetid) + .ExecuteScalarInt64(m_logQueries); + + m_rtr.Commit(); + + return true; } /// @@ -501,9 +497,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the hashkey for the file /// The ID for the metadata /// The transaction to use for insertion, or null for no transaction - public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction? transaction) + public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID) { - var fileidobj = m_findfilesetCommand.SetTransaction(transaction) + var fileidobj = m_findfilesetCommand.SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetID) .SetParameterValue("@MetadataId", metadataID) .SetParameterValue("@Path", filename) @@ -512,19 +508,16 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist if (fileidobj == -1) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - fileidobj = m_insertfileCommand.SetTransaction(tr.Parent) - .SetParameterValue("@PrefixId", pathprefixid) - .SetParameterValue("@Path", filename) - .SetParameterValue("@BlocksetId", blocksetID) - .SetParameterValue("@MetadataId", metadataID) - .ExecuteScalarInt64(m_logQueries); - tr.Commit(); - } + fileidobj = m_insertfileCommand.SetTransaction(m_rtr) + .SetParameterValue("@PrefixId", pathprefixid) + .SetParameterValue("@Path", filename) + .SetParameterValue("@BlocksetId", blocksetID) + .SetParameterValue("@MetadataId", metadataID) + .ExecuteScalarInt64(m_logQueries); + m_rtr.Commit(); } - AddKnownFile(fileidobj, lastmodified, transaction); + AddKnownFile(fileidobj, lastmodified); } /// @@ -535,10 +528,10 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the hashkey for the file /// The ID for the metadata /// The transaction to use for insertion, or null for no transaction - public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction? transaction) + public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID) { var split = SplitIntoPrefixAndName(filename); - AddFile(GetOrCreatePathPrefix(split.Key, transaction), split.Value, lastmodified, blocksetID, metadataID, transaction); + AddFile(GetOrCreatePathPrefix(split.Key), split.Value, lastmodified, blocksetID, metadataID); } /// @@ -547,30 +540,30 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Id of the file /// The time the file was modified /// The transaction to use for insertion, or null for no transaction - public void AddKnownFile(long fileid, DateTime lastmodified, IDbTransaction? transaction = null) + public void AddKnownFile(long fileid, DateTime lastmodified) { - m_insertfileOperationCommand.SetTransaction(transaction) + m_insertfileOperationCommand.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", m_filesetId) .SetParameterValue("@FileId", fileid) .SetParameterValue("@LastModified", lastmodified.ToUniversalTime().Ticks) .ExecuteNonQuery(m_logQueries); } - public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction? transaction = null) + public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified) { - AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID, transaction); + AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID); } - public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction? transaction = null) + public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified) { - AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction); + AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID); } - public long GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, IDbTransaction? transaction = null) + public long GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, out DateTime oldModified, out long length) { if (includeLength) { - m_selectfilelastmodifiedWithSizeCommand.SetTransaction(transaction) + m_selectfilelastmodifiedWithSizeCommand.SetTransaction(m_rtr) .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); @@ -584,7 +577,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } else { - m_selectfilelastmodifiedCommand.SetTransaction(transaction) + m_selectfilelastmodifiedCommand.SetTransaction(m_rtr) .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); @@ -603,9 +596,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } - public long GetFileEntry(long prefixid, string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string? oldMetahash, out long oldMetasize, IDbTransaction transaction) + public long GetFileEntry(long prefixid, string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string? oldMetahash, out long oldMetasize) { - m_findfileCommand.SetTransaction(transaction) + m_findfileCommand.SetTransaction(m_rtr) .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); @@ -629,9 +622,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - public (long Size, string MetadataHash)? GetMetadataHashAndSizeForFile(long fileid, IDbTransaction transaction) + public (long Size, string MetadataHash)? GetMetadataHashAndSizeForFile(long fileid) { - m_selectfilemetadatahashandsizeCommand.SetTransaction(transaction) + m_selectfilemetadatahashandsizeCommand.SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid); using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader()) @@ -642,9 +635,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } - public string? GetFileHash(long fileid, IDbTransaction transaction) + public string? GetFileHash(long fileid) { - var r = m_selectfileHashCommand.SetTransaction(transaction) + var r = m_selectfileHashCommand.SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid) .ExecuteScalar(m_logQueries, null); if (r == null || r == DBNull.Value) @@ -669,23 +662,23 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist base.Dispose(); } - public long GetLastWrittenDBlockVolumeSize(IDbTransaction? transaction) + public long GetLastWrittenDBlockVolumeSize() { - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand(m_rtr)) return cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""RemoteVolume"" WHERE ""State"" = @State AND ""Type"" = @Type ORDER BY ""ID"" DESC LIMIT 1") .SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExecuteScalarInt64(-1); } - private long GetPreviousFilesetID(IDbCommand cmd) + private long GetPreviousFilesetID(SqliteCommand cmd) { - return GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId, cmd.Transaction); + return GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId); } - private long GetPreviousFilesetID(IDbCommand cmd, DateTime timestamp, long filesetid, IDbTransaction? transaction) + private long GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid) { - return cmd.SetTransaction(transaction) + return cmd.SetTransaction(m_rtr) .SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < @Timestamp AND ""ID"" != @FilesetId ORDER BY ""Timestamp"" DESC ") .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) .SetParameterValue("@FilesetId", filesetid) @@ -713,9 +706,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - internal void UpdateChangeStatistics(BackupResults results, IDbTransaction transaction) + internal void UpdateChangeStatistics(BackupResults results) { - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand(m_rtr)) { var prevFileSetId = GetPreviousFilesetID(cmd); ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId); @@ -728,9 +721,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// /// Transaction /// List of deleted paths, or null - public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable? deleted = null) + public void AppendFilesFromPreviousSet(IEnumerable? deleted = null) { - AppendFilesFromPreviousSet(transaction, deleted, m_filesetId, -1, OperationTimestamp); + AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp); } /// @@ -742,15 +735,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Current file-set ID /// Source file-set ID /// If filesetid == -1, used to locate previous file-set - public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) + public void AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) { using (var cmd = m_connection.CreateCommand()) using (var cmdDelete = m_connection.CreateCommand()) - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { long lastFilesetId = prevId < 0 ? GetPreviousFilesetID(cmd, timestamp, filesetid, tr.Parent) : prevId; - cmd.SetTransaction(tr.Parent) + cmd.SetTransaction(m_rtr) .SetCommandAndParameters(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @CurrentFilesetId AS ""FilesetID"", ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId)) ") .SetParameterValue("@CurrentFilesetId", filesetid) .SetParameterValue("@PreviousFilesetId", lastFilesetId) @@ -759,14 +751,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist if (deleted != null) { using var tmplist = new TemporaryDbValueList(m_connection, tr.Parent, deleted); - cmdDelete.SetTransaction(tr.Parent) + cmdDelete.SetTransaction(m_rtr) .SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId AND ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" IN (@Paths)) ") .SetParameterValue("@FilesetId", filesetid) .ExpandInClauseParameter("@Paths", tmplist) .ExecuteNonQuery(m_logQueries); } - tr.Commit(); + m_rtr.Commit(); } } @@ -777,9 +769,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// /// Transaction /// Optional exclusion predicate (true = exclude file) - public void AppendFilesFromPreviousSetWithPredicate(IDbTransaction transaction, Func exclusionPredicate) + public void AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate) { - AppendFilesFromPreviousSetWithPredicate(transaction, exclusionPredicate, m_filesetId, -1, OperationTimestamp); + AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp); } /// @@ -792,24 +784,22 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Current fileset ID /// Source fileset ID /// If prevFileSetId == -1, used to locate previous fileset - public void AppendFilesFromPreviousSetWithPredicate(IDbTransaction transaction, - Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) + public void AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) { if (exclusionPredicate == null) { - AppendFilesFromPreviousSet(transaction, null, fileSetId, prevFileSetId, timestamp); + AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp); return; } using (var cmd = m_connection.CreateCommand()) using (var cmdDelete = m_connection.CreateCommand()) - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId, tr.Parent) : prevFileSetId; + long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId; // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.Transaction = tr.Parent; + cmd.SetTransaction(m_rtr) cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId))")) .SetParameterValue("@PreviousFilesetId", lastFilesetId) .SetParameterValue("@CurrentFilesetId", fileSetId) @@ -817,11 +807,10 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist // now we need to remove, from the above, any entries that were enumerated by the // UNC-driven backup - cmdDelete.SetTransaction(tr.Parent) + cmdDelete.SetTransaction(m_rtr) .SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = @FileId")); // enumerate files from new temporary file set, and remove any entries handled by UNC - cmd.Transaction = tr.Parent; foreach (var row in cmd.ExecuteReaderEnumerable(FormatInvariant( $@"SELECT f.""Path"", fs.""FileID"", fs.""Lastmodified"", COALESCE(bs.""Length"", -1) FROM (SELECT DISTINCT ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"") AS fs @@ -842,7 +831,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist .SetParameterValue("@FilesetId", fileSetId) .ExecuteNonQuery(); - tr.Commit(); + m_rtr.Commit(); } } @@ -852,14 +841,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the fileset volume to update /// The timestamp of the operation to create /// An optional external transaction - public override long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction? transaction = null) + public override long CreateFileset(long volumeid, DateTime timestamp) { - return m_filesetId = base.CreateFileset(volumeid, timestamp, transaction); + return m_filesetId = base.CreateFileset(volumeid, timestamp); } - public IEnumerable GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction? transaction = null) + public IEnumerable GetTemporaryFilelistVolumeNames(bool latestOnly) { - var incompleteFilesetIDs = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).Select(x => x.Key).ToArray(); + var incompleteFilesetIDs = GetIncompleteFilesets().OrderBy(x => x.Value).Select(x => x.Key).ToArray(); if (!incompleteFilesetIDs.Any()) return Enumerable.Empty(); @@ -874,9 +863,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist return volumeNames; } - public IEnumerable GetMissingIndexFiles(IDbTransaction transaction) + public IEnumerable GetMissingIndexFiles() { - using var cmd = m_connection.CreateCommand(transaction) + using var cmd = m_connection.CreateCommand(m_rtr) .SetCommandAndParameters(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND NOT ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"") AND ""State"" IN (@States)") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); @@ -886,12 +875,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } - public void MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid, IDbTransaction transaction) + public void MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid) { using (var cmd = m_connection.CreateCommand()) { - cmd.Transaction = transaction; var c = cmd.SetCommandAndParameters(@"UPDATE ""Block"" SET ""VolumeID"" = @NewVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" = @PreviousVolumeId ") + .SetTransaction(m_rtr) .SetParameterValue("@NewVolumeId", targetvolumeid) .SetParameterValue("@Hash", blockkey) .SetParameterValue("@Size", size) @@ -902,11 +891,11 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - public void SafeDeleteRemoteVolume(string name, IDbTransaction transaction) + public void SafeDeleteRemoteVolume(string name) { - var volumeid = GetRemoteVolumeID(name, transaction); + var volumeid = GetRemoteVolumeID(name); - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand(m_rtr)) { var c = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", volumeid) @@ -914,14 +903,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist if (c != 0) throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); - RemoveRemoteVolume(name, transaction); + RemoveRemoteVolume(name); } } - public string[] GetBlocklistHashes(string name, IDbTransaction transaction) + public string[] GetBlocklistHashes(string name) { - var volumeid = GetRemoteVolumeID(name, transaction); - using (var cmd = m_connection.CreateCommand(transaction)) + var volumeid = GetRemoteVolumeID(name); + using (var cmd = m_connection.CreateCommand(m_rtr)) { // Grab the strings and return as array to avoid concurrent access to the IEnumerable cmd.SetCommandAndParameters(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE ""Block"".""VolumeID"" = @VolumeId AND ""Block"".""Hash"" IN (SELECT ""Hash"" FROM ""BlocklistHash"")") @@ -977,31 +966,27 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// /// Data to add /// An optional external transaction - public void CreateChangeJournalData(IEnumerable data, IDbTransaction? transaction = null) + public void CreateChangeJournalData(IEnumerable data) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + foreach (var entry in data) { - foreach (var entry in data) + using (var cmd = m_connection.CreateCommand(m_rtr)) { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = tr.Parent; - var c = cmd.SetCommandAndParameters( - @"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (@FilesetId, @VolumeName, @JournalId, @NextUsn, @ConfigHash);") - .SetParameterValue("@FilesetId", m_filesetId) - .SetParameterValue("@VolumeName", entry.Volume) - .SetParameterValue("@JournalId", entry.JournalId) - .SetParameterValue("@NextUsn", entry.NextUsn) - .SetParameterValue("@ConfigHash", entry.ConfigHash) - .ExecuteNonQuery(); + var c = cmd.SetCommandAndParameters( + @"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (@FilesetId, @VolumeName, @JournalId, @NextUsn, @ConfigHash);") + .SetParameterValue("@FilesetId", m_filesetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@ConfigHash", entry.ConfigHash) + .ExecuteNonQuery(); - if (c != 1) - throw new Exception("Unable to add change journal entry"); - } + if (c != 1) + throw new Exception("Unable to add change journal entry"); } - - tr.Commit(); } + + m_rtr.Commit(); } /// @@ -1010,27 +995,24 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Data to add /// Existing file set to update /// An optional external transaction - public void UpdateChangeJournalData(IEnumerable data, long fileSetId, IDbTransaction? transaction = null) + public void UpdateChangeJournalData(IEnumerable data, long fileSetId) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + foreach (var entry in data) { - foreach (var entry in data) + using (var cmd = m_connection.CreateCommand()) { - using (var cmd = m_connection.CreateCommand()) - { - cmd.SetTransaction(tr.Parent) - .SetCommandAndParameters( + cmd.SetCommandAndParameters( @"UPDATE ""ChangeJournalData"" SET ""NextUSN"" = @NextUsn WHERE ""FilesetID"" = @FilesetId AND ""VolumeName"" = @VolumeName AND ""JournalID"" = @JournalId;") - .SetParameterValue("@NextUsn", entry.NextUsn) - .SetParameterValue("@FilesetId", fileSetId) - .SetParameterValue("@VolumeName", entry.Volume) - .SetParameterValue("@JournalId", entry.JournalId) - .ExecuteNonQuery(); - } + .SetTransaction(m_rtr) + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@FilesetId", fileSetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .ExecuteNonQuery(); } - - tr.Commit(); } + + m_rtr.Commit(); } /// @@ -1039,9 +1021,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The hash to check /// An optional external transaction /// True if the hash is known, false otherwise - public bool IsBlocklistHashKnown(string hash, IDbTransaction transaction) + public bool IsBlocklistHashKnown(string hash) { - var res = m_getfirstfilesetwithblockinblockset.SetTransaction(transaction) + var res = m_getfirstfilesetwithblockinblockset.SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .ExecuteScalarInt64(); if (res != -1 && res != m_filesetId) From 973ba1ad9c21faad38c3143e1d6c48f3eca2efa5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 14:40:28 +0200 Subject: [PATCH 059/362] Preliminary work on implicit (from an external view) transactions --- .../Library/Main/Database/LocalDatabase.cs | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index dd3c3ccea..8df7f3650 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -394,19 +394,19 @@ namespace Duplicati.Library.Main.Database return Library.Utility.Utility.EPOCH.AddSeconds(seconds); } - public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, SqliteTransaction transaction) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash) { - await UpdateRemoteVolume(name, state, size, hash, false, transaction); + await UpdateRemoteVolume(name, state, size, hash, false); } - public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, SqliteTransaction transaction) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup) { - await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null, transaction); + await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null); } - public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, SqliteTransaction transaction) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived) { - var c = await m_updateremotevolumeCommand.SetTransaction(transaction) + var c = await m_updateremotevolumeCommand.SetTransaction(m_rtr) .SetParameterValue("@OperationID", m_operationid) .SetParameterValue("@State", state.ToString()) .SetParameterValue("@Hash", hash) @@ -426,7 +426,7 @@ namespace Duplicati.Library.Main.Database SET ""DeleteGraceTime"" = @DeleteGraceTime WHERE ""Name"" = @Name "); - c = await cmd.SetTransaction(transaction) + c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)) .SetParameterValue("@Name", name) .ExecuteNonQueryAsync(); @@ -442,7 +442,7 @@ namespace Duplicati.Library.Main.Database SET ""ArchiveTime"" = @ArchiveTime WHERE ""Name"" = @Name "); - c = await cmd.SetTransaction(transaction) + c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0) .SetParameterValue("@Name", name) .ExecuteNonQueryAsync(); @@ -453,7 +453,7 @@ namespace Duplicati.Library.Main.Database if (!suppressCleanup && state == RemoteVolumeState.Deleted) { - await RemoveRemoteVolume(name, transaction); + await RemoveRemoteVolume(name); } } @@ -526,10 +526,10 @@ namespace Duplicati.Library.Main.Database return (query.ToString(), args); } - public async Task GetRemoteVolumeID(string file, SqliteTransaction transaction) + public async Task GetRemoteVolumeID(string file) { return await m_selectremotevolumeIdCommand - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Name", file) .ExecuteScalarInt64Async(-1); } @@ -630,10 +630,10 @@ namespace Duplicati.Library.Main.Database /// The message type /// The message /// An optional exception - public async Task LogMessage(string type, string message, Exception? exception, SqliteTransaction transaction) + public async Task LogMessage(string type, string message, Exception? exception) { await m_insertlogCommand - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@OperationID", m_operationid) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) .SetParameterValue("@Type", type) @@ -661,16 +661,16 @@ namespace Duplicati.Library.Main.Database await transaction.CommitAsync(); } - public async Task RemoveRemoteVolume(string name, SqliteTransaction transaction) + public async Task RemoveRemoteVolume(string name) { - await RemoveRemoteVolumes([name], transaction); + await RemoveRemoteVolumes([name]); } - public async Task RemoveRemoteVolumes(IEnumerable names, SqliteTransaction transaction) + public async Task RemoveRemoteVolumes(IEnumerable names) { if (names == null || !names.Any()) return; - using var deletecmd = m_connection.CreateCommand(); + using var deletecmd = m_connection.CreateCommand(m_rtr); string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var volidstable = "DelVolSetIds-" + temptransguid; var blocksetidstable = "DelBlockSetIds-" + temptransguid; @@ -921,12 +921,13 @@ namespace Duplicati.Library.Main.Database catch { /* Ignore, will be deleted on close anyway. */ } m_removeremotevolumeCommand - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); foreach (var name in names) { await m_removeremotevolumeCommand + .SetTransaction(m_rtr) .SetParameterValue("@Name", name) .ExecuteNonQueryAsync(); } @@ -942,7 +943,7 @@ namespace Duplicati.Library.Main.Database if (nonAttachedFiles > 0) throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public async Task Vacuum() @@ -2087,7 +2088,7 @@ namespace Duplicati.Library.Main.Database /// The ID of the fileset volume to update /// The timestamp of the operation to create /// An optional external transaction - public virtual async Task CreateFileset(long volumeid, DateTime timestamp, SqliteTransaction transaction) + public virtual async Task CreateFileset(long volumeid, DateTime timestamp) { using var cmd = await m_connection.CreateCommandAsync(@" INSERT INTO ""Fileset"" ( @@ -2104,14 +2105,14 @@ namespace Duplicati.Library.Main.Database ); SELECT last_insert_rowid(); "); - var id = await cmd.SetTransaction(transaction) + var id = await cmd.SetTransaction(m_rtr) .SetParameterValue("@OperationId", m_operationid) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP) .ExecuteScalarInt64Async(-1); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); return id; } @@ -2246,13 +2247,13 @@ namespace Duplicati.Library.Main.Database /// /// The transaction to use /// The last incomplete fileset or default - public async Task GetLastIncompleteFilesetVolume(SqliteTransaction transaction) + public async Task GetLastIncompleteFilesetVolume() { - var candidates = GetIncompleteFilesets(transaction) + var candidates = GetIncompleteFilesets() .OrderBy(x => x.Value); if (await candidates.AnyAsync()) - return await GetRemoteVolumeFromFilesetID((await candidates.LastAsync()).Key, transaction); + return await GetRemoteVolumeFromFilesetID((await candidates.LastAsync()).Key); return default; } @@ -2262,7 +2263,7 @@ namespace Duplicati.Library.Main.Database /// /// An optional transaction /// A list of fileset IDs and timestamps - public async IAsyncEnumerable> GetIncompleteFilesets(SqliteTransaction transaction) + public async IAsyncEnumerable> GetIncompleteFilesets() { using var cmd = await m_connection.CreateCommandAsync(@$" SELECT DISTINCT @@ -2283,7 +2284,7 @@ namespace Duplicati.Library.Main.Database ) "); - using var rd = await cmd.SetTransaction(transaction) + using var rd = await cmd.SetTransaction(m_rtr) .ExecuteReaderAsync(); while (await rd.ReadAsync()) { @@ -2301,7 +2302,7 @@ namespace Duplicati.Library.Main.Database /// The fileset ID /// An optional transaction /// The remote volume entry or default - public async Task GetRemoteVolumeFromFilesetID(long filesetID, SqliteTransaction transaction) + public async Task GetRemoteVolumeFromFilesetID(long filesetID) { using var cmd = await m_connection.CreateCommandAsync(@" SELECT @@ -2321,7 +2322,7 @@ namespace Duplicati.Library.Main.Database AND ""Fileset"".""ID"" = @FilesetId "); - using var rd = await cmd.SetTransaction(transaction) + using var rd = await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetID) .ExecuteReaderAsync(); if (await rd.ReadAsync()) @@ -2481,7 +2482,6 @@ namespace Duplicati.Library.Main.Database var serializer = new JsonFormatSerializer(); await LogMessage("Result", serializer.SerializeResults(result), - null, null ); } @@ -2502,7 +2502,7 @@ namespace Duplicati.Library.Main.Database /// The path prefix ID. /// The path to get the prefix for. /// The transaction to use for insertion, or null for no transaction - public async Task GetOrCreatePathPrefix(string prefix, SqliteTransaction transaction) + public async Task GetOrCreatePathPrefix(string prefix) { // Ring-buffer style lookup for (var i = 0; i < m_pathPrefixLookup.Length; i++) @@ -2512,13 +2512,13 @@ namespace Duplicati.Library.Main.Database return m_pathPrefixLookup[ix].Value; } - var id = await m_findpathprefixCommand.SetTransaction(transaction) + var id = await m_findpathprefixCommand.SetTransaction(m_rtr) .SetParameterValue("@Prefix", prefix) .ExecuteScalarInt64Async(); if (id < 0) { - id = await m_insertpathprefixCommand.SetTransaction(transaction) + id = await m_insertpathprefixCommand.SetTransaction(m_rtr) .SetParameterValue("@Prefix", prefix) .ExecuteScalarInt64Async(); } From 41462ba7e30f147c8515f2471d3a109d7b8a968b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 14:41:16 +0200 Subject: [PATCH 060/362] Added default value to the synchronous ReusableTransaction.Commit(), as nothing should be a valid parameter --- Duplicati/Library/Main/Database/ReusableTransaction.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 5d2179e6e..4d303967c 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -70,7 +70,7 @@ internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transact /// The log message to use. /// True if the transaction should be restarted. /// If the transaction is already Disposed. - public void Commit(string? message, bool restart = true) + public void Commit(string? message = null, bool restart = true) { CommitAsync(message, restart).Await(); } From c848ca68199c38ba555cce5bef9d6be3d420355b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 15:43:21 +0200 Subject: [PATCH 061/362] Added more extension methods as they're required by LocalBackupDatabase --- .../Library/Main/Database/ExtensionMethods.cs | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 247912a58..0be32a1f8 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -88,6 +88,20 @@ public static partial class ExtensionMethods return self.CreateCommand(rtr.Transaction); } + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog) + { + return await ExecuteNonQueryAsync(self, writeLog, null); + } + + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", string.Format("ExecuteNonQueryAsync: {0}", self.GetPrintableCommandText())) : null) + return await self.ExecuteNonQueryAsync(); + } + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmdtext) { self.SetCommandAndParameters(cmdtext); @@ -110,7 +124,7 @@ public static partial class ExtensionMethods if (values != null && values.Count > 0) self.SetParameterValues(values); - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", string.Format("ExecuteNonQueryAsync: {0}", self.GetPrintableCommandText())) : null) return await self.ExecuteNonQueryAsync(); } @@ -127,6 +141,15 @@ public static partial class ExtensionMethods // return self.ExecuteReader(); //} + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmd) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReaderAsync", string.Format("ExecuteReaderAsync: {0}", self.GetPrintableCommandText())) : null) + return await self.ExecuteReaderAsync(); + } + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext) { self.SetCommandAndParameters(cmdtext); @@ -172,6 +195,15 @@ public static partial class ExtensionMethods return await self.ExecuteScalarAsync().ConfigureAwait(false); } + public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarAsync", string.Format("ExecuteScalarAsync: {0}", self.GetPrintableCommandText())) : null) + return self.ExecuteScalar(); + } + public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext) { self.SetCommandAndParameters(cmdtext); @@ -197,6 +229,24 @@ public static partial class ExtensionMethods // return defaultvalue; //} + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue = -1) + { + return await ExecuteScalarInt64Async(self, writeLog, null, defaultvalue); + } + + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd, long defaultvalue) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64Async", string.Format("ExecuteScalarInt64Async: {0}", self.GetPrintableCommandText())) : null) + using (var rd = await self.ExecuteReaderAsync()) + if (await rd.ReadAsync()) + return ConvertValueToInt64(rd, 0, defaultvalue); + + return defaultvalue; + } + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue = -1) { using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) @@ -241,7 +291,7 @@ public static partial class ExtensionMethods return cmd; } - internal static async Task ExpandInClauseParameter(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) + internal static async Task ExpandInClauseParameterAsync(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); From d73a555647f11f9a870ad0b0df66bbb7795f0188 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 15:43:55 +0200 Subject: [PATCH 062/362] Restructured the SQL queries in LocalBackupDatabase and made sure it's properly async --- .../Main/Database/LocalBackupDatabase.cs | 679 ++++++++++++------ 1 file changed, 474 insertions(+), 205 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index ec0a1eb0d..4d7c986bb 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -24,6 +24,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database @@ -60,7 +61,7 @@ namespace Duplicati.Library.Main.Database private SqliteCommand m_selectfilemetadatahashandsizeCommand = null!; private SqliteCommand m_getfirstfilesetwithblockinblockset = null!; - private HashSet m_blocklistHashes; + private HashSet m_blocklistHashes = []; /// /// The temporary table with deleted blocks that can be re-used; null if not table is used @@ -136,12 +137,12 @@ namespace Duplicati.Library.Main.Database .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted, RemoteVolumeState.Deleting]) .ExecuteNonQuery(); - var deletedBlocks = cmd.ExecuteScalarInt64(@$"SELECT COUNT(*) FROM ""{dbnew.m_tempDeletedBlockTable}""", 0); + var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$"SELECT COUNT(*) FROM ""{dbnew.m_tempDeletedBlockTable}""", 0); // There are no deleted blocks, so we can drop the table if (deletedBlocks == 0) { - cmd.ExecuteNonQuery($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}"""); dbnew.m_tempDeletedBlockTable = null; } @@ -149,8 +150,8 @@ namespace Duplicati.Library.Main.Database else if (deletedBlocks <= deletedBlockCacheSizeLong) { dbnew.m_deletedBlockLookup = new Dictionary>(); - using (var reader = cmd.ExecuteReader(@$"SELECT ""ID"", ""Hash"", ""Size"" FROM ""{dbnew.m_tempDeletedBlockTable}""")) - while (reader.Read()) + using (var reader = await cmd.ExecuteReaderAsync(@$"SELECT ""ID"", ""Hash"", ""Size"" FROM ""{dbnew.m_tempDeletedBlockTable}""")) + while (await reader.ReadAsync()) { var id = reader.ConvertValueToInt64(0); var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null"); @@ -161,15 +162,15 @@ namespace Duplicati.Library.Main.Database sizes[size] = id; } - cmd.ExecuteNonQuery($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}"""); dbnew.m_tempDeletedBlockTable = null; } // The deleted blocks are too large to fit in memory, so we use a temporary table else { - cmd.ExecuteNonQuery($@"CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" ON ""{dbnew.m_tempDeletedBlockTable}"" (""Hash"", ""Size"")"); - dbnew.m_findindeletedCommand = dbnew.m_connection.CreateCommand($@"SELECT ""ID"" FROM ""{dbnew.m_tempDeletedBlockTable}"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - dbnew.m_moveblockfromdeletedCommand = dbnew.m_connection.CreateCommand(string.Join(";", + await cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" ON ""{dbnew.m_tempDeletedBlockTable}"" (""Hash"", ""Size"")"); + dbnew.m_findindeletedCommand = await dbnew.m_connection.CreateCommandAsync($@"SELECT ""ID"" FROM ""{dbnew.m_tempDeletedBlockTable}"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(string.Join(";", @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", @$"DELETE FROM {dbnew.m_tempDeletedBlockTable} WHERE ""ID"" = @DeletedBlockId", @@ -179,7 +180,7 @@ namespace Duplicati.Library.Main.Database if (deletedBlocks > 0) { - dbnew.m_moveblockfromdeletedCommand = dbnew.m_connection.CreateCommand(string.Join(";", + dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(string.Join(";", @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", "SELECT last_insert_rowid()")); @@ -286,12 +287,13 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The block key /// The size of the block /// True if the block should be added to the current output - public long FindBlockID(string key, long size) + public async Task FindBlockID(string key, long size) { - return m_findblockCommand.SetTransaction(m_rtr) + return await m_findblockCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(m_logQueries, -1); + .ExecuteScalarInt64Async(m_logQueries, -1); } /// @@ -300,9 +302,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The block key /// The size of the block /// True if the block should be added to the current output - public bool AddBlock(string key, long size, long volumeid) + public async Task AddBlock(string key, long size, long volumeid) { - var r = FindBlockID(key, size); + var r = await FindBlockID(key, size); if (r == -1L) { if (m_moveblockfromdeletedCommand != null) @@ -312,9 +314,10 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist if (m_deletedBlockLookup.TryGetValue(key, out var sizes)) if (sizes.TryGetValue(size, out var id)) { - m_moveblockfromdeletedCommand.SetTransaction(m_rtr) + await m_moveblockfromdeletedCommand + .SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries); sizes.Remove(size); if (sizes.Count == 0) @@ -325,16 +328,17 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist else if (m_findindeletedCommand != null) { // No transaction on the temporary table - var id = m_findindeletedCommand + var id = await m_findindeletedCommand .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(m_logQueries, -1); + .ExecuteScalarInt64Async(m_logQueries, -1); if (id != -1) { - var c = m_moveblockfromdeletedCommand.SetTransaction(m_rtr) + var c = await m_moveblockfromdeletedCommand + .SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries); if (c != 2) throw new Exception($"Failed to move block {key} with size {size}, result count: {c}"); @@ -345,11 +349,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - var ins = m_insertblockCommand.SetTransaction(m_rtr) + var ins = await m_insertblockCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@Size", size) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries); if (ins != 1) throw new Exception($"Failed to insert block {key} with size {size}, result count: {ins}"); @@ -371,38 +376,44 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The list of hashes /// The id of the blockset, new or old /// True if the blockset was created, false otherwise - public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid) + public async Task<(bool, long)> AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes) { - blocksetid = m_findblocksetCommand.SetTransaction(m_rtr) + long blocksetid = await m_findblocksetCommand + .SetTransaction(m_rtr) .SetParameterValue("@Fullhash", filehash) .SetParameterValue("@Length", size) - .ExecuteScalarInt64(m_logQueries, null, -1); - if (blocksetid != -1) - return false; //Found it + .ExecuteScalarInt64Async(m_logQueries, null, -1); - blocksetid = m_insertblocksetCommand.SetTransaction(m_rtr) + if (blocksetid != -1) + return (false, blocksetid); //Found it + + blocksetid = await m_insertblocksetCommand + .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@Fullhash", filehash) - .ExecuteScalarInt64(m_logQueries); + .ExecuteScalarInt64Async(m_logQueries); long ix = 0; if (blocklistHashes != null) { - m_insertblocklistHashesCommand.SetTransaction(m_rtr) + m_insertblocklistHashesCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid); foreach (var bh in blocklistHashes) { - var c = m_insertblocklistHashesCommand.SetParameterValue("@Index", ix) + var c = await m_insertblocklistHashesCommand + .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", bh) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries); if (c != 1) throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}"); ix++; } } - m_insertblocksetentryCommand.SetTransaction(m_rtr) + m_insertblocksetentryCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid); ix = 0; @@ -410,23 +421,35 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist foreach (var h in hashes) { var exsize = remainsize < blocksize ? remainsize : blocksize; - var c = m_insertblocksetentryCommand.SetParameterValue("@Index", ix) + var c = await m_insertblocksetentryCommand + .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", h) .SetParameterValue("@Size", exsize) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries); if (c != 1) { Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size); using (var cmd = m_connection.CreateCommand(m_rtr)) { - var bid = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash") + var bid = await cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Block"" + WHERE ""Hash"" = @Hash + ") .SetParameterValue("@Hash", h) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1); + if (bid == -1) throw new Exception($"Could not find any blocks with the given hash: {h}"); - cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = @Hash") + + cmd.SetCommandAndParameters(@" + SELECT ""Size"" + FROM ""Block"" + WHERE ""Hash"" = @Hash + ") .SetParameterValue("@Hash", h); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); } @@ -437,9 +460,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist remainsize -= blocksize; } - m_rtr.Commit(); + await m_rtr.CommitAsync(); - return true; + return (true, blocksetid); } /// @@ -450,19 +473,22 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The size of the metadata. /// The ID of the metadataset. /// An optional transaction. - public bool GetMetadatasetID(string filehash, long size, out long metadataid) + public async Task<(bool, long)> GetMetadatasetID(string filehash, long size) { + long metadataid; + if (size > 0) { - metadataid = m_findmetadatasetCommand.SetTransaction(m_rtr) + metadataid = await m_findmetadatasetCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", filehash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(m_logQueries, null, -1); - return metadataid != -1; + .ExecuteScalarInt64Async(m_logQueries, null, -1); + return (metadataid != -1, metadataid); } metadataid = -2; - return false; + return (false, metadataid); } /// @@ -474,18 +500,20 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The id of the blockset to add /// The id of the metadata set /// True if the set was added to the database, false otherwise - public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid) + public async Task<(bool, long)> AddMetadataset(string filehash, long size, long blocksetid) { - if (GetMetadatasetID(filehash, size, out metadataid)) - return false; + var (metadatafound, metadataid) = await GetMetadatasetID(filehash, size); + if (metadatafound) + return (false, metadataid); - metadataid = m_insertmetadatasetCommand.SetTransaction(m_rtr) + metadataid = await m_insertmetadatasetCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64(m_logQueries); + .ExecuteScalarInt64Async(m_logQueries); m_rtr.Commit(); - return true; + return (true, metadataid); } /// @@ -497,27 +525,29 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the hashkey for the file /// The ID for the metadata /// The transaction to use for insertion, or null for no transaction - public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID) + public async Task AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID) { - var fileidobj = m_findfilesetCommand.SetTransaction(m_rtr) + var fileidobj = await m_findfilesetCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetID) .SetParameterValue("@MetadataId", metadataID) .SetParameterValue("@Path", filename) .SetParameterValue("@PrefixId", pathprefixid) - .ExecuteScalarInt64(m_logQueries); + .ExecuteScalarInt64Async(m_logQueries); if (fileidobj == -1) { - fileidobj = m_insertfileCommand.SetTransaction(m_rtr) + fileidobj = await m_insertfileCommand.SetTransaction(m_rtr) .SetParameterValue("@PrefixId", pathprefixid) .SetParameterValue("@Path", filename) .SetParameterValue("@BlocksetId", blocksetID) .SetParameterValue("@MetadataId", metadataID) - .ExecuteScalarInt64(m_logQueries); - m_rtr.Commit(); + .ExecuteScalarInt64Async(m_logQueries); + + await m_rtr.CommitAsync(); } - AddKnownFile(fileidobj, lastmodified); + await AddKnownFile(fileidobj, lastmodified); } /// @@ -528,10 +558,17 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the hashkey for the file /// The ID for the metadata /// The transaction to use for insertion, or null for no transaction - public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID) + public async Task AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID) { var split = SplitIntoPrefixAndName(filename); - AddFile(GetOrCreatePathPrefix(split.Key), split.Value, lastmodified, blocksetID, metadataID); + + await AddFile( + await GetOrCreatePathPrefix(split.Key), + split.Value, + lastmodified, + blocksetID, + metadataID + ); } /// @@ -540,39 +577,44 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Id of the file /// The time the file was modified /// The transaction to use for insertion, or null for no transaction - public void AddKnownFile(long fileid, DateTime lastmodified) + public async Task AddKnownFile(long fileid, DateTime lastmodified) { - m_insertfileOperationCommand.SetTransaction(m_rtr) + await m_insertfileOperationCommand + .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", m_filesetId) .SetParameterValue("@FileId", fileid) .SetParameterValue("@LastModified", lastmodified.ToUniversalTime().Ticks) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries); } - public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified) + public async Task AddDirectoryEntry(string path, long metadataID, DateTime lastmodified) { - AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID); + await AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID); } - public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified) + public async Task AddSymlinkEntry(string path, long metadataID, DateTime lastmodified) { - AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID); + await AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID); } - public long GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, out DateTime oldModified, out long length) + public async Task<(long, DateTime, long)> GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength) { + DateTime oldModified; + long length; if (includeLength) { - m_selectfilelastmodifiedWithSizeCommand.SetTransaction(m_rtr) + m_selectfilelastmodifiedWithSizeCommand + .SetTransaction(m_rtr) .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = m_selectfilelastmodifiedWithSizeCommand.ExecuteReader(m_logQueries, null)) - if (rd.Read()) + + using (var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null)) + if (await rd.ReadAsync()) { oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); length = rd.ConvertValueToInt64(2); - return rd.ConvertValueToInt64(0); + return (rd.ConvertValueToInt64(0), oldModified, length); } } else @@ -581,36 +623,49 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null)) - if (rd.Read()) + + using (var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries, null)) + if (await rd.ReadAsync()) { length = -1; oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - return rd.ConvertValueToInt64(0); + return (rd.ConvertValueToInt64(0), oldModified, length); } - } + oldModified = new DateTime(0, DateTimeKind.Utc); length = -1; - return -1; + return (-1, oldModified, length); } - public long GetFileEntry(long prefixid, string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string? oldMetahash, out long oldMetasize) + public async Task<(long, DateTime, long, string?, long)> GetFileEntry(long prefixid, string path, long filesetid) { - m_findfileCommand.SetTransaction(m_rtr) + DateTime oldModified; + long lastFileSize; + string? oldMetahash; + long oldMetasize; + + m_findfileCommand + .SetTransaction(m_rtr) .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = m_findfileCommand.ExecuteReader()) - if (rd.Read()) + using (var rd = await m_findfileCommand.ExecuteReaderAsync()) + if (await rd.ReadAsync()) { oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); lastFileSize = rd.ConvertValueToInt64(2); oldMetahash = rd.ConvertValueToString(3); oldMetasize = rd.ConvertValueToInt64(4); - return rd.ConvertValueToInt64(0); + return ( + rd.ConvertValueToInt64(0), + oldModified, + lastFileSize, + oldMetahash, + oldMetasize + ); } else { @@ -618,28 +673,40 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist lastFileSize = -1; oldMetahash = null; oldMetasize = -1; - return -1; + return ( + -1, + oldModified, + lastFileSize, + oldMetahash, + oldMetasize + ); } } - public (long Size, string MetadataHash)? GetMetadataHashAndSizeForFile(long fileid) + public async Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFile(long fileid) { - m_selectfilemetadatahashandsizeCommand.SetTransaction(m_rtr) + m_selectfilemetadatahashandsizeCommand + .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid); - using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader()) - if (rd.Read()) - return (rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null")); + using (var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync()) + if (await rd.ReadAsync()) + return ( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null") + ); return null; } - public string? GetFileHash(long fileid) + public async Task GetFileHash(long fileid) { - var r = m_selectfileHashCommand.SetTransaction(m_rtr) + var r = await m_selectfileHashCommand + .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid) - .ExecuteScalar(m_logQueries, null); + .ExecuteScalarAsync(m_logQueries, null); + if (r == null || r == DBNull.Value) return null; @@ -647,12 +714,17 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } public override void Dispose() + { + DisposeAsync().Await(); + } + + public override async Task DisposeAsync() { if (!string.IsNullOrWhiteSpace(m_tempDeletedBlockTable)) try { using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{m_tempDeletedBlockTable}""")); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}"""); } catch (Exception ex) { @@ -662,55 +734,102 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist base.Dispose(); } - public long GetLastWrittenDBlockVolumeSize() + public async Task GetLastWrittenDBlockVolumeSize() { using (var cmd = m_connection.CreateCommand(m_rtr)) - return cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""RemoteVolume"" WHERE ""State"" = @State AND ""Type"" = @Type ORDER BY ""ID"" DESC LIMIT 1") + return await cmd.SetCommandAndParameters(@" + SELECT ""Size"" + FROM ""RemoteVolume"" + WHERE + ""State"" = @State + AND ""Type"" = @Type + ORDER BY ""ID"" DESC + LIMIT 1 + ") .SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1); } - private long GetPreviousFilesetID(SqliteCommand cmd) + private async Task GetPreviousFilesetID(SqliteCommand cmd) { - return GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId); + return await GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId); } - private long GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid) + private async Task GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid) { - return cmd.SetTransaction(m_rtr) - .SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < @Timestamp AND ""ID"" != @FilesetId ORDER BY ""Timestamp"" DESC ") + return await cmd + .SetTransaction(m_rtr) + .SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + WHERE + ""Timestamp"" < @Timestamp + AND ""ID"" != @FilesetId + ORDER BY ""Timestamp"" DESC + ") .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) .SetParameterValue("@FilesetId", filesetid) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1); } - internal Tuple GetLastBackupFileCountAndSize() + internal async Task> GetLastBackupFileCountAndSize() { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_rtr)) { - var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1"); - var count = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""BlocksetID"" NOT IN (@FolderBlocksetId, @SymlinkBlocksetId)") - .SetParameterValue("@FilesetId", lastFilesetId) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64(-1); + var lastFilesetId = await cmd.ExecuteScalarInt64Async(@" + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + LIMIT 1 + "); - var size = cmd.SetCommandAndParameters(@"SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", ""FilesetEntry"", ""Blockset"" WHERE ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""BlocksetID"" NOT IN (@FolderBlocksetId, @SymlinkBlocksetId)") + var count = await cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""FileLookup"" + INNER JOIN ""FilesetEntry"" + ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + WHERE + ""FilesetEntry"".""FilesetID"" = @FilesetId + AND ""FileLookup"".""BlocksetID"" NOT IN ( + @FolderBlocksetId, + @SymlinkBlocksetId + ) + ") .SetParameterValue("@FilesetId", lastFilesetId) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1); + + var size = await cmd.SetCommandAndParameters(@" + SELECT SUM(""Blockset"".""Length"") + FROM + ""FileLookup"", + ""FilesetEntry"", + ""Blockset"" + WHERE + ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND + ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND + ""FilesetEntry"".""FilesetID"" = @FilesetId AND + ""FileLookup"".""BlocksetID"" NOT IN ( + @FolderBlocksetId, + @SymlinkBlocksetId + ) + ") + .SetParameterValue("@FilesetId", lastFilesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64Async(-1); return new Tuple(count, size); } } - internal void UpdateChangeStatistics(BackupResults results) + internal async Task UpdateChangeStatistics(BackupResults results) { using (var cmd = m_connection.CreateCommand(m_rtr)) { - var prevFileSetId = GetPreviousFilesetID(cmd); + var prevFileSetId = await GetPreviousFilesetID(cmd); ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId); } } @@ -721,9 +840,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// /// Transaction /// List of deleted paths, or null - public void AppendFilesFromPreviousSet(IEnumerable? deleted = null) + public async Task AppendFilesFromPreviousSet(IEnumerable? deleted = null) { - AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp); + await AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp); } /// @@ -735,30 +854,63 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Current file-set ID /// Source file-set ID /// If filesetid == -1, used to locate previous file-set - public void AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) + public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) { using (var cmd = m_connection.CreateCommand()) using (var cmdDelete = m_connection.CreateCommand()) { - long lastFilesetId = prevId < 0 ? GetPreviousFilesetID(cmd, timestamp, filesetid, tr.Parent) : prevId; + long lastFilesetId = prevId < 0 ? await GetPreviousFilesetID(cmd, timestamp, filesetid) : prevId; - cmd.SetTransaction(m_rtr) - .SetCommandAndParameters(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @CurrentFilesetId AS ""FilesetID"", ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId)) ") + await cmd.SetTransaction(m_rtr) + .SetCommandAndParameters(@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + SELECT + @CurrentFilesetId AS ""FilesetID"", + ""FileID"", + ""Lastmodified"" + FROM ( + SELECT DISTINCT + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @PreviousFilesetId + AND ""FileID"" NOT IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @CurrentFilesetId + ) + ) + ") .SetParameterValue("@CurrentFilesetId", filesetid) .SetParameterValue("@PreviousFilesetId", lastFilesetId) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries); if (deleted != null) { - using var tmplist = new TemporaryDbValueList(m_connection, tr.Parent, deleted); - cmdDelete.SetTransaction(m_rtr) - .SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId AND ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" IN (@Paths)) ") + using var tmplist = await TemporaryDbValueList.CreateAsync(this, deleted); + await (await cmdDelete.SetTransaction(m_rtr) + .SetCommandAndParameters(@" + DELETE FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @FilesetId + AND ""FileID"" IN ( + SELECT ""ID"" + FROM ""File"" + WHERE ""Path"" IN (@Paths) + ) + ") .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameter("@Paths", tmplist) - .ExecuteNonQuery(m_logQueries); + .ExpandInClauseParameterAsync("@Paths", tmplist)) + .ExecuteNonQueryAsync(m_logQueries); } - m_rtr.Commit(); + await m_rtr.CommitAsync(); } } @@ -769,9 +921,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// /// Transaction /// Optional exclusion predicate (true = exclude file) - public void AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate) + public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate) { - AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp); + await AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp); } /// @@ -784,54 +936,101 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Current fileset ID /// Source fileset ID /// If prevFileSetId == -1, used to locate previous fileset - public void AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) + public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) { if (exclusionPredicate == null) { - AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp); + await AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp); return; } using (var cmd = m_connection.CreateCommand()) using (var cmdDelete = m_connection.CreateCommand()) { - long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId; + long lastFilesetId = prevFileSetId < 0 ? await GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId; // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.SetTransaction(m_rtr) - cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId))")) + await cmd + .SetTransaction(m_rtr) + .SetCommandAndParameters($@" + CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS + SELECT + ""FileID"", + ""Lastmodified"" + FROM ( + SELECT DISTINCT + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @PreviousFilesetId + AND ""FileID"" NOT IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @CurrentFilesetId + ) + ) + ") .SetParameterValue("@PreviousFilesetId", lastFilesetId) .SetParameterValue("@CurrentFilesetId", fileSetId) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); // now we need to remove, from the above, any entries that were enumerated by the // UNC-driven backup cmdDelete.SetTransaction(m_rtr) - .SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = @FileId")); + .SetCommandAndParameters($@" + DELETE FROM ""{tempFileSetTable}"" + WHERE ""FileID"" = @FileId + "); // enumerate files from new temporary file set, and remove any entries handled by UNC - foreach (var row in cmd.ExecuteReaderEnumerable(FormatInvariant( - $@"SELECT f.""Path"", fs.""FileID"", fs.""Lastmodified"", COALESCE(bs.""Length"", -1) - FROM (SELECT DISTINCT ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"") AS fs - LEFT JOIN ""File"" AS f ON fs.""FileID"" = f.""ID"" - LEFT JOIN ""Blockset"" AS bs ON f.""BlocksetID"" = bs.""ID"";"))) + cmd.SetCommandAndParameters($@" + SELECT + f.""Path"", + fs.""FileID"", + fs.""Lastmodified"", + COALESCE(bs.""Length"", -1) + FROM ( + SELECT DISTINCT + ""FileID"", + ""Lastmodified"" + FROM ""{tempFileSetTable}"" + ) AS fs + LEFT JOIN ""File"" AS f + ON fs.""FileID"" = f.""ID"" + LEFT JOIN ""Blockset"" AS bs + ON f.""BlocksetID"" = bs.""ID""; + "); + + await foreach (var row in cmd.ExecuteReaderEnumerableAsync()) { var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path"); var size = row.ConvertValueToInt64(3); if (exclusionPredicate(path, size)) - cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) - .ExecuteNonQuery(); + await cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) + .ExecuteNonQueryAsync(); } // now copy the temporary table into the FileSetEntry table - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") - SELECT @FilesetId, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}""")) + await cmd.SetCommandAndParameters($@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + SELECT + @FilesetId, + ""FileID"", + ""Lastmodified"" + FROM ""{tempFileSetTable}"" + ") .SetParameterValue("@FilesetId", fileSetId) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); - m_rtr.Commit(); + await m_rtr.CommitAsync(); } } @@ -841,91 +1040,132 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the fileset volume to update /// The timestamp of the operation to create /// An optional external transaction - public override long CreateFileset(long volumeid, DateTime timestamp) + public override async Task CreateFileset(long volumeid, DateTime timestamp) { - return m_filesetId = base.CreateFileset(volumeid, timestamp); + return m_filesetId = await base.CreateFileset(volumeid, timestamp); } - public IEnumerable GetTemporaryFilelistVolumeNames(bool latestOnly) + public async Task> GetTemporaryFilelistVolumeNames(bool latestOnly) { - var incompleteFilesetIDs = GetIncompleteFilesets().OrderBy(x => x.Value).Select(x => x.Key).ToArray(); + var incompleteFilesetIDs = GetIncompleteFilesets().OrderBy(x => x.Value).Select(x => x.Key); - if (!incompleteFilesetIDs.Any()) - return Enumerable.Empty(); + if (!await incompleteFilesetIDs.AnyAsync()) + return []; if (latestOnly) - incompleteFilesetIDs = new long[] { incompleteFilesetIDs.Last() }; + incompleteFilesetIDs = new long[] { await incompleteFilesetIDs.LastAsync() }.ToAsyncEnumerable(); var volumeNames = new List(); - foreach (var filesetID in incompleteFilesetIDs) - volumeNames.Add(GetRemoteVolumeFromFilesetID(filesetID).Name); + await foreach (var filesetID in incompleteFilesetIDs) + volumeNames.Add((await GetRemoteVolumeFromFilesetID(filesetID)).Name); return volumeNames; } - public IEnumerable GetMissingIndexFiles() + public async IAsyncEnumerable GetMissingIndexFiles() { using var cmd = m_connection.CreateCommand(m_rtr) - .SetCommandAndParameters(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND NOT ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"") AND ""State"" IN (@States)") + .SetCommandAndParameters(@" + SELECT ""Name"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND NOT ""ID"" IN ( + SELECT ""BlockVolumeID"" + FROM ""IndexBlockLink"" + ) + AND ""State"" IN (@States) + ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + .ExpandInClauseParameter("@States", [ + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString() + ]); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } - public void MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid) + public async Task MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_rtr)) { - var c = cmd.SetCommandAndParameters(@"UPDATE ""Block"" SET ""VolumeID"" = @NewVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" = @PreviousVolumeId ") - .SetTransaction(m_rtr) + var c = await cmd.SetCommandAndParameters(@" + UPDATE ""Block"" + SET ""VolumeID"" = @NewVolumeId + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""VolumeID"" = @PreviousVolumeId + ") .SetParameterValue("@NewVolumeId", targetvolumeid) .SetParameterValue("@Hash", blockkey) .SetParameterValue("@Size", size) .SetParameterValue("@PreviousVolumeId", sourcevolumeid) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); + if (c != 1) throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}"); } } - public void SafeDeleteRemoteVolume(string name) + public async Task SafeDeleteRemoteVolume(string name) { var volumeid = GetRemoteVolumeID(name); using (var cmd = m_connection.CreateCommand(m_rtr)) { - var c = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = @VolumeId ") + var c = await cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""Block"" + WHERE ""VolumeID"" = @VolumeId + ") .SetParameterValue("@VolumeId", volumeid) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1); + if (c != 0) throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); - RemoveRemoteVolume(name); + await RemoveRemoteVolume(name); } } - public string[] GetBlocklistHashes(string name) + public async Task GetBlocklistHashes(string name) { var volumeid = GetRemoteVolumeID(name); using (var cmd = m_connection.CreateCommand(m_rtr)) { // Grab the strings and return as array to avoid concurrent access to the IEnumerable - cmd.SetCommandAndParameters(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE ""Block"".""VolumeID"" = @VolumeId AND ""Block"".""Hash"" IN (SELECT ""Hash"" FROM ""BlocklistHash"")") + cmd.SetCommandAndParameters(@" + SELECT DISTINCT ""Block"".""Hash"" + FROM ""Block"" + WHERE + ""Block"".""VolumeID"" = @VolumeId + AND ""Block"".""Hash"" IN ( + SELECT ""Hash"" + FROM ""BlocklistHash"" + ) + ") .SetParameterValue("@VolumeId", volumeid); - return cmd.ExecuteReaderEnumerable() + + return await cmd.ExecuteReaderEnumerableAsync() .Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash")) - .ToArray(); + .ToArrayAsync(); } } - public string? GetFirstPath() + public async Task GetFirstPath() { - using (var cmd = m_connection.CreateCommand(@"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")) + using (var cmd = m_connection.CreateCommand(m_rtr)) { - var v0 = cmd.ExecuteScalar(); + var v0 = await cmd.ExecuteScalarAsync(@" + SELECT ""Path"" + FROM ""File"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + "); + if (v0 == null || v0 == DBNull.Value) return null; @@ -937,28 +1177,35 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Retrieves change journal data for file set /// /// Fileset-ID - public IEnumerable GetChangeJournalData(long fileSetId) + public async IAsyncEnumerable GetChangeJournalData(long fileSetId) { var data = new List(); - using var cmd = m_connection.CreateCommand(@"SELECT ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"" FROM ""ChangeJournalData"" WHERE ""FilesetID"" = @FilesetId") + using var cmd = m_connection.CreateCommand(@" + SELECT + ""VolumeName"", + ""JournalID"", + ""NextUSN"", + ""ConfigHash"" + FROM ""ChangeJournalData"" + WHERE ""FilesetID"" = @FilesetId + ") + .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId); - using (var rd = cmd.ExecuteReader()) + + using (var rd = await cmd.ExecuteReaderAsync()) { - while (rd.Read()) + while (await rd.ReadAsync()) { - data.Add(new Interface.USNJournalDataEntry + yield return new Interface.USNJournalDataEntry { Volume = rd.ConvertValueToString(0), JournalId = rd.ConvertValueToInt64(1), NextUsn = rd.ConvertValueToInt64(2), ConfigHash = rd.ConvertValueToString(3) - }); + }; } - } - - return data; } /// @@ -966,27 +1213,41 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// /// Data to add /// An optional external transaction - public void CreateChangeJournalData(IEnumerable data) + public async Task CreateChangeJournalData(IEnumerable data) { foreach (var entry in data) { using (var cmd = m_connection.CreateCommand(m_rtr)) { - var c = cmd.SetCommandAndParameters( - @"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (@FilesetId, @VolumeName, @JournalId, @NextUsn, @ConfigHash);") - .SetParameterValue("@FilesetId", m_filesetId) - .SetParameterValue("@VolumeName", entry.Volume) - .SetParameterValue("@JournalId", entry.JournalId) - .SetParameterValue("@NextUsn", entry.NextUsn) - .SetParameterValue("@ConfigHash", entry.ConfigHash) - .ExecuteNonQuery(); + var c = await cmd.SetCommandAndParameters(@" + INSERT INTO ""ChangeJournalData"" ( + ""FilesetID"", + ""VolumeName"", + ""JournalID"", + ""NextUSN"", + ""ConfigHash"" + ) + VALUES ( + @FilesetId, + @VolumeName, + @JournalId, + @NextUsn, + @ConfigHash + ); + ") + .SetParameterValue("@FilesetId", m_filesetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@ConfigHash", entry.ConfigHash) + .ExecuteNonQueryAsync(); if (c != 1) throw new Exception("Unable to add change journal entry"); } } - m_rtr.Commit(); + await m_rtr.CommitAsync(); } /// @@ -995,24 +1256,30 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Data to add /// Existing file set to update /// An optional external transaction - public void UpdateChangeJournalData(IEnumerable data, long fileSetId) + public async Task UpdateChangeJournalData(IEnumerable data, long fileSetId) { foreach (var entry in data) { using (var cmd = m_connection.CreateCommand()) { - cmd.SetCommandAndParameters( - @"UPDATE ""ChangeJournalData"" SET ""NextUSN"" = @NextUsn WHERE ""FilesetID"" = @FilesetId AND ""VolumeName"" = @VolumeName AND ""JournalID"" = @JournalId;") + await cmd.SetCommandAndParameters(@" + UPDATE ""ChangeJournalData"" + SET ""NextUSN"" = @NextUsn + WHERE + ""FilesetID"" = @FilesetId + AND ""VolumeName"" = @VolumeName + AND ""JournalID"" = @JournalId; + ") .SetTransaction(m_rtr) .SetParameterValue("@NextUsn", entry.NextUsn) .SetParameterValue("@FilesetId", fileSetId) .SetParameterValue("@VolumeName", entry.Volume) .SetParameterValue("@JournalId", entry.JournalId) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); } } - m_rtr.Commit(); + await m_rtr.CommitAsync(); } /// @@ -1021,11 +1288,13 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The hash to check /// An optional external transaction /// True if the hash is known, false otherwise - public bool IsBlocklistHashKnown(string hash) + public async Task IsBlocklistHashKnown(string hash) { - var res = m_getfirstfilesetwithblockinblockset.SetTransaction(m_rtr) + var res = await m_getfirstfilesetwithblockinblockset + .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) - .ExecuteScalarInt64(); + .ExecuteScalarInt64Async(); + if (res != -1 && res != m_filesetId) return true; else From f3f00faf8e115035018e1a170cf47f4e10e5aea1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 15:45:31 +0200 Subject: [PATCH 063/362] Incorrectly called the synchronous version of a function --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 8df7f3650..1940f9903 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -545,7 +545,7 @@ namespace Duplicati.Library.Main.Database "); cmd.SetTransaction(transaction); using var tmptable = await TemporaryDbValueList.CreateAsync(this, files); - await cmd.ExpandInClauseParameter("@Name", tmptable); + await cmd.ExpandInClauseParameterAsync("@Name", tmptable); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) From d08bcbba3a1d884b8a3602f6f50b2b42556ce64a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 15:56:16 +0200 Subject: [PATCH 064/362] Updated ChangeStatistics to use Microsoft.Data.Sqlite and be async --- .../Library/Main/Database/ChangeStatistics.cs | 224 +++++++++++------- 1 file changed, 141 insertions(+), 83 deletions(-) diff --git a/Duplicati/Library/Main/Database/ChangeStatistics.cs b/Duplicati/Library/Main/Database/ChangeStatistics.cs index 49f69ec11..67a8e3ff3 100644 --- a/Duplicati/Library/Main/Database/ChangeStatistics.cs +++ b/Duplicati/Library/Main/Database/ChangeStatistics.cs @@ -1,27 +1,28 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.Data; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database; @@ -42,57 +43,85 @@ public static class ChangeStatistics /// The results object to update with the statistics /// The ID of the current fileset /// The ID of the previous fileset - internal static void UpdateChangeStatistics(IDbCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId) + internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId) { var tmpName = $"TmpFileState_{Guid.NewGuid():N}"; try { // Create temp table - cmd.SetCommandAndParameters(LocalDatabase.FormatInvariant($@" - CREATE TEMP TABLE ""{tmpName}"" AS - SELECT - FL.""PrefixID"", - FL.""Path"", - FL.""BlocksetID"", - BS_Meta.""Fullhash"" AS ""Metahash"", - CASE FE.""FilesetID"" - WHEN @LastFilesetId THEN 0 - WHEN @CurrentFilesetId THEN 1 - END AS Source - FROM ""FileLookup"" FL - JOIN ""FilesetEntry"" FE ON FL.""ID"" = FE.""FileID"" - LEFT JOIN ""Blockset"" BS_Data ON FL.""BlocksetID"" = BS_Data.""ID"" - LEFT JOIN ""Metadataset"" M ON FL.""MetadataID"" = M.""ID"" - LEFT JOIN ""Blockset"" BS_Meta ON M.""BlocksetID"" = BS_Meta.""ID"" - WHERE FE.""FilesetID"" IN (@LastFilesetId, @CurrentFilesetId); -")) -.SetParameterValue("@LastFilesetId", previousFilesetId) -.SetParameterValue("@CurrentFilesetId", currentFilesetId) -.ExecuteNonQuery(); + await cmd.SetCommandAndParameters($@" + CREATE TEMP TABLE ""{tmpName}"" AS + SELECT + FL.""PrefixID"", + FL.""Path"", + FL.""BlocksetID"", + BS_Meta.""Fullhash"" AS ""Metahash"", + CASE FE.""FilesetID"" + WHEN @LastFilesetId THEN 0 + WHEN @CurrentFilesetId THEN 1 + END AS Source + FROM ""FileLookup"" FL + JOIN ""FilesetEntry"" FE + ON FL.""ID"" = FE.""FileID"" + LEFT JOIN ""Blockset"" BS_Data + ON FL.""BlocksetID"" = BS_Data.""ID"" + LEFT JOIN ""Metadataset"" M + ON FL.""MetadataID"" = M.""ID"" + LEFT JOIN ""Blockset"" BS_Meta + ON M.""BlocksetID"" = BS_Meta.""ID"" + WHERE FE.""FilesetID"" IN ( + @LastFilesetId, + @CurrentFilesetId + ); + ") + .SetParameterValue("@LastFilesetId", previousFilesetId) + .SetParameterValue("@CurrentFilesetId", currentFilesetId) + .ExecuteNonQueryAsync(); // Index for fast comparison - cmd.ExecuteNonQuery(LocalDatabase.FormatInvariant($@"CREATE INDEX ""idx_{tmpName}"" ON ""{tmpName}"" (""PrefixID"", ""Path"")")); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""idx_{tmpName}"" + ON ""{tmpName}"" ( + ""PrefixID"", + ""Path"" + ) + "); // Added - results.AddedFolders = CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.AddedSymlinks = CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); - results.AddedFiles = CountAdded(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID }); + results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); + results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); + results.AddedFiles = await CountAdded(cmd, tmpName, null, [ + LocalDatabase.FOLDER_BLOCKSET_ID, + LocalDatabase.SYMLINK_BLOCKSET_ID + ]); // Deleted - results.DeletedFolders = CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.DeletedSymlinks = CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); - results.DeletedFiles = CountDeleted(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID }); + results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); + results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); + results.DeletedFiles = await CountDeleted(cmd, tmpName, null, [ + LocalDatabase.FOLDER_BLOCKSET_ID, + LocalDatabase.SYMLINK_BLOCKSET_ID + ]); // Modified - results.ModifiedFolders = CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.ModifiedSymlinks = CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); - results.ModifiedFiles = CountModified(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID }); + results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); + results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); + results.ModifiedFiles = await CountModified(cmd, tmpName, null, [ + LocalDatabase.FOLDER_BLOCKSET_ID, + LocalDatabase.SYMLINK_BLOCKSET_ID + ]); } finally { - try { cmd.ExecuteNonQuery(LocalDatabase.FormatInvariant($@"DROP TABLE IF EXISTS ""{tmpName}"";")); } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}"); } + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";"); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}"); + } } } @@ -104,16 +133,21 @@ public static class ChangeStatistics /// The ID of the blockset to count, or null for all /// The blocksets to exclude from the count /// The number of added files or folders - private static long CountAdded(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + private static async Task CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) { - var conditions = LocalDatabase.FormatInvariant($@"Source = 1 AND NOT EXISTS ( - SELECT 1 FROM ""{tmpName}"" B - WHERE B.Source = 0 - AND B.PrefixID = A.PrefixID - AND B.Path = A.Path - )"); + var conditions = $@" + Source = 1 + AND NOT EXISTS ( + SELECT 1 + FROM ""{tmpName}"" B + WHERE + B.Source = 0 + AND B.PrefixID = A.PrefixID + AND B.Path = A.Path + ) + "; - return CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets); + return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets); } /// @@ -124,16 +158,21 @@ public static class ChangeStatistics /// The ID of the blockset to count, or null for all /// The blocksets to exclude from the count /// The number of deleted files or folders - private static long CountDeleted(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + private static async Task CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) { - var conditions = @"Source = 0 AND NOT EXISTS ( - SELECT 1 FROM """ + tmpName + @""" B - WHERE B.Source = 1 - AND B.PrefixID = A.PrefixID - AND B.Path = A.Path - )"; + var conditions = @$" + Source = 0 + AND NOT EXISTS ( + SELECT 1 + FROM ""{tmpName}"" B + WHERE + B.Source = 1 + AND B.PrefixID = A.PrefixID + AND B.Path = A.Path + ) + "; - return CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets); + return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets); } /// @@ -145,23 +184,32 @@ public static class ChangeStatistics /// The hash column to compare /// The blocksets to exclude from the count /// The number of modified files or folders - private static long CountModified(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + private static async Task CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) { string conditions; if (blocksetId == LocalDatabase.FOLDER_BLOCKSET_ID || blocksetId == LocalDatabase.SYMLINK_BLOCKSET_ID) { conditions = @" - A.Source = 0 AND B.Source = 1 - AND A.PrefixID = B.PrefixID AND A.Path = B.Path - AND A.Metahash IS NOT B.Metahash"; + A.Source = 0 + AND B.Source = 1 + AND A.PrefixID = B.PrefixID + AND A.Path = B.Path + AND A.Metahash IS NOT B.Metahash + "; } else { conditions = @" - A.Source = 0 AND B.Source = 1 - AND A.PrefixID = B.PrefixID AND A.Path = B.Path - AND (A.BlocksetID IS NOT B.BlocksetID OR A.Metahash IS NOT B.Metahash)"; + A.Source = 0 + AND B.Source = 1 + AND A.PrefixID = B.PrefixID + AND A.Path = B.Path + AND ( + A.BlocksetID IS NOT B.BlocksetID + OR A.Metahash IS NOT B.Metahash + ) + "; } string blocksetCondition = GetBlocksetCondition("A", blocksetId, excludeBlocksets); @@ -169,12 +217,16 @@ public static class ChangeStatistics conditions += $" AND {blocksetCondition}"; string sql = $@" - SELECT COUNT(*) - FROM ""{tmpName}"" A - JOIN ""{tmpName}"" B ON A.PrefixID = B.PrefixID AND A.Path = B.Path - WHERE {conditions}"; + SELECT COUNT(*) + FROM ""{tmpName}"" A + JOIN ""{tmpName}"" B + ON A.PrefixID = B.PrefixID + AND A.Path = B.Path + WHERE {conditions} + "; - return cmd.SetCommandAndParameters(sql).ExecuteScalarInt64(0); + return await cmd.SetCommandAndParameters(sql) + .ExecuteScalarInt64Async(0); } /// @@ -187,15 +239,21 @@ public static class ChangeStatistics /// The ID of the blockset to count, or null for all /// The blocksets to exclude from the count /// The number of files or folders that match the condition - private static long CountWithCondition(IDbCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets) + private static async Task CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets) { var fullCondition = baseCondition; var blocksetCondition = GetBlocksetCondition(alias, blocksetId, excludeBlocksets); if (!string.IsNullOrEmpty(blocksetCondition)) fullCondition += " AND " + blocksetCondition; - var sql = LocalDatabase.FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName}"" {alias} WHERE {fullCondition}"); - return cmd.SetCommandAndParameters(sql).ExecuteScalarInt64(0); + var sql = $@" + SELECT COUNT(*) + FROM ""{tmpName}"" {alias} + WHERE {fullCondition} + "; + + return await cmd.SetCommandAndParameters(sql) + .ExecuteScalarInt64Async(0); } /// @@ -208,9 +266,9 @@ public static class ChangeStatistics private static string GetBlocksetCondition(string alias, long? blocksetId, long[]? exclude) { if (blocksetId.HasValue) - return LocalDatabase.FormatInvariant($"{alias}.\"BlocksetID\" = {blocksetId.Value}"); + return @$"{alias}.""BlocksetID"" = {blocksetId.Value}"; if (exclude?.Length > 0) - return LocalDatabase.FormatInvariant($"{alias}.\"BlocksetID\" NOT IN ({string.Join(",", exclude)})"); + return @$"{alias}.""BlocksetID"" NOT IN ({string.Join(",", exclude)})"; return string.Empty; } } \ No newline at end of file From f2f28f7eea6d0d57461f745079be214064886d36 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 20:30:59 +0200 Subject: [PATCH 065/362] Removed whitespace --- .../Main/Database/LocalBugReportDatabase.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 73a2ba126..29fab2f73 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From 940d4077ffcfc99494eb6fac87cd8978089c70ca Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 20:31:37 +0200 Subject: [PATCH 066/362] Changed LocalBugReportDatabase to use factory construction --- .../Main/Database/LocalBugReportDatabase.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 29fab2f73..0e5eec8f6 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -22,6 +22,7 @@ #nullable enable using System; +using System.Threading.Tasks; using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Database @@ -33,16 +34,16 @@ namespace Duplicati.Library.Main.Database /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalBugReportDatabase)); - public LocalBugReportDatabase(string path, long pagecachesize) - : base(path, "BugReportCreate", false, pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize) { - ShouldCloseConnection = true; + var db = new LocalBugReportDatabase(); + + db = (LocalBugReportDatabase)await CreateLocalDatabaseAsync(db, path, "BugReportCreate", false, pagecachesize); + db.ShouldCloseConnection = true; + + return db; } - public void Fix() - { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = tr; var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); From 13194bbd9816d81362cff51e7f465c16f457389d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 20:38:10 +0200 Subject: [PATCH 067/362] Restructured SQL queries in LocalBugReportDatabase and ensured it's async --- .../Main/Database/LocalBugReportDatabase.cs | 91 +++++++++++++++---- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 0e5eec8f6..5eee2187c 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -44,23 +44,49 @@ namespace Duplicati.Library.Main.Database return db; } + public async Task Fix() + { + using (var cmd = m_connection.CreateCommand(m_rtr)) { - cmd.Transaction = tr; var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); // TODO: Rewrite this to use PathPrefix // TODO: Needs to be much faster - using (var upcmd = m_connection.CreateCommand()) + using (var upcmd = m_connection.CreateCommand(m_rtr)) { + await upcmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tablename}"" ( + ""ID"" INTEGER PRIMARY KEY, + ""RealPath"" TEXT NOT NULL, + ""Obfuscated"" TEXT NULL + ) + "); - upcmd.Transaction = tr; - upcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL)")); - upcmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" ")); - upcmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Obfuscated"" = @StartPath || length(""RealPath"") || @DirSep || ""ID"" || (CASE WHEN substr(""RealPath"", length(""RealPath"")) = @DirSep THEN @DirSep ELSE @FileExt END) ")) + await upcmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{tablename}"" (""RealPath"") + SELECT DISTINCT ""Path"" + FROM ""File"" + ORDER BY ""Path"" + "); + + await upcmd.SetCommandAndParameters($@" + UPDATE ""{tablename}"" + SET ""Obfuscated"" = + @StartPath + || length(""RealPath"") + || @DirSep + || ""ID"" + || ( + CASE WHEN substr(""RealPath"", length(""RealPath"")) = @DirSep + THEN @DirSep + ELSE @FileExt + END + ) + ") .SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\") .SetParameterValue("@DirSep", Util.DirectorySeparatorString) .SetParameterValue("@FileExt", ".bin") - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); /*long id = 1; using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename))) @@ -75,24 +101,51 @@ namespace Duplicati.Library.Main.Database */ } - cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = 'ERASED!' WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' "); - cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = 'ERASED!' WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' "); + await cmd.ExecuteNonQueryAsync(@" + UPDATE ""LogData"" + SET ""Message"" = 'ERASED!' + WHERE + ""Message"" LIKE '%/%' + OR ""Message"" LIKE '%:\%' + "); - cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' "); + await cmd.ExecuteNonQueryAsync(@" + UPDATE ""LogData"" + SET ""Exception"" = 'ERASED!' + WHERE + ""Exception"" LIKE '%/%' + OR ""Exception"" LIKE '%:\%' + "); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{tablename}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ")); - cmd.ExecuteNonQuery(@"DROP VIEW ""File"" "); - cmd.ExecuteNonQuery(@"DROP TABLE ""FileLookup"" "); - cmd.ExecuteNonQuery(@"DROP TABLE ""PathPrefix"" "); + await cmd.ExecuteNonQueryAsync(@" + UPDATE ""Configuration"" + SET ""Value"" = 'ERASED!' + WHERE ""Key"" = 'passphrase' + "); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); + await cmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""FixedFile"" AS + SELECT + ""B"".""ID"" AS ""ID"", + ""A"".""Obfuscated"" AS ""Path"", + ""B"".""BlocksetID"" AS ""BlocksetID"", + ""B"".""MetadataID"" AS ""MetadataID"" + FROM + ""{tablename}"" ""A"", + ""File"" ""B"" + WHERE ""A"".""RealPath"" = ""B"".""Path"" + "); + + await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" "); + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" "); + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" "); using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) - tr.Commit(); + await m_rtr.CommitAsync(); - cmd.Transaction = null; - - cmd.ExecuteNonQuery("VACUUM"); + // TODO maybe this needs a transaction. + await cmd.ExecuteNonQueryAsync("VACUUM"); } } } From af2b1565b53c4c1a4c7fee4728735e530c0ce05c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 20:38:34 +0200 Subject: [PATCH 068/362] Removed whitespace --- .../Main/Database/LocalDeleteDatabase.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 20a2cd912..af8074606 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From 2c9bc49b3dff6ad1aa3a779dc6551cbb24cfd3f6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 20:43:45 +0200 Subject: [PATCH 069/362] Changed LocalDeleteDatabase to use factory construction --- .../Main/Database/LocalDeleteDatabase.cs | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index af8074606..deff17bbd 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -23,9 +23,10 @@ using System; using System.Collections.Generic; -using System.Data; using System.Linq; +using System.Threading.Tasks; using Duplicati.Library.Interface; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -43,18 +44,26 @@ namespace Duplicati.Library.Main.Database private const string REGISTER_COMMAND = @"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""ID"", @VolumeId FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "; - private IDbCommand m_registerDuplicateBlockCommand; + private SqliteCommand m_registerDuplicateBlockCommand = null!; - public LocalDeleteDatabase(string path, string operation, long pagecachesize) - : base(path, operation, true, pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize) { - m_registerDuplicateBlockCommand = m_connection.CreateCommand(REGISTER_COMMAND); + var db = new LocalDeleteDatabase(); + + db = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(db, path, "BugReportCreate", false, pagecachesize); + + return db; } - public LocalDeleteDatabase(LocalDatabase db) - : base(db) + public static async Task CreateAsync(LocalDatabase dbparent) { - m_registerDuplicateBlockCommand = m_connection.CreateCommand(REGISTER_COMMAND); + var dbnew = new LocalDeleteDatabase(); + + dbnew = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + + dbnew.m_registerDuplicateBlockCommand = await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND); + + return dbnew; } /// @@ -480,7 +489,7 @@ namespace Duplicati.Library.Main.Database { if (lookupIndexfiles.TryGetValue(sh.Name, out var backref)) { - //If this is the last reference, + //If this is the last reference, // remove the index file as well if (backref.Remove(r.Name) && backref.Count == 0) yield return sh; From ed6cee24d1c3f290080410f181485f7f036dd421 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 21:03:40 +0200 Subject: [PATCH 070/362] Changed a heavy property in LocalDeleteDatabase to a function to convey the heavyness --- .../Main/Database/LocalDeleteDatabase.cs | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index deff17bbd..0d2e9a78d 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -124,27 +124,36 @@ namespace Duplicati.Library.Main.Database /// exposed to the user. This is in contrast to other cases where the Version is the ID in the /// Fileset table. /// - internal IEnumerable FilesetsWithBackupVersion + internal async IAsyncEnumerable FilesetsWithBackupVersion() { - get + using (var cmd = m_connection.CreateCommand(m_rtr)) { - var filesets = new List(); - using (var cmd = m_connection.CreateCommand()) + // TODO check if this is still the case? (shouldn't be with new sqlite driver): + // We can also use the ROW_NUMBER() window function to generate the backup versions, + // but this requires at least SQLite 3.25, which is not available in some common + // distributions (e.g., Debian) currently. + cmd.SetCommandAndParameters(@" + SELECT + ""IsFullBackup"", + ""Timestamp"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); + + using (var reader = await cmd.ExecuteReaderAsync()) { - // We can also use the ROW_NUMBER() window function to generate the backup versions, - // but this requires at least SQLite 3.25, which is not available in some common - // distributions (e.g., Debian) currently. - using (IDataReader reader = cmd.ExecuteReader(@"SELECT ""IsFullBackup"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC")) + int version = 0; + while (await reader.ReadAsync()) { - int version = 0; - while (reader.Read()) - { - filesets.Add(new ListResultFileset(version++, reader.GetInt32(0), ParseFromEpochSeconds(reader.ConvertValueToInt64(1)).ToLocalTime(), -1L, -1L)); - } + yield return new ListResultFileset( + version++, + reader.GetInt32(0), + ParseFromEpochSeconds(reader.ConvertValueToInt64(1)).ToLocalTime(), + -1L, + -1L + ); } } - - return filesets; } } From 88db9676b4f1ed4a74d11667729c897af8d6ab5a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 21:51:28 +0200 Subject: [PATCH 071/362] Restructured SQL queries in LocalDeleteDatabase and made everything async where applicable --- .../Library/Main/Database/LocalDatabase.cs | 4 +- .../Main/Database/LocalDeleteDatabase.cs | 528 +++++++++++++++--- 2 files changed, 445 insertions(+), 87 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 1940f9903..e78f21bdb 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -552,10 +552,10 @@ namespace Duplicati.Library.Main.Database yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } - public async Task GetRemoteVolume(string file, SqliteTransaction transaction) + public async Task GetRemoteVolume(string file) { m_selectremotevolumeCommand - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Name", file); using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync()) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 0d2e9a78d..4ac17ffa4 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -72,50 +72,164 @@ namespace Duplicati.Library.Main.Database /// The fileset entries to delete /// The transaction to execute the commands in /// A list of filesets to delete - public IEnumerable> DropFilesetsFromTable(DateTime[] toDelete, IDbTransaction transaction) + public async IAsyncEnumerable> DropFilesetsFromTable(DateTime[] toDelete) { - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand(m_rtr)) { var deleted = 0; - using (var tempTable = new TemporaryDbValueList(m_connection, transaction, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds))) - deleted += cmd.SetCommandAndParameters(@"DELETE FROM ""Fileset"" WHERE ""Timestamp"" IN (@Timestamps) ") - .ExpandInClauseParameter("@Timestamps", tempTable) - .ExecuteNonQuery(); + using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds))) + deleted += await (await cmd.SetCommandAndParameters(@" + DELETE FROM ""Fileset"" + WHERE ""Timestamp"" IN (@Timestamps) + ") + .ExpandInClauseParameterAsync("@Timestamps", tempTable)) + .ExecuteNonQueryAsync(); if (deleted != toDelete.Length) throw new Exception($"Unexpected number of deleted filesets {deleted} vs {toDelete.Length}"); //Then we delete anything that is no longer being referenced - cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); - cmd.ExecuteNonQuery(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); - cmd.ExecuteNonQuery(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""FileLookup"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""FileLookup"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") "); + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetID"" NOT IN ( + SELECT DISTINCT ""ID"" + FROM ""Fileset"" + ) + "); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""ChangeJournalData"" + WHERE ""FilesetID"" NOT IN ( + SELECT DISTINCT ""ID"" + FROM ""Fileset"" + ) + "); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""FileLookup"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""FileID"" + FROM ""FilesetEntry"" + ) + "); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""Metadataset"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""MetadataID"" + FROM ""FileLookup"" + ) + "); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""Blockset"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""BlocksetID"" + FROM ""FileLookup"" + UNION + SELECT DISTINCT ""BlocksetID"" + FROM ""Metadataset"" + ) + "); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""BlocksetEntry"" + WHERE ""BlocksetID"" NOT IN ( + SELECT DISTINCT ""ID"" + FROM ""Blockset"" + ) + "); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""BlocklistHash"" + WHERE ""BlocksetID"" NOT IN ( + SELECT DISTINCT ""ID"" + FROM ""Blockset"" + ) + "); //We save the block info for the remote files, before we delete it - cmd.ExecuteNonQuery(@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""Block"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlockID"" AS ""BlockID"" FROM ""BlocksetEntry"" UNION SELECT DISTINCT ""ID"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Block"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlockID"" FROM ""BlocksetEntry"" UNION SELECT DISTINCT ""ID"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"") "); + await cmd.ExecuteNonQueryAsync(@" + INSERT INTO ""DeletedBlock"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""Block"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""BlockID"" AS ""BlockID"" + FROM ""BlocksetEntry"" + UNION + SELECT DISTINCT ""ID"" + FROM + ""Block"", + ""BlocklistHash"" + WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" + ) + "); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""Block"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""BlockID"" + FROM ""BlocksetEntry"" + UNION + SELECT DISTINCT ""ID"" + FROM + ""Block"", + ""BlocklistHash"" + WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" + ) + "); //Find all remote filesets that are no longer required, and mark them as deleting - var updated = cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""State"" = @NewState WHERE ""Type"" = @CurrentType AND ""State"" IN (@AllowedStates) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"") ") + var updated = await cmd.SetCommandAndParameters(@" + UPDATE ""RemoteVolume"" + SET ""State"" = @NewState + WHERE + ""Type"" = @CurrentType + AND ""State"" IN (@AllowedStates) + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) + ") .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) .SetParameterValue("@CurrentType", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@AllowedStates", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Deleting.ToString()]) - .ExecuteNonQuery(); + .ExpandInClauseParameter("@AllowedStates", [ + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString(), + RemoteVolumeState.Temporary.ToString(), + RemoteVolumeState.Deleting.ToString() + ]) + .ExecuteNonQueryAsync(); if (deleted != updated) throw new Exception($"Unexpected number of remote volumes marked as deleted. Found {deleted} filesets, but {updated} volumes"); - cmd.SetCommandAndParameters(@"SELECT ""Name"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" = @State ") + cmd.SetCommandAndParameters(@" + SELECT + ""Name"", + ""Size"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" = @State + ") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) .SetParameterValue("@State", RemoteVolumeState.Deleting.ToString()); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) + yield return new KeyValuePair( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1) + ); } } @@ -179,39 +293,171 @@ namespace Duplicati.Library.Main.Database /// The sizes are the uncompressed values. /// /// A list of tuples with name, datasize, wastedbytes. - private IEnumerable GetWastedSpaceReport(IDbTransaction transaction) + private async IAsyncEnumerable GetWastedSpaceReport() { var tmptablename = "UsageReport-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var usedBlocks = @"SELECT SUM(""Block"".""Size"") AS ""ActiveSize"", ""Block"".""VolumeID"" AS ""VolumeID"" FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Block"".""ID"" NOT IN (SELECT ""Block"".""ID"" FROM ""Block"",""DeletedBlock"" WHERE ""Block"".""Hash"" = ""DeletedBlock"".""Hash"" AND ""Block"".""Size"" = ""DeletedBlock"".""Size"" AND ""Block"".""VolumeID"" = ""DeletedBlock"".""VolumeID"") GROUP BY ""Block"".""VolumeID"" "; - var lastmodifiedFile = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""FileLookup"", ""BlocksetEntry"", ""Block"" WHERE ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" "; - var lastmodifiedMetadata = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""FileLookup"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" "; - var scantime = @"SELECT ""VolumeID"" AS ""VolumeID"", MIN(""Sorttime"") AS ""Sorttime"" FROM (" + lastmodifiedFile + @" UNION " + lastmodifiedMetadata + @") GROUP BY ""VolumeID"" "; - var active = @"SELECT ""A"".""ActiveSize"" AS ""ActiveSize"", 0 AS ""InactiveSize"", ""A"".""VolumeID"" AS ""VolumeID"", CASE WHEN ""B"".""Sorttime"" IS NULL THEN 0 ELSE ""B"".""Sorttime"" END AS ""Sorttime"" FROM (" + usedBlocks + @") A LEFT OUTER JOIN (" + scantime + @") B ON ""B"".""VolumeID"" = ""A"".""VolumeID"" "; + var usedBlocks = @" + SELECT + SUM(""Block"".""Size"") AS ""ActiveSize"", + ""Block"".""VolumeID"" AS ""VolumeID"" FROM ""Block"", + ""Remotevolume"" + WHERE + ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Block"".""ID"" NOT IN ( + SELECT ""Block"".""ID"" + FROM + ""Block"", + ""DeletedBlock"" + WHERE + ""Block"".""Hash"" = ""DeletedBlock"".""Hash"" + AND ""Block"".""Size"" = ""DeletedBlock"".""Size"" + AND ""Block"".""VolumeID"" = ""DeletedBlock"".""VolumeID"" + ) + GROUP BY ""Block"".""VolumeID"" + "; - var inactive = @"SELECT 0 AS ""ActiveSize"", SUM(""Size"") AS ""InactiveSize"", ""VolumeID"" AS ""VolumeID"", 0 AS ""SortScantime"" FROM ""DeletedBlock"" GROUP BY ""VolumeID"" "; - var empty = @"SELECT 0 AS ""ActiveSize"", 0 AS ""InactiveSize"", ""Remotevolume"".""ID"" AS ""VolumeID"", 0 AS ""SortScantime"" FROM ""Remotevolume"" WHERE ""Remotevolume"".""Type"" = @Type AND ""Remotevolume"".""State"" IN (@AllowedStates) AND ""Remotevolume"".""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Block"") "; + var lastmodifiedFile = @" + SELECT + ""Block"".""VolumeID"" AS ""VolumeID"", + ""Fileset"".""Timestamp"" AS ""Sorttime"" + FROM + ""Fileset"", + ""FilesetEntry"", + ""FileLookup"", + ""BlocksetEntry"", + ""Block"" + WHERE + ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + AND ""FileLookup"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + "; - var combined = active + " UNION " + inactive + " UNION " + empty; - var collected = @"SELECT ""VolumeID"" AS ""VolumeID"", SUM(""ActiveSize"") AS ""ActiveSize"", SUM(""InactiveSize"") AS ""InactiveSize"", MAX(""Sorttime"") AS ""Sorttime"" FROM (" + combined + @") GROUP BY ""VolumeID"" "; - var createtable = FormatInvariant($"{@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS "}{collected}"); + var lastmodifiedMetadata = @" + SELECT + ""Block"".""VolumeID"" AS ""VolumeID"", + ""Fileset"".""Timestamp"" AS ""Sorttime"" + FROM + ""Fileset"", + ""FilesetEntry"", + ""FileLookup"", + ""BlocksetEntry"", + ""Block"", + ""Metadataset"" + WHERE + ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + AND ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID"" + AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + "; - using (var cmd = m_connection.CreateCommand(transaction)) + var scantime = @$" + SELECT + ""VolumeID"" AS ""VolumeID"", + MIN(""Sorttime"") AS ""Sorttime"" + FROM ( + {lastmodifiedFile} + UNION {lastmodifiedMetadata} + ) + GROUP BY ""VolumeID"" + "; + + var active = @$" + SELECT + ""A"".""ActiveSize"" AS ""ActiveSize"", + 0 AS ""InactiveSize"", + ""A"".""VolumeID"" AS ""VolumeID"", + CASE + WHEN ""B"".""Sorttime"" IS NULL + THEN 0 + ELSE ""B"".""Sorttime"" + END AS ""Sorttime"" + FROM ({usedBlocks}) A + LEFT OUTER JOIN ({scantime}) B + ON ""B"".""VolumeID"" = ""A"".""VolumeID"" + "; + + var inactive = @" + SELECT + 0 AS ""ActiveSize"", + SUM(""Size"") AS ""InactiveSize"", + ""VolumeID"" AS ""VolumeID"", + 0 AS ""SortScantime"" + FROM ""DeletedBlock"" + GROUP BY ""VolumeID"" + "; + + var empty = @" + SELECT + 0 AS ""ActiveSize"", + 0 AS ""InactiveSize"", + ""Remotevolume"".""ID"" AS ""VolumeID"", + 0 AS ""SortScantime"" + FROM ""Remotevolume"" + WHERE + ""Remotevolume"".""Type"" = @Type + AND ""Remotevolume"".""State"" IN (@AllowedStates) + AND ""Remotevolume"".""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Block"" + ) + "; + + var combined = $"{active} UNION {inactive} UNION {empty}"; + var collected = @$" + SELECT + ""VolumeID"" AS ""VolumeID"", + SUM(""ActiveSize"") AS ""ActiveSize"", + SUM(""InactiveSize"") AS ""InactiveSize"", + MAX(""Sorttime"") AS ""Sorttime"" + FROM ({combined}) + GROUP BY ""VolumeID"" + "; + + var createtable = $"{@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS "}{collected}"; + + using (var cmd = m_connection.CreateCommand(m_rtr)) { try { - cmd.SetCommandAndParameters(createtable) + await cmd.SetCommandAndParameters(createtable) .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameter("@AllowedStates", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]) - .ExecuteNonQuery(); + .ExpandInClauseParameter("@AllowedStates", [ + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString() + ]) + .ExecuteNonQueryAsync(); - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""A"".""Name"", ""B"".""ActiveSize"", ""B"".""InactiveSize"", ""A"".""Size"" FROM ""Remotevolume"" A, ""{tmptablename}"" B WHERE ""A"".""ID"" = ""B"".""VolumeID"" ORDER BY ""B"".""Sorttime"" ASC "))) - while (rd.Read()) - yield return new VolumeUsage(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), rd.ConvertValueToInt64(2, 0), rd.ConvertValueToInt64(3, 0)); + cmd.SetCommandAndParameters($@" + SELECT + ""A"".""Name"", + ""B"".""ActiveSize"", + ""B"".""InactiveSize"", + ""A"".""Size"" + FROM + ""Remotevolume"" A, + ""{tmptablename}"" B + WHERE ""A"".""ID"" = ""B"".""VolumeID"" + ORDER BY ""B"".""Sorttime"" ASC + "); + + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) + yield return new VolumeUsage( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), + rd.ConvertValueToInt64(2, 0), + rd.ConvertValueToInt64(3, 0) + ); } finally { - try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmptablename}"" ")); } + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"" "); + } catch { } } } @@ -315,9 +561,9 @@ namespace Duplicati.Library.Main.Database } } - public ICompactReport GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, IDbTransaction transaction) + public async Task GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount) { - return new CompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, GetWastedSpaceReport(transaction).ToList()); + return new CompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, await GetWastedSpaceReport().ToListAsync()); } @@ -331,25 +577,50 @@ namespace Duplicati.Library.Main.Database /// The volume ID to check, or -1 to check all volumes /// The transaction to execute the command in /// True if the block is in use - bool UseBlock(string hash, long size, long volumeId, IDbTransaction? transaction); + Task UseBlock(string hash, long size, long volumeId); } private class BlockQuery : IBlockQuery { - private IDbCommand m_command; + private LocalDatabase m_db = null!; - public BlockQuery(IDbConnection con, IDbTransaction transaction) + private SqliteCommand m_command = null!; + + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync() instead.")] + public BlockQuery(SqliteConnection connection, SqliteTransaction? transaction) { - m_command = con.CreateCommand(transaction, @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); + throw new NotImplementedException("Use CreateAsync() instead"); + } + + private BlockQuery() + { + // Private constructor to prevent instantiation without CreateAsync + } + + public static async Task CreateAsync(LocalDatabase db) + { + return new BlockQuery + { + m_db = db, + m_command = await db.Connection.CreateCommandAsync(@" + SELECT ""VolumeID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ") + }; } /// - public bool UseBlock(string hash, long size, long volumeId, IDbTransaction? transaction) + public async Task UseBlock(string hash, long size, long volumeId) { - var r = m_command.SetTransaction(transaction) + var r = await m_command + .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1); + if (r == -1) { return false; @@ -373,13 +644,14 @@ namespace Duplicati.Library.Main.Database } } + // TODO signature has been changed. /// /// Builds a lookup table to enable faster response to block queries /// - public IBlockQuery CreateBlockQueryHelper(IDbTransaction transaction) - { - return new BlockQuery(m_connection, transaction); - } + //public IBlockQuery CreateBlockQueryHelper(IDbTransaction transaction) + //{ + // return new BlockQuery(m_connection, transaction); + //} /// /// Registers a block as moved to a new volume @@ -387,15 +659,15 @@ namespace Duplicati.Library.Main.Database /// The hash of the block /// The size of the block /// The new volume ID - /// The transaction to execute the command in - public void RegisterDuplicatedBlock(string hash, long size, long volumeID, IDbTransaction tr) + public async Task RegisterDuplicatedBlock(string hash, long size, long volumeID) { - m_registerDuplicateBlockCommand.Transaction = tr; - m_registerDuplicateBlockCommand.SetParameterValue("@VolumeId", volumeID); - m_registerDuplicateBlockCommand.SetParameterValue("@Hash", hash); - m_registerDuplicateBlockCommand.SetParameterValue("@Size", size); // Using INSERT OR IGNORE to avoid duplicate entries, result may be 1 or 0 - m_registerDuplicateBlockCommand.ExecuteNonQuery(); + await m_registerDuplicateBlockCommand + .SetTransaction(m_rtr) + .SetParameterValue("@VolumeId", volumeID) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(); } /// @@ -404,49 +676,114 @@ namespace Duplicati.Library.Main.Database /// The file to remove /// The volume IDs that will be removed /// The transaction to execute the command in - public void PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved, IDbTransaction transaction) + public async Task PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved) { - var deletedVolume = GetRemoteVolume(filename, transaction); + var deletedVolume = await GetRemoteVolume(filename); if (deletedVolume.Type != RemoteVolumeType.Blocks) return; - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand(m_rtr)) { var updatedBlocks = "BlocksToUpdate-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var replacementBlocks = "ReplacementBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try { - cmd.SetCommandAndParameters(FormatInvariant($@"CREATE {TEMPORARY} TABLE ""{updatedBlocks}"" AS SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId")) + await cmd.SetCommandAndParameters($@" + CREATE {TEMPORARY} TABLE ""{updatedBlocks}"" AS + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" = @VolumeId + ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); - using (var tempTable = new TemporaryDbValueList(m_connection, transaction, volumeIdsToBeRemoved)) - cmd.SetCommandAndParameters(FormatInvariant($@"CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS SELECT ""BlockID"", MAX(""VolumeID"") AS ""VolumeID"" FROM ""DuplicateBlock"" WHERE ""VolumeID"" NOT IN (@VolumeIds) AND ""BlockID"" IN (SELECT ""ID"" FROM ""{updatedBlocks}"") GROUP BY ""BlockID"" ")) - .ExpandInClauseParameter("@VolumeIds", tempTable) - .ExecuteNonQuery(); + using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved)) + await (await cmd.SetCommandAndParameters($@" + CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS + SELECT + ""BlockID"", + MAX(""VolumeID"") AS ""VolumeID"" + FROM ""DuplicateBlock"" + WHERE + ""VolumeID"" NOT IN (@VolumeIds) + AND ""BlockID"" IN ( + SELECT ""ID"" + FROM ""{updatedBlocks}"" + ) + GROUP BY ""BlockID"" + ") + .ExpandInClauseParameterAsync("@VolumeIds", tempTable)) + .ExecuteNonQueryAsync(); + + var targetCount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{updatedBlocks}"" + "); - var targetCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{updatedBlocks}"" ")); if (targetCount == 0) return; - var replacementCount = cmd.ExecuteScalarInt64($@"SELECT COUNT(*) FROM ""{replacementBlocks}"" "); - var updateCount = cmd.SetCommandAndParameters(FormatInvariant(@$"UPDATE ""Block"" SET ""VolumeID"" = (SELECT ""VolumeID"" FROM ""{replacementBlocks}"" WHERE ""{replacementBlocks}"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" = @VolumeId) WHERE ""Block"".""VolumeID"" = @VolumeId ")) + var replacementCount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{replacementBlocks}"" + "); + + var updateCount = await cmd.SetCommandAndParameters(@$" + UPDATE ""Block"" + SET ""VolumeID"" = ( + SELECT ""VolumeID"" + FROM ""{replacementBlocks}"" + WHERE + ""{replacementBlocks}"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" = @VolumeId + ) + WHERE ""Block"".""VolumeID"" = @VolumeId + ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQuery(); - var deleteCount = cmd.ExecuteNonQuery(FormatInvariant(@$"DELETE FROM ""DuplicateBlock"" WHERE (""DuplicateBlock"".""BlockID"" || ':' || ""DuplicateBlock"".""VolumeID"") IN (SELECT ""RB"".""BlockID"" || ':' || ""RB"".""VolumeID"" FROM ""{replacementBlocks}"" RB)")); - if (targetCount != updateCount || replacementCount != deleteCount || updateCount != deleteCount) + .ExecuteNonQueryAsync(); + + var deleteCount = await cmd.ExecuteNonQueryAsync(@$" + DELETE FROM ""DuplicateBlock"" + WHERE + ( + ""DuplicateBlock"".""BlockID"" + || ':' + || ""DuplicateBlock"".""VolumeID"" + ) IN ( + SELECT + ""RB"".""BlockID"" + || ':' + || ""RB"".""VolumeID"" + FROM ""{replacementBlocks}"" RB + ) + "); + + if (targetCount != updateCount + || replacementCount != deleteCount + || updateCount != deleteCount) + { throw new Exception($"Unexpected number of rows updated. Expected {targetCount} but got updated {updateCount}, deleted {deleteCount}, and replaced {replacementCount}"); + } // Remove knowledge of any old blocks - cmd.SetCommandAndParameters(FormatInvariant(@$"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" = @VolumeId")) + await cmd.SetCommandAndParameters(@$" + DELETE FROM ""DuplicateBlock"" + WHERE ""VolumeID"" = @VolumeId + ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); } finally { - try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ")); } + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" "); + } catch { } - try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ")); } + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" "); + } catch { } } } @@ -458,9 +795,9 @@ namespace Duplicati.Library.Main.Database /// Block volumes slated for deletion. /// The transaction to execute the command in /// The deletable volumes. - public IEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes, IDbTransaction transaction) + public async IAsyncEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes) { - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand(m_rtr)) { // Although the generated index volumes are always in pairs, // this code handles many-to-many relations between @@ -468,8 +805,25 @@ namespace Duplicati.Library.Main.Database var lookupBlock = new Dictionary>(); var lookupIndexfiles = new Dictionary>(); - using (var rd = cmd.ExecuteReader(@"SELECT ""C"".""Name"", ""B"".""Name"", ""B"".""Hash"", ""B"".""Size"" FROM ""IndexBlockLink"" A, ""RemoteVolume"" B, ""RemoteVolume"" C WHERE ""A"".""IndexVolumeID"" = ""B"".""ID"" AND ""A"".""BlockVolumeID"" = ""C"".""ID"" AND ""B"".""Hash"" IS NOT NULL AND ""B"".""Size"" IS NOT NULL ")) - while (rd.Read()) + cmd.SetCommandAndParameters(@" + SELECT + ""C"".""Name"", + ""B"".""Name"", + ""B"".""Hash"", + ""B"".""Size"" + FROM + ""IndexBlockLink"" A, + ""RemoteVolume"" B, + ""RemoteVolume"" C + WHERE + ""A"".""IndexVolumeID"" = ""B"".""ID"" + AND ""A"".""BlockVolumeID"" = ""C"".""ID"" + AND ""B"".""Hash"" IS NOT NULL + AND ""B"".""Size"" IS NOT NULL + "); + + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) { var name = rd.ConvertValueToString(0) ?? ""; if (!lookupBlock.TryGetValue(name, out var indexfileList)) @@ -478,7 +832,11 @@ namespace Duplicati.Library.Main.Database lookupBlock.Add(name, indexfileList); } - var v = new RemoteVolume(rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToInt64(3)); + var v = new RemoteVolume( + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToInt64(3) + ); indexfileList.Add(v); if (!lookupIndexfiles.TryGetValue(v.Name, out var blockList)) From 0b508d25756bef72ae821e3812248b06a4416264 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 19 May 2025 21:59:44 +0200 Subject: [PATCH 072/362] Started work on restructuring the SQL queries in LocalBackupDatabase. --- .../Main/Database/LocalBackupDatabase.cs | 200 ++++++++++++++++-- 1 file changed, 186 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 4d7c986bb..7b22dfa65 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -18,6 +18,7 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + #nullable enable using System; @@ -102,20 +103,191 @@ namespace Duplicati.Library.Main.Database dbnew.m_logQueries = options.ProfileAllDatabaseQueries; - dbnew.m_findblockCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @hash AND ""Size"" = @Size"); - dbnew.m_findblocksetCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = @Fullhash AND ""Length"" = @Length"); - dbnew.m_findmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = @Hash AND ""C"".""Size"" = @Size"); - dbnew.m_findfilesetCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""Path"" = @Path AND ""PrefixID"" = @PrefixId"); - dbnew.m_insertblockCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (@Hash, @VolumeId, @Size); SELECT last_insert_rowid();"); - dbnew.m_insertfileOperationCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId, @FileId, @LastModified)"); - dbnew.m_insertfileCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (@PrefixId, @Path, @BlocksetId, @MetadataId); SELECT last_insert_rowid();"); - dbnew.m_insertblocksetCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length, @Fullhash); SELECT last_insert_rowid();"); - dbnew.m_insertblocksetentryCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT @BlocksetId AS A, @Index AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - dbnew.m_insertblocklistHashesCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash)"); - dbnew.m_insertmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); - dbnew.m_selectfilelastmodifiedCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId"); - dbnew.m_selectfilelastmodifiedWithSizeCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" "); - dbnew.m_selectfilemetadatahashandsizeCommand = await dbnew.Connection.CreateCommandAsync(@"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = @FileId AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" "); + dbnew.m_findblockCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Block"" + WHERE + ""Hash"" = @hash + AND ""Size"" = @Size + "); + + dbnew.m_findblocksetCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Blockset"" + WHERE + ""Fullhash"" = @Fullhash + AND ""Length"" = @Length + "); + + dbnew.m_findmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT ""A"".""ID"" + FROM + ""Metadataset"" A, + ""BlocksetEntry"" B, + ""Block"" C + WHERE + ""A"".""BlocksetID"" = ""B"".""BlocksetID"" + AND ""B"".""BlockID"" = ""C"".""ID"" + AND ""C"".""Hash"" = @Hash + AND ""C"".""Size"" = @Size + "); + + dbnew.m_findfilesetCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""FileLookup"" + WHERE + ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + AND ""Path"" = @Path + AND ""PrefixID"" = @PrefixId + "); + + dbnew.m_insertblockCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""Block"" ( + ""Hash"", + ""VolumeID"", + ""Size"" + ) + VALUES ( + @Hash, + @VolumeId, + @Size + ); + SELECT last_insert_rowid(); + "); + + dbnew.m_insertfileOperationCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + VALUES ( + @FilesetId, + @FileId, + @LastModified + ) + "); + + dbnew.m_insertfileCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""FileLookup"" ( + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + VALUES ( + @PrefixId, + @Path, + @BlocksetId, + @MetadataId + ); + SELECT last_insert_rowid(); + "); + + dbnew.m_insertblocksetCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""Blockset"" ( + ""Length"", + ""FullHash"" + ) + VALUES ( + @Length, + @Fullhash + ); + SELECT last_insert_rowid();"); + + dbnew.m_insertblocksetentryCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""BlocksetEntry"" ( + ""BlocksetID"", + ""Index"", + ""BlockID"" + ) + SELECT + @BlocksetId AS A, + @Index AS B, + ""ID"" + FROM ""Block"" + WHERE ""Hash"" = @Hash + AND ""Size"" = @Size + "); + + dbnew.m_insertblocklistHashesCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""BlocklistHash"" ( + ""BlocksetID"", + ""Index"", + ""Hash"" + ) + VALUES ( + @BlocksetId, + @Index, + @Hash + ) + "); + + dbnew.m_insertmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""Metadataset"" (""BlocksetID"") + VALUES (@BlocksetId); + SELECT last_insert_rowid(); + "); + + dbnew.m_selectfilelastmodifiedCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT + ""A"".""ID"", + ""B"".""LastModified"" + FROM ( + SELECT ""ID"" + FROM ""FileLookup"" + WHERE ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + ) ""A"" + CROSS JOIN ""FilesetEntry"" ""B"" + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""B"".""FilesetID"" = @FilesetId + "); + + dbnew.m_selectfilelastmodifiedWithSizeCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT + ""C"".""ID"", + ""C"".""LastModified"", + ""D"".""Length"" + FROM + ( + SELECT + ""A"".""ID"", + ""B"".""LastModified"", + ""A"".""BlocksetID"" + FROM ( + SELECT + ""ID"", + ""BlocksetID"" + FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + ) ""A"" + CROSS JOIN ""FilesetEntry"" ""B"" + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""B"".""FilesetID"" = @FilesetId + ) AS ""C"", + ""Blockset"" AS ""D"" + WHERE ""C"".""BlocksetID"" == ""D"".""ID"" + "); + + dbnew.m_selectfilemetadatahashandsizeCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT + ""Blockset"".""Length"", + ""Blockset"".""FullHash"" + FROM + ""Blockset"", + ""Metadataset"", + ""File"" + WHERE + ""File"".""ID"" = @FileId + AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" + AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" + "); // Experimental toggling of the deleted block cache // If the value is less than zero, the lookup is disabled From 3e3a9b5aa9697ddce28ee68dfb973e5d433cbc0f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:07:45 +0200 Subject: [PATCH 073/362] More SQL query restructuring --- .../Main/Database/LocalBackupDatabase.cs | 287 ++++++++++++++---- .../Main/Database/LocalBugReportDatabase.cs | 7 +- .../Library/Main/Database/LocalDatabase.cs | 6 +- .../Main/Database/LocalRepairDatabase.cs | 6 +- .../Main/Database/LocalRestoreDatabase.cs | 104 ++++++- 5 files changed, 327 insertions(+), 83 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 7b22dfa65..98bddc904 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -304,12 +304,32 @@ namespace Duplicati.Library.Main.Database using (var cmd = dbnew.Connection.CreateCommand()) { dbnew.m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.SetCommandAndParameters($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS " - + @$"SELECT MAX(""ID"") AS ""ID"", ""Hash"", ""Size"" FROM ""DeletedBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""State"" NOT IN (@States)) GROUP BY ""Hash"", ""Size""") - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted, RemoteVolumeState.Deleting]) - .ExecuteNonQuery(); + await cmd.SetCommandAndParameters($@" + CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS + SELECT + MAX(""ID"") AS ""ID"", + ""Hash"", + ""Size"" + FROM ""DeletedBlock"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""State"" NOT IN (@States) + ) + GROUP BY + ""Hash"", + ""Size"" + ") + .ExpandInClauseParameter("@States", [ + RemoteVolumeState.Deleted, + RemoteVolumeState.Deleting + ]) + .ExecuteNonQueryAsync(); - var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$"SELECT COUNT(*) FROM ""{dbnew.m_tempDeletedBlockTable}""", 0); + var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$" + SELECT COUNT(*) + FROM ""{dbnew.m_tempDeletedBlockTable}"" + ", 0); // There are no deleted blocks, so we can drop the table if (deletedBlocks == 0) @@ -322,7 +342,15 @@ namespace Duplicati.Library.Main.Database else if (deletedBlocks <= deletedBlockCacheSizeLong) { dbnew.m_deletedBlockLookup = new Dictionary>(); - using (var reader = await cmd.ExecuteReaderAsync(@$"SELECT ""ID"", ""Hash"", ""Size"" FROM ""{dbnew.m_tempDeletedBlockTable}""")) + cmd.SetCommandAndParameters(@$" + SELECT + ""ID"", + ""Hash"", + ""Size"" + FROM ""{dbnew.m_tempDeletedBlockTable}"" + "); + + using (var reader = await cmd.ExecuteReaderAsync()) while (await reader.ReadAsync()) { var id = reader.ConvertValueToInt64(0); @@ -340,22 +368,66 @@ namespace Duplicati.Library.Main.Database // The deleted blocks are too large to fit in memory, so we use a temporary table else { - await cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" ON ""{dbnew.m_tempDeletedBlockTable}"" (""Hash"", ""Size"")"); - dbnew.m_findindeletedCommand = await dbnew.m_connection.CreateCommandAsync($@"SELECT ""ID"" FROM ""{dbnew.m_tempDeletedBlockTable}"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(string.Join(";", - @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", - @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", - @$"DELETE FROM {dbnew.m_tempDeletedBlockTable} WHERE ""ID"" = @DeletedBlockId", - "SELECT last_insert_rowid()")); + await cmd.ExecuteNonQueryAsync($@" + CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" + ON ""{dbnew.m_tempDeletedBlockTable}"" ( + ""Hash"", + ""Size"" + ) + "); + + dbnew.m_findindeletedCommand = await dbnew.Connection.CreateCommandAsync($@" + SELECT ""ID"" + FROM ""{dbnew.m_tempDeletedBlockTable}"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + "); + + dbnew.m_moveblockfromdeletedCommand = await dbnew.Connection.CreateCommandAsync(@$" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId LIMIT 1; + + DELETE FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId; + + DELETE FROM {dbnew.m_tempDeletedBlockTable} + WHERE ""ID"" = @DeletedBlockId; + + SELECT last_insert_rowid() + "); } if (deletedBlocks > 0) { - dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(string.Join(";", - @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", - @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", - "SELECT last_insert_rowid()")); + dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(@$" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId LIMIT 1; + + DELETE FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId; + + SELECT last_insert_rowid() + "); } } @@ -383,70 +455,161 @@ namespace Duplicati.Library.Main.Database { // The query used in Duplicati until 2.0.3.9 case 1: - findQuery = - @" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + - @" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " + - @" WHERE ""FileLookup"".""PrefixID"" = @PrefixId AND ""FileLookup"".""Path"" = @Path " + - @" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " + - @" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " + - @" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " + - @" AND @FilesetId IS NOT NULL" + - @" ORDER BY ""Fileset"".""Timestamp"" DESC " + - @" LIMIT 1 "; + findQuery = @" + SELECT + ""FileLookup"".""ID"" AS ""FileID"", + ""FilesetEntry"".""Lastmodified"", + ""FileBlockset"".""Length"", + ""MetaBlockset"".""Fullhash"" AS ""Metahash"", + ""MetaBlockset"".""Length"" AS ""Metasize"" + FROM + ""FileLookup"", + ""FilesetEntry"", + ""Fileset"", + ""Blockset"" ""FileBlockset"", + ""Metadataset"", + ""Blockset"" ""MetaBlockset"" + WHERE + ""FileLookup"".""PrefixID"" = @PrefixId + AND ""FileLookup"".""Path"" = @Path + AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" + AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" + AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" + AND @FilesetId IS NOT NULL + ORDER BY ""Fileset"".""Timestamp"" DESC + LIMIT 1 + "; break; // The fastest reported query in Duplicati 2.0.3.10, but with "LIMIT 1" added default: case 2: - var getLastFileEntryForPath = - @"SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"", ""A"".""MetadataID"" " + - @" FROM (SELECT ""ID"", ""BlocksetID"", ""MetadataID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" " + - @" CROSS JOIN ""FilesetEntry"" ""B"" " + - @" WHERE ""A"".""ID"" = ""B"".""FileID"" " + - @" AND ""B"".""FilesetID"" = @FilesetId "; + var getLastFileEntryForPath = @" + SELECT + ""A"".""ID"", + ""B"".""LastModified"", + ""A"".""BlocksetID"", + ""A"".""MetadataID"" + FROM ( + SELECT + ""ID"", + ""BlocksetID"", + ""MetadataID"" + FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + ) ""A"" + CROSS JOIN ""FilesetEntry"" ""B"" + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""B"".""FilesetID"" = @FilesetId + "; - findQuery = $@"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" -FROM ({getLastFileEntryForPath}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" -WHERE ""C"".""BlocksetID"" == ""D"".""ID"" AND ""C"".""MetadataID"" == ""F"".""ID"" AND ""F"".""BlocksetID"" = ""E"".""ID"" -LIMIT 1"; + findQuery = $@" + SELECT + ""C"".""ID"" AS ""FileID"", + ""C"".""LastModified"", + ""D"".""Length"", + ""E"".""FullHash"" as ""Metahash"", + ""E"".""Length"" AS ""Metasize"" + FROM + ({getLastFileEntryForPath}) AS ""C"", + ""Blockset"" AS ""D"", + ""Blockset"" AS ""E"", + ""Metadataset"" ""F"" + WHERE + ""C"".""BlocksetID"" == ""D"".""ID"" + AND ""C"".""MetadataID"" == ""F"".""ID"" + AND ""F"".""BlocksetID"" = ""E"".""ID"" + LIMIT 1 + "; break; // Potentially faster query: https://forum.duplicati.com/t/release-2-0-3-10-canary-2018-08-30/4497/25 case 3: - findQuery = - @" SELECT FileLookup.ID as FileID, FilesetEntry.Lastmodified, FileBlockset.Length, " + - @" MetaBlockset.FullHash AS Metahash, MetaBlockset.Length as Metasize " + - @" FROM FilesetEntry " + - @"INNER JOIN Fileset ON (FileSet.ID = FilesetEntry.FilesetID) " + - @"INNER JOIN FileLookup ON (FileLookup.ID = FilesetEntry.FileID) " + - @"INNER JOIN Metadataset ON (Metadataset.ID = FileLookup.MetadataID) " + - @"INNER JOIN Blockset AS MetaBlockset ON (MetaBlockset.ID = Metadataset.BlocksetID) " + - @" LEFT JOIN Blockset AS FileBlockset ON (FileBlockset.ID = FileLookup.BlocksetID) " + - @" WHERE FileLookup.PrefixID = @PrefixId AND FileLookup.Path = @Path AND FilesetID = @FilesetId " + - @" LIMIT 1 "; + findQuery = @" + SELECT + FileLookup.ID as FileID, + FilesetEntry.Lastmodified, + FileBlockset.Length, + MetaBlockset.FullHash AS Metahash, + MetaBlockset.Length as Metasize + FROM FilesetEntry + INNER JOIN Fileset + ON (FileSet.ID = FilesetEntry.FilesetID) + INNER JOIN FileLookup + ON (FileLookup.ID = FilesetEntry.FileID) + INNER JOIN Metadataset + ON (Metadataset.ID = FileLookup.MetadataID) + INNER JOIN Blockset AS MetaBlockset + ON (MetaBlockset.ID = Metadataset.BlocksetID) + LEFT JOIN Blockset AS FileBlockset + ON (FileBlockset.ID = FileLookup.BlocksetID) + WHERE + FileLookup.PrefixID = @PrefixId + AND FileLookup.Path = @Path + AND FilesetID = @FilesetId + LIMIT 1 + "; break; // The slow query used in Duplicati 2.0.3.10, but with "LIMIT 1" added case 4: - findQuery = - @" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + - @" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " + - @" WHERE ""FileLookup"".""PrefixID"" = @PrefixId AND ""FileLookup"".""Path"" = @Path " + - @" AND ""Fileset"".""ID"" = @FilesetId " + - @" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " + - @" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " + - @" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " + - @" LIMIT 1 "; + findQuery = @" + SELECT + ""FileLookup"".""ID"" AS ""FileID"", + ""FilesetEntry"".""Lastmodified"", + ""FileBlockset"".""Length"", + ""MetaBlockset"".""Fullhash"" AS ""Metahash"", + ""MetaBlockset"".""Length"" AS ""Metasize"" + FROM + ""FileLookup"", + ""FilesetEntry"", + ""Fileset"", + ""Blockset"" ""FileBlockset"", + ""Metadataset"", + ""Blockset"" ""MetaBlockset"" + WHERE + ""FileLookup"".""PrefixID"" = @PrefixId + AND ""FileLookup"".""Path"" = @Path + AND ""Fileset"".""ID"" = @FilesetId + AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" + AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" + AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" + LIMIT 1 + "; break; } dbnew.m_findfileCommand = dbnew.m_connection.CreateCommand(findQuery); - dbnew.m_selectfileHashCommand = dbnew.m_connection.CreateCommand(@"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = @FileId "); - dbnew.m_getfirstfilesetwithblockinblockset = dbnew.m_connection.CreateCommand(@"SELECT MIN(""FilesetEntry"".""FilesetID"") FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" IN ( -SELECT ""File"".""ID"" FROM ""File"" WHERE ""File"".""BlocksetID"" IN( -SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""Hash"" = @Hash))"); + dbnew.m_selectfileHashCommand = dbnew.m_connection.CreateCommand(@" + SELECT ""Blockset"".""Fullhash"" + FROM ""Blockset"", ""FileLookup"" + WHERE + ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" + AND ""FileLookup"".""ID"" = @FileId + "); + + dbnew.m_getfirstfilesetwithblockinblockset = dbnew.m_connection.CreateCommand(@" + SELECT MIN(""FilesetEntry"".""FilesetID"") + FROM ""FilesetEntry"" + WHERE ""FilesetEntry"".""FileID"" IN ( + SELECT ""File"".""ID"" + FROM ""File"" + WHERE ""File"".""BlocksetID"" IN( + SELECT ""BlocklistHash"".""BlocksetID"" + FROM ""BlocklistHash"" + WHERE ""BlocklistHash"".""Hash"" = @Hash + ) + ) + "); dbnew.m_blocklistHashes = new HashSet(); @@ -1002,7 +1165,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist using (var cmd = m_connection.CreateCommand(m_rtr)) { var prevFileSetId = await GetPreviousFilesetID(cmd); - ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId); + await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId); } } diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 5eee2187c..f5b959b2d 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -77,9 +77,10 @@ namespace Duplicati.Library.Main.Database || @DirSep || ""ID"" || ( - CASE WHEN substr(""RealPath"", length(""RealPath"")) = @DirSep - THEN @DirSep - ELSE @FileExt + CASE + WHEN substr(""RealPath"", length(""RealPath"")) = @DirSep + THEN @DirSep + ELSE @FileExt END ) ") diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index e78f21bdb..0ee8a2e49 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1309,7 +1309,11 @@ namespace Duplicati.Library.Main.Database SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", - CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" + CASE + WHEN ""G"".""BlocklistHashCount"" IS NULL + THEN 0 + ELSE ""G"".""BlocklistHashCount"" + END AS ""BlocklistHashCountActual"" FROM ( SELECT ""BlocksetID"", diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index a2d9bc5f2..675c938ee 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1159,7 +1159,11 @@ namespace Duplicati.Library.Main.Database SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", - CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" + CASE + WHEN ""G"".""BlocklistHashCount"" IS NULL + THEN 0 + ELSE ""G"".""BlocklistHashCount"" + END AS ""BlocklistHashCountActual"" FROM ( SELECT ""BlocksetID"", diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index eabe64069..9901fda4a 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -156,8 +156,26 @@ namespace Duplicati.Library.Main.Database ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size""), 0), - IFNULL(COUNT(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""ID"" ELSE NULL END), 0), - IFNULL(SUM(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""Size"" ELSE 0 END), 0) + IFNULL( + COUNT( + CASE ""B"".""Restored"" + WHEN 1 + THEN ""B"".""ID"" + ELSE NULL + END + ), + 0 + ), + IFNULL( + SUM( + CASE ""B"".""Restored"" + WHEN 1 + THEN ""B"".""Size"" + ELSE 0 + END + ), + 0 + ) FROM ""{m_tempfiletable}"" ""F"" LEFT JOIN ""{m_tempblocktable}"" ""B"" ON ""B"".""FileID"" = ""F"".""ID"" @@ -180,9 +198,30 @@ namespace Duplicati.Library.Main.Database IFNULL(COUNT(""P"".""FileId""), 0), IFNULL(SUM(""P"".""TotalBlocks""), 0), IFNULL(SUM(""P"".""TotalSize""), 0), - IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" THEN 1 ELSE NULL END), 0), - IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" BETWEEN 1 AND ""P"".""TotalBlocks"" - 1 THEN 1 ELSE NULL END), 0), - IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) + IFNULL( + COUNT( + CASE + WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" + THEN 1 + ELSE NULL + END + ), + 0 + ), + IFNULL( + COUNT( + CASE + WHEN + ""P"".""BlocksRestored"" BETWEEN 1 + AND ""P"".""TotalBlocks"" - 1 + THEN 1 + ELSE NULL + END + ), + 0 + ), + IFNULL(SUM(""P"".""BlocksRestored""), 0), + IFNULL(SUM(""P"".""SizeRestored""), 0) FROM ""{m_fileprogtable}"" ""P"" "); @@ -201,8 +240,12 @@ namespace Duplicati.Library.Main.Database AND NEW.""Metadata"" = 0 BEGIN UPDATE ""{m_fileprogtable}"" SET - ""BlocksRestored"" = ""{m_fileprogtable}"".""BlocksRestored"" + (NEW.""Restored"" - OLD.""Restored""), - ""SizeRestored"" = ""{m_fileprogtable}"".""SizeRestored"" + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) + ""BlocksRestored"" = + ""{m_fileprogtable}"".""BlocksRestored"" + + (NEW.""Restored"" - OLD.""Restored""), + ""SizeRestored"" = + ""{m_fileprogtable}"".""SizeRestored"" + + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" ; END "); @@ -213,14 +256,42 @@ namespace Duplicati.Library.Main.Database AFTER UPDATE ON ""{m_fileprogtable}"" BEGIN UPDATE ""{m_totalprogtable}"" SET - ""FilesFullyRestored"" = ""{m_totalprogtable}"".""FilesFullyRestored"" - + (CASE WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" THEN 1 ELSE 0 END) - - (CASE WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" THEN 1 ELSE 0 END), - ""FilesPartiallyRestored"" = ""{m_totalprogtable}"".""FilesPartiallyRestored"" - + (CASE WHEN NEW.""BlocksRestored"" BETWEEN 1 AND NEW.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) - - (CASE WHEN OLD.""BlocksRestored"" BETWEEN 1 AND OLD.""TotalBlocks"" - 1 THEN 1 ELSE 0 END), - ""BlocksRestored"" = ""{m_totalprogtable}"".""BlocksRestored"" + NEW.""BlocksRestored"" - OLD.""BlocksRestored"", - ""SizeRestored"" = ""{m_totalprogtable}"".""SizeRestored"" + NEW.""SizeRestored"" - OLD.""SizeRestored"" + ""FilesFullyRestored"" = + ""{m_totalprogtable}"".""FilesFullyRestored"" + + (CASE + WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" + THEN 1 + ELSE 0 + END) + - (CASE + WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" + THEN 1 + ELSE 0 + END), + ""FilesPartiallyRestored"" = + ""{m_totalprogtable}"".""FilesPartiallyRestored"" + + (CASE + WHEN + NEW.""BlocksRestored"" BETWEEN 1 + AND NEW.""TotalBlocks"" - 1 + THEN 1 + ELSE 0 + END) + - (CASE + WHEN + OLD.""BlocksRestored"" BETWEEN 1 + AND OLD.""TotalBlocks"" - 1 + THEN 1 + ELSE 0 + END), + ""BlocksRestored"" = + ""{m_totalprogtable}"".""BlocksRestored"" + + NEW.""BlocksRestored"" + - OLD.""BlocksRestored"", + ""SizeRestored"" = + ""{m_totalprogtable}"".""SizeRestored"" + + NEW.""SizeRestored"" + - OLD.""SizeRestored"" ; END "); @@ -650,7 +721,8 @@ namespace Duplicati.Library.Main.Database WHEN SUBSTR(""Path"", 1, 1) == '/' THEN @Path || SUBSTR(""Path"", 2) ELSE ""Path"" - END") + END + ") .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) .ExecuteNonQueryAsync(); } From 044110318ba4245cd43e000f17b98c50df25fa40 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:11:41 +0200 Subject: [PATCH 074/362] Changed LocalListAffectedDatabase to use factory construction --- .../Database/LocalListAffectedDatabase.cs | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index c60f26e5d..1dd586ab1 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -24,15 +24,20 @@ using System; using System.Linq; using System.Collections.Generic; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Database { internal class LocalListAffectedDatabase : LocalDatabase { - public LocalListAffectedDatabase(string path, long pagecachesize) - : base(path, "ListAffected", false, pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize) { - ShouldCloseConnection = true; + var db = new LocalListAffectedDatabase(); + + db = (LocalListAffectedDatabase)await CreateLocalDatabaseAsync(db, path, "ListAffected", false, pagecachesize); + db.ShouldCloseConnection = true; + + return db; } private class ListResultFileset : Interface.IListResultFileset @@ -91,9 +96,9 @@ SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""Remo { var sql = FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ( SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names)))) - UNION + UNION SELECT ""Path"" FROM ""File"" WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names))))) - UNION + UNION SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))) ) ORDER BY ""Path"""); @@ -116,7 +121,7 @@ SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""Fileset { var sql = FormatInvariant($@" SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {string.Join(" OR ", slice.Select((x, i) => @$"""Message"" LIKE @Message{i}"))} -UNION +UNION SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Paths)"); cmd.SetCommandAndParameters(sql); @@ -136,9 +141,9 @@ SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Path public IEnumerable GetVolumes(IEnumerable items) { - var sql = FormatInvariant($@"SELECT DISTINCT ""Name"" FROM ( -SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))))) -UNION + var sql = FormatInvariant($@"SELECT DISTINCT ""Name"" FROM ( +SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))))) +UNION SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))))))) )"); From dadbd9ed76aab9bc9343019e5e160f3044d566e0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:15:56 +0200 Subject: [PATCH 075/362] Made LocalListAffected Database async --- .../Database/LocalListAffectedDatabase.cs | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 1dd586ab1..2d58975c1 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -66,22 +66,22 @@ namespace Duplicati.Library.Main.Database public required string Name { get; set; } } - public IEnumerable GetFilesets(IEnumerable items) + public async IAsyncEnumerable GetFilesets(IEnumerable items) { - var filesets = FilesetTimes.ToArray(); + var filesets = await FilesetTimes().ToArrayAsync(); var dict = new Dictionary(); for (var i = 0; i < filesets.Length; i++) dict[filesets[i].Key] = i; - var sql = FormatInvariant($@"SELECT DISTINCT ""FilesetID"" FROM ( + var sql = $@"SELECT DISTINCT ""FilesetID"" FROM ( SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))) UNION SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)) -)"); - using (var tmptable = new TemporaryDbValueList(m_connection, null, items)) - using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", tmptable)) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) +)"; + using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items)) + using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterAsync("@Names", tmptable)) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) { var v = dict[rd.ConvertValueToInt64(0)]; yield return new ListResultFileset() @@ -92,20 +92,20 @@ SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""Remo } } - public IEnumerable GetFiles(IEnumerable items) + public async IAsyncEnumerable GetFiles(IEnumerable items) { - var sql = FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ( + var sql = $@"SELECT DISTINCT ""Path"" FROM ( SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names)))) UNION SELECT ""Path"" FROM ""File"" WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names))))) UNION SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))) -) ORDER BY ""Path"""); +) ORDER BY ""Path"""; - using (var tmptable = new TemporaryDbValueList(m_connection, null, items)) - using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", tmptable)) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items)) + using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterAsync("@Names", tmptable)) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new ListResultFile() { Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"), @@ -113,24 +113,24 @@ SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""Fileset }; } - public IEnumerable GetLogLines(IEnumerable items) + public async IAsyncEnumerable GetLogLines(IEnumerable items) { using var cmd = m_connection.CreateCommand(); foreach (var slice in items.Chunk(CHUNK_SIZE / 2)) { - var sql = FormatInvariant($@" + var sql = $@" SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {string.Join(" OR ", slice.Select((x, i) => @$"""Message"" LIKE @Message{i}"))} UNION -SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Paths)"); +SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Paths)"; - cmd.SetCommandAndParameters(sql); - cmd.ExpandInClauseParameter("@Paths", items.ToArray()); + cmd.SetCommandAndParameters(sql) + .ExpandInClauseParameter("@Paths", items.ToArray()); foreach ((var x, var i) in items.Select((x, i) => (x, i))) cmd.SetParameterValue($"@Message{i}", "%" + x + "%"); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new ListResultRemoteLog() { Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)), @@ -139,17 +139,17 @@ SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Path } } - public IEnumerable GetVolumes(IEnumerable items) + public async IAsyncEnumerable GetVolumes(IEnumerable items) { - var sql = FormatInvariant($@"SELECT DISTINCT ""Name"" FROM ( + var sql = $@"SELECT DISTINCT ""Name"" FROM ( SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))))) UNION SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))))))) -)"); +)"; using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", items.ToArray())) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new ListResultRemoteVolume() { Name = rd.ConvertValueToString(0) ?? "" From 654b80fee395b8d865c2a4c67eaa7a02f5ecc724 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:35:17 +0200 Subject: [PATCH 076/362] Restructured SQL queries in LocalListAffectedDatabase --- .../Database/LocalListAffectedDatabase.cs | 196 ++++++++++++++++-- 1 file changed, 175 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 2d58975c1..19cd16441 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -73,13 +73,41 @@ namespace Duplicati.Library.Main.Database for (var i = 0; i < filesets.Length; i++) dict[filesets[i].Key] = i; - var sql = $@"SELECT DISTINCT ""FilesetID"" FROM ( -SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))) -UNION -SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)) -)"; + var sql = $@" + SELECT DISTINCT ""FilesetID"" + FROM ( + SELECT ""FilesetID"" + FROM ""FilesetEntry"" + WHERE ""FileID"" IN ( + SELECT ""ID"" + FROM ""FileLookup"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"" + WHERE ""BlockID"" IN ( + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + UNION + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + "; + using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items)) - using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterAsync("@Names", tmptable)) + using (var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterAsync("@Names", tmptable)) using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) { @@ -94,13 +122,63 @@ SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""Remo public async IAsyncEnumerable GetFiles(IEnumerable items) { - var sql = $@"SELECT DISTINCT ""Path"" FROM ( -SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names)))) - UNION -SELECT ""Path"" FROM ""File"" WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names))))) - UNION -SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))) -) ORDER BY ""Path"""; + var sql = $@" + SELECT DISTINCT ""Path"" + FROM ( + SELECT ""Path"" + FROM ""File"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"" + WHERE ""BlockID"" IN ( + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + UNION + SELECT ""Path"" + FROM ""File"" + WHERE ""MetadataID"" IN ( + SELECT ""ID"" + FROM ""Metadataset"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"" + WHERE ""BlockID"" IN ( + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + UNION + SELECT ""Path"" + FROM ""File"" + WHERE ""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + ORDER BY ""Path"" + "; using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items)) using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterAsync("@Names", tmptable)) @@ -120,9 +198,26 @@ SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""Fileset foreach (var slice in items.Chunk(CHUNK_SIZE / 2)) { var sql = $@" -SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {string.Join(" OR ", slice.Select((x, i) => @$"""Message"" LIKE @Message{i}"))} -UNION -SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Paths)"; + SELECT + ""TimeStamp"", + ""Message"" + || ' ' + || + CASE + WHEN ""Exception"" IS NULL + THEN '' + ELSE ""Exception"" + END + FROM ""LogData"" + WHERE + {string.Join(" OR ", slice.Select((x, i) => @$"""Message"" LIKE @Message{i}"))} + UNION + SELECT + ""Timestamp"", + ""Data"" + FROM ""RemoteOperation"" + WHERE ""Path"" IN (@Paths) + "; cmd.SetCommandAndParameters(sql) .ExpandInClauseParameter("@Paths", items.ToArray()); @@ -141,11 +236,70 @@ SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Path public async IAsyncEnumerable GetVolumes(IEnumerable items) { - var sql = $@"SELECT DISTINCT ""Name"" FROM ( -SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))))) -UNION -SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))))))) -)"; + var sql = $@" + SELECT DISTINCT ""Name"" + FROM ( + SELECT ""Name"" + FROM ""Remotevolume"" + WHERE ""ID"" IN ( + SELECT ""VolumeID"" + FROM ""Block"" + WHERE ""ID"" IN ( + SELECT ""BlockID"" + FROM ""BlocksetEntry"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""FileLookup"" + WHERE ""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + ) + ) + UNION + SELECT ""Name"" + FROM ""Remotevolume"" + WHERE ""ID"" IN ( + SELECT ""VolumeID"" + FROM ""Block"" + WHERE ""ID"" IN ( + SELECT ""BlockID"" + FROM ""BlocksetEntry"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""Metadataset"" + WHERE ""ID"" IN ( + SELECT ""MetadataID"" + FROM ""FileLookup"" + WHERE ""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + ) + ) + ) + ) + "; using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", items.ToArray())) using (var rd = await cmd.ExecuteReaderAsync()) From b292756eb9a4d1bd693e203f9ce1ef3d0f92e332 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:36:54 +0200 Subject: [PATCH 077/362] Removed whitespace --- .../Database/LocalListBrokenFilesDatabase.cs | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 8a2b6ad18..5a895485d 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -44,7 +44,7 @@ SELECT DISTINCT ""ID"" FROM ( UNION SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID"" ) -WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN +WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN ( SELECT DISTINCT ""BlocksetID"" FROM ( @@ -178,19 +178,19 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction) { using var cmd = m_connection.CreateCommand(transaction, @" -UPDATE ""Metadataset"" -SET ""BlocksetID"" = @EmptyBlocksetID -WHERE +UPDATE ""Metadataset"" +SET ""BlocksetID"" = @EmptyBlocksetID +WHERE ""ID"" IN ( - SELECT ""FileLookup"".""MetadataID"" + SELECT ""FileLookup"".""MetadataID"" FROM ""FileLookup"", ""FilesetEntry"" - WHERE - ""FilesetEntry"".""FilesetId"" = @FilesetId + WHERE + ""FilesetEntry"".""FilesetId"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" ) AND ""BlocksetID"" NOT IN ( - SELECT ""BlocksetID"" - FROM ""BlocksetEntry"", ""Block"" + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" )") .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) From 7cdee45c30706ece5c42231bc0126935147130a3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:38:55 +0200 Subject: [PATCH 078/362] Changed LocalListBrokenFilesDatabase to use factory construction --- .../Database/LocalListBrokenFilesDatabase.cs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 5a895485d..8a47ce242 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -25,6 +25,7 @@ using System; using System.Collections.Generic; using System.Data; using System.Linq; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Database { @@ -67,16 +68,24 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); - public LocalListBrokenFilesDatabase(string path, long pagecachesize) - : base(path, "ListBrokenFiles", false, pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize) { - ShouldCloseConnection = true; + var db = new LocalListBrokenFilesDatabase(); + + db = (LocalListBrokenFilesDatabase)await CreateLocalDatabaseAsync(db, path, "ListBrokenFiles", false, pagecachesize); + db.ShouldCloseConnection = true; + + return db; } - public LocalListBrokenFilesDatabase(LocalDatabase parent) - : base(parent) + public static async Task CreateAsync(LocalDatabase dbparent) { - ShouldCloseConnection = false; + var dbnew = new LocalListBrokenFilesDatabase(); + + dbnew = (LocalListBrokenFilesDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + dbnew.ShouldCloseConnection = false; + + return dbnew; } public IEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) From 5622547abb8d43a2c04673b2ec61b8c58702c706 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:41:51 +0200 Subject: [PATCH 079/362] Removed calls to FormatInvariant as the extension methods handle this now --- .../Database/LocalListBrokenFilesDatabase.cs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 8a47ce242..316edb51c 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -31,7 +31,7 @@ namespace Duplicati.Library.Main.Database { internal class LocalListBrokenFilesDatabase : LocalDatabase { - private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"); + private static readonly string BLOCK_VOLUME_IDS = $@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"; // Invalid blocksets include those that: // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) @@ -39,7 +39,7 @@ namespace Duplicati.Library.Main.Database // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. - private static readonly string BROKEN_FILE_IDS = FormatInvariant($@" + private static readonly string BROKEN_FILE_IDS = $@" SELECT DISTINCT ""ID"" FROM ( SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {FOLDER_BLOCKSET_ID} AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID} UNION @@ -61,12 +61,12 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN ) WHERE ""BlocksetID"" NOT IN (SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == 0) ) -"); - private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"); +"; + private static readonly string BROKEN_FILE_SETS = $@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"; - private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); + private static readonly string BROKEN_FILE_NAMES = $@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"; - private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); + private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => $@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"; public static async Task CreateAsync(string path, long pagecachesize) { @@ -93,7 +93,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN var query = BROKEN_FILE_SETS; var clause = GetFilelistWhereClause(time, versions); if (!string.IsNullOrWhiteSpace(clause.Item1)) - query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"); + query += $@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"; query += @" GROUP BY ""A"".""FilesetID"""; @@ -118,7 +118,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// >All index files that are orphaned. public IEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) { - using var cmd = Connection.CreateCommand(transaction, FormatInvariant($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")")); + using var cmd = Connection.CreateCommand(transaction, $@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")"); foreach (var rd in cmd.ExecuteReaderEnumerable()) yield return new RemoteVolume( @@ -219,29 +219,29 @@ WHERE var volidstable = "DelVolSetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + deletecmd.ExecuteNonQuery($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"); using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names)) - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)")) + deletecmd.SetCommandAndParameters($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)") .ExpandInClauseParameter("@Names", tmptable) .ExecuteNonQuery(); - var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); + var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" "; deletecmd.Parameters.Clear(); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})"); + deletecmd.ExecuteNonQuery($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); + deletecmd.ExecuteNonQuery($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); + deletecmd.ExecuteNonQuery($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); + deletecmd.ExecuteNonQuery($@"DELETE FROM ""{volidstable}"" "); try { deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); + deletecmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{volidstable}"" "); } catch { /* Ignore, will be deleted on close anyway. */ } } From f7a0e17110304c595b431374a0cd39cc1ec88c6e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:46:45 +0200 Subject: [PATCH 080/362] Made LocalListBrokenFilesDatabase async --- .../Database/LocalListBrokenFilesDatabase.cs | 69 ++++++++++--------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 316edb51c..c60c7f886 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -88,27 +88,34 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN return dbnew; } - public IEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) + public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) { var query = BROKEN_FILE_SETS; - var clause = GetFilelistWhereClause(time, versions); + var clause = await GetFilelistWhereClause(time, versions); if (!string.IsNullOrWhiteSpace(clause.Item1)) query += $@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"; query += @" GROUP BY ""A"".""FilesetID"""; using (var cmd = Connection.CreateCommand(transaction)) - foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable()) + await foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerableAsync()) if (!rd.IsDBNull(0)) - yield return (ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0)); + yield return ( + ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), + rd.ConvertValueToInt64(1, -1), + rd.ConvertValueToInt64(2, 0) + ); } - public IEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) + public async IAsyncEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) { using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) if (!rd.IsDBNull(0)) - yield return new Tuple(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1)); + yield return new Tuple( + rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), + rd.ConvertValueToInt64(1) + ); } /// @@ -116,11 +123,11 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// /// Transaction to use for the query. /// >All index files that are orphaned. - public IEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) + public async IAsyncEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) { using var cmd = Connection.CreateCommand(transaction, $@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")"); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), @@ -135,11 +142,11 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// The name of the table to insert into /// The name of the ID field in the table /// The transaction to use for the query - public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) + public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) { using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname)) .SetParameterValue("@FilesetId", filesetid); - cmd.ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync(); } /// @@ -151,27 +158,27 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// The size of the empty blockset /// The transaction to use for the query /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found - public long GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction transaction) + public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction transaction) { using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) .SetParameterValue("@EmptyHash", emptyHash) .SetParameterValue("@EmptyHashSize", emptyHashSize); - var res = cmd.ExecuteScalarInt64(-1); + var res = await cmd.ExecuteScalarInt64Async(-1); // No empty block found, try to find a zero-length block instead if (res < 0 && emptyHashSize != 0) - res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + res = await cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) .SetParameterValue("@EmptyHashSize", 0) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1); // No empty block found, pick the smallest one if (res < 0) - res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") + res = await cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1); return res; } @@ -184,7 +191,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// The empty blockset ID to replace with /// The transaction to use for the query /// The number of rows affected - public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction) + public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction) { using var cmd = m_connection.CreateCommand(transaction, @" UPDATE ""Metadataset"" @@ -204,10 +211,10 @@ WHERE )") .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) .SetParameterValue("@FilesetId", filesetId); - return cmd.ExecuteNonQuery(); + return await cmd.ExecuteNonQueryAsync(); } - public void RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) + public async Task RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) { if (names == null || !names.Any()) return; if (transaction == null) @@ -221,37 +228,37 @@ WHERE // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. deletecmd.ExecuteNonQuery($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"); - using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names)) - deletecmd.SetCommandAndParameters($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)") + using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names)) + await deletecmd.SetCommandAndParameters($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)") .ExpandInClauseParameter("@Names", tmptable) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" "; deletecmd.Parameters.Clear(); - deletecmd.ExecuteNonQuery($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})"); - deletecmd.ExecuteNonQuery($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); - deletecmd.ExecuteNonQuery($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); - deletecmd.ExecuteNonQuery($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery($@"DELETE FROM ""{volidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" "); try { deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{volidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" "); } catch { /* Ignore, will be deleted on close anyway. */ } } } - public long GetFilesetFileCount(long filesetid, IDbTransaction transaction) + public async Task GetFilesetFileCount(long filesetid, IDbTransaction transaction) { using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") .SetParameterValue("@FilesetId", filesetid); - return cmd.ExecuteScalarInt64(0); + return await cmd.ExecuteScalarInt64Async(0); } } } From ef717048e8ce0738267ed2c691afb0403bb0e5f7 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 06:54:09 +0200 Subject: [PATCH 081/362] Purged IDb from LocalListBrokenFilesDatabase --- .../Database/LocalListBrokenFilesDatabase.cs | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index c60c7f886..a566bc556 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -23,7 +23,6 @@ using System; using System.Collections.Generic; -using System.Data; using System.Linq; using System.Threading.Tasks; @@ -88,7 +87,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN return dbnew; } - public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) + public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[] versions) { var query = BROKEN_FILE_SETS; var clause = await GetFilelistWhereClause(time, versions); @@ -97,7 +96,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN query += @" GROUP BY ""A"".""FilesetID"""; - using (var cmd = Connection.CreateCommand(transaction)) + using (var cmd = Connection.CreateCommand(m_rtr)) await foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerableAsync()) if (!rd.IsDBNull(0)) yield return ( @@ -107,9 +106,9 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN ); } - public async IAsyncEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) + public async IAsyncEnumerable> GetBrokenFilenames(long filesetid) { - using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) + using (var cmd = Connection.CreateCommand(m_rtr).SetCommandAndParameters(BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) if (!rd.IsDBNull(0)) yield return new Tuple( @@ -123,9 +122,10 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// /// Transaction to use for the query. /// >All index files that are orphaned. - public async IAsyncEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) + public async IAsyncEnumerable GetOrphanedIndexFiles() { - using var cmd = Connection.CreateCommand(transaction, $@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")"); + using var cmd = Connection.CreateCommand($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")") + .SetTransaction(m_rtr); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) yield return new RemoteVolume( @@ -142,10 +142,11 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// The name of the table to insert into /// The name of the ID field in the table /// The transaction to use for the query - public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) + public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname) { - using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname)) - .SetParameterValue("@FilesetId", filesetid); + using var cmd = Connection.CreateCommand(m_rtr) + .SetCommandAndParameters(INSERT_BROKEN_IDS(tablename, IDfieldname)) + .SetParameterValue("@FilesetId", filesetid); await cmd.ExecuteNonQueryAsync(); } @@ -158,26 +159,28 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// The size of the empty blockset /// The transaction to use for the query /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found - public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction transaction) + public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize) { - using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .SetParameterValue("@EmptyHash", emptyHash) - .SetParameterValue("@EmptyHashSize", emptyHashSize); + // TODO the casting to object is a typing fix for the extension method for now. Should be fixed properly. + using var cmd = Connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .SetTransaction(m_rtr) + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) + .SetParameterValue("@EmptyHash", emptyHash) + .SetParameterValue("@EmptyHashSize", emptyHashSize); var res = await cmd.ExecuteScalarInt64Async(-1); // No empty block found, try to find a zero-length block instead if (res < 0 && emptyHashSize != 0) res = await cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) .SetParameterValue("@EmptyHashSize", 0) .ExecuteScalarInt64Async(-1); // No empty block found, pick the smallest one if (res < 0) res = await cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) .ExecuteScalarInt64Async(-1); return res; @@ -191,9 +194,9 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// The empty blockset ID to replace with /// The transaction to use for the query /// The number of rows affected - public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction) + public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId) { - using var cmd = m_connection.CreateCommand(transaction, @" + using var cmd = m_connection.CreateCommand(@" UPDATE ""Metadataset"" SET ""BlocksetID"" = @EmptyBlocksetID WHERE @@ -209,28 +212,27 @@ WHERE FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" )") - .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) - .SetParameterValue("@FilesetId", filesetId); + .SetTransaction(m_rtr) + .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) + .SetParameterValue("@FilesetId", filesetId); return await cmd.ExecuteNonQueryAsync(); } - public async Task RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) + public async Task RemoveMissingBlocks(IEnumerable names) { if (names == null || !names.Any()) return; - if (transaction == null) - throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state"); - using (var deletecmd = m_connection.CreateCommand(transaction)) + using (var deletecmd = m_connection.CreateCommand(m_rtr)) { var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var volidstable = "DelVolSetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"); + await deletecmd.ExecuteNonQueryAsync($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"); using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names)) - await deletecmd.SetCommandAndParameters($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)") - .ExpandInClauseParameter("@Names", tmptable) + await (await deletecmd.SetCommandAndParameters($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)") + .ExpandInClauseParameterAsync("@Names", tmptable)) .ExecuteNonQueryAsync(); var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" "; @@ -254,10 +256,11 @@ WHERE } } - public async Task GetFilesetFileCount(long filesetid, IDbTransaction transaction) + public async Task GetFilesetFileCount(long filesetid) { - using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetid); + using var cmd = m_connection.CreateCommand(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetid); return await cmd.ExecuteScalarInt64Async(0); } } From caf5bddc0cffc1a0885bb90eab6d03e58137778e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 07:07:44 +0200 Subject: [PATCH 082/362] Restructured SQL queries in LocalListBrokenFilesDatabase --- .../Database/LocalListBrokenFilesDatabase.cs | 316 ++++++++++++++---- 1 file changed, 246 insertions(+), 70 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index a566bc556..a1354d57f 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -30,7 +30,11 @@ namespace Duplicati.Library.Main.Database { internal class LocalListBrokenFilesDatabase : LocalDatabase { - private static readonly string BLOCK_VOLUME_IDS = $@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"; + private static readonly string BLOCK_VOLUME_IDS = $@" + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Type"" = '{RemoteVolumeType.Blocks}' + "; // Invalid blocksets include those that: // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) @@ -39,33 +43,100 @@ namespace Duplicati.Library.Main.Database // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. private static readonly string BROKEN_FILE_IDS = $@" -SELECT DISTINCT ""ID"" FROM ( - SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {FOLDER_BLOCKSET_ID} AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID} -UNION - SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID"" -) -WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN - ( - SELECT DISTINCT ""BlocksetID"" FROM - ( - SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" NOT IN - (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN - ({BLOCK_VOLUME_IDS})) - UNION - SELECT ""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Hash"" NOT IN - (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN - ({BLOCK_VOLUME_IDS})) - UNION - SELECT ""A"".""ID"" AS ""BlocksetID"" FROM ""Blockset"" A LEFT JOIN ""BlocksetEntry"" B ON ""A"".""ID"" = ""B"".""BlocksetID"" WHERE ""A"".""Length"" > 0 AND ""B"".""BlocksetID"" IS NULL - ) - WHERE ""BlocksetID"" NOT IN (SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == 0) - ) -"; - private static readonly string BROKEN_FILE_SETS = $@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"; + SELECT DISTINCT ""ID"" + FROM ( + SELECT + ""ID"" AS ""ID"", + ""BlocksetID"" AS ""BlocksetID"" + FROM ""FileLookup"" + WHERE + ""BlocksetID"" != {FOLDER_BLOCKSET_ID} + AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID} + UNION + SELECT + ""A"".""ID"" AS ""ID"", + ""B"".""BlocksetID"" AS ""BlocksetID"" + FROM ""FileLookup"" A + LEFT JOIN ""Metadataset"" B + ON ""A"".""MetadataID"" = ""B"".""ID"" + ) + WHERE + ""BlocksetID"" IS NULL + OR ""BlocksetID"" IN ( + SELECT DISTINCT ""BlocksetID"" + FROM ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"" + WHERE ""BlockID"" NOT IN ( + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" IN ({BLOCK_VOLUME_IDS}) + ) + UNION + SELECT ""BlocksetID"" + FROM ""BlocklistHash"" + WHERE ""Hash"" NOT IN ( + SELECT ""Hash"" + FROM ""Block"" + WHERE ""VolumeID"" IN ({BLOCK_VOLUME_IDS}) + ) + UNION + SELECT ""A"".""ID"" AS ""BlocksetID"" + FROM ""Blockset"" A + LEFT JOIN ""BlocksetEntry"" B + ON ""A"".""ID"" = ""B"".""BlocksetID"" + WHERE + ""A"".""Length"" > 0 + AND ""B"".""BlocksetID"" IS NULL + ) + WHERE ""BlocksetID"" NOT IN ( + SELECT ""ID"" + FROM ""Blockset"" + WHERE ""Length"" == 0 + ) + ) + "; - private static readonly string BROKEN_FILE_NAMES = $@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"; + private static readonly string BROKEN_FILE_SETS = $@" + SELECT DISTINCT + ""B"".""Timestamp"", + ""A"".""FilesetID"", + COUNT(""A"".""FileID"") AS ""FileCount"" + FROM + ""FilesetEntry"" A, + ""Fileset"" B + WHERE + ""A"".""FilesetID"" = ""B"".""ID"" + AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS}) + "; - private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => $@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"; + private static readonly string BROKEN_FILE_NAMES = $@" + SELECT + ""A"".""Path"", + ""B"".""Length"" + FROM ""File"" A + LEFT JOIN ""Blockset"" B + ON (""A"".""BlocksetID"" = ""B"".""ID"") + WHERE + ""A"".""ID"" IN ({BROKEN_FILE_IDS}) + AND ""A"".""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ) + "; + + private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => $@" + INSERT INTO ""{tablename}"" ( + ""{IDfieldname}"" + ) + {BROKEN_FILE_IDS} + AND ""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ) + "; public static async Task CreateAsync(string path, long pagecachesize) { @@ -91,30 +162,42 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN { var query = BROKEN_FILE_SETS; var clause = await GetFilelistWhereClause(time, versions); - if (!string.IsNullOrWhiteSpace(clause.Item1)) - query += $@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"; + if (!string.IsNullOrWhiteSpace(clause.Query)) + query += $@" + AND ""A"".""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + {clause.Query} + ) + "; query += @" GROUP BY ""A"".""FilesetID"""; - using (var cmd = Connection.CreateCommand(m_rtr)) - await foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerableAsync()) - if (!rd.IsDBNull(0)) - yield return ( - ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), - rd.ConvertValueToInt64(1, -1), - rd.ConvertValueToInt64(2, 0) - ); + using var cmd = Connection.CreateCommand(m_rtr) + .SetCommandAndParameters(query) + .SetParameterValues(clause.Values); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + if (!rd.IsDBNull(0)) + yield return ( + ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), + rd.ConvertValueToInt64(1, -1), + rd.ConvertValueToInt64(2, 0) + ); } public async IAsyncEnumerable> GetBrokenFilenames(long filesetid) { - using (var cmd = Connection.CreateCommand(m_rtr).SetCommandAndParameters(BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) - if (!rd.IsDBNull(0)) - yield return new Tuple( - rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), - rd.ConvertValueToInt64(1) - ); + using var cmd = Connection.CreateCommand(m_rtr) + .SetCommandAndParameters(BROKEN_FILE_NAMES) + .SetParameterValue("@FilesetId", filesetid); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + if (!rd.IsDBNull(0)) + yield return new Tuple( + rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), + rd.ConvertValueToInt64(1) + ); } /// @@ -124,7 +207,19 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN /// >All index files that are orphaned. public async IAsyncEnumerable GetOrphanedIndexFiles() { - using var cmd = Connection.CreateCommand($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")") + using var cmd = Connection.CreateCommand($@" + SELECT + ""Name"", + ""Hash"", + ""Size"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = '{RemoteVolumeType.Index}' + AND ""ID"" NOT IN ( + SELECT ""IndexVolumeID"" + FROM ""IndexBlockLink"" + ) + ") .SetTransaction(m_rtr); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -147,6 +242,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN using var cmd = Connection.CreateCommand(m_rtr) .SetCommandAndParameters(INSERT_BROKEN_IDS(tablename, IDfieldname)) .SetParameterValue("@FilesetId", filesetid); + await cmd.ExecuteNonQueryAsync(); } @@ -162,7 +258,22 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize) { // TODO the casting to object is a typing fix for the extension method for now. Should be fixed properly. - using var cmd = Connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + using var cmd = Connection.CreateCommand(@" + SELECT ""ID"" + FROM ""Blockset"" + WHERE + ""FullHash"" = @EmptyHash + AND ""Length"" == @EmptyHashSize + AND ""ID"" NOT IN ( + SELECT ""BlocksetID"" + FROM + ""BlocksetEntry"", + ""Block"" + WHERE + ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) + ) + ") .SetTransaction(m_rtr) .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) .SetParameterValue("@EmptyHash", emptyHash) @@ -172,14 +283,42 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN // No empty block found, try to find a zero-length block instead if (res < 0 && emptyHashSize != 0) - res = await cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + res = await cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Blockset"" + WHERE + ""Length"" == @EmptyHashSize + AND ""ID"" NOT IN ( + SELECT ""BlocksetID"" + FROM + ""BlocksetEntry"", + ""Block"" + WHERE + ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) + ) + ") .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) .SetParameterValue("@EmptyHashSize", 0) .ExecuteScalarInt64Async(-1); // No empty block found, pick the smallest one if (res < 0) - res = await cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") + res = await cmd.SetCommandAndParameters(@" + SELECT ""Blockset"".""ID"" + FROM + ""BlocksetEntry"", + ""Blockset"", + ""Metadataset"", + ""Block"" + WHERE + ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" + AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) + ORDER BY ""Blockset"".""Length"" ASC + LIMIT 1 + ") .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) .ExecuteScalarInt64Async(-1); @@ -197,21 +336,24 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId) { using var cmd = m_connection.CreateCommand(@" -UPDATE ""Metadataset"" -SET ""BlocksetID"" = @EmptyBlocksetID -WHERE - ""ID"" IN ( - SELECT ""FileLookup"".""MetadataID"" - FROM ""FileLookup"", ""FilesetEntry"" - WHERE - ""FilesetEntry"".""FilesetId"" = @FilesetId - AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" - ) - AND ""BlocksetID"" NOT IN ( - SELECT ""BlocksetID"" - FROM ""BlocksetEntry"", ""Block"" - WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - )") + UPDATE ""Metadataset"" + SET ""BlocksetID"" = @EmptyBlocksetID + WHERE + ""ID"" IN ( + SELECT ""FileLookup"".""MetadataID"" + FROM + ""FileLookup"", + ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FilesetId"" = @FilesetId + AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + ) + AND ""BlocksetID"" NOT IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"", ""Block"" + WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + ) + ") .SetTransaction(m_rtr) .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) .SetParameterValue("@FilesetId", filesetId); @@ -228,20 +370,49 @@ WHERE var volidstable = "DelVolSetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - await deletecmd.ExecuteNonQueryAsync($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"); + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TEMP TABLE ""{volidstable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + "); using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names)) - await (await deletecmd.SetCommandAndParameters($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)") + await (await deletecmd.SetCommandAndParameters($@" + INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ") .ExpandInClauseParameterAsync("@Names", tmptable)) .ExecuteNonQueryAsync(); - var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" "; + var volIdsSubQuery = $@" + SELECT ""ID"" + FROM ""{volidstable}"" + "; deletecmd.Parameters.Clear(); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})"); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"); + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""IndexBlockLink"" + WHERE + ""BlockVolumeID"" IN ({volIdsSubQuery}) + OR ""IndexVolumeID"" IN ({volIdsSubQuery}) + "); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Block"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + "); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""DeletedBlock"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + "); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""DuplicateBlock"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + "); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) @@ -258,9 +429,14 @@ WHERE public async Task GetFilesetFileCount(long filesetid) { - using var cmd = m_connection.CreateCommand(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") + using var cmd = m_connection.CreateCommand(@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ") .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetid); + return await cmd.ExecuteScalarInt64Async(0); } } From 75c77bdaa61fd742289e6565aa9b13f819c626d2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 07:08:01 +0200 Subject: [PATCH 083/362] Fixed async method not being async --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 0be32a1f8..99b40f17c 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -201,7 +201,7 @@ public static partial class ExtensionMethods self.SetCommandAndParameters(cmd); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarAsync", string.Format("ExecuteScalarAsync: {0}", self.GetPrintableCommandText())) : null) - return self.ExecuteScalar(); + return await self.ExecuteScalarAsync(); } public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext) From 49663b9d55143b158fe758c48a0722a47e81ad9d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 07:25:12 +0200 Subject: [PATCH 084/362] Changed LocalListChangesDatabase to use factory construction --- .../Main/Database/LocalListChangesDatabase.cs | 80 +++++++++++-------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index e334ffb3d..d9facfae7 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -24,16 +24,22 @@ using System; using System.Data; using System.Collections.Generic; +using System.Threading.Tasks; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Database { internal class LocalListChangesDatabase : LocalDatabase { - public LocalListChangesDatabase(string path, long pagecachesize) - : base(path, "ListChanges", false, pagecachesize) + + public static async Task CreateAsync(string path, long pagecachesize) { - ShouldCloseConnection = true; + var db = new LocalListChangesDatabase(); + + db = (LocalListChangesDatabase)await CreateLocalDatabaseAsync(db, path, "ListChanges", false, pagecachesize); + db.ShouldCloseConnection = true; + + return db; } public interface IStorageHelper : IDisposable @@ -97,31 +103,39 @@ namespace Duplicati.Library.Main.Database private class StorageHelper : IStorageHelper { - private readonly IDbConnection m_connection; - private IDbTransaction m_transaction; + private IDbConnection m_connection = null!; + private IDbTransaction m_transaction = null!; - private IDbCommand m_insertPreviousElementCommand; - private IDbCommand m_insertCurrentElementCommand; + private IDbCommand m_insertPreviousElementCommand = null!; + private IDbCommand m_insertCurrentElementCommand = null!; - private string m_previousTable; - private string m_currentTable; + private string m_previousTable = null!; + private string m_currentTable = null!; + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public StorageHelper(IDbConnection con) { - m_connection = con; - m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + } - m_transaction = m_connection.BeginTransactionSafe(); + private StorageHelper() { } - using (var cmd = m_connection.CreateCommand(m_transaction)) + public static async Task CreateAsync(LocalDatabase db) + { + var sh = new StorageHelper(); + sh.m_connection = db.Connection; + sh.m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + sh.m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + using (var cmd = sh.m_connection.CreateCommand(db.Transaction)) { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); } - m_insertPreviousElementCommand = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); - m_insertCurrentElementCommand = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); + sh.m_insertPreviousElementCommand = sh.m_connection.CreateCommand(sh.m_transaction, FormatInvariant($@"INSERT INTO ""{sh.m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); + sh.m_insertCurrentElementCommand = sh.m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{sh.m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); + + return sh; } public void AddFromDb(long filesetId, bool asNew, IFilter filter) From def0be53d2b73f4df2dfc2fe4d52a3e974afd437 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 07:33:57 +0200 Subject: [PATCH 085/362] Purged IDb from LocalListChangesDatabase --- .../Main/Database/LocalListChangesDatabase.cs | 55 +++++++++---------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index d9facfae7..c4e6f18e6 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -22,10 +22,10 @@ #nullable enable using System; -using System.Data; using System.Collections.Generic; using System.Threading.Tasks; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -103,17 +103,16 @@ namespace Duplicati.Library.Main.Database private class StorageHelper : IStorageHelper { - private IDbConnection m_connection = null!; - private IDbTransaction m_transaction = null!; + private LocalDatabase m_db = null!; - private IDbCommand m_insertPreviousElementCommand = null!; - private IDbCommand m_insertCurrentElementCommand = null!; + private SqliteCommand m_insertPreviousElementCommand = null!; + private SqliteCommand m_insertCurrentElementCommand = null!; private string m_previousTable = null!; private string m_currentTable = null!; [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] - public StorageHelper(IDbConnection con) + public StorageHelper(SqliteConnection con) { } @@ -122,18 +121,18 @@ namespace Duplicati.Library.Main.Database public static async Task CreateAsync(LocalDatabase db) { var sh = new StorageHelper(); - sh.m_connection = db.Connection; + sh.m_db = db; sh.m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); sh.m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = sh.m_connection.CreateCommand(db.Transaction)) + using (var cmd = sh.m_db.Connection.CreateCommand(db.Transaction)) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); } - sh.m_insertPreviousElementCommand = sh.m_connection.CreateCommand(sh.m_transaction, FormatInvariant($@"INSERT INTO ""{sh.m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); - sh.m_insertCurrentElementCommand = sh.m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{sh.m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); + sh.m_insertPreviousElementCommand = sh.m_db.Connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{sh.m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); + sh.m_insertCurrentElementCommand = sh.m_db.Connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{sh.m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); return sh; } @@ -148,7 +147,7 @@ namespace Duplicati.Library.Main.Database var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")"; - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { if (filter == null || filter.Empty) { @@ -224,7 +223,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(); } - private static IEnumerable ReaderToStringList(IDataReader rd) + private static IEnumerable ReaderToStringList(SqliteDataReader rd) { using (rd) while (rd.Read()) @@ -253,7 +252,7 @@ namespace Duplicati.Library.Main.Database var deleted = sqls.Deleted; //var modified = sqls.Modified; - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { var result = new ChangeSizeReport(); @@ -274,7 +273,7 @@ namespace Duplicati.Library.Main.Database var deleted = @"SELECT COUNT(*) FROM (" + sqls.Deleted + ")"; var modified = @"SELECT COUNT(*) FROM (" + sqls.Modified + ")"; - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { var result = new ChangeCountReport(); result.AddedFolders = cmd.SetCommandAndParameters(added) @@ -315,7 +314,7 @@ namespace Duplicati.Library.Main.Database { var sqls = GetSqls(false); - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { var elTypes = new[] { Interface.ListChangesElementType.Folder, @@ -323,7 +322,7 @@ namespace Duplicati.Library.Main.Database Interface.ListChangesElementType.File }; - IEnumerable> BuildResult(IDbCommand cmd, string sql, Interface.ListChangesChangeType changeType) + IEnumerable> BuildResult(SqliteCommand cmd, string sql, Interface.ListChangesChangeType changeType) { cmd.SetCommandAndParameters(sql); foreach (var type in elTypes) @@ -341,6 +340,11 @@ namespace Duplicati.Library.Main.Database } public void Dispose() + { + DisposeAsync().Await(); + } + + public async Task DisposeAsync() { if (m_insertPreviousElementCommand != null) { @@ -356,24 +360,15 @@ namespace Duplicati.Library.Main.Database finally { m_insertCurrentElementCommand = null!; } } - if (m_transaction != null) + try { await m_db.Transaction.RollBackAsync(); } + catch { } + finally { - try { m_transaction.Rollback(); } - catch { } - finally - { - m_previousTable = null!; - m_currentTable = null!; - m_transaction = null!; - } + m_previousTable = null!; + m_currentTable = null!; } } } - - public IStorageHelper CreateStorageHelper() - { - return new StorageHelper(m_connection); - } } } From 96d4906a73efa3fc51755da18374172d3552e555 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 07:34:14 +0200 Subject: [PATCH 086/362] Added rollback to ReusableTransaction --- .../Main/Database/ReusableTransaction.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 4d303967c..dc0253089 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -118,4 +118,23 @@ internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transact await m_transaction.DisposeAsync(); } } + + public async Task RollBackAsync(string? message = null, bool restart = true) + { + if (m_disposed) + throw new InvalidOperationException("Transaction is already disposed"); + + using (var timer = new Logging.Timer(LOGTAG, message, $"RollbackTransaction: {message}")) + await m_transaction.RollbackAsync(); + await m_transaction.DisposeAsync(); + + if (restart) + { + m_transaction = m_db.Connection.BeginTransaction(); + } + else + { + m_disposed = true; + } + } } From 4db6d247f441db6f1aa8c0714f8b6874bdf80b4e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 07:38:03 +0200 Subject: [PATCH 087/362] Removed FormatInvariant as it's now handled by the extension methods --- .../Main/Database/LocalListChangesDatabase.cs | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index c4e6f18e6..c8da22690 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -120,19 +120,21 @@ namespace Duplicati.Library.Main.Database public static async Task CreateAsync(LocalDatabase db) { - var sh = new StorageHelper(); - sh.m_db = db; - sh.m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - sh.m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var sh = new StorageHelper + { + m_db = db, + m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()), + m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) + }; using (var cmd = sh.m_db.Connection.CreateCommand(db.Transaction)) { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); + cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "); + cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "); } - sh.m_insertPreviousElementCommand = sh.m_db.Connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{sh.m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); - sh.m_insertCurrentElementCommand = sh.m_db.Connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{sh.m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); + sh.m_insertPreviousElementCommand = sh.m_db.Connection.CreateCommand($@"INSERT INTO ""{sh.m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"); + sh.m_insertCurrentElementCommand = sh.m_db.Connection.CreateCommand($@"INSERT INTO ""{sh.m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"); return sh; } @@ -141,9 +143,9 @@ namespace Duplicati.Library.Main.Database { var tablename = asNew ? m_currentTable : m_previousTable; - var folders = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); - var symlinks = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); - var files = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "); + var folders = $@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "; + var symlinks = $@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "; + var files = $@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "; var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")"; @@ -152,7 +154,7 @@ namespace Duplicati.Library.Main.Database if (filter == null || filter.Empty) { // Simple case, select everything - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId ")) + cmd.SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQuery(); } @@ -163,8 +165,8 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) ")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)")); + cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "); + cmd.SetCommandAndParameters($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)"); foreach (var s in p) cmd.SetParameterValue("@Path", s) @@ -174,26 +176,26 @@ namespace Duplicati.Library.Main.Database if (expression.Result) { // Include filter - whereClause = FormatInvariant($@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"); + whereClause = $@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"; } else { // Exclude filter - whereClause = FormatInvariant($@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"); + whereClause = $@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"; } - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} ")) + cmd.SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} ") .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQuery(); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" ")); + cmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); } else { // Do row-wise iteration var values = new object[5]; - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId")) + cmd.SetCommandAndParameters($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId") .SetParameterValue("@FilesetId", filesetId); - using (var cmd2 = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"))) + using (var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction).SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")) using (var rd = cmd.ExecuteReader()) while (rd.Read()) { @@ -239,9 +241,9 @@ namespace Duplicati.Library.Main.Database private (string Added, string Deleted, string Modified) GetSqls(bool allTypes) { return ( - FormatInvariant($@"SELECT ""Path"" FROM ""{m_currentTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_currentTable}"".""Type"" = @Type AND "))} ""{m_currentTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_previousTable}"")"), - FormatInvariant($@"SELECT ""Path"" FROM ""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_previousTable}"".""Type"" = @Type AND "))} ""{m_previousTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_currentTable}"")"), - FormatInvariant($@"SELECT ""{m_currentTable}"".""Path"" FROM ""{m_currentTable}"",""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant($@" ""{m_currentTable}"".""Type"" = @Type AND "))} ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" AND (""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"") ") + $@"SELECT ""Path"" FROM ""{m_currentTable}"" WHERE {(allTypes ? "" : @$" ""{m_currentTable}"".""Type"" = @Type AND ")} ""{m_currentTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_previousTable}"")", + $@"SELECT ""Path"" FROM ""{m_previousTable}"" WHERE {(allTypes ? "" : @$" ""{m_previousTable}"".""Type"" = @Type AND ")} ""{m_previousTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_currentTable}"")", + $@"SELECT ""{m_currentTable}"".""Path"" FROM ""{m_currentTable}"",""{m_previousTable}"" WHERE {(allTypes ? "" : $@" ""{m_currentTable}"".""Type"" = @Type AND ")} ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" AND (""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"") " ); } @@ -256,11 +258,11 @@ namespace Duplicati.Library.Main.Database { var result = new ChangeSizeReport(); - result.PreviousSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" "), 0); - result.CurrentSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" "), 0); + result.PreviousSize = cmd.ExecuteScalarInt64($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" ", 0); + result.CurrentSize = cmd.ExecuteScalarInt64($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" ", 0); - result.AddedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) "), 0); - result.DeletedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) "), 0); + result.AddedSize = cmd.ExecuteScalarInt64($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) ", 0); + result.DeletedSize = cmd.ExecuteScalarInt64($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) ", 0); return result; } From 9f7a82ff37f18b36f1d140bdbb10a02c278c9a0c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 07:48:14 +0200 Subject: [PATCH 088/362] Made LocalListChangesDatabase async --- .../Main/Database/LocalListChangesDatabase.cs | 136 +++++++++--------- 1 file changed, 70 insertions(+), 66 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index c8da22690..ea9661bdd 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -44,13 +44,13 @@ namespace Duplicati.Library.Main.Database public interface IStorageHelper : IDisposable { - void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew); + Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew); - void AddFromDb(long filesetId, bool asNew, IFilter filter); + Task AddFromDb(long filesetId, bool asNew, IFilter filter); - IChangeCountReport CreateChangeCountReport(); - IChangeSizeReport CreateChangeSizeReport(); - IEnumerable> CreateChangedFileReport(); + Task CreateChangeCountReport(); + Task CreateChangeSizeReport(); + IAsyncEnumerable> CreateChangedFileReport(); } public interface IChangeCountReport @@ -129,17 +129,17 @@ namespace Duplicati.Library.Main.Database using (var cmd = sh.m_db.Connection.CreateCommand(db.Transaction)) { - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "); - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "); } - sh.m_insertPreviousElementCommand = sh.m_db.Connection.CreateCommand($@"INSERT INTO ""{sh.m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"); - sh.m_insertCurrentElementCommand = sh.m_db.Connection.CreateCommand($@"INSERT INTO ""{sh.m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"); + sh.m_insertPreviousElementCommand = await sh.m_db.Connection.CreateCommandAsync($@"INSERT INTO ""{sh.m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"); + sh.m_insertCurrentElementCommand = await sh.m_db.Connection.CreateCommandAsync($@"INSERT INTO ""{sh.m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"); return sh; } - public void AddFromDb(long filesetId, bool asNew, IFilter filter) + public async Task AddFromDb(long filesetId, bool asNew, IFilter filter) { var tablename = asNew ? m_currentTable : m_previousTable; @@ -154,9 +154,9 @@ namespace Duplicati.Library.Main.Database if (filter == null || filter.Empty) { // Simple case, select everything - cmd.SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId ") + await cmd.SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { @@ -165,12 +165,13 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "); - cmd.SetCommandAndParameters($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)"); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "); + await cmd.SetCommandAndParameters($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)") + .PrepareAsync(); foreach (var s in p) - cmd.SetParameterValue("@Path", s) - .ExecuteNonQuery(); + await cmd.SetParameterValue("@Path", s) + .ExecuteNonQueryAsync(); string whereClause; if (expression.Result) @@ -183,52 +184,55 @@ namespace Duplicati.Library.Main.Database // Exclude filter whereClause = $@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"; } - cmd.SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} ") + await cmd.SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); - cmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); } else { // Do row-wise iteration var values = new object[5]; - cmd.SetCommandAndParameters($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId); + await cmd.SetCommandAndParameters($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId") + .SetParameterValue("@FilesetId", filesetId) + .PrepareAsync(); using (var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction).SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) { rd.GetValues(values); if (values[0] != null && values[0] != DBNull.Value && FilterExpression.Matches(filter, values[0].ToString())) { - cmd2.SetParameterValue("@Path", values[0]) + await cmd2 + .SetParameterValue("@Path", values[0]) .SetParameterValue("@FileHash", values[1]) .SetParameterValue("@MetaHash", values[2]) .SetParameterValue("@Size", values[3]) .SetParameterValue("@Type", values[4]) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); } } } } } - public void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew) + public async Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew) { var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; - cmd.SetParameterValue("@Path", path); - cmd.SetParameterValue("@FileHash", filehash); - cmd.SetParameterValue("@MetaHash", metahash); - cmd.SetParameterValue("@Size", size); - cmd.SetParameterValue("@Type", (int)type); - cmd.ExecuteNonQuery(); + await cmd + .SetParameterValue("@Path", path) + .SetParameterValue("@FileHash", filehash) + .SetParameterValue("@MetaHash", metahash) + .SetParameterValue("@Size", size) + .SetParameterValue("@Type", (int)type) + .ExecuteNonQueryAsync(); } - private static IEnumerable ReaderToStringList(SqliteDataReader rd) + private static async IAsyncEnumerable ReaderToStringList(SqliteDataReader rd) { using (rd) - while (rd.Read()) + while (await rd.ReadAsync()) { var v = rd.GetValue(0); if (v == null || v == DBNull.Value) @@ -247,7 +251,7 @@ namespace Duplicati.Library.Main.Database ); } - public IChangeSizeReport CreateChangeSizeReport() + public async Task CreateChangeSizeReport() { var sqls = GetSqls(true); var added = sqls.Added; @@ -258,17 +262,17 @@ namespace Duplicati.Library.Main.Database { var result = new ChangeSizeReport(); - result.PreviousSize = cmd.ExecuteScalarInt64($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" ", 0); - result.CurrentSize = cmd.ExecuteScalarInt64($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" ", 0); + result.PreviousSize = await cmd.ExecuteScalarInt64Async($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" ", 0); + result.CurrentSize = await cmd.ExecuteScalarInt64Async($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" ", 0); - result.AddedSize = cmd.ExecuteScalarInt64($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) ", 0); - result.DeletedSize = cmd.ExecuteScalarInt64($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) ", 0); + result.AddedSize = await cmd.ExecuteScalarInt64Async($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) ", 0); + result.DeletedSize = await cmd.ExecuteScalarInt64Async($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) ", 0); return result; } } - public IChangeCountReport CreateChangeCountReport() + public async Task CreateChangeCountReport() { var sqls = GetSqls(false); var added = @"SELECT COUNT(*) FROM (" + sqls.Added + ")"; @@ -278,41 +282,41 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { var result = new ChangeCountReport(); - result.AddedFolders = cmd.SetCommandAndParameters(added) + result.AddedFolders = await cmd.SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64(0); - result.AddedSymlinks = cmd.SetCommandAndParameters(added) + .ExecuteScalarInt64Async(0); + result.AddedSymlinks = await cmd.SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64(0); - result.AddedFiles = cmd.SetCommandAndParameters(added) + .ExecuteScalarInt64Async(0); + result.AddedFiles = await cmd.SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64(0); + .ExecuteScalarInt64Async(0); - result.DeletedFolders = cmd.SetCommandAndParameters(deleted) + result.DeletedFolders = await cmd.SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64(0); - result.DeletedSymlinks = cmd.SetCommandAndParameters(deleted) + .ExecuteScalarInt64Async(0); + result.DeletedSymlinks = await cmd.SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64(0); - result.DeletedFiles = cmd.SetCommandAndParameters(deleted) + .ExecuteScalarInt64Async(0); + result.DeletedFiles = await cmd.SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64(0); + .ExecuteScalarInt64Async(0); - result.ModifiedFolders = cmd.SetCommandAndParameters(modified) + result.ModifiedFolders = await cmd.SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64(0); - result.ModifiedSymlinks = cmd.SetCommandAndParameters(modified) + .ExecuteScalarInt64Async(0); + result.ModifiedSymlinks = await cmd.SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64(0); - result.ModifiedFiles = cmd.SetCommandAndParameters(modified) + .ExecuteScalarInt64Async(0); + result.ModifiedFiles = await cmd.SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64(0); + .ExecuteScalarInt64Async(0); return result; } } - public IEnumerable> CreateChangedFileReport() + public async IAsyncEnumerable> CreateChangedFileReport() { var sqls = GetSqls(false); @@ -324,19 +328,19 @@ namespace Duplicati.Library.Main.Database Interface.ListChangesElementType.File }; - IEnumerable> BuildResult(SqliteCommand cmd, string sql, Interface.ListChangesChangeType changeType) + async IAsyncEnumerable> BuildResult(SqliteCommand cmd, string sql, Interface.ListChangesChangeType changeType) { cmd.SetCommandAndParameters(sql); foreach (var type in elTypes) - foreach (var s in ReaderToStringList(cmd.SetParameterValue("@Type", (int)type).ExecuteReader())) + await foreach (var s in ReaderToStringList(cmd.SetParameterValue("@Type", (int)type).ExecuteReader())) yield return new Tuple(changeType, type, s ?? ""); } - foreach (var r in BuildResult(cmd, sqls.Added, Interface.ListChangesChangeType.Added)) + await foreach (var r in BuildResult(cmd, sqls.Added, Interface.ListChangesChangeType.Added)) yield return r; - foreach (var r in BuildResult(cmd, sqls.Deleted, Interface.ListChangesChangeType.Deleted)) + await foreach (var r in BuildResult(cmd, sqls.Deleted, Interface.ListChangesChangeType.Deleted)) yield return r; - foreach (var r in BuildResult(cmd, sqls.Modified, Interface.ListChangesChangeType.Modified)) + await foreach (var r in BuildResult(cmd, sqls.Modified, Interface.ListChangesChangeType.Modified)) yield return r; } } @@ -350,14 +354,14 @@ namespace Duplicati.Library.Main.Database { if (m_insertPreviousElementCommand != null) { - try { m_insertPreviousElementCommand.Dispose(); } + try { await m_insertPreviousElementCommand.DisposeAsync(); } catch { } finally { m_insertPreviousElementCommand = null!; } } if (m_insertCurrentElementCommand != null) { - try { m_insertCurrentElementCommand.Dispose(); } + try { await m_insertCurrentElementCommand.DisposeAsync(); } catch { } finally { m_insertCurrentElementCommand = null!; } } From 293e329a2df10f2d76426cc18da0ca892c3242b4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 08:03:12 +0200 Subject: [PATCH 089/362] Restructured SQL queries in LocalListChangesDatabase --- .../Main/Database/LocalListChangesDatabase.cs | 387 ++++++++++++++---- 1 file changed, 315 insertions(+), 72 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index ea9661bdd..d9d9af826 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -129,12 +129,60 @@ namespace Duplicati.Library.Main.Database using (var cmd = sh.m_db.Connection.CreateCommand(db.Transaction)) { - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "); - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" ( + ""Path"" TEXT NOT NULL, + ""FileHash"" TEXT NULL, + ""MetaHash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Type"" INTEGER NOT NULL + ) + "); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" ( + ""Path"" TEXT NOT NULL, + ""FileHash"" TEXT NULL, + ""MetaHash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Type"" INTEGER NOT NULL + ) + "); } - sh.m_insertPreviousElementCommand = await sh.m_db.Connection.CreateCommandAsync($@"INSERT INTO ""{sh.m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"); - sh.m_insertCurrentElementCommand = await sh.m_db.Connection.CreateCommandAsync($@"INSERT INTO ""{sh.m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"); + sh.m_insertPreviousElementCommand = await sh.m_db.Connection.CreateCommandAsync($@" + INSERT INTO ""{sh.m_previousTable}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + VALUES ( + @Path, + @FileHash, + @MetaHash, + @Size, + @Type + ) + "); + + sh.m_insertCurrentElementCommand = await sh.m_db.Connection.CreateCommandAsync($@" + INSERT INTO ""{sh.m_currentTable}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + VALUES ( + @Path, + @FileHash, + @MetaHash, + @Size, + @Type + ) + "); return sh; } @@ -143,18 +191,91 @@ namespace Duplicati.Library.Main.Database { var tablename = asNew ? m_currentTable : m_previousTable; - var folders = $@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "; - var symlinks = $@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "; - var files = $@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "; - var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")"; + var folders = $@" + SELECT + ""File"".""Path"" AS ""Path"", + NULL AS ""FileHash"", + ""Blockset"".""Fullhash"" AS ""MetaHash"", + -1 AS ""Size"", + {(int)Interface.ListChangesElementType.Folder} AS ""Type"", + ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" + FROM + ""File"", + ""FilesetEntry"", + ""Metadataset"", + ""Blockset"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""File"".""BlocksetID"" = -100 + AND ""Metadataset"".""ID""=""File"".""MetadataID"" + AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + "; + var symlinks = $@" + SELECT + ""File"".""Path"" AS ""Path"", + NULL AS ""FileHash"", + ""Blockset"".""Fullhash"" AS ""MetaHash"", + -1 AS ""Size"", + {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", + ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" + FROM + ""File"", + ""FilesetEntry"", + ""Metadataset"", + ""Blockset"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""File"".""BlocksetID"" = -200 + AND ""Metadataset"".""ID""=""File"".""MetadataID"" + AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + "; + + var files = $@" + SELECT + ""File"".""Path"" AS ""Path"", + ""FB"".""FullHash"" AS ""FileHash"", + ""MB"".""Fullhash"" AS ""MetaHash"", + ""FB"".""Length"" AS ""Size"", + {(int)Interface.ListChangesElementType.File} AS ""Type"", + ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" + FROM + ""File"", + ""FilesetEntry"", + ""Metadataset"", + ""Blockset"" MB, + ""Blockset"" FB + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""File"".""BlocksetID"" >= 0 + AND ""Metadataset"".""ID""=""File"".""MetadataID"" + AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" + AND ""File"".""BlocksetID"" = ""FB"".""ID"" "; + + var combined = $"({folders} UNION {symlinks} UNION {files})"; using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { if (filter == null || filter.Empty) { // Simple case, select everything - await cmd.SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId ") + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{tablename}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + SELECT + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + FROM {combined} A + WHERE ""A"".""FilesetID"" = @FilesetId + ") .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQueryAsync(); } @@ -165,8 +286,16 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "); - await cmd.SetCommandAndParameters($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)") + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{filenamestable}"" ( + ""Path"" TEXT NOT NULL + ) + "); + + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{filenamestable}"" (""Path"") + VALUES (@Path) + ") .PrepareAsync(); foreach (var s in p) @@ -177,14 +306,42 @@ namespace Duplicati.Library.Main.Database if (expression.Result) { // Include filter - whereClause = $@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"; + whereClause = $@" + ""A"".""FilesetID"" = @FilesetId + AND ""A"".""Path"" IN ( + SELECT DISTINCT ""Path"" + FROM ""{filenamestable}"" + ) + "; } else { // Exclude filter - whereClause = $@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"; + whereClause = $@" + ""A"".""FilesetID"" = @FilesetId + AND ""A"".""Path"" NOT IN ( + SELECT DISTINCT ""Path"" + FROM ""{filenamestable}"" + ) + "; } - await cmd.SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} ") + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{tablename}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + SELECT + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + FROM {combined} A + WHERE {whereClause} + ") .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQueryAsync(); @@ -194,25 +351,53 @@ namespace Duplicati.Library.Main.Database { // Do row-wise iteration var values = new object[5]; - await cmd.SetCommandAndParameters($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId") + await cmd.SetCommandAndParameters($@" + SELECT + ""A"".""Path"", + ""A"".""FileHash"", + ""A"".""MetaHash"", + ""A"".""Size"", + ""A"".""Type"" + FROM {combined} A + WHERE ""A"".""FilesetID"" = @FilesetId + ") .SetParameterValue("@FilesetId", filesetId) .PrepareAsync(); - using (var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction).SetCommandAndParameters($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")) - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + + using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) + .SetCommandAndParameters($@" + INSERT INTO ""{tablename}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + VALUES ( + @Path, + @FileHash, + @MetaHash, + @Size, + @Type + ) + "); + await cmd2.PrepareAsync(); + + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + { + rd.GetValues(values); + if (values[0] != null && values[0] != DBNull.Value && FilterExpression.Matches(filter, values[0].ToString())) { - rd.GetValues(values); - if (values[0] != null && values[0] != DBNull.Value && FilterExpression.Matches(filter, values[0].ToString())) - { - await cmd2 - .SetParameterValue("@Path", values[0]) - .SetParameterValue("@FileHash", values[1]) - .SetParameterValue("@MetaHash", values[2]) - .SetParameterValue("@Size", values[3]) - .SetParameterValue("@Type", values[4]) - .ExecuteNonQueryAsync(); - } + await cmd2 + .SetParameterValue("@Path", values[0]) + .SetParameterValue("@FileHash", values[1]) + .SetParameterValue("@MetaHash", values[2]) + .SetParameterValue("@Size", values[3]) + .SetParameterValue("@Type", values[4]) + .ExecuteNonQueryAsync(); } + } } } } @@ -245,28 +430,73 @@ namespace Duplicati.Library.Main.Database private (string Added, string Deleted, string Modified) GetSqls(bool allTypes) { return ( - $@"SELECT ""Path"" FROM ""{m_currentTable}"" WHERE {(allTypes ? "" : @$" ""{m_currentTable}"".""Type"" = @Type AND ")} ""{m_currentTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_previousTable}"")", - $@"SELECT ""Path"" FROM ""{m_previousTable}"" WHERE {(allTypes ? "" : @$" ""{m_previousTable}"".""Type"" = @Type AND ")} ""{m_previousTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_currentTable}"")", - $@"SELECT ""{m_currentTable}"".""Path"" FROM ""{m_currentTable}"",""{m_previousTable}"" WHERE {(allTypes ? "" : $@" ""{m_currentTable}"".""Type"" = @Type AND ")} ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" AND (""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"") " + $@" + SELECT ""Path"" + FROM ""{m_currentTable}"" + WHERE + {(allTypes ? "" : @$" ""{m_currentTable}"".""Type"" = @Type AND ")} + ""{m_currentTable}"".""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_previousTable}"" + ) + ", + + $@" + SELECT ""Path"" + FROM ""{m_previousTable}"" + WHERE + {(allTypes ? "" : @$" ""{m_previousTable}"".""Type"" = @Type AND ")} + ""{m_previousTable}"".""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_currentTable}"" + ) + ", + + $@" + SELECT ""{m_currentTable}"".""Path"" + FROM ""{m_currentTable}"",""{m_previousTable}"" + WHERE + {(allTypes ? "" : $@" ""{m_currentTable}"".""Type"" = @Type AND ")} + ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" + AND ( + ""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" + OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" + OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"" + ) + " ); } public async Task CreateChangeSizeReport() { - var sqls = GetSqls(true); - var added = sqls.Added; - var deleted = sqls.Deleted; - //var modified = sqls.Modified; + var (Added, Deleted, Modified) = GetSqls(true); using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { - var result = new ChangeSizeReport(); + var result = new ChangeSizeReport + { + PreviousSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_previousTable}"" + ", 0), - result.PreviousSize = await cmd.ExecuteScalarInt64Async($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" ", 0); - result.CurrentSize = await cmd.ExecuteScalarInt64Async($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" ", 0); + CurrentSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_currentTable}"" + ", 0), - result.AddedSize = await cmd.ExecuteScalarInt64Async($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) ", 0); - result.DeletedSize = await cmd.ExecuteScalarInt64Async($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) ", 0); + AddedSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_currentTable}"" + WHERE ""{m_currentTable}"".""Path"" IN ({Added}) + ", 0), + + DeletedSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_previousTable}"" + WHERE ""{m_previousTable}"".""Path"" IN ({Deleted}) + ", 0) + }; return result; } @@ -274,51 +504,64 @@ namespace Duplicati.Library.Main.Database public async Task CreateChangeCountReport() { - var sqls = GetSqls(false); - var added = @"SELECT COUNT(*) FROM (" + sqls.Added + ")"; - var deleted = @"SELECT COUNT(*) FROM (" + sqls.Deleted + ")"; - var modified = @"SELECT COUNT(*) FROM (" + sqls.Modified + ")"; + var (Added, Deleted, Modified) = GetSqls(false); - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + var added = @$" + SELECT COUNT(*) + FROM ({Added}) + "; + + var deleted = @$" + SELECT COUNT(*) + FROM ({Deleted}) + "; + + var modified = @$" + SELECT COUNT(*) + FROM ({Modified}) + "; + + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + + var result = new ChangeCountReport { - var result = new ChangeCountReport(); - result.AddedFolders = await cmd.SetCommandAndParameters(added) + AddedFolders = await cmd.SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0); - result.AddedSymlinks = await cmd.SetCommandAndParameters(added) + .ExecuteScalarInt64Async(0), + AddedSymlinks = await cmd.SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0); - result.AddedFiles = await cmd.SetCommandAndParameters(added) + .ExecuteScalarInt64Async(0), + AddedFiles = await cmd.SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0), - result.DeletedFolders = await cmd.SetCommandAndParameters(deleted) + DeletedFolders = await cmd.SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0); - result.DeletedSymlinks = await cmd.SetCommandAndParameters(deleted) + .ExecuteScalarInt64Async(0), + DeletedSymlinks = await cmd.SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0); - result.DeletedFiles = await cmd.SetCommandAndParameters(deleted) + .ExecuteScalarInt64Async(0), + DeletedFiles = await cmd.SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0), - result.ModifiedFolders = await cmd.SetCommandAndParameters(modified) + ModifiedFolders = await cmd.SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0); - result.ModifiedSymlinks = await cmd.SetCommandAndParameters(modified) + .ExecuteScalarInt64Async(0), + ModifiedSymlinks = await cmd.SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0); - result.ModifiedFiles = await cmd.SetCommandAndParameters(modified) + .ExecuteScalarInt64Async(0), + ModifiedFiles = await cmd.SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0) + }; - return result; - } + return result; } public async IAsyncEnumerable> CreateChangedFileReport() { - var sqls = GetSqls(false); + var (Added, Deleted, Modified) = GetSqls(false); using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { @@ -332,15 +575,15 @@ namespace Duplicati.Library.Main.Database { cmd.SetCommandAndParameters(sql); foreach (var type in elTypes) - await foreach (var s in ReaderToStringList(cmd.SetParameterValue("@Type", (int)type).ExecuteReader())) + await foreach (var s in ReaderToStringList(await cmd.SetParameterValue("@Type", (int)type).ExecuteReaderAsync())) yield return new Tuple(changeType, type, s ?? ""); } - await foreach (var r in BuildResult(cmd, sqls.Added, Interface.ListChangesChangeType.Added)) + await foreach (var r in BuildResult(cmd, Added, Interface.ListChangesChangeType.Added)) yield return r; - await foreach (var r in BuildResult(cmd, sqls.Deleted, Interface.ListChangesChangeType.Deleted)) + await foreach (var r in BuildResult(cmd, Deleted, Interface.ListChangesChangeType.Deleted)) yield return r; - await foreach (var r in BuildResult(cmd, sqls.Modified, Interface.ListChangesChangeType.Modified)) + await foreach (var r in BuildResult(cmd, Modified, Interface.ListChangesChangeType.Modified)) yield return r; } } From f1f6924fe00593c790ad97a4c90db42ae535353d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 08:10:53 +0200 Subject: [PATCH 090/362] Changed LocalListDatabase to use factory construction --- .../Main/Database/LocalListDatabase.cs | 92 +++++++++++-------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 2dad4dafa..fd0037d9e 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -25,6 +25,7 @@ using System; using System.Data; using System.Linq; using System.Collections.Generic; +using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Utility; using Duplicati.Library.Interface; @@ -33,10 +34,14 @@ namespace Duplicati.Library.Main.Database { internal class LocalListDatabase : LocalDatabase { - public LocalListDatabase(string path, long pagecachesize) - : base(path, "List", false, pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize) { - ShouldCloseConnection = true; + var db = new LocalListDatabase(); + + db = (LocalListDatabase)await CreateLocalDatabaseAsync(db, path, "List", false, pagecachesize); + db.ShouldCloseConnection = true; + + return db; } public interface IFileversion @@ -66,24 +71,37 @@ namespace Duplicati.Library.Main.Database private class FileSets : IFileSets { - private readonly IDbConnection m_connection; - private string m_tablename; - private readonly KeyValuePair[] m_filesets; + private LocalDatabase m_db = null!; + private string m_tablename = null!; + private KeyValuePair[] m_filesets = []; + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public FileSets(LocalListDatabase owner, DateTime time, long[] versions) { - m_connection = owner.m_connection; - m_filesets = owner.FilesetTimes.ToArray(); - m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var tmp = owner.GetFilelistWhereClause(time, versions, m_filesets); + throw new NotSupportedException("Use CreateAsync instead."); + } + + private FileSets() { } + + public async Task CreateAsync(LocalListDatabase owner, DateTime time, long[] versions) + { + var fs = new FileSets + { + m_db = owner, + m_filesets = await owner.FilesetTimes().ToArrayAsync(), + m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) + }; + var tmp = await owner.GetFilelistWhereClause(time, versions, m_filesets); var query = tmp.Item1; var args = tmp.Item2; - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_db.Connection.CreateCommand()) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}"), args); cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tablename}_FilesetIDTimestampIndex"" ON ""{m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)")); } + + return fs; } private class Fileset : IFileset @@ -432,23 +450,23 @@ namespace Duplicati.Library.Main.Database public IEnumerable ListFilesetsExtended() { const string query = @" -SELECT +SELECT f.""ID"" AS ""FilesetID"", f.""Timestamp"", f.""IsFullBackup"", COUNT(fe.""FileID"") AS ""NumberOfFiles"", COALESCE(SUM(b.""Length""), 0) AS ""TotalFileSize"" -FROM +FROM ""Fileset"" f -LEFT JOIN +LEFT JOIN ""FilesetEntry"" fe ON f.""ID"" = fe.""FilesetID"" -LEFT JOIN +LEFT JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" -LEFT JOIN +LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" -GROUP BY +GROUP BY f.""ID"" -ORDER BY +ORDER BY f.""Timestamp"" DESC; "; @@ -496,7 +514,7 @@ ORDER BY // Then query the matching files cmd.SetCommandAndParameters($@" - SELECT + SELECT pp.""Prefix"" || fl.""Path"" AS ""FullPath"", b.""Length"", (CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END) AS ""IsDirectory"", @@ -581,7 +599,7 @@ ORDER BY // Fetch entries cmd.SetCommandAndParameters(@" - SELECT + SELECT pp.""Prefix"" || fl.""Path"" AS ""FullPath"", b.""Length"", CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", @@ -783,7 +801,7 @@ ORDER BY // Then fetch the actual data var fetchSql = @" - SELECT + SELECT fe.""FilesetID"", f.""Timestamp"", fl.""Path"", @@ -994,7 +1012,7 @@ ORDER BY // Fetch results var fetchSql = $@" - SELECT + SELECT fe.""FilesetID"", f.""Timestamp"", pp.""Prefix"" || fl.""Path"" AS ""FullPath"", From 9c4eb5a2220400957c62ff2ab8baef6e6eb1ae32 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 08:14:12 +0200 Subject: [PATCH 091/362] FilteredFilenameTable.CreateAsync wasn't static --- .../Library/Main/Database/LocalDatabase.cs | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 0ee8a2e49..e8b01d7f5 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1919,12 +1919,12 @@ namespace Duplicati.Library.Main.Database public class FilteredFilenameTable : IDisposable { public string Tablename { get; private set; } - private readonly SqliteConnection m_connection; + private readonly LocalDatabase m_db; - private FilteredFilenameTable(SqliteConnection connection) + private FilteredFilenameTable(LocalDatabase db) { Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_connection = connection; + m_db = db; } [Obsolete("Calling this constructor will throw an exception. Use the Create method instead.")] @@ -1933,9 +1933,9 @@ namespace Duplicati.Library.Main.Database throw new NotImplementedException("Use the Create method instead."); } - public async Task CreateFilteredFilenameTableAsync(SqliteConnection connection, IFilter filter, SqliteTransaction transaction) + public static async Task CreateFilteredFilenameTableAsync(LocalDatabase db, IFilter filter) { - var ftt = new FilteredFilenameTable(connection); + var ftt = new FilteredFilenameTable(db); // TODO factory var type = FilterType.Regexp; @@ -1948,12 +1948,12 @@ namespace Duplicati.Library.Main.Database if (filter.Empty) { - using var cmd = await m_connection.CreateCommandAsync($@" - CREATE TEMPORARY TABLE ""{Tablename}"" AS + using var cmd = await db.Connection.CreateCommandAsync($@" + CREATE TEMPORARY TABLE ""{ftt.Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" "); - await cmd.SetTransaction(transaction) + await cmd.SetTransaction(db.Transaction) .ExecuteNonQueryAsync(); return ftt; } @@ -1961,23 +1961,23 @@ namespace Duplicati.Library.Main.Database if (type == FilterType.Regexp || type == FilterType.Group) { // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full - using var cmd = await m_connection.CreateCommandAsync($@" - CREATE TEMPORARY TABLE ""{Tablename}"" ( + using var cmd = await db.Connection.CreateCommandAsync($@" + CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( ""Path"" TEXT NOT NULL ) "); - await cmd.SetTransaction(transaction) + await cmd.SetTransaction(db.Transaction) .ExecuteNonQueryAsync(); cmd.SetCommandAndParameters($@" - INSERT INTO ""{Tablename}"" (""Path"") + INSERT INTO ""{ftt.Tablename}"" (""Path"") VALUES (@Path) "); - using var c2 = await m_connection.CreateCommandAsync(@" + using var c2 = await db.Connection.CreateCommandAsync(@" SELECT DISTINCT ""Path"" FROM ""File"" "); - c2.SetTransaction(transaction); + c2.SetTransaction(db.Transaction); using (var rd = await c2.ExecuteReaderAsync()) while (await rd.ReadAsync()) { @@ -1989,7 +1989,7 @@ namespace Duplicati.Library.Main.Database } } - await transaction.CommitAsync(); + await db.Transaction.CommitAsync(); } else { @@ -2013,20 +2013,23 @@ namespace Duplicati.Library.Main.Database } } - using var cmd = m_connection.CreateCommand(); - cmd.SetTransaction(transaction); + using var cmd = db.Connection.CreateCommand(); + cmd.SetTransaction(db.Transaction); + await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TABLE ""{Tablename}"" ( + CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( ""Path"" TEXT NOT NULL ) "); + await cmd.ExecuteNonQueryAsync($@" - INSERT INTO ""{Tablename}"" + INSERT INTO ""{ftt.Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb} ", args); - await transaction.CommitAsync(); + + await db.Transaction.CommitAsync(); } return ftt; @@ -2042,7 +2045,7 @@ namespace Duplicati.Library.Main.Database if (Tablename != null) try { - using var cmd = m_connection.CreateCommand(); + using var cmd = m_db.Connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@$" DROP TABLE IF EXISTS ""{Tablename}"" "); From 388ed0d2ea4309a422fcaf261ee751e52a49176e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 08:30:32 +0200 Subject: [PATCH 092/362] Purged IDb from LocalListDatabase --- .../Library/Main/Database/LocalListDatabase.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index fd0037d9e..b89f5fc3c 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -22,13 +22,13 @@ #nullable enable using System; -using System.Data; using System.Linq; using System.Collections.Generic; using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Utility; using Duplicati.Library.Interface; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -124,11 +124,11 @@ namespace Duplicati.Library.Main.Database private class Fileversion : IFileversion { - private readonly IDataReader m_reader; + private readonly SqliteDataReader m_reader; public string? Path { get; private set; } public bool More { get; private set; } - public Fileversion(IDataReader reader) + public Fileversion(SqliteDataReader reader) { m_reader = reader; Path = reader.ConvertValueToString(0); @@ -159,9 +159,9 @@ namespace Duplicati.Library.Main.Database return GetLargestPrefix(filter, null); } - private IEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) + private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) { - using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) + using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter)) using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets @@ -222,7 +222,7 @@ namespace Duplicati.Library.Main.Database } } - private IEnumerable SelectFolderEntries(IDbCommand cmd, string prefix, string table) + private IEnumerable SelectFolderEntries(SqliteCommand cmd, string prefix, string table) { if (!string.IsNullOrEmpty(prefix)) prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); @@ -263,7 +263,7 @@ namespace Duplicati.Library.Main.Database if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); - using (var tmpnames = new FilteredFilenameTable(m_connection, new FilterExpression(new string[] { pathprefix + "*" }, true), null)) + using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true))) using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets From af49f7332adf9b9b3cb0fdf303dd8305cabfcf58 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 08:34:03 +0200 Subject: [PATCH 093/362] Removed FormatInvariant as it's now handled by the extension methods --- .../Main/Database/LocalListDatabase.cs | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index b89f5fc3c..c76114e98 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -97,8 +97,8 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_db.Connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}"), args); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tablename}_FilesetIDTimestampIndex"" ON ""{m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)")); + cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}", args); + cmd.ExecuteNonQuery($@"CREATE INDEX ""{m_tablename}_FilesetIDTimestampIndex"" ON ""{m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)"); } return fs; @@ -165,27 +165,27 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); + cmd.ExecuteNonQuery($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); //If we have a prefix rule, apply it if (!string.IsNullOrWhiteSpace(prefixrule)) - cmd.SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule")) + cmd.SetCommandAndParameters($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule") .SetParameterValue("@Rule", prefixrule) .ExecuteNonQuery(); // Then we recursively find the largest prefix - var v0 = cmd.ExecuteScalar(FormatInvariant($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); + var v0 = cmd.ExecuteScalar($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); var maxpath = ""; if (v0 != null) maxpath = v0.ToString() ?? ""; var dirsep = Util.GuessDirSeparator(maxpath); - var filecount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"""), 0); + var filecount = cmd.ExecuteScalarInt64($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}""", 0); var foundfiles = -1L; //TODO: Handle FS case-sensitive? - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix")); + cmd.SetCommandAndParameters($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix"); while (filecount != foundfiles && maxpath.Length > 0) { @@ -208,7 +208,7 @@ namespace Duplicati.Library.Main.Database // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) { - var paths = cmd.ExecuteReaderEnumerable(FormatInvariant($@"SELECT Path FROM ""{tmpnames.Tablename}""")).Select(x => x.ConvertValueToString(0) ?? "").ToArray(); + var paths = cmd.ExecuteReaderEnumerable($@"SELECT Path FROM ""{tmpnames.Tablename}""").Select(x => x.ConvertValueToString(0) ?? "").ToArray(); var roots = paths.Select(x => x.Substring(0, 1)).Distinct().Where(x => x != "\\").ToArray(); //unc path like \\server.domain\ @@ -228,7 +228,7 @@ namespace Duplicati.Library.Main.Database prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); var ppl = prefix.Length; - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ""{table}"" "))) + using (var rd = cmd.ExecuteReader($@"SELECT DISTINCT ""Path"" FROM ""{table}"" ")) while (rd.Read()) { var s = rd.ConvertValueToString(0) ?? ""; @@ -267,7 +267,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); + cmd.ExecuteNonQuery($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); // If we had instr support this would work: /*var distinctPaths = @"SELECT DISTINCT :1 || " + @@ -278,23 +278,23 @@ namespace Duplicati.Library.Main.Database @" FROM (" + cartesianPathFileset + @")";*/ // Instead we manually iterate the paths - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tbname}"" (""Path"" TEXT NOT NULL)")); + cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{tbname}"" (""Path"" TEXT NOT NULL)"); using (var c2 = m_connection.CreateCommand()) { - c2.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tbname}"" (""Path"") VALUES (@Path)")); + c2.SetCommandAndParameters($@"INSERT INTO ""{tbname}"" (""Path"") VALUES (@Path)"); foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) c2.SetParameterValue("@Path", n) .ExecuteNonQuery(); - c2.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"")")); + c2.ExecuteNonQuery($@"CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"")"); } //Then we select the matching results - var filesets = FormatInvariant($@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"); - var cartesianPathFileset = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tbname}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"); + var filesets = $@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"; + var cartesianPathFileset = $@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tbname}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"; - var filesWithSizes = FormatInvariant($@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "); + var filesWithSizes = $@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "; var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; using (var rd = cmd.ExecuteReader(query)) @@ -323,7 +323,7 @@ namespace Duplicati.Library.Main.Database try { using (var c = m_connection.CreateCommand()) - c.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tbname}""")); + c.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{tbname}"""); } catch { @@ -337,12 +337,12 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); + cmd.ExecuteNonQuery($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); //Then we select the matching results - var filesets = FormatInvariant($@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"); - var cartesianPathFileset = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tmpnames.Tablename}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"); - var filesWithSizes = FormatInvariant($@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "); + var filesets = $@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"; + var cartesianPathFileset = $@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tmpnames.Tablename}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"; + var filesWithSizes = $@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "; var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; using (var rd = cmd.ExecuteReader(query)) if (rd.Read()) @@ -361,7 +361,7 @@ namespace Duplicati.Library.Main.Database public void TakeFirst() { using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )")); + cmd.ExecuteNonQuery($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )"); } public IEnumerable QuickSets @@ -392,12 +392,12 @@ namespace Duplicati.Library.Main.Database for (var i = 0; i < m_filesets.Length; i++) dict[m_filesets[i].Key] = i; - var summation = FormatInvariant( - $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "); + var summation = + $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "; using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(FormatInvariant( - $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ")) + using (var rd = cmd.ExecuteReader( + $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ") ) while (rd.Read()) { @@ -419,7 +419,7 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ")); + cmd.ExecuteNonQuery(@$"DROP TABLE IF EXISTS ""{m_tablename}"" "); } catch { } finally { m_tablename = null!; } From b15b1db3335c22bf32a19d56a0b3fd1278243a05 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 08:49:32 +0200 Subject: [PATCH 094/362] Changed heavy async properties in LocalListDatabase.FileSets into methods --- .../Main/Database/LocalListDatabase.cs | 74 +++++++++---------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index c76114e98..49af7e1e6 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -61,8 +61,8 @@ namespace Duplicati.Library.Main.Database public interface IFileSets : IDisposable { - IEnumerable Sets { get; } - IEnumerable QuickSets { get; } + IAsyncEnumerable Sets(); + IAsyncEnumerable QuickSets(); IEnumerable SelectFiles(IFilter filter); IEnumerable GetLargestPrefix(IFilter filter); IEnumerable SelectFolderContents(IFilter filter); @@ -364,52 +364,46 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )"); } - public IEnumerable QuickSets + public async IAsyncEnumerable QuickSets() { - get - { - var dict = new Dictionary(); - for (var i = 0; i < m_filesets.Length; i++) - dict[m_filesets[i].Key] = i; + var dict = new Dictionary(); + for (var i = 0; i < m_filesets.Length; i++) + dict[m_filesets[i].Key] = i; - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) - while (rd.Read()) - { - var id = rd.ConvertValueToInt64(0); - var backupType = rd.GetInt32(1); - var e = dict[id]; - yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); - } - } + using (var cmd = m_db.Connection.CreateCommand()) + using (var rd = await cmd.ExecuteReaderAsync(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) + while (await rd.ReadAsync()) + { + var id = rd.ConvertValueToInt64(0); + var backupType = rd.GetInt32(1); + var e = dict[id]; + yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); + } } - public IEnumerable Sets + public async IAsyncEnumerable Sets() { - get - { - var dict = new Dictionary(); - for (var i = 0; i < m_filesets.Length; i++) - dict[m_filesets[i].Key] = i; + var dict = new Dictionary(); + for (var i = 0; i < m_filesets.Length; i++) + dict[m_filesets[i].Key] = i; - var summation = - $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "; + var summation = + $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "; - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader( - $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ") - ) - while (rd.Read()) - { - var id = rd.ConvertValueToInt64(0); - var isFullBackup = rd.GetInt32(1); - var e = dict[id]; - var filecount = rd.ConvertValueToInt64(2, -1L); - var filesizes = rd.ConvertValueToInt64(3, -1L); + using (var cmd = m_db.Connection.CreateCommand()) + using (var rd = await cmd.ExecuteReaderAsync( + $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ") + ) + while (await rd.ReadAsync()) + { + var id = rd.ConvertValueToInt64(0); + var isFullBackup = rd.GetInt32(1); + var e = dict[id]; + var filecount = rd.ConvertValueToInt64(2, -1L); + var filesizes = rd.ConvertValueToInt64(3, -1L); - yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); - } - } + yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); + } } public void Dispose() From e0dece7fab87726109a957c75ec2cdef15d593c3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 09:00:45 +0200 Subject: [PATCH 095/362] Made LocalListDatabase async --- .../Main/Database/LocalListDatabase.cs | 212 ++++++++++-------- 1 file changed, 119 insertions(+), 93 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 49af7e1e6..60c689d6c 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -63,10 +63,10 @@ namespace Duplicati.Library.Main.Database { IAsyncEnumerable Sets(); IAsyncEnumerable QuickSets(); - IEnumerable SelectFiles(IFilter filter); - IEnumerable GetLargestPrefix(IFilter filter); - IEnumerable SelectFolderContents(IFilter filter); - void TakeFirst(); + IAsyncEnumerable SelectFiles(IFilter filter); + IAsyncEnumerable GetLargestPrefix(IFilter filter); + IAsyncEnumerable SelectFolderContents(IFilter filter); + Task TakeFirst(); } private class FileSets : IFileSets @@ -83,7 +83,7 @@ namespace Duplicati.Library.Main.Database private FileSets() { } - public async Task CreateAsync(LocalListDatabase owner, DateTime time, long[] versions) + public static async Task CreateAsync(LocalListDatabase owner, DateTime time, long[] versions) { var fs = new FileSets { @@ -91,14 +91,14 @@ namespace Duplicati.Library.Main.Database m_filesets = await owner.FilesetTimes().ToArrayAsync(), m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) }; - var tmp = await owner.GetFilelistWhereClause(time, versions, m_filesets); + var tmp = await owner.GetFilelistWhereClause(time, versions, fs.m_filesets); var query = tmp.Item1; var args = tmp.Item2; - using (var cmd = m_db.Connection.CreateCommand()) + using (var cmd = fs.m_db.Connection.CreateCommand()) { - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}", args); - cmd.ExecuteNonQuery($@"CREATE INDEX ""{m_tablename}_FilesetIDTimestampIndex"" ON ""{m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)"); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{fs.m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}", args); + await cmd.ExecuteNonQueryAsync($@"CREATE INDEX ""{fs.m_tablename}_FilesetIDTimestampIndex"" ON ""{fs.m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)"); } return fs; @@ -154,7 +154,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable Sizes { get { return new long[0]; } } } - public IEnumerable GetLargestPrefix(IFilter filter) + public IAsyncEnumerable GetLargestPrefix(IFilter filter) { return GetLargestPrefix(filter, null); } @@ -162,10 +162,10 @@ namespace Duplicati.Library.Main.Database private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) { using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter)) - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); + await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); //If we have a prefix rule, apply it if (!string.IsNullOrWhiteSpace(prefixrule)) @@ -174,25 +174,25 @@ namespace Duplicati.Library.Main.Database .ExecuteNonQuery(); // Then we recursively find the largest prefix - var v0 = cmd.ExecuteScalar($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); + var v0 = await cmd.ExecuteScalarAsync($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); var maxpath = ""; if (v0 != null) maxpath = v0.ToString() ?? ""; var dirsep = Util.GuessDirSeparator(maxpath); - var filecount = cmd.ExecuteScalarInt64($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}""", 0); + var filecount = await cmd.ExecuteScalarInt64Async($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}""", 0); var foundfiles = -1L; //TODO: Handle FS case-sensitive? - cmd.SetCommandAndParameters($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix"); + await cmd.SetCommandAndParameters($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix").PrepareAsync(); while (filecount != foundfiles && maxpath.Length > 0) { var mp = Util.AppendDirSeparator(maxpath, dirsep); - foundfiles = cmd.SetParameterValue("@PrefixLength", mp.Length) + foundfiles = await cmd.SetParameterValue("@PrefixLength", mp.Length) .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64(0); + .ExecuteScalarInt64Async(0); if (filecount != foundfiles) { @@ -208,28 +208,32 @@ namespace Duplicati.Library.Main.Database // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) { - var paths = cmd.ExecuteReaderEnumerable($@"SELECT Path FROM ""{tmpnames.Tablename}""").Select(x => x.ConvertValueToString(0) ?? "").ToArray(); - var roots = paths.Select(x => x.Substring(0, 1)).Distinct().Where(x => x != "\\").ToArray(); + var paths = cmd.ExecuteReaderEnumerableAsync($@"SELECT Path FROM ""{tmpnames.Tablename}""").Select(x => x.ConvertValueToString(0) ?? ""); + var roots = paths.Select(x => x.Substring(0, 1)).Distinct().Where(x => x != "\\"); //unc path like \\server.domain\ var regexUNCPrefix = new System.Text.RegularExpressions.Regex(@"^\\\\.*?\\"); - var rootsUNC = paths.Select(x => regexUNCPrefix.Match(x)).Where(x => x.Success).Select(x => x.Value).Distinct().ToArray(); + var rootsUNC = paths.Select(x => regexUNCPrefix.Match(x)).Where(x => x.Success).Select(x => x.Value).Distinct(); - return roots.Concat(rootsUNC).Select(x => GetLargestPrefix(filter, x).First()).Distinct().ToArray(); + await foreach (var el in roots.Concat(rootsUNC).Select(x => GetLargestPrefix(filter, x).FirstAsync()).Distinct()) + yield return await el; } - return [new FileversionFixed { Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) }]; + yield return new FileversionFixed + { + Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) + }; } } - private IEnumerable SelectFolderEntries(SqliteCommand cmd, string prefix, string table) + private async IAsyncEnumerable SelectFolderEntries(SqliteCommand cmd, string prefix, string table) { if (!string.IsNullOrEmpty(prefix)) prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); var ppl = prefix.Length; - using (var rd = cmd.ExecuteReader($@"SELECT DISTINCT ""Path"" FROM ""{table}"" ")) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync($@"SELECT DISTINCT ""Path"" FROM ""{table}"" ")) + while (await rd.ReadAsync()) { var s = rd.ConvertValueToString(0) ?? ""; if (!s.StartsWith(prefix, StringComparison.Ordinal)) @@ -245,7 +249,7 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable SelectFolderContents(IFilter filter) + public async IAsyncEnumerable SelectFolderContents(IFilter filter) { var tbname = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try @@ -264,10 +268,10 @@ namespace Duplicati.Library.Main.Database pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true))) - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); + await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); // If we had instr support this would work: /*var distinctPaths = @"SELECT DISTINCT :1 || " + @@ -278,16 +282,16 @@ namespace Duplicati.Library.Main.Database @" FROM (" + cartesianPathFileset + @")";*/ // Instead we manually iterate the paths - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{tbname}"" (""Path"" TEXT NOT NULL)"); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tbname}"" (""Path"" TEXT NOT NULL)"); - using (var c2 = m_connection.CreateCommand()) + using (var c2 = m_db.Connection.CreateCommand()) { c2.SetCommandAndParameters($@"INSERT INTO ""{tbname}"" (""Path"") VALUES (@Path)"); - foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) - c2.SetParameterValue("@Path", n) - .ExecuteNonQuery(); + await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) + await c2.SetParameterValue("@Path", n) + .ExecuteNonQueryAsync(); - c2.ExecuteNonQuery($@"CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"")"); + await c2.ExecuteNonQueryAsync($@"CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"")"); } //Then we select the matching results @@ -297,8 +301,8 @@ namespace Duplicati.Library.Main.Database var filesWithSizes = $@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "; var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; - using (var rd = cmd.ExecuteReader(query)) - if (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync(query)) + if (await rd.ReadAsync()) { bool more; do @@ -322,8 +326,8 @@ namespace Duplicati.Library.Main.Database { try { - using (var c = m_connection.CreateCommand()) - c.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{tbname}"""); + using (var c = m_db.Connection.CreateCommand()) + await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}"""); } catch { @@ -331,21 +335,21 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable SelectFiles(IFilter filter) + public async IAsyncEnumerable SelectFiles(IFilter filter) { - using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) - using (var cmd = m_connection.CreateCommand()) + using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter)) + using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); + await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); //Then we select the matching results var filesets = $@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"; var cartesianPathFileset = $@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tmpnames.Tablename}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"; var filesWithSizes = $@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "; var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; - using (var rd = cmd.ExecuteReader(query)) - if (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync(query)) + if (await rd.ReadAsync()) { bool more; do @@ -358,10 +362,10 @@ namespace Duplicati.Library.Main.Database } } - public void TakeFirst() + public async Task TakeFirst() { - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )"); + using var cmd = m_db.Connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )"); } public async IAsyncEnumerable QuickSets() @@ -407,24 +411,28 @@ namespace Duplicati.Library.Main.Database } public void Dispose() + { + DisposeAsync().Await(); + } + + public async Task DisposeAsync() { if (m_tablename != null) { try { - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(@$"DROP TABLE IF EXISTS ""{m_tablename}"" "); + using (var cmd = m_db.Connection.CreateCommand()) + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" "); } catch { } finally { m_tablename = null!; } } - } } - public IFileSets SelectFileSets(DateTime time, long[] versions) + public async Task SelectFileSets(DateTime time, long[] versions) { - return new FileSets(this, time, versions); + return await FileSets.CreateAsync(this, time, versions); } /// @@ -441,7 +449,7 @@ namespace Duplicati.Library.Main.Database /// Lists all filesets with summary data /// /// A collection of fileset entries - public IEnumerable ListFilesetsExtended() + public async IAsyncEnumerable ListFilesetsExtended() { const string query = @" SELECT @@ -465,14 +473,20 @@ ORDER BY "; using (var cmd = m_connection.CreateCommand(query)) - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { var id = rd.ConvertValueToInt64(0); var time = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); var isFullBackup = rd.GetInt32(2) == BackupType.FULL_BACKUP; var filecount = rd.ConvertValueToInt64(3, -1L); var filesizes = rd.ConvertValueToInt64(4, -1L); - yield return new FilesetEntry(id, isFullBackup, time, filecount, filesizes); + yield return new FilesetEntry( + id, + isFullBackup, + time, + filecount, + filesizes + ); } } @@ -494,7 +508,7 @@ ORDER BY /// The offset for pagination. /// >The limit for pagination. /// A paginated result set of folder entries. - public IPaginatedResults ListFolder(IEnumerable prefixIds, long filesetid, long offset, long limit) + public async Task> ListFolder(IEnumerable prefixIds, long filesetid, long offset, long limit) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); @@ -524,13 +538,13 @@ ORDER BY ") .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExpandInClauseParameter("@PrefixIds", prefixIds) + .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) .SetParameterValue("@filesetid", filesetid) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); var results = new List(); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); @@ -538,7 +552,13 @@ ORDER BY var isSymlink = rd.GetInt32(3) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(4, 0), DateTimeKind.Utc); - results.Add(new FolderEntry(path, size, isDir, isSymlink, lastModified)); + results.Add(new FolderEntry( + path, + size, + isDir, + isSymlink, + lastModified + )); } long totalCount; @@ -555,16 +575,16 @@ ORDER BY else { // Get the total count of files for the given fileset and prefix IDs - totalCount = cmd.SetCommandAndParameters($@" + totalCount = await cmd.SetCommandAndParameters($@" SELECT COUNT(*) FROM FilesetEntry fe INNER JOIN FileLookup fl ON fe.FileID = fl.ID WHERE fe.FilesetID = @filesetid AND fl.PrefixID IN (@PrefixIds) ") - .ExpandInClauseParameter("@PrefixIds", prefixIds) + .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) .SetParameterValue("@filesetid", filesetid) - .ExecuteScalarInt64(0); + .ExecuteScalarInt64Async(0); } // Return paginated results @@ -579,7 +599,7 @@ ORDER BY /// The offset for pagination. /// >The limit for pagination. /// A paginated result set of folder entries. - public IPaginatedResults ListFilesetEntries(IEnumerable prefixIds, long filesetid, long offset, long limit) + public async Task> ListFilesetEntries(IEnumerable prefixIds, long filesetid, long offset, long limit) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); @@ -611,12 +631,12 @@ ORDER BY .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameter("@PrefixIds", prefixIds) + .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); var results = new List(); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); @@ -640,7 +660,7 @@ ORDER BY } else { - totalCount = cmd.SetCommandAndParameters(@" + totalCount = await cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" @@ -648,8 +668,8 @@ ORDER BY AND fl.""PrefixId"" IN (@PrefixIds) ") .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameter("@PrefixIds", prefixIds) - .ExecuteScalarInt64(0); + .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) + .ExecuteScalarInt64Async(0); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; @@ -661,7 +681,7 @@ ORDER BY /// /// The prefixes to get the IDs for. /// >A list of prefix IDs. - public IEnumerable GetPrefixIds(IEnumerable prefixes) + public IAsyncEnumerable GetPrefixIds(IEnumerable prefixes) { using var cmd = m_connection.CreateCommand(); @@ -669,9 +689,8 @@ ORDER BY cmd.SetCommandAndParameters($@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" IN (@Prefixes)") .ExpandInClauseParameter("@Prefixes", prefixes); - return cmd.ExecuteReaderEnumerable() - .Select(x => x.ConvertValueToInt64(0)) - .ToList(); + return cmd.ExecuteReaderEnumerableAsync() + .Select(x => x.ConvertValueToInt64(0)); } /// @@ -679,7 +698,7 @@ ORDER BY /// /// The filesetId to get the prefixes for /// A list of root prefixes - public IEnumerable GetRootPrefixes(long filesetid) + public IAsyncEnumerable GetRootPrefixes(long filesetid) { using var cmd = m_connection.CreateCommand(); @@ -704,10 +723,9 @@ ORDER BY ") .SetParameterValue("@filesetid", filesetid); - return cmd.ExecuteReaderEnumerable() + return cmd.ExecuteReaderEnumerableAsync() .Select(x => x.ConvertValueToInt64(0, -1)) - .Where(x => x != -1) - .ToList(); + .Where(x => x != -1); } /// @@ -715,7 +733,7 @@ ORDER BY /// /// The filesetId to get the prefixes for /// A list of (PrefixID, ParentPrefixID) pairs - public IEnumerable<(long PrefixId, long ParentPrefixId)> GetRootPrefixesWithParents(long filesetid) + public IAsyncEnumerable<(long PrefixId, long ParentPrefixId)> GetRootPrefixesWithParents(long filesetid) { using var cmd = m_connection.CreateCommand(); @@ -756,7 +774,7 @@ ORDER BY .SetParameterValue("@dirSeparator", dirSeparator) .SetParameterValue("@filesetid", filesetid); - return cmd.ExecuteReaderEnumerable() + return cmd.ExecuteReaderEnumerableAsync() .Select(x => ( PrefixId: x.ConvertValueToInt64(0, -1), ParentPrefixId: x.ConvertValueToInt64(1, -1) @@ -771,7 +789,7 @@ ORDER BY /// Pagination offset. /// Pagination limit. /// All matching file versions ordered by path and then version time. - public IPaginatedResults ListFileVersions(IEnumerable paths, long[]? filesetIds, long offset, long limit) + public async Task> ListFileVersions(IEnumerable paths, long[]? filesetIds, long offset, long limit) { if (paths == null || !paths.Any()) return new PaginatedResults(0, (int)(limit > 0 ? limit : long.MaxValue), 0, 0, Enumerable.Empty()); @@ -789,9 +807,9 @@ ORDER BY cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"); - var versionMap = cmd.ExecuteReaderEnumerable() + var versionMap = await cmd.ExecuteReaderEnumerableAsync() .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) - .ToDictionary(x => x.FilesetId, x => x.Version); + .ToDictionaryAsync(x => x.FilesetId, x => x.Version); // Then fetch the actual data var fetchSql = @" @@ -819,10 +837,10 @@ ORDER BY .SetParameterValue("@offset", offset); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); + cmd.ExpandInClauseParameter("@FilesetIds", filesetIds.Cast()); var results = new List(); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); @@ -832,7 +850,15 @@ ORDER BY var isSymlink = rd.GetInt32(5) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc); - results.Add(new ListFileVersion(version, time, path, size, isDirectory, isSymlink, lastModified)); + results.Add(new ListFileVersion( + version, + time, + path, + size, + isDirectory, + isSymlink, + lastModified + )); } long totalCount; @@ -859,9 +885,9 @@ ORDER BY .ExpandInClauseParameter("@Paths", paths); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); + cmd.ExpandInClauseParameter("@FilesetIds", filesetIds.Cast()); - totalCount = cmd.ExecuteScalarInt64(0); + totalCount = await cmd.ExecuteScalarInt64Async(0); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; @@ -885,7 +911,7 @@ ORDER BY /// The offset for pagination. /// The limit for pagination. /// All matching file versions ordered by path and then version time. - public IPaginatedResults SearchEntries(IEnumerable? pathprefixes, IFilter filter, long[]? filesetIds, long offset, long limit) + public async Task> SearchEntries(IEnumerable? pathprefixes, IFilter filter, long[]? filesetIds, long offset, long limit) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); @@ -1000,9 +1026,9 @@ ORDER BY cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"); - var versionMap = cmd.ExecuteReaderEnumerable() + var versionMap = await cmd.ExecuteReaderEnumerableAsync() .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) - .ToDictionary(x => x.FilesetId, x => x.Version); + .ToDictionaryAsync(x => x.FilesetId, x => x.Version); // Fetch results var fetchSql = $@" @@ -1036,10 +1062,10 @@ ORDER BY cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); + cmd.ExpandInClauseParameter("@FilesetIds", filesetIds.Cast()); var results = new List(); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); @@ -1084,9 +1110,9 @@ ORDER BY cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); + cmd.ExpandInClauseParameter("@FilesetIds", filesetIds.Cast()); - totalCount = cmd.ExecuteScalarInt64(0); + totalCount = await cmd.ExecuteScalarInt64Async(0); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; From aa70a1e41cc0a4e027b91eeadf05ae123d60fc15 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 09:28:52 +0200 Subject: [PATCH 096/362] Restructured SQL queries in LocalListDatabase --- .../Main/Database/LocalListDatabase.cs | 822 +++++++++++++----- 1 file changed, 582 insertions(+), 240 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 60c689d6c..c89fadc77 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -97,8 +97,21 @@ namespace Duplicati.Library.Main.Database using (var cmd = fs.m_db.Connection.CreateCommand()) { - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{fs.m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}", args); - await cmd.ExecuteNonQueryAsync($@"CREATE INDEX ""{fs.m_tablename}_FilesetIDTimestampIndex"" ON ""{fs.m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)"); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{fs.m_tablename}"" AS + SELECT DISTINCT + ""ID"" AS ""FilesetID"", + ""IsFullBackup"" AS ""IsFullBackup"", + ""Timestamp"" AS ""Timestamp"" + FROM ""Fileset"" {query}", args); + + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{fs.m_tablename}_FilesetIDTimestampIndex"" + ON ""{fs.m_tablename}"" ( + ""FilesetID"", + ""Timestamp"" DESC + ) + "); } return fs; @@ -165,32 +178,65 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""{tmpnames.Tablename}"" + WHERE ""Path"" NOT IN ( + SELECT DISTINCT ""Path"" + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND ""FilesetEntry"".""FilesetID"" IN ( + SELECT ""FilesetID"" + FROM ""{m_tablename}"" + ) + ) + "); //If we have a prefix rule, apply it if (!string.IsNullOrWhiteSpace(prefixrule)) - cmd.SetCommandAndParameters($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule") + await cmd.SetCommandAndParameters($@" + DELETE FROM ""{tmpnames.Tablename}"" + WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule + ") .SetParameterValue("@Rule", prefixrule) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); // Then we recursively find the largest prefix - var v0 = await cmd.ExecuteScalarAsync($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); + var v0 = await cmd.ExecuteScalarAsync($@" + SELECT ""Path"" + FROM ""{tmpnames.Tablename}"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + "); + var maxpath = ""; if (v0 != null) maxpath = v0.ToString() ?? ""; var dirsep = Util.GuessDirSeparator(maxpath); - var filecount = await cmd.ExecuteScalarInt64Async($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}""", 0); + var filecount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{tmpnames.Tablename}"" + ", 0); + var foundfiles = -1L; //TODO: Handle FS case-sensitive? - await cmd.SetCommandAndParameters($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix").PrepareAsync(); + await cmd.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ""{tmpnames.Tablename}"" + WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix + ") + .PrepareAsync(); while (filecount != foundfiles && maxpath.Length > 0) { var mp = Util.AppendDirSeparator(maxpath, dirsep); - foundfiles = await cmd.SetParameterValue("@PrefixLength", mp.Length) + foundfiles = await cmd + .SetParameterValue("@PrefixLength", mp.Length) .SetParameterValue("@Prefix", mp) .ExecuteScalarInt64Async(0); @@ -208,14 +254,32 @@ namespace Duplicati.Library.Main.Database // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) { - var paths = cmd.ExecuteReaderEnumerableAsync($@"SELECT Path FROM ""{tmpnames.Tablename}""").Select(x => x.ConvertValueToString(0) ?? ""); - var roots = paths.Select(x => x.Substring(0, 1)).Distinct().Where(x => x != "\\"); + var paths = cmd.ExecuteReaderEnumerableAsync($@" + SELECT Path + FROM ""{tmpnames.Tablename}"" + ") + .Select(x => x.ConvertValueToString(0) ?? ""); + + var roots = paths + .Select(x => x.Substring(0, 1)) + .Distinct() + .Where(x => x != "\\"); //unc path like \\server.domain\ var regexUNCPrefix = new System.Text.RegularExpressions.Regex(@"^\\\\.*?\\"); - var rootsUNC = paths.Select(x => regexUNCPrefix.Match(x)).Where(x => x.Success).Select(x => x.Value).Distinct(); + var rootsUNC = paths + .Select(x => regexUNCPrefix.Match(x)) + .Where(x => x.Success) + .Select(x => x.Value) + .Distinct(); - await foreach (var el in roots.Concat(rootsUNC).Select(x => GetLargestPrefix(filter, x).FirstAsync()).Distinct()) + var result = roots + .Concat(rootsUNC) + .Select(x => GetLargestPrefix(filter, x) + .FirstAsync()) + .Distinct(); + + await foreach (var el in result) yield return await el; } @@ -232,7 +296,11 @@ namespace Duplicati.Library.Main.Database prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); var ppl = prefix.Length; - using (var rd = await cmd.ExecuteReaderAsync($@"SELECT DISTINCT ""Path"" FROM ""{table}"" ")) + cmd.SetCommandAndParameters($@" + SELECT DISTINCT ""Path"" + FROM ""{table}"" + "); + using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) { var s = rd.ConvertValueToString(0) ?? ""; @@ -245,6 +313,7 @@ namespace Duplicati.Library.Main.Database var ix = s.IndexOf(dirsep, StringComparison.Ordinal); if (ix > 0 && ix != s.Length - 1) s = s.Substring(0, ix + 1); + yield return prefix + s; } } @@ -257,7 +326,9 @@ namespace Duplicati.Library.Main.Database string pathprefix; if (filter == null || filter.Empty) pathprefix = ""; - else if (filter as FilterExpression == null || ((FilterExpression)filter).Type != FilterType.Simple || ((FilterExpression)filter).GetSimpleList().Length != 1) + else if (filter as FilterExpression == null + || ((FilterExpression)filter).Type != FilterType.Simple + || ((FilterExpression)filter).GetSimpleList().Length != 1) throw new ArgumentException("Filter for list-folder-contents must be a path prefix with no wildcards", nameof(filter)); else pathprefix = ((FilterExpression)filter).GetSimpleList().First(); @@ -271,7 +342,21 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""{tmpnames.Tablename}"" + WHERE ""Path"" NOT IN ( + SELECT DISTINCT ""Path"" + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND ""FilesetEntry"".""FilesetID"" IN ( + SELECT ""FilesetID"" + FROM ""{m_tablename}"" + ) + ) + "); // If we had instr support this would work: /*var distinctPaths = @"SELECT DISTINCT :1 || " + @@ -282,44 +367,99 @@ namespace Duplicati.Library.Main.Database @" FROM (" + cartesianPathFileset + @")";*/ // Instead we manually iterate the paths - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tbname}"" (""Path"" TEXT NOT NULL)"); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tbname}"" ( + ""Path"" TEXT NOT NULL + ) + "); using (var c2 = m_db.Connection.CreateCommand()) { - c2.SetCommandAndParameters($@"INSERT INTO ""{tbname}"" (""Path"") VALUES (@Path)"); + c2.SetCommandAndParameters($@" + INSERT INTO ""{tbname}"" (""Path"") + VALUES (@Path) + "); + await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) - await c2.SetParameterValue("@Path", n) + await c2 + .SetParameterValue("@Path", n) .ExecuteNonQueryAsync(); - await c2.ExecuteNonQueryAsync($@"CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"")"); + await c2.ExecuteNonQueryAsync($@" + CREATE INDEX ""{tbname}_PathIndex"" + ON ""{tbname}"" (""Path"") + "); } //Then we select the matching results - var filesets = $@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"; - var cartesianPathFileset = $@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tbname}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"; + var filesets = $@" + SELECT + ""FilesetID"", + ""Timestamp"" + FROM ""{m_tablename}"" + ORDER BY ""Timestamp"" DESC + "; - var filesWithSizes = $@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "; - var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; + var cartesianPathFileset = $@" + SELECT + ""A"".""Path"", + ""B"".""FilesetID"" + FROM + ""{tbname}"" A, + ({filesets}) B + ORDER BY + ""A"".""Path"" ASC, + ""B"".""Timestamp"" DESC + "; - using (var rd = await cmd.ExecuteReaderAsync(query)) - if (await rd.ReadAsync()) + var filesWithSizes = $@" + SELECT + ""Length"", + ""FilesetEntry"".""FilesetID"", + ""File"".""Path"" + FROM + ""Blockset"", + ""FilesetEntry"", + ""File"" + WHERE + ""File"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND FilesetEntry.""FilesetID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{m_tablename}"" + ) + "; + + var query = @$" + SELECT + ""C"".""Path"", + ""D"".""Length"", + ""C"".""FilesetID"" + FROM ({cartesianPathFileset}) C + LEFT OUTER JOIN ({filesWithSizes}) D + ON ""C"".""FilesetID"" = ""D"".""FilesetID"" + AND ""C"".""Path"" = ""D"".""Path"" + "; + + using var rd = await cmd.ExecuteReaderAsync(query); + if (await rd.ReadAsync()) + { + bool more; + do { - bool more; - do + var f = new Fileversion(rd); + if (!(string.IsNullOrWhiteSpace(f.Path) || f.Path == pathprefix)) { - var f = new Fileversion(rd); - if (!(string.IsNullOrWhiteSpace(f.Path) || f.Path == pathprefix)) - { - yield return f; - more = f.More; - } - else - { - more = rd.Read(); - } + yield return f; + more = f.More; + } + else + { + more = rd.Read(); + } - } while (more); - } + } while (more); + } } } finally @@ -341,31 +481,98 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) "); + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""{tmpnames.Tablename}"" + WHERE ""Path"" NOT IN ( + SELECT DISTINCT ""Path"" + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND ""FilesetEntry"".""FilesetID"" IN ( + SELECT ""FilesetID"" + FROM ""{m_tablename}"" + ) + ) + "); //Then we select the matching results - var filesets = $@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"; - var cartesianPathFileset = $@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tmpnames.Tablename}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"; - var filesWithSizes = $@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "; - var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; - using (var rd = await cmd.ExecuteReaderAsync(query)) - if (await rd.ReadAsync()) + var filesets = $@" + SELECT + ""FilesetID"", + ""Timestamp"" + FROM ""{m_tablename}"" + ORDER BY ""Timestamp"" DESC + "; + + var cartesianPathFileset = $@" + SELECT + ""A"".""Path"", + ""B"".""FilesetID"" + FROM + ""{tmpnames.Tablename}"" A, + ({filesets}) B + ORDER BY + ""A"".""Path"" ASC, + ""B"".""Timestamp"" DESC + "; + + var filesWithSizes = $@" + SELECT + ""Length"", + ""FilesetEntry"".""FilesetID"", + ""File"".""Path"" + FROM + ""Blockset"", + ""FilesetEntry"", + ""File"" + WHERE + ""File"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND FilesetEntry.""FilesetID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{m_tablename}"" + ) + "; + + var query = @$" + SELECT + ""C"".""Path"", + ""D"".""Length"", + ""C"".""FilesetID"" + FROM ({cartesianPathFileset}) C + LEFT OUTER JOIN ({filesWithSizes}) D + ON ""C"".""FilesetID"" = ""D"".""FilesetID"" + AND ""C"".""Path"" = ""D"".""Path"" + "; + + using var rd = await cmd.ExecuteReaderAsync(query); + if (await rd.ReadAsync()) + { + bool more; + do { - bool more; - do - { - var f = new Fileversion(rd); - yield return f; - more = f.More; - } while (more); - } + var f = new Fileversion(rd); + yield return f; + more = f.More; + } while (more); + } } } public async Task TakeFirst() { using var cmd = m_db.Connection.CreateCommand(); - await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )"); + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""{m_tablename}"" + WHERE ""FilesetID"" NOT IN ( + SELECT ""FilesetID"" + FROM ""{m_tablename}"" + ORDER BY ""Timestamp"" DESC + LIMIT 1 + ) + "); } public async IAsyncEnumerable QuickSets() @@ -375,12 +582,19 @@ namespace Duplicati.Library.Main.Database dict[m_filesets[i].Key] = i; using (var cmd = m_db.Connection.CreateCommand()) - using (var rd = await cmd.ExecuteReaderAsync(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) + using (var rd = await cmd.ExecuteReaderAsync(@" + SELECT DISTINCT + ""ID"", + ""IsFullBackup"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + ")) while (await rd.ReadAsync()) { var id = rd.ConvertValueToInt64(0); var backupType = rd.GetInt32(1); var e = dict[id]; + yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); } } @@ -391,23 +605,47 @@ namespace Duplicati.Library.Main.Database for (var i = 0; i < m_filesets.Length; i++) dict[m_filesets[i].Key] = i; - var summation = - $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "; + var summation = $@" + SELECT + ""A"".""FilesetID"" AS ""FilesetID"", + COUNT(*) AS ""FileCount"", + SUM(""C"".""Length"") AS ""FileSizes"" + FROM + ""FilesetEntry"" A, + ""File"" B, + ""Blockset"" C + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""BlocksetID"" = ""C"".""ID"" + AND ""A"".""FilesetID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{m_tablename}"") + GROUP BY ""A"".""FilesetID"" + "; - using (var cmd = m_db.Connection.CreateCommand()) - using (var rd = await cmd.ExecuteReaderAsync( - $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ") - ) - while (await rd.ReadAsync()) - { - var id = rd.ConvertValueToInt64(0); - var isFullBackup = rd.GetInt32(1); - var e = dict[id]; - var filecount = rd.ConvertValueToInt64(2, -1L); - var filesizes = rd.ConvertValueToInt64(3, -1L); + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + using var rd = await cmd.ExecuteReaderAsync($@" + SELECT DISTINCT + ""A"".""FilesetID"", + ""A"".""IsFullBackup"", + ""B"".""FileCount"", + ""B"".""FileSizes"" + FROM ""{m_tablename}"" A + LEFT OUTER JOIN ( {summation} ) B + ON ""A"".""FilesetID"" = ""B"".""FilesetID"" + ORDER BY ""A"".""Timestamp"" DESC + "); - yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); - } + while (await rd.ReadAsync()) + { + var id = rd.ConvertValueToInt64(0); + var isFullBackup = rd.GetInt32(1); + var e = dict[id]; + var filecount = rd.ConvertValueToInt64(2, -1L); + var filesizes = rd.ConvertValueToInt64(3, -1L); + + yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); + } } public void Dispose() @@ -452,25 +690,22 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable ListFilesetsExtended() { const string query = @" -SELECT - f.""ID"" AS ""FilesetID"", - f.""Timestamp"", - f.""IsFullBackup"", - COUNT(fe.""FileID"") AS ""NumberOfFiles"", - COALESCE(SUM(b.""Length""), 0) AS ""TotalFileSize"" -FROM - ""Fileset"" f -LEFT JOIN - ""FilesetEntry"" fe ON f.""ID"" = fe.""FilesetID"" -LEFT JOIN - ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" -LEFT JOIN - ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" -GROUP BY - f.""ID"" -ORDER BY - f.""Timestamp"" DESC; -"; + SELECT + f.""ID"" AS ""FilesetID"", + f.""Timestamp"", + f.""IsFullBackup"", + COUNT(fe.""FileID"") AS ""NumberOfFiles"", + COALESCE(SUM(b.""Length""), 0) AS ""TotalFileSize"" + FROM ""Fileset"" f + LEFT JOIN ""FilesetEntry"" fe + ON f.""ID"" = fe.""FilesetID"" + LEFT JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + LEFT JOIN ""Blockset"" b + ON fl.""BlocksetID"" = b.""ID"" + GROUP BY f.""ID"" + ORDER BY f.""Timestamp"" DESC; + "; using (var cmd = m_connection.CreateCommand(query)) await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -522,20 +757,36 @@ ORDER BY // Then query the matching files cmd.SetCommandAndParameters($@" - SELECT - pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - b.""Length"", - (CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END) AS ""IsDirectory"", - (CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END) AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" - WHERE fe.""FilesetID"" = @filesetid AND fl.""PrefixID"" IN (@PrefixIds) - ORDER BY pp.""Prefix"" ASC, fl.""Path"" ASC - LIMIT @limit OFFSET @offset - ") + SELECT + pp.""Prefix"" || fl.""Path"" AS ""FullPath"", + b.""Length"", + (CASE + WHEN fl.""BlocksetID"" = @FolderBlocksetId + THEN 1 + ELSE 0 + END) AS ""IsDirectory"", + (CASE + WHEN fl.""BlocksetID"" = @SymlinkBlocksetId + THEN 1 + ELSE 0 + END) AS ""IsSymlink"", + fe.""Lastmodified"" + FROM ""FilesetEntry"" fe + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + INNER JOIN ""PathPrefix"" pp + ON fl.""PrefixID"" = pp.""ID"" + LEFT JOIN ""Blockset"" b + ON fl.""BlocksetID"" = b.""ID"" + WHERE + fe.""FilesetID"" = @filesetid + AND fl.""PrefixID"" IN (@PrefixIds) + ORDER BY + pp.""Prefix"" ASC, + fl.""Path"" ASC + LIMIT @limit + OFFSET @offset + ") .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) @@ -578,9 +829,11 @@ ORDER BY totalCount = await cmd.SetCommandAndParameters($@" SELECT COUNT(*) FROM FilesetEntry fe - INNER JOIN FileLookup fl ON fe.FileID = fl.ID - WHERE fe.FilesetID = @filesetid - AND fl.PrefixID IN (@PrefixIds) + INNER JOIN FileLookup fl + ON fe.FileID = fl.ID + WHERE + fe.FilesetID = @filesetid + AND fl.PrefixID IN (@PrefixIds) ") .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) .SetParameterValue("@filesetid", filesetid) @@ -613,27 +866,42 @@ ORDER BY // Fetch entries cmd.SetCommandAndParameters(@" - SELECT - pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - b.""Length"", - CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", - CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" - WHERE fe.""FilesetId"" = @FilesetId - AND fl.""PrefixId"" IN (@PrefixIds) - ORDER BY pp.""Prefix"" ASC, fl.""Path"" ASC - LIMIT @limit OFFSET @offset - ") - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) - .SetParameterValue("@limit", limit) - .SetParameterValue("@offset", offset); + SELECT + pp.""Prefix"" || fl.""Path"" AS ""FullPath"", + b.""Length"", + CASE + WHEN fl.""BlocksetID"" = @FolderBlocksetId + THEN 1 + ELSE 0 + END AS ""IsDirectory"", + CASE + WHEN fl.""BlocksetID"" = @SymlinkBlocksetId + THEN 1 + ELSE 0 + END AS ""IsSymlink"", + fe.""Lastmodified"" + FROM ""FilesetEntry"" fe + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + INNER JOIN ""PathPrefix"" pp + ON fl.""PrefixID"" = pp.""ID"" + LEFT JOIN ""Blockset"" b + ON fl.""BlocksetID"" = b.""ID"" + WHERE + fe.""FilesetId"" = @FilesetId + AND fl.""PrefixId"" IN (@PrefixIds) + ORDER BY + pp.""Prefix"" ASC, + fl.""Path"" ASC + LIMIT @limit + OFFSET @offset + ") + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .SetParameterValue("@FilesetId", filesetid) + .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) + .SetParameterValue("@limit", limit) + .SetParameterValue("@offset", offset); var results = new List(); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -663,13 +931,15 @@ ORDER BY totalCount = await cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - WHERE fe.""FilesetId"" = @FilesetId - AND fl.""PrefixId"" IN (@PrefixIds) + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + WHERE + fe.""FilesetId"" = @FilesetId + AND fl.""PrefixId"" IN (@PrefixIds) ") - .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) - .ExecuteScalarInt64Async(0); + .SetParameterValue("@FilesetId", filesetid) + .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) + .ExecuteScalarInt64Async(0); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; @@ -686,7 +956,11 @@ ORDER BY using var cmd = m_connection.CreateCommand(); // Map folders to prefix IDs - cmd.SetCommandAndParameters($@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" IN (@Prefixes)") + cmd.SetCommandAndParameters($@" + SELECT ""ID"" + FROM ""PathPrefix"" + WHERE ""Prefix"" IN (@Prefixes) + ") .ExpandInClauseParameter("@Prefixes", prefixes); return cmd.ExecuteReaderEnumerableAsync() @@ -703,25 +977,30 @@ ORDER BY using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters(@" - WITH Prefixes AS ( - SELECT DISTINCT fl.""PrefixID"", pp.""Prefix"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - WHERE fe.""FilesetID"" = @filesetid - ) - SELECT p1.""PrefixID"" - FROM Prefixes p1 - WHERE NOT EXISTS ( - SELECT 1 - FROM Prefixes p2 - WHERE p2.""PrefixID"" != p1.""PrefixID"" - AND p1.""Prefix"" LIKE p2.""Prefix"" || '%' - AND LENGTH(p1.""Prefix"") > LENGTH(p2.""Prefix"") - ) - ORDER BY p1.""Prefix"" ASC - ") - .SetParameterValue("@filesetid", filesetid); + WITH Prefixes AS ( + SELECT DISTINCT + fl.""PrefixID"", + pp.""Prefix"" + FROM ""FilesetEntry"" fe + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + INNER JOIN ""PathPrefix"" pp + ON fl.""PrefixID"" = pp.""ID"" + WHERE fe.""FilesetID"" = @filesetid + ) + SELECT p1.""PrefixID"" + FROM Prefixes p1 + WHERE NOT EXISTS ( + SELECT 1 + FROM Prefixes p2 + WHERE + p2.""PrefixID"" != p1.""PrefixID"" + AND p1.""Prefix"" LIKE p2.""Prefix"" || '%' + AND LENGTH(p1.""Prefix"") > LENGTH(p2.""Prefix"") + ) + ORDER BY p1.""Prefix"" ASC + ") + .SetParameterValue("@filesetid", filesetid); return cmd.ExecuteReaderEnumerableAsync() .Select(x => x.ConvertValueToInt64(0, -1)) @@ -733,46 +1012,54 @@ ORDER BY /// /// The filesetId to get the prefixes for /// A list of (PrefixID, ParentPrefixID) pairs - public IAsyncEnumerable<(long PrefixId, long ParentPrefixId)> GetRootPrefixesWithParents(long filesetid) + public async Task> GetRootPrefixesWithParents(long filesetid) { using var cmd = m_connection.CreateCommand(); - var dirSeparator = Util.GuessDirSeparator(cmd.SetCommandAndParameters(@"SELECT ""Prefix"" FROM ""PathPrefix"" LIMIT 1") - .ExecuteScalar()?.ToString()); + var dirSeparator = Util.GuessDirSeparator((await cmd.SetCommandAndParameters(@" + SELECT ""Prefix"" + FROM ""PathPrefix"" + LIMIT 1 + ") + .ExecuteScalarAsync())?.ToString()); cmd.SetCommandAndParameters(@" - WITH Prefixes AS ( - SELECT DISTINCT fl.""PrefixID"", pp.""Prefix"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - WHERE fe.""FilesetID"" = @filesetid - ), - PrefixParents AS ( - SELECT - child.""PrefixID"" AS ChildPrefixID, - parent.""PrefixID"" AS ParentPrefixID, - LENGTH(parent.""Prefix"") AS ParentLength - FROM Prefixes child - JOIN Prefixes parent - ON LENGTH(child.""Prefix"") > LENGTH(parent.""Prefix"") - AND SUBSTR(child.""Prefix"", 1, LENGTH(parent.""Prefix"")) = parent.""Prefix"" - AND (SUBSTR(child.""Prefix"", LENGTH(parent.""Prefix"") + 1, 1) = @dirSeparator) - ) - SELECT - child.""PrefixID"", - ( - SELECT ParentPrefixID - FROM PrefixParents pp - WHERE pp.ChildPrefixID = child.""PrefixID"" - ORDER BY pp.ParentLength DESC - LIMIT 1 - ) AS ParentPrefixID - FROM Prefixes child - ORDER BY child.""Prefix"" ASC - ") - .SetParameterValue("@dirSeparator", dirSeparator) - .SetParameterValue("@filesetid", filesetid); + WITH Prefixes AS ( + SELECT DISTINCT + fl.""PrefixID"", + pp.""Prefix"" + FROM ""FilesetEntry"" fe + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + INNER JOIN ""PathPrefix"" pp + ON fl.""PrefixID"" = pp.""ID"" + WHERE fe.""FilesetID"" = @filesetid + ), + PrefixParents AS ( + SELECT + child.""PrefixID"" AS ChildPrefixID, + parent.""PrefixID"" AS ParentPrefixID, + LENGTH(parent.""Prefix"") AS ParentLength + FROM Prefixes child + JOIN Prefixes parent + ON LENGTH(child.""Prefix"") > LENGTH(parent.""Prefix"") + AND SUBSTR(child.""Prefix"", 1, LENGTH(parent.""Prefix"")) = parent.""Prefix"" + AND (SUBSTR(child.""Prefix"", LENGTH(parent.""Prefix"") + 1, 1) = @dirSeparator) + ) + SELECT + child.""PrefixID"", + ( + SELECT ParentPrefixID + FROM PrefixParents pp + WHERE pp.ChildPrefixID = child.""PrefixID"" + ORDER BY pp.ParentLength DESC + LIMIT 1 + ) AS ParentPrefixID + FROM Prefixes child + ORDER BY child.""Prefix"" ASC + ") + .SetParameterValue("@dirSeparator", dirSeparator) + .SetParameterValue("@filesetid", filesetid); return cmd.ExecuteReaderEnumerableAsync() .Select(x => ( @@ -805,29 +1092,48 @@ ORDER BY if (filesetIds != null && filesetIds.Length > 0) countWhere += " AND fe.\"FilesetID\" IN (@FilesetIds)"; - cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"); + cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); var versionMap = await cmd.ExecuteReaderEnumerableAsync() .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) .ToDictionaryAsync(x => x.FilesetId, x => x.Version); // Then fetch the actual data - var fetchSql = @" - SELECT - fe.""FilesetID"", - f.""Timestamp"", - fl.""Path"", - COALESCE(b.""Length"", 0) AS ""Size"", - CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", - CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""Fileset"" f ON fe.""FilesetID"" = f.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" - " + countWhere + @" - ORDER BY fl.""Path"" ASC, f.""Timestamp"" ASC - LIMIT @limit OFFSET @offset"; + var fetchSql = @$" + SELECT + fe.""FilesetID"", + f.""Timestamp"", + fl.""Path"", + COALESCE(b.""Length"", 0) AS ""Size"", + CASE + WHEN fl.""BlocksetID"" = @FolderBlocksetId + THEN 1 + ELSE 0 + END AS ""IsDirectory"", + CASE + WHEN fl.""BlocksetID"" = @SymlinkBlocksetId + THEN 1 + ELSE 0 + END AS ""IsSymlink"", + fe.""Lastmodified"" + FROM ""FilesetEntry"" fe + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + INNER JOIN ""Fileset"" f + ON fe.""FilesetID"" = f.""ID"" + LEFT JOIN ""Blockset"" b + ON fl.""BlocksetID"" = b.""ID"" + {countWhere} + ORDER BY + fl.""Path"" ASC, + f.""Timestamp"" ASC + LIMIT @limit + OFFSET @offset + "; cmd.SetCommandAndParameters(fetchSql) .ExpandInClauseParameter("@Paths", paths) @@ -878,7 +1184,8 @@ ORDER BY var countSql = @" SELECT COUNT(*) FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" "; cmd.SetCommandAndParameters(countSql + "\n" + countWhere) @@ -970,9 +1277,15 @@ ORDER BY var propName = $"@Part{filterProps.Count}"; filterProps[propName] = part; if (caseSensitive) - caseWhenParts.Add($@"WHEN instr(LOWER(pp.""Prefix"" || fl.""Path""), LOWER({propName})) > 0 THEN {(filterExpression.Result ? 0 : 1)}"); + caseWhenParts.Add($@" + WHEN instr(LOWER(pp.""Prefix"" || fl.""Path""), LOWER({propName})) > 0 + THEN {(filterExpression.Result ? 0 : 1)} + "); else - caseWhenParts.Add($@"WHEN instr(pp.""Prefix"" || fl.""Path"", {propName}) > 0 THEN {(filterExpression.Result ? 0 : 1)}"); + caseWhenParts.Add($@" + WHEN instr(pp.""Prefix"" || fl.""Path"", {propName}) > 0 + THEN {(filterExpression.Result ? 0 : 1)} + "); } } else if (filterExpression.Type == FilterType.Wildcard) @@ -983,9 +1296,15 @@ ORDER BY var propName = $"@Part{filterProps.Count}"; filterProps[propName] = part; if (caseSensitive) - caseWhenParts.Add($@"WHEN LOWER(pp.""Prefix"" || fl.""Path"") GLOB LOWER({propName}) THEN {(filterExpression.Result ? 0 : 1)}"); + caseWhenParts.Add($@" + WHEN LOWER(pp.""Prefix"" || fl.""Path"") GLOB LOWER({propName}) + THEN {(filterExpression.Result ? 0 : 1)} + "); else - caseWhenParts.Add($@"WHEN pp.""Prefix"" || fl.""Path"" GLOB {propName} THEN {(filterExpression.Result ? 0 : 1)}"); + caseWhenParts.Add($@" + WHEN pp.""Prefix"" || fl.""Path"" GLOB {propName} + THEN {(filterExpression.Result ? 0 : 1)} + "); } } else @@ -1004,13 +1323,15 @@ ORDER BY if (caseWhenParts.Any()) { - whereClauses.Add($@"COALESCE( - CASE - {string.Join("\n ", caseWhenParts)} - ELSE NULL - END, - @DefaultBehavior - ) = 0"); + whereClauses.Add($@" + COALESCE( + CASE + {string.Join("\n ", caseWhenParts)} + ELSE NULL + END, + @DefaultBehavior + ) = 0 + "); } else { @@ -1024,7 +1345,11 @@ ORDER BY var whereClause = string.Join("\n AND ", whereClauses); - cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"); + cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); var versionMap = await cmd.ExecuteReaderEnumerableAsync() .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) @@ -1032,23 +1357,39 @@ ORDER BY // Fetch results var fetchSql = $@" - SELECT - fe.""FilesetID"", - f.""Timestamp"", - pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - COALESCE(b.""Length"", 0) AS ""Size"", - CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", - CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""Fileset"" f ON fe.""FilesetID"" = f.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - WHERE {whereClause} - ORDER BY pp.""Prefix"" ASC, fl.""Path"" ASC, f.""Timestamp"" ASC - LIMIT @limit OFFSET @offset - "; + SELECT + fe.""FilesetID"", + f.""Timestamp"", + pp.""Prefix"" || fl.""Path"" AS ""FullPath"", + COALESCE(b.""Length"", 0) AS ""Size"", + CASE + WHEN fl.""BlocksetID"" = @FolderBlocksetId + THEN 1 + ELSE 0 + END AS ""IsDirectory"", + CASE + WHEN fl.""BlocksetID"" = @SymlinkBlocksetId + THEN 1 + ELSE 0 + END AS ""IsSymlink"", + fe.""Lastmodified"" + FROM ""FilesetEntry"" fe + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + INNER JOIN ""Fileset"" f + ON fe.""FilesetID"" = f.""ID"" + LEFT JOIN ""Blockset"" b + ON fl.""BlocksetID"" = b.""ID"" + INNER JOIN ""PathPrefix"" pp + ON fl.""PrefixID"" = pp.""ID"" + WHERE {whereClause} + ORDER BY + pp.""Prefix"" ASC, + fl.""Path"" ASC, + f.""Timestamp"" ASC + LIMIT @limit + OFFSET @offset + "; cmd.SetCommandAndParameters(fetchSql) .SetParameterValues(filterProps) @@ -1097,8 +1438,10 @@ ORDER BY var countSql = $@" SELECT COUNT(*) FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" + INNER JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + INNER JOIN ""PathPrefix"" pp + ON fl.""PrefixID"" = pp.""ID"" WHERE {whereClause} "; @@ -1128,4 +1471,3 @@ ORDER BY } } } - From 2e570a58629fb4b983206efef225b229160aa8b5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 15:54:23 +0200 Subject: [PATCH 097/362] Changed LocalPurgeDatabase to use factory construction --- .../Main/Database/LocalPurgeDatabase.cs | 90 +++++++++++-------- 1 file changed, 54 insertions(+), 36 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 2bdb15209..59b7ba40c 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -24,21 +24,30 @@ using System; using System.Data; using System.Collections.Generic; +using System.Threading.Tasks; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Database { internal class LocalPurgeDatabase : LocalDeleteDatabase { - public LocalPurgeDatabase(string path, long pagecachesize) - : base(path, "Purge", pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize) { - ShouldCloseConnection = true; + var db = new LocalPurgeDatabase(); + + db = (LocalPurgeDatabase)await CreateLocalDatabaseAsync(db, path, "ListBrokenFiles", false, pagecachesize); + db.ShouldCloseConnection = true; + + return db; } - public LocalPurgeDatabase(LocalDatabase db) - : base(db) + public static async Task CreateAsync(LocalDatabase dbparent) { + var dbnew = new LocalPurgeDatabase(); + + dbnew = (LocalPurgeDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + + return dbnew; } public ITemporaryFileset CreateTemporaryFileset(long parentid, IDbTransaction transaction) @@ -82,26 +91,35 @@ namespace Duplicati.Library.Main.Database private class TemporaryFileset : ITemporaryFileset { - private readonly IDbConnection m_connection; - private readonly IDbTransaction m_transaction; - private readonly string m_tablename; - private readonly LocalPurgeDatabase m_parentdb; + private string m_tablename = null!; + private LocalDatabase m_db = null!; public long ParentID { get; private set; } public long RemovedFileCount { get; private set; } public long RemovedFileSize { get; private set; } public long UpdatedFileCount { get; private set; } + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, IDbConnection connection, IDbTransaction transaction) { - ParentID = parentid; - m_parentdb = parentdb; - m_connection = connection; - m_transaction = transaction; - m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + throw new NotImplementedException("Use CreateAsync instead"); + } - using (var cmd = m_connection.CreateCommand(m_transaction)) + private TemporaryFileset() { } + + public static async Task CreateAsync(long parentid, LocalDatabase db) + { + var tempf = new TemporaryFileset() + { + ParentID = parentid, + m_db = db, + m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) + }; + + using (var cmd = db.Connection.CreateCommand()) cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) ")); + + return tempf; } public void ApplyFilter(Func filtercommand) @@ -169,13 +187,13 @@ namespace Duplicati.Library.Main.Database UpdatedFileCount = updated; RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0); RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@" - SELECT - SUM(""C"".""Length"") + SELECT + SUM(""C"".""Length"") FROM ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C, ""Metadataset"" D WHERE ""A"".""FileID"" = ""B"".""ID"" - AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID"")) + AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID"")) "), 0); var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0); if (filesetcount == RemovedFileCount) @@ -185,9 +203,9 @@ namespace Duplicati.Library.Main.Database public Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup) { - var remotevolid = m_parentdb.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction); - var filesetid = m_parentdb.CreateFileset(remotevolid, timestamp, m_transaction); - m_parentdb.UpdateFullBackupStateInFileset(filesetid, isFullBackup); + var remotevolid = m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction); + var filesetid = m_db.CreateFileset(remotevolid, timestamp, m_transaction); + m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); using (var cmd = m_connection.CreateCommand(m_transaction)) cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" ")) From 1ddce07d0fbddfb5f9a30abaf2cdee5c13a6863f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 15:59:07 +0200 Subject: [PATCH 098/362] Purged IDb from LocalPurgeDatabase --- .../Main/Database/LocalPurgeDatabase.cs | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 59b7ba40c..0455b2d0a 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -22,10 +22,10 @@ #nullable enable using System; -using System.Data; using System.Collections.Generic; using System.Threading.Tasks; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -50,14 +50,9 @@ namespace Duplicati.Library.Main.Database return dbnew; } - public ITemporaryFileset CreateTemporaryFileset(long parentid, IDbTransaction transaction) + public string GetRemoteVolumeNameForFileset(long id) { - return new TemporaryFileset(parentid, this, m_connection, transaction); - } - - public string GetRemoteVolumeNameForFileset(long id, IDbTransaction transaction) - { - using var cmd = m_connection.CreateCommand(transaction, @"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = @FilesetId ") + using var cmd = m_connection.CreateCommand(@"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = @FilesetId ") .SetParameterValue("@FilesetId", id); using (var rd = cmd.ExecuteReader()) if (!rd.Read()) @@ -66,9 +61,9 @@ namespace Duplicati.Library.Main.Database return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); } - internal long CountOrphanFiles(IDbTransaction transaction) + internal long CountOrphanFiles() { - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand()) using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")")) if (rd.Read()) return rd.ConvertValueToInt64(0, 0); @@ -84,7 +79,7 @@ namespace Duplicati.Library.Main.Database long UpdatedFileCount { get; } void ApplyFilter(Library.Utility.IFilter filter); - void ApplyFilter(Func filtercommand); + void ApplyFilter(Func filtercommand); Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); IEnumerable> ListAllDeletedFiles(); } @@ -100,7 +95,7 @@ namespace Duplicati.Library.Main.Database public long UpdatedFileCount { get; private set; } [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] - public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, IDbConnection connection, IDbTransaction transaction) + public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, SqliteConnection connection, SqliteTransaction transaction) { throw new NotImplementedException("Use CreateAsync instead"); } @@ -122,10 +117,10 @@ namespace Duplicati.Library.Main.Database return tempf; } - public void ApplyFilter(Func filtercommand) + public void ApplyFilter(Func filtercommand) { int updated; - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_connection.CreateCommand()) updated = filtercommand(cmd, ParentID, m_tablename); PostFilterChecks(updated); @@ -140,7 +135,7 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_connection.CreateCommand()) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) ")); cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)")); @@ -158,8 +153,8 @@ namespace Duplicati.Library.Main.Database { // Do row-wise iteration var values = new object[2]; - using (var cmd = m_connection.CreateCommand(m_transaction)) - using (var cmd2 = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_connection.CreateCommand()) + using (var cmd2 = m_connection.CreateCommand()) { cmd2.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId)")); cmd.SetCommandAndParameters(@"SELECT ""B"".""Path"", ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" ") @@ -182,7 +177,7 @@ namespace Duplicati.Library.Main.Database private void PostFilterChecks(int updated) { - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_connection.CreateCommand()) { UpdatedFileCount = updated; RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0); @@ -203,11 +198,11 @@ namespace Duplicati.Library.Main.Database public Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup) { - var remotevolid = m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction); - var filesetid = m_db.CreateFileset(remotevolid, timestamp, m_transaction); + var remotevolid = m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary); + var filesetid = m_db.CreateFileset(remotevolid, timestamp); m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_connection.CreateCommand()) cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" ")) .SetParameterValue("@TargetFilesetId", filesetid) .SetParameterValue("@SourceFilesetId", ParentID) @@ -218,7 +213,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable> ListAllDeletedFiles() { - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_connection.CreateCommand()) using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "))) while (rd.Read()) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); @@ -228,7 +223,7 @@ namespace Duplicati.Library.Main.Database { try { - using (var cmd = m_connection.CreateCommand(m_transaction)) + using (var cmd = m_connection.CreateCommand()) cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); } catch From 5f65c877d90dc82d3e4d74fe0f6a43396d6da354 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 16:03:12 +0200 Subject: [PATCH 099/362] Removed FormatInvariant as it's now handled by the extension methods --- .../Main/Database/LocalPurgeDatabase.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 0455b2d0a..2a55fc66d 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -112,7 +112,7 @@ namespace Duplicati.Library.Main.Database }; using (var cmd = db.Connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) ")); + cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) "); return tempf; } @@ -137,16 +137,16 @@ namespace Duplicati.Library.Main.Database var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) ")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)")); + cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "); + cmd.SetCommandAndParameters($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)"); foreach (var s in p) cmd.SetParameterValue("@Path", s) .ExecuteNonQuery(); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}""")) + cmd.SetCommandAndParameters($@"INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}""") .SetParameterValue("@FilesetId", ParentID) .ExecuteNonQuery(); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" ")); + cmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); } } else @@ -156,7 +156,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) using (var cmd2 = m_connection.CreateCommand()) { - cmd2.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId)")); + cmd2.SetCommandAndParameters($@"INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId)"); cmd.SetCommandAndParameters(@"SELECT ""B"".""Path"", ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" ") .SetParameterValue("@FilesetId", ParentID); using (var rd = cmd.ExecuteReader()) @@ -180,8 +180,8 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { UpdatedFileCount = updated; - RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0); - RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@" + RemovedFileCount = cmd.ExecuteScalarInt64($@"SELECT COUNT(*) FROM ""{m_tablename}""", 0); + RemovedFileSize = cmd.ExecuteScalarInt64($@" SELECT SUM(""C"".""Length"") FROM @@ -190,7 +190,7 @@ namespace Duplicati.Library.Main.Database ""A"".""FileID"" = ""B"".""ID"" AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID"")) "), 0); - var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0); + var filesetcount = cmd.ExecuteScalarInt64($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}", 0); if (filesetcount == RemovedFileCount) throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } @@ -203,7 +203,7 @@ namespace Duplicati.Library.Main.Database m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); using (var cmd = m_connection.CreateCommand()) - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" ")) + cmd.SetCommandAndParameters($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" ") .SetParameterValue("@TargetFilesetId", filesetid) .SetParameterValue("@SourceFilesetId", ParentID) .ExecuteNonQuery(); @@ -214,7 +214,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable> ListAllDeletedFiles() { using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "))) + using (var rd = cmd.ExecuteReader($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ")) while (rd.Read()) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } @@ -224,7 +224,7 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); + cmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{m_tablename}"""); } catch { From a720a692912b87d131fb1fc5c1f64c6827bc9277 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 16:15:37 +0200 Subject: [PATCH 100/362] Made LocalPurgeDatabase async --- .../Main/Database/LocalPurgeDatabase.cs | 108 +++++++++--------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 2a55fc66d..e7bc1cc04 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -31,7 +31,7 @@ namespace Duplicati.Library.Main.Database { internal class LocalPurgeDatabase : LocalDeleteDatabase { - public static async Task CreateAsync(string path, long pagecachesize) + public static new async Task CreateAsync(string path, long pagecachesize) { var db = new LocalPurgeDatabase(); @@ -41,7 +41,7 @@ namespace Duplicati.Library.Main.Database return db; } - public static async Task CreateAsync(LocalDatabase dbparent) + public static new async Task CreateAsync(LocalDatabase dbparent) { var dbnew = new LocalPurgeDatabase(); @@ -50,22 +50,22 @@ namespace Duplicati.Library.Main.Database return dbnew; } - public string GetRemoteVolumeNameForFileset(long id) + public async Task GetRemoteVolumeNameForFileset(long id) { using var cmd = m_connection.CreateCommand(@"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = @FilesetId ") .SetParameterValue("@FilesetId", id); - using (var rd = cmd.ExecuteReader()) - if (!rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + if (!await rd.ReadAsync()) throw new Exception($"No remote volume found for fileset with id {id}"); else return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); } - internal long CountOrphanFiles() + internal async Task CountOrphanFiles() { using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")")) - if (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")")) + if (await rd.ReadAsync()) return rd.ConvertValueToInt64(0, 0); else return 0; @@ -78,10 +78,10 @@ namespace Duplicati.Library.Main.Database long RemovedFileSize { get; } long UpdatedFileCount { get; } - void ApplyFilter(Library.Utility.IFilter filter); - void ApplyFilter(Func filtercommand); - Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); - IEnumerable> ListAllDeletedFiles(); + Task ApplyFilter(Library.Utility.IFilter filter); + Task ApplyFilter(Func filtercommand); + Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); + IAsyncEnumerable> ListAllDeletedFiles(); } private class TemporaryFileset : ITemporaryFileset @@ -112,21 +112,21 @@ namespace Duplicati.Library.Main.Database }; using (var cmd = db.Connection.CreateCommand()) - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) "); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tempf.m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) "); return tempf; } - public void ApplyFilter(Func filtercommand) + public async Task ApplyFilter(Func filtercommand) { int updated; - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_db.Connection.CreateCommand()) updated = filtercommand(cmd, ParentID, m_tablename); - PostFilterChecks(updated); + await PostFilterChecks(updated); } - public void ApplyFilter(Library.Utility.IFilter filter) + public async Task ApplyFilter(Library.Utility.IFilter filter) { if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { @@ -135,53 +135,54 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_db.Connection.CreateCommand()) { - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "); - cmd.SetCommandAndParameters($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)"); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "); + await cmd.SetCommandAndParameters($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)") + .PrepareAsync(); foreach (var s in p) - cmd.SetParameterValue("@Path", s) - .ExecuteNonQuery(); + await cmd.SetParameterValue("@Path", s) + .ExecuteNonQueryAsync(); - cmd.SetCommandAndParameters($@"INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}""") + await cmd.SetCommandAndParameters($@"INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}""") .SetParameterValue("@FilesetId", ParentID) - .ExecuteNonQuery(); - cmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); + .ExecuteNonQueryAsync(); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); } } else { // Do row-wise iteration var values = new object[2]; - using (var cmd = m_connection.CreateCommand()) - using (var cmd2 = m_connection.CreateCommand()) + using (var cmd = m_db.Connection.CreateCommand()) + using (var cmd2 = m_db.Connection.CreateCommand()) { cmd2.SetCommandAndParameters($@"INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId)"); cmd.SetCommandAndParameters(@"SELECT ""B"".""Path"", ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" ") .SetParameterValue("@FilesetId", ParentID); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) { rd.GetValues(values); if (values[0] != null && values[0] != DBNull.Value && Library.Utility.FilterExpression.Matches(filter, values[0].ToString())) { - cmd2.SetParameterValue("@FileId", values[1]) - .ExecuteNonQuery(); + await cmd2.SetParameterValue("@FileId", values[1]) + .ExecuteNonQueryAsync(); } } } } - PostFilterChecks(0); + await PostFilterChecks(0); } - private void PostFilterChecks(int updated) + private async Task PostFilterChecks(int updated) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_db.Connection.CreateCommand()) { UpdatedFileCount = updated; - RemovedFileCount = cmd.ExecuteScalarInt64($@"SELECT COUNT(*) FROM ""{m_tablename}""", 0); - RemovedFileSize = cmd.ExecuteScalarInt64($@" + RemovedFileCount = await cmd.ExecuteScalarInt64Async($@"SELECT COUNT(*) FROM ""{m_tablename}""", 0); + RemovedFileSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""C"".""Length"") FROM @@ -189,42 +190,47 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""FileID"" = ""B"".""ID"" AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID"")) - "), 0); - var filesetcount = cmd.ExecuteScalarInt64($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}", 0); + ", 0); + var filesetcount = await cmd.ExecuteScalarInt64Async($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}", 0); if (filesetcount == RemovedFileCount) throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } } - public Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup) + public async Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup) { - var remotevolid = m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary); - var filesetid = m_db.CreateFileset(remotevolid, timestamp); - m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); + var remotevolid = await m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary); + var filesetid = await m_db.CreateFileset(remotevolid, timestamp); + await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); - using (var cmd = m_connection.CreateCommand()) - cmd.SetCommandAndParameters($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" ") + using (var cmd = m_db.Connection.CreateCommand()) + await cmd.SetCommandAndParameters($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" ") .SetParameterValue("@TargetFilesetId", filesetid) .SetParameterValue("@SourceFilesetId", ParentID) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); return new Tuple(remotevolid, filesetid); } - public IEnumerable> ListAllDeletedFiles() + public async IAsyncEnumerable> ListAllDeletedFiles() { - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ")) - while (rd.Read()) + using (var cmd = m_db.Connection.CreateCommand()) + using (var rd = await cmd.ExecuteReaderAsync($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ")) + while (await rd.ReadAsync()) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } public void Dispose() + { + DisposeAsync().Await(); + } + + public async Task DisposeAsync() { try { - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{m_tablename}"""); + using (var cmd = m_db.Connection.CreateCommand()) + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}"""); } catch { From 4ca147d0f11f94cf2a2e24567ed4e65bf8b030ce Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 16:27:07 +0200 Subject: [PATCH 101/362] More preliminary work towards implicit reusable transaction --- .../Library/Main/Database/LocalDatabase.cs | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index e8b01d7f5..4c3e3a587 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -956,24 +956,23 @@ namespace Duplicati.Library.Main.Database public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state) { - var transaction = m_connection.BeginTransaction(); - return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), transaction); + return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0)); } - public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, SqliteTransaction transaction) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state) { - return await RegisterRemoteVolume(name, type, state, new TimeSpan(0), transaction); + return await RegisterRemoteVolume(name, type, state, new TimeSpan(0)); } - public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, SqliteTransaction transaction) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime) { - return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime, transaction); + return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime); } - public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, SqliteTransaction transaction) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime) { var r = await m_createremotevolumeCommand - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@OperationId", m_operationid) .SetParameterValue("@Name", name) .SetParameterValue("@Type", type.ToString()) @@ -984,7 +983,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@ArchiveTime", 0) .ExecuteScalarInt64Async(); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); return r; } @@ -2055,7 +2054,7 @@ namespace Duplicati.Library.Main.Database } } - public async Task RenameRemoteFile(string oldname, string newname, SqliteTransaction transaction) + public async Task RenameRemoteFile(string oldname, string newname) { //Rename the old entry, to preserve ID links using var cmd = await m_connection.CreateCommandAsync(@" @@ -2063,7 +2062,7 @@ namespace Duplicati.Library.Main.Database SET ""Name"" = @Newname WHERE ""Name"" = @Oldname "); - var c = await cmd.SetTransaction(transaction) + var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@Newname", newname) .SetParameterValue("@Oldname", oldname) .ExecuteNonQueryAsync(); @@ -2084,9 +2083,9 @@ namespace Duplicati.Library.Main.Database //Create a fake new entry with the old name and mark as deleting // as this ensures we will remove it, if it shows up in some later listing - await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, transaction); + await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); } /// @@ -2218,19 +2217,19 @@ namespace Duplicati.Library.Main.Database /// Existing file set to update /// Full backup state /// An optional external transaction - public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup, SqliteTransaction transaction) + public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup) { using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""Fileset"" SET ""IsFullBackup"" = @IsFullBackup WHERE ""ID"" = @FilesetId "); - await cmd.SetTransaction(transaction) + await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId) .SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP) .ExecuteNonQueryAsync(); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); } /// From 6a5214f72d13d3728e1e55ec4fc9d2c9bd1a9e7b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 16:27:33 +0200 Subject: [PATCH 102/362] Restructered SQL queries in LocalPurgeDatabase --- .../Main/Database/LocalPurgeDatabase.cs | 164 ++++++++++++++---- 1 file changed, 132 insertions(+), 32 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index e7bc1cc04..24c6c272f 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -35,7 +35,7 @@ namespace Duplicati.Library.Main.Database { var db = new LocalPurgeDatabase(); - db = (LocalPurgeDatabase)await CreateLocalDatabaseAsync(db, path, "ListBrokenFiles", false, pagecachesize); + db = (LocalPurgeDatabase)await CreateLocalDatabaseAsync(db, path, "Purge", false, pagecachesize); db.ShouldCloseConnection = true; return db; @@ -52,8 +52,18 @@ namespace Duplicati.Library.Main.Database public async Task GetRemoteVolumeNameForFileset(long id) { - using var cmd = m_connection.CreateCommand(@"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = @FilesetId ") - .SetParameterValue("@FilesetId", id); + using var cmd = m_connection.CreateCommand(@" + SELECT + ""B"".""Name"" + FROM + ""Fileset"" A, + ""RemoteVolume"" B + WHERE + ""A"".""VolumeID"" = ""B"".""ID"" + AND ""A"".""ID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", id); + using (var rd = await cmd.ExecuteReaderAsync()) if (!await rd.ReadAsync()) throw new Exception($"No remote volume found for fileset with id {id}"); @@ -63,12 +73,20 @@ namespace Duplicati.Library.Main.Database internal async Task CountOrphanFiles() { - using (var cmd = m_connection.CreateCommand()) - using (var rd = await cmd.ExecuteReaderAsync(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")")) - if (await rd.ReadAsync()) - return rd.ConvertValueToInt64(0, 0); - else - return 0; + using var cmd = m_connection.CreateCommand(m_rtr); + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""FileLookup"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""FileID"" + FROM ""FilesetEntry"" + ) + "); + using var rd = await cmd.ExecuteReaderAsync(); + if (await rd.ReadAsync()) + return rd.ConvertValueToInt64(0, 0); + else + return 0; } public interface ITemporaryFileset : IDisposable @@ -112,7 +130,11 @@ namespace Duplicati.Library.Main.Database }; using (var cmd = db.Connection.CreateCommand()) - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tempf.m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tempf.m_tablename}"" ( + ""FileID"" INTEGER PRIMARY KEY + ) + "); return tempf; } @@ -137,16 +159,36 @@ namespace Duplicati.Library.Main.Database var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_db.Connection.CreateCommand()) { - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "); - await cmd.SetCommandAndParameters($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)") + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{filenamestable}"" ( + ""Path"" TEXT NOT NULL + ) + "); + + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{filenamestable}"" (""Path"") + VALUES (@Path) + ") .PrepareAsync(); + foreach (var s in p) await cmd.SetParameterValue("@Path", s) .ExecuteNonQueryAsync(); - await cmd.SetCommandAndParameters($@"INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}""") + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_tablename}"" (""FileID"") + SELECT DISTINCT ""A"".""FileID"" + FROM + ""FilesetEntry"" A, + ""File"" B + WHERE + ""A"".""FilesetID"" = @FilesetId + AND ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""Path"" IN ""{filenamestable}"" + ") .SetParameterValue("@FilesetId", ParentID) .ExecuteNonQueryAsync(); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); } } @@ -157,16 +199,33 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_db.Connection.CreateCommand()) using (var cmd2 = m_db.Connection.CreateCommand()) { - cmd2.SetCommandAndParameters($@"INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId)"); - cmd.SetCommandAndParameters(@"SELECT ""B"".""Path"", ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" ") + await cmd2.SetCommandAndParameters($@" + INSERT INTO ""{m_tablename}"" (""FileID"") + VALUES (@FileId) + ") + .PrepareAsync(); + + cmd.SetCommandAndParameters(@" + SELECT + ""B"".""Path"", + ""A"".""FileID"" + FROM + ""FilesetEntry"" A, + ""File"" B + WHERE + ""A"".""FilesetID"" = @FilesetId + AND ""A"".""FileID"" = ""B"".""ID"" + ") .SetParameterValue("@FilesetId", ParentID); + using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) { rd.GetValues(values); if (values[0] != null && values[0] != DBNull.Value && Library.Utility.FilterExpression.Matches(filter, values[0].ToString())) { - await cmd2.SetParameterValue("@FileId", values[1]) + await cmd2 + .SetParameterValue("@FileId", values[1]) .ExecuteNonQueryAsync(); } } @@ -181,17 +240,36 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_db.Connection.CreateCommand()) { UpdatedFileCount = updated; - RemovedFileCount = await cmd.ExecuteScalarInt64Async($@"SELECT COUNT(*) FROM ""{m_tablename}""", 0); + + RemovedFileCount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{m_tablename}"" + ", 0); + RemovedFileSize = await cmd.ExecuteScalarInt64Async($@" - SELECT - SUM(""C"".""Length"") + SELECT SUM(""C"".""Length"") FROM - ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C, ""Metadataset"" D + ""{m_tablename}"" A, + ""FileLookup"" B, + ""Blockset"" C, + ""Metadataset"" D WHERE ""A"".""FileID"" = ""B"".""ID"" - AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID"")) + AND ( + ""B"".""BlocksetID"" = ""C"".""ID"" + OR ( + ""B"".""MetadataID"" = ""D"".""ID"" + AND ""D"".""BlocksetID"" = ""C"".""ID"" + ) + ) ", 0); - var filesetcount = await cmd.ExecuteScalarInt64Async($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}", 0); + + var filesetcount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = {ParentID} + ", 0); + if (filesetcount == RemovedFileCount) throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } @@ -203,11 +281,25 @@ namespace Duplicati.Library.Main.Database var filesetid = await m_db.CreateFileset(remotevolid, timestamp); await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); - using (var cmd = m_db.Connection.CreateCommand()) - await cmd.SetCommandAndParameters($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" ") - .SetParameterValue("@TargetFilesetId", filesetid) - .SetParameterValue("@SourceFilesetId", ParentID) - .ExecuteNonQueryAsync(); + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + await cmd.SetCommandAndParameters($@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + SELECT + @TargetFilesetId, + ""FileID"", + ""LastModified"" + FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @SourceFilesetId + AND ""FileID"" NOT IN ""{m_tablename}"" + ") + .SetParameterValue("@TargetFilesetId", filesetid) + .SetParameterValue("@SourceFilesetId", ParentID) + .ExecuteNonQueryAsync(); return new Tuple(remotevolid, filesetid); } @@ -215,7 +307,18 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable> ListAllDeletedFiles() { using (var cmd = m_db.Connection.CreateCommand()) - using (var rd = await cmd.ExecuteReaderAsync($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ")) + using (var rd = await cmd.ExecuteReaderAsync($@" + SELECT + ""B"".""Path"", + ""C"".""Length"" + FROM + ""{m_tablename}"" A, + ""File"" B, + ""Blockset"" C + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""BlocksetID"" = ""C"".""ID"" + ")) while (await rd.ReadAsync()) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } @@ -232,11 +335,8 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_db.Connection.CreateCommand()) await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}"""); } - catch - { - } + catch { } } } - } } From 9776e8ba3371279a1c4ad2dd0d7ba32136e26248 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 20 May 2025 21:41:47 +0200 Subject: [PATCH 103/362] Found inconsintensies in how the purge and delete databases are constructed now, through the create methods compared to the previous constructors. --- Duplicati/Library/Main/Database/LocalDeleteDatabase.cs | 4 ++-- Duplicati/Library/Main/Database/LocalPurgeDatabase.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 4ac17ffa4..cd58083c1 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -46,11 +46,11 @@ namespace Duplicati.Library.Main.Database private SqliteCommand m_registerDuplicateBlockCommand = null!; - public static async Task CreateAsync(string path, long pagecachesize) + public static async Task CreateAsync(string path, string operation, long pagecachesize) { var db = new LocalDeleteDatabase(); - db = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(db, path, "BugReportCreate", false, pagecachesize); + db = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(db, path, operation, true, pagecachesize); return db; } diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 24c6c272f..305d0ecad 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -33,9 +33,9 @@ namespace Duplicati.Library.Main.Database { public static new async Task CreateAsync(string path, long pagecachesize) { - var db = new LocalPurgeDatabase(); + // TODO double check whether this is possible. + var db = (LocalPurgeDatabase)await LocalDeleteDatabase.CreateAsync(path, "Purge", pagecachesize); - db = (LocalPurgeDatabase)await CreateLocalDatabaseAsync(db, path, "Purge", false, pagecachesize); db.ShouldCloseConnection = true; return db; From d11b4e58e72646fe1a2b9194a4b65135a1f27604 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 07:24:36 +0200 Subject: [PATCH 104/362] Reworked the factory construction chains to forward an instantiated object where applicable --- .../Main/Database/LocalBackupDatabase.cs | 14 ++-- .../Main/Database/LocalBugReportDatabase.cs | 10 +-- .../Library/Main/Database/LocalDatabase.cs | 75 +++++++++---------- .../Main/Database/LocalDeleteDatabase.cs | 15 ++-- .../Database/LocalListAffectedDatabase.cs | 10 +-- .../Database/LocalListBrokenFilesDatabase.cs | 14 ++-- .../Main/Database/LocalListChangesDatabase.cs | 10 +-- .../Main/Database/LocalListDatabase.cs | 10 +-- .../Main/Database/LocalPurgeDatabase.cs | 17 +++-- 9 files changed, 89 insertions(+), 86 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 98bddc904..f3da98fa1 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -85,19 +85,19 @@ namespace Duplicati.Library.Main.Database private bool m_logQueries; - public static async Task CreateAsync(string path, Options options) + public static async Task CreateAsync(string path, Options options, LocalBackupDatabase? dbnew = null) { - var db = new LocalBackupDatabase(); + dbnew ??= new LocalBackupDatabase(); - db = (LocalBackupDatabase)await CreateLocalDatabaseAsync(db, path, "Backup", false, options.SqlitePageCache); - db.ShouldCloseConnection = true; + dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(path, "Backup", false, options.SqlitePageCache, dbnew); + dbnew.ShouldCloseConnection = true; - return await CreateAsync(db, options); + return await CreateAsync(dbnew, options); } - public static async Task CreateAsync(LocalBackupDatabase dbparent, Options options) + public static async Task CreateAsync(LocalDatabase dbparent, Options options, LocalBackupDatabase? dbnew = null) { - var dbnew = new LocalBackupDatabase(); + dbnew ??= new LocalBackupDatabase(); dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index f5b959b2d..338967313 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -34,14 +34,14 @@ namespace Duplicati.Library.Main.Database /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalBugReportDatabase)); - public static async Task CreateAsync(string path, long pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize, LocalBugReportDatabase? dbnew = null) { - var db = new LocalBugReportDatabase(); + dbnew ??= new LocalBugReportDatabase(); - db = (LocalBugReportDatabase)await CreateLocalDatabaseAsync(db, path, "BugReportCreate", false, pagecachesize); - db.ShouldCloseConnection = true; + dbnew = (LocalBugReportDatabase)await CreateLocalDatabaseAsync(path, "BugReportCreate", false, pagecachesize, dbnew); + dbnew.ShouldCloseConnection = true; - return db; + return dbnew; } public async Task Fix() diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 4c3e3a587..1209511fc 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -147,17 +147,12 @@ namespace Duplicati.Library.Main.Database // return CreateLocalDatabaseAsync(connection).Await(); // } - public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize) + public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize, LocalDatabase? db = null) { - var db = new LocalDatabase(); + db ??= new LocalDatabase(); - return await CreateLocalDatabaseAsync(db, path, operation, shouldclose, pagecachesize); - } - - public static async Task CreateLocalDatabaseAsync(LocalDatabase db, string path, string operation, bool shouldclose, long pagecachesize) - { var connection = await CreateConnectionAsync(path, pagecachesize); - db = await CreateLocalDatabaseAsync(db, connection, operation); + db = await CreateLocalDatabaseAsync(connection, operation, db); db.ShouldCloseConnection = shouldclose; db.m_pagecachesize = pagecachesize; @@ -165,9 +160,11 @@ namespace Duplicati.Library.Main.Database return db; } - public static async Task CreateLocalDatabaseAsync(LocalDatabase dbparent, LocalDatabase dbnew) + public static async Task CreateLocalDatabaseAsync(LocalDatabase dbparent, LocalDatabase? dbnew = null) { - dbnew = await CreateLocalDatabaseAsync(dbnew, dbparent.m_connection); + dbnew ??= new LocalDatabase(); + + dbnew = await CreateLocalDatabaseAsync(dbparent.m_connection, dbnew); dbnew.OperationTimestamp = dbparent.OperationTimestamp; dbnew.m_connection = dbparent.m_connection; @@ -177,21 +174,22 @@ namespace Duplicati.Library.Main.Database return dbnew; } - public static async Task CreateLocalDatabaseAsync(LocalDatabase db, SqliteConnection connection, string operation) + public static async Task CreateLocalDatabaseAsync(SqliteConnection connection, string operation, LocalDatabase? dbnew = null) { - db = await CreateLocalDatabaseAsync(db, connection); + dbnew ??= new LocalDatabase(); + dbnew = await CreateLocalDatabaseAsync(connection, dbnew); - db.OperationTimestamp = DateTime.UtcNow; + dbnew.OperationTimestamp = DateTime.UtcNow; - if (db.m_connection.State != ConnectionState.Open) - await db.m_connection.OpenAsync(); + if (dbnew.m_connection.State != ConnectionState.Open) + await dbnew.m_connection.OpenAsync(); - using var cmd = db.m_connection.CreateCommand(); - using var transaction = db.m_connection.BeginTransaction(); + using var cmd = dbnew.m_connection.CreateCommand(); + using var transaction = dbnew.m_connection.BeginTransaction(); cmd.Transaction = transaction; if (operation != null) { - db.m_operationid = await cmd.SetCommandAndParameters(@" + dbnew.m_operationid = await cmd.SetCommandAndParameters(@" INSERT INTO ""Operation"" ( ""Description"", ""Timestamp"" @@ -203,7 +201,7 @@ namespace Duplicati.Library.Main.Database SELECT last_insert_rowid(); ") .SetParameterValue("@Description", operation) - .SetParameterValue("@Timestamp", db.OperationTimestamp) + .SetParameterValue("@Timestamp", dbnew.OperationTimestamp) .ExecuteScalarInt64Async(-1); } else @@ -221,16 +219,17 @@ namespace Duplicati.Library.Main.Database if (!await rd.ReadAsync()) throw new Exception("LocalDatabase does not contain a previous operation."); - db.m_operationid = rd.ConvertValueToInt64(0); - db.OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); + dbnew.m_operationid = rd.ConvertValueToInt64(0); + dbnew.OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); } - return db; + return dbnew; } - private static async Task CreateLocalDatabaseAsync(LocalDatabase db, SqliteConnection connection) + private static async Task CreateLocalDatabaseAsync(SqliteConnection connection, LocalDatabase? dbnew = null) { - db.m_connection = connection; + dbnew ??= new LocalDatabase(); + dbnew.m_connection = connection; var selectremotevolumes_sql = @" SELECT @@ -245,7 +244,7 @@ namespace Duplicati.Library.Main.Database FROM ""Remotevolume"" "; - db.m_insertlogCommand = await connection.CreateCommandAsync(@" + dbnew.m_insertlogCommand = await connection.CreateCommandAsync(@" INSERT INTO ""LogData"" ( ""OperationID"", ""Timestamp"", @@ -262,7 +261,7 @@ namespace Duplicati.Library.Main.Database ) "); - db.m_insertremotelogCommand = await connection.CreateCommandAsync(@" + dbnew.m_insertremotelogCommand = await connection.CreateCommandAsync(@" INSERT INTO ""RemoteOperation"" ( ""OperationID"", ""Timestamp"", @@ -279,7 +278,7 @@ namespace Duplicati.Library.Main.Database ) "); - db.m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" + dbnew.m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" UPDATE ""Remotevolume"" SET ""OperationID"" = @OperationID, @@ -289,14 +288,14 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name "); - db.m_selectremotevolumesCommand = await connection.CreateCommandAsync(selectremotevolumes_sql); + dbnew.m_selectremotevolumesCommand = await connection.CreateCommandAsync(selectremotevolumes_sql); - db.m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$" + dbnew.m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$" {selectremotevolumes_sql} WHERE ""Name"" = @Name "); - db.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" + dbnew.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE @@ -308,7 +307,7 @@ namespace Duplicati.Library.Main.Database AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') "); - db.m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" + dbnew.m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" DELETE FROM ""Remotevolume"" WHERE ""Name"" = @Name @@ -319,7 +318,7 @@ namespace Duplicati.Library.Main.Database "); // >12 is to handle removal of old records that were in ticks - db.m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@" + dbnew.m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@" DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted}' @@ -329,13 +328,13 @@ namespace Duplicati.Library.Main.Database ) "); - db.m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" + dbnew.m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = @Name "); - db.m_createremotevolumeCommand = await connection.CreateCommandAsync(@" + dbnew.m_createremotevolumeCommand = await connection.CreateCommandAsync(@" INSERT INTO ""Remotevolume"" ( ""OperationID"", ""Name"", @@ -359,7 +358,7 @@ namespace Duplicati.Library.Main.Database SELECT last_insert_rowid(); "); - db.m_insertIndexBlockLink = await connection.CreateCommandAsync(@" + dbnew.m_insertIndexBlockLink = await connection.CreateCommandAsync(@" INSERT INTO ""IndexBlockLink"" ( ""IndexVolumeID"", ""BlockVolumeID"" @@ -370,19 +369,19 @@ namespace Duplicati.Library.Main.Database ) "); - db.m_findpathprefixCommand = await connection.CreateCommandAsync(@" + dbnew.m_findpathprefixCommand = await connection.CreateCommandAsync(@" SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = @Prefix "); - db.m_insertpathprefixCommand = await connection.CreateCommandAsync(@" + dbnew.m_insertpathprefixCommand = await connection.CreateCommandAsync(@" INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (@Prefix); SELECT last_insert_rowid(); "); - return db; + return dbnew; } /// diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index cd58083c1..64f11506d 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -46,18 +46,21 @@ namespace Duplicati.Library.Main.Database private SqliteCommand m_registerDuplicateBlockCommand = null!; - public static async Task CreateAsync(string path, string operation, long pagecachesize) + public static async Task CreateAsync(string path, string operation, long pagecachesize, LocalDeleteDatabase? dbnew = null) { - var db = new LocalDeleteDatabase(); + dbnew ??= new LocalDeleteDatabase(); - db = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(db, path, operation, true, pagecachesize); + dbnew = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(path, operation, true, pagecachesize, dbnew); - return db; + dbnew.m_registerDuplicateBlockCommand = await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND); + + return dbnew; } - public static async Task CreateAsync(LocalDatabase dbparent) + // TODO Second argumnet is optional, as the original constructor call chain will have initialized the object if the constructor is called from another constructor. If not, the constructor will instantiate a new object. + public static async Task CreateAsync(LocalDatabase dbparent, LocalDeleteDatabase? dbnew = null) { - var dbnew = new LocalDeleteDatabase(); + dbnew ??= new LocalDeleteDatabase(); dbnew = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 19cd16441..5ba622a94 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -30,14 +30,14 @@ namespace Duplicati.Library.Main.Database { internal class LocalListAffectedDatabase : LocalDatabase { - public static async Task CreateAsync(string path, long pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize, LocalListAffectedDatabase? dbnew = null) { - var db = new LocalListAffectedDatabase(); + dbnew ??= new LocalListAffectedDatabase(); - db = (LocalListAffectedDatabase)await CreateLocalDatabaseAsync(db, path, "ListAffected", false, pagecachesize); - db.ShouldCloseConnection = true; + dbnew = (LocalListAffectedDatabase)await CreateLocalDatabaseAsync(path, "ListAffected", false, pagecachesize, dbnew); + dbnew.ShouldCloseConnection = true; - return db; + return dbnew; } private class ListResultFileset : Interface.IListResultFileset diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index a1354d57f..962fb28e0 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -138,19 +138,19 @@ namespace Duplicati.Library.Main.Database ) "; - public static async Task CreateAsync(string path, long pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize, LocalListBrokenFilesDatabase? dbnew = null) { - var db = new LocalListBrokenFilesDatabase(); + dbnew ??= new LocalListBrokenFilesDatabase(); - db = (LocalListBrokenFilesDatabase)await CreateLocalDatabaseAsync(db, path, "ListBrokenFiles", false, pagecachesize); - db.ShouldCloseConnection = true; + dbnew = (LocalListBrokenFilesDatabase)await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, pagecachesize, dbnew); + dbnew.ShouldCloseConnection = true; - return db; + return dbnew; } - public static async Task CreateAsync(LocalDatabase dbparent) + public static async Task CreateAsync(LocalDatabase dbparent, LocalListBrokenFilesDatabase? dbnew = null) { - var dbnew = new LocalListBrokenFilesDatabase(); + dbnew ??= new LocalListBrokenFilesDatabase(); dbnew = (LocalListBrokenFilesDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); dbnew.ShouldCloseConnection = false; diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index d9d9af826..ff499fbb8 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -32,14 +32,14 @@ namespace Duplicati.Library.Main.Database internal class LocalListChangesDatabase : LocalDatabase { - public static async Task CreateAsync(string path, long pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize, LocalListChangesDatabase? dbnew = null) { - var db = new LocalListChangesDatabase(); + dbnew ??= new LocalListChangesDatabase(); - db = (LocalListChangesDatabase)await CreateLocalDatabaseAsync(db, path, "ListChanges", false, pagecachesize); - db.ShouldCloseConnection = true; + dbnew = (LocalListChangesDatabase)await CreateLocalDatabaseAsync(path, "ListChanges", false, pagecachesize, dbnew); + dbnew.ShouldCloseConnection = true; - return db; + return dbnew; } public interface IStorageHelper : IDisposable diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index c89fadc77..ee0eac7d8 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -34,14 +34,14 @@ namespace Duplicati.Library.Main.Database { internal class LocalListDatabase : LocalDatabase { - public static async Task CreateAsync(string path, long pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize, LocalListDatabase? dbnew = null) { - var db = new LocalListDatabase(); + dbnew ??= new LocalListDatabase(); - db = (LocalListDatabase)await CreateLocalDatabaseAsync(db, path, "List", false, pagecachesize); - db.ShouldCloseConnection = true; + dbnew = (LocalListDatabase)await CreateLocalDatabaseAsync(path, "List", false, pagecachesize, dbnew); + dbnew.ShouldCloseConnection = true; - return db; + return dbnew; } public interface IFileversion diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 305d0ecad..75afe24cb 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -31,21 +31,22 @@ namespace Duplicati.Library.Main.Database { internal class LocalPurgeDatabase : LocalDeleteDatabase { - public static new async Task CreateAsync(string path, long pagecachesize) + public static new async Task CreateAsync(string path, long pagecachesize, LocalPurgeDatabase? dbnew = null) { - // TODO double check whether this is possible. - var db = (LocalPurgeDatabase)await LocalDeleteDatabase.CreateAsync(path, "Purge", pagecachesize); + dbnew ??= new LocalPurgeDatabase(); - db.ShouldCloseConnection = true; + dbnew = (LocalPurgeDatabase)await LocalDeleteDatabase.CreateAsync(path, "Purge", pagecachesize, dbnew); - return db; + dbnew.ShouldCloseConnection = true; + + return dbnew; } - public static new async Task CreateAsync(LocalDatabase dbparent) + public static new async Task CreateAsync(LocalDatabase dbparent, LocalPurgeDatabase? dbnew = null) { - var dbnew = new LocalPurgeDatabase(); + dbnew ??= new LocalPurgeDatabase(); - dbnew = (LocalPurgeDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + dbnew = (LocalPurgeDatabase)await LocalDeleteDatabase.CreateAsync(dbparent, dbnew); return dbnew; } From 89a5f5d4f729f7ba857046f227dbb88b0b3485bb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 07:36:44 +0200 Subject: [PATCH 105/362] Changed LocalRecreateDatabase to use factory construction --- .../Main/Database/LocalRecreateDatabase.cs | 94 ++++++++++--------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 6eea3579a..d823ba51c 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -51,26 +51,27 @@ namespace Duplicati.Library.Main.Database } } - private readonly SqliteCommand m_insertFileCommand; - private readonly SqliteCommand m_insertFilesetEntryCommand; - private readonly SqliteCommand m_insertMetadatasetCommand; - private readonly SqliteCommand m_insertBlocksetCommand; - private readonly SqliteCommand m_insertBlocklistHashCommand; - private readonly SqliteCommand m_updateBlockVolumeCommand; - private readonly SqliteCommand m_insertTempBlockListHash; - private readonly SqliteCommand m_insertSmallBlockset; - private readonly SqliteCommand m_findBlocksetCommand; - private readonly SqliteCommand m_findMetadatasetCommand; - private readonly SqliteCommand m_findFilesetCommand; - private readonly SqliteCommand m_findTempBlockListHashCommand; - private readonly SqliteCommand m_findHashBlockCommand; - private readonly SqliteCommand m_insertBlockCommand; - private readonly SqliteCommand m_insertDuplicateBlockCommand; + private SqliteCommand m_insertFileCommand = null!; + private SqliteCommand m_insertFilesetEntryCommand = null!; + private SqliteCommand m_insertMetadatasetCommand = null!; + private SqliteCommand m_insertBlocksetCommand = null!; + private SqliteCommand m_insertBlocklistHashCommand = null!; + private SqliteCommand m_updateBlockVolumeCommand = null!; + private SqliteCommand m_insertTempBlockListHash = null!; + private SqliteCommand m_insertSmallBlockset = null!; + private SqliteCommand m_findBlocksetCommand = null!; + private SqliteCommand m_findMetadatasetCommand = null!; + private SqliteCommand m_findFilesetCommand = null!; + private SqliteCommand m_findTempBlockListHashCommand = null!; + private SqliteCommand m_findHashBlockCommand = null!; + private SqliteCommand m_insertBlockCommand = null!; + private SqliteCommand m_insertDuplicateBlockCommand = null!; - private string m_tempblocklist; - private string m_tempsmalllist; + private string m_tempblocklist = null!; + private string m_tempsmalllist = null!; - private new readonly SqliteConnection m_connection; + // TODO why was this "new" ? + //private new readonly SqliteConnection m_connection; //public new bool RepairInProgress { get; set; } = false; @@ -125,38 +126,43 @@ namespace Duplicati.Library.Main.Database ""FullIndex"" "); - public LocalRecreateDatabase(LocalDatabase parentdb, Options options) - : base(parentdb) + public static async Task CreateAsync(LocalDatabase parentdb, Options options, LocalRecreateDatabase? dbnew = null) { - m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + dbnew ??= new LocalRecreateDatabase(); - if (m_connection == null) + dbnew = (LocalRecreateDatabase)await CreateLocalDatabaseAsync(parentdb, dbnew); + + dbnew.m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + dbnew.m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + if (dbnew.m_connection == null) throw new Exception("Connection is null"); - using var cmd = m_connection.CreateCommand(); + using var cmd = dbnew.m_connection.CreateCommand(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)")).Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""Index_{m_tempblocklist}"" ON ""{m_tempblocklist}"" (""BlockListHash"");")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" ON ""{dbnew.m_tempblocklist}"" (""BlockListHash"");")).Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)")).Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_File_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""FileHash"", ""BlockSize"");")).Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_Block_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockSize"");")).Await(); + cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");")).Await(); - m_insertFileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (@PrefixId,@Path,@BlocksetId,@MetadataId); SELECT last_insert_rowid();"); - m_insertFilesetEntryCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId,@FileId,@LastModified)"); - m_insertMetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); - m_insertBlocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length,@FullHash); SELECT last_insert_rowid();"); - m_insertBlocklistHashCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId,@Index,@Hash)"); - m_updateBlockVolumeCommand = m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @VolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - m_insertTempBlockListHash = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (@BlocklistHash,@BlockHash,@Index) ")); - m_insertSmallBlockset = m_connection.CreateCommand(FormatInvariant($@"INSERT OR IGNORE INTO ""{m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (@FileHash,@BlockHash,@BlockSize) ")); - m_findBlocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash "); - m_findMetadatasetCommand = m_connection.CreateCommand(@"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length "); - m_findFilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId "); - m_findTempBlockListHashCommand = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""BlockListHash"" FROM ""{m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash ")); - m_findHashBlockCommand = m_connection.CreateCommand(@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); - m_insertBlockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (@Hash,@Size,@VolumeId)"); - m_insertDuplicateBlockCommand = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size), @VolumeId)"); + dbnew.m_insertFileCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (@PrefixId,@Path,@BlocksetId,@MetadataId); SELECT last_insert_rowid();"); + dbnew.m_insertFilesetEntryCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId,@FileId,@LastModified)"); + dbnew.m_insertMetadatasetCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); + dbnew.m_insertBlocksetCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length,@FullHash); SELECT last_insert_rowid();"); + dbnew.m_insertBlocklistHashCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId,@Index,@Hash)"); + dbnew.m_updateBlockVolumeCommand = dbnew.m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @VolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + dbnew.m_insertTempBlockListHash = dbnew.m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{dbnew.m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (@BlocklistHash,@BlockHash,@Index) ")); + dbnew.m_insertSmallBlockset = dbnew.m_connection.CreateCommand(FormatInvariant($@"INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (@FileHash,@BlockHash,@BlockSize) ")); + dbnew.m_findBlocksetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash "); + dbnew.m_findMetadatasetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length "); + dbnew.m_findFilesetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId "); + dbnew.m_findTempBlockListHashCommand = dbnew.m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""BlockListHash"" FROM ""{dbnew.m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash ")); + dbnew.m_findHashBlockCommand = dbnew.m_connection.CreateCommand(@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); + dbnew.m_insertBlockCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (@Hash,@Size,@VolumeId)"); + dbnew.m_insertDuplicateBlockCommand = dbnew.m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size), @VolumeId)"); + + return dbnew; } public async Task FindMissingBlocklistHashes(long hashsize, long blocksize, SqliteTransaction transaction) From 712d75a3d3f05b4af801ad3363de9b035095a594 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 07:40:39 +0200 Subject: [PATCH 106/362] Removed FormatInvariant as it's now handeld by the extension methods --- .../Main/Database/LocalRecreateDatabase.cs | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index d823ba51c..b242a0eb5 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -87,7 +87,7 @@ namespace Duplicati.Library.Main.Database // {2} --> Temp-Table // {3} --> FullBlocklist-BlockCount [equals ({0} / {1}), if SQLite pays respect to ints] private static string SELECT_BLOCKLIST_ENTRIES(long blocksize, long blockhashsize, string temptable, long fullBlockListBlockCount) => - FormatInvariant($@" + $@" SELECT ""E"".""BlocksetID"", ""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount}) AS ""FullIndex"", @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""E"".""BlocksetID"", ""FullIndex"" -"); +"; public static async Task CreateAsync(LocalDatabase parentdb, Options options, LocalRecreateDatabase? dbnew = null) { @@ -139,12 +139,12 @@ namespace Duplicati.Library.Main.Database throw new Exception("Connection is null"); using var cmd = dbnew.m_connection.CreateCommand(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)")).Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" ON ""{dbnew.m_tempblocklist}"" (""BlockListHash"");")).Await(); + cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)").Await(); + cmd.ExecuteNonQueryAsync($@"CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" ON ""{dbnew.m_tempblocklist}"" (""BlockListHash"");").Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)")).Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockSize"");")).Await(); - cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");")).Await(); + cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)").Await(); + cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockSize"");").Await(); + cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");").Await(); dbnew.m_insertFileCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (@PrefixId,@Path,@BlocksetId,@MetadataId); SELECT last_insert_rowid();"); dbnew.m_insertFilesetEntryCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId,@FileId,@LastModified)"); @@ -152,12 +152,12 @@ namespace Duplicati.Library.Main.Database dbnew.m_insertBlocksetCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length,@FullHash); SELECT last_insert_rowid();"); dbnew.m_insertBlocklistHashCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId,@Index,@Hash)"); dbnew.m_updateBlockVolumeCommand = dbnew.m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @VolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - dbnew.m_insertTempBlockListHash = dbnew.m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{dbnew.m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (@BlocklistHash,@BlockHash,@Index) ")); - dbnew.m_insertSmallBlockset = dbnew.m_connection.CreateCommand(FormatInvariant($@"INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (@FileHash,@BlockHash,@BlockSize) ")); + dbnew.m_insertTempBlockListHash = dbnew.m_connection.CreateCommand($@"INSERT INTO ""{dbnew.m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (@BlocklistHash,@BlockHash,@Index) "); + dbnew.m_insertSmallBlockset = dbnew.m_connection.CreateCommand($@"INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (@FileHash,@BlockHash,@BlockSize) "); dbnew.m_findBlocksetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash "); dbnew.m_findMetadatasetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length "); dbnew.m_findFilesetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId "); - dbnew.m_findTempBlockListHashCommand = dbnew.m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""BlockListHash"" FROM ""{dbnew.m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash ")); + dbnew.m_findTempBlockListHashCommand = dbnew.m_connection.CreateCommand($@"SELECT DISTINCT ""BlockListHash"" FROM ""{dbnew.m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash "); dbnew.m_findHashBlockCommand = dbnew.m_connection.CreateCommand(@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); dbnew.m_insertBlockCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (@Hash,@Size,@VolumeId)"); dbnew.m_insertDuplicateBlockCommand = dbnew.m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size), @VolumeId)"); @@ -171,16 +171,16 @@ namespace Duplicati.Library.Main.Database cmd.Transaction = transaction; //Update all small blocklists and matching blocks - var selectSmallBlocks = FormatInvariant($@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""); + var selectSmallBlocks = $@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""; - var selectBlockHashes = FormatInvariant($@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} )"); + var selectBlockHashes = $@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} )"; var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; - var selectNewBlocks = FormatInvariant($@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" + var selectNewBlocks = $@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({selectAllBlocks}) A LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" ) -WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); +WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "; var insertBlocksCommand = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + @@ -189,14 +189,14 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); // Insert all known blocks into block table with volumeid = -1 await cmd.ExecuteNonQueryAsync(insertBlocksCommand); - var selectBlocklistBlocksetEntries = FormatInvariant( + var selectBlocklistBlocksetEntries = // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati // this could be used instead //@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" "; $@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" " - ); + ; - var selectBlocksetEntries = FormatInvariant($@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{m_tempsmalllist}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {blocksize} "); + var selectBlocksetEntries = $@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{m_tempsmalllist}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {blocksize} "; var selectAllBlocksetEntries = selectBlocklistBlocksetEntries + @@ -221,8 +221,8 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); using (var fixcmd = m_connection.CreateCommand()) { - await fixcmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); - await fixcmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); + await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" "); + await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" "); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -249,7 +249,7 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); { using var cmd = m_connection.CreateCommand(); cmd.Transaction = transaction; - var insertBlocksCommand = FormatInvariant($@"INSERT INTO BLOCK (Hash, Size, VolumeID) + var insertBlocksCommand = $@"INSERT INTO BLOCK (Hash, Size, VolumeID) SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM ( SELECT NB.BlockHash, @@ -260,29 +260,29 @@ FROM ( WHERE B.Hash IS NULL ) NB JOIN BlocklistHash BH ON (BH.Hash = NB.BlocklistHash) -JOIN Blockset BS ON (BS.ID = BH.Blocksetid) "); +JOIN Blockset BS ON (BS.ID = BH.Blocksetid) "; if (!hashOnly) { - insertBlocksCommand += FormatInvariant($@" UNION + insertBlocksCommand += $@" UNION SELECT TS.BlockHash, TS.BlockSize FROM {m_tempsmalllist} TS WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE B.Hash = TS.BlockHash AND B.Size = TS.BlockSize) -)"); +)"; } - var insertBlocksetEntriesCommand = FormatInvariant($@"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) + var insertBlocksetEntriesCommand = $@"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) SELECT DISTINCT BH.blocksetid, (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" as FullIndex, BK.ID AS BlockID FROM {m_tempblocklist} TBL JOIN blocklisthash BH ON (BH.hash = TBL.blocklisthash) JOIN block BK ON (BK.Hash = TBL.BlockHash) LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlockSetID = BH.BlocksetID AND BE.""Index"" = (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"") WHERE -BE.BlockSetID IS NULL "); +BE.BlockSetID IS NULL "; if (!hashOnly) { - insertBlocksetEntriesCommand += FormatInvariant($@" UNION + insertBlocksetEntriesCommand += $@" UNION SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID FROM {m_tempsmalllist} TS JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND @@ -292,7 +292,7 @@ FROM {m_tempsmalllist} TS BL.Size = TS.BlockSize) LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) WHERE -BE.BlocksetID IS NULL "); +BE.BlocksetID IS NULL "; } try @@ -308,8 +308,8 @@ BE.BlocksetID IS NULL "); using (var fixcmd = m_connection.CreateCommand()) { - await fixcmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); - await fixcmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); + await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" "); + await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" "); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -523,22 +523,22 @@ BE.BlocksetID IS NULL "); var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume"""; var missingBlocklistEntries = - FormatInvariant($@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {blocksize / hashsize}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL"); + $@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {blocksize / hashsize}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL"; var missingBlockInfo = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 AND SIZE > 0"; - var missingBlocklistVolumes = FormatInvariant( + var missingBlocklistVolumes = $@"SELECT ""VolumeID"" FROM ""Block"", ({missingBlocklistEntries}) A WHERE ""A"".""Hash"" = ""Block"".""Hash"" " - ); + ; - var countMissingInformation = FormatInvariant($@"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({missingBlockInfo} UNION {missingBlocklistVolumes}))"); + var countMissingInformation = $@"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({missingBlockInfo} UNION {missingBlocklistVolumes}))"; if (passNo == 0) { // On the first pass, we select all the volumes we know we need, // which may be an empty list - cmd.CommandText = FormatInvariant($@"{selectCommand} WHERE ""ID"" IN ({missingBlocklistVolumes})"); + cmd.CommandText = $@"{selectCommand} WHERE ""ID"" IN ({missingBlocklistVolumes})"; // Reset the list m_proccessedVolumes.Clear(); @@ -559,7 +559,7 @@ BE.BlocksetID IS NULL "); var mentionedVolumes = @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; - cmd.CommandText = FormatInvariant($@"{selectCommand} WHERE ""ID"" IN ({mentionedVolumes}) AND ""Type"" = @Type "); + cmd.CommandText = $@"{selectCommand} WHERE ""ID"" IN ({mentionedVolumes}) AND ""Type"" = @Type "; cmd.Parameters.Clear(); cmd.Parameters.AddWithValue("@Type", RemoteVolumeType.Blocks.ToString()); @@ -569,7 +569,7 @@ BE.BlocksetID IS NULL "); { // On the final pass, we select all volumes // the filter will ensure that we do not download anything twice - cmd.CommandText = FormatInvariant($@"{selectCommand} WHERE ""Type"" = @Type"); + cmd.CommandText = $@"{selectCommand} WHERE ""Type"" = @Type"; cmd.Parameters.Clear(); cmd.Parameters.AddWithValue("@Type", RemoteVolumeType.Blocks.ToString()); } @@ -603,7 +603,7 @@ BE.BlocksetID IS NULL "); // existing ones (as recorded in the DuplicateBlock table) // The second part removes references to the non-present remote files, // and marks the index files that pointed to them, such that they will be removed later on - var sql = FormatInvariant($@" + var sql = $@" CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" @@ -642,12 +642,12 @@ DROP TABLE ""{tablename}""; DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""); -"); +"; // We could delete these, but we don't have to, so we keep them around until the next compact is done // UPDATE ""RemoteVolume"" SET ""State"" = ""{3}"" WHERE ""Type"" = ""{5}"" AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink""); - var countsql = FormatInvariant($@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' "); + var countsql = $@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' "; using (var cmd = m_connection.CreateCommand()) { @@ -683,15 +683,15 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" using var cmd = m_connection.CreateCommand(); cmd.Transaction = m_connection.BeginTransaction(); // 1. Select blocks not used by any file and not as a blocklist into temporary table - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmptablename}"" + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tmptablename}"" AS SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""ID"" NOT IN (SELECT ""BlocksetEntry"".""BlockID"" FROM ""BlocksetEntry"") - AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")")); + AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")"); // 2. Insert blocks into DeletedBlock table - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{tmptablename}""")); + await cmd.ExecuteNonQueryAsync($@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{tmptablename}"""); // 3. Remove blocks from Block table - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{tmptablename}"")")); - await cmd.ExecuteNonQueryAsync(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmptablename}""")); + await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{tmptablename}"")"); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"""); await cmd.Transaction.CommitAsync(); } @@ -702,7 +702,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempblocklist != null) try { - cmd.ExecuteNonQueryAsync(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""")).Wait(); + cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""").Wait(); } catch (Exception ex) { @@ -716,7 +716,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempsmalllist != null) try { - cmd.ExecuteNonQueryAsync(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""")).Wait(); + cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""").Wait(); } catch (Exception ex) { From 003faef8bf081b1fe9ce1e01410909e19714f7ad Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 07:57:13 +0200 Subject: [PATCH 107/362] Ensured all calls to sqlite in LocalRecreateDatabase uses extension methods and are properly async --- .../Main/Database/LocalRecreateDatabase.cs | 251 ++++++++++-------- 1 file changed, 137 insertions(+), 114 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index b242a0eb5..3b492aec1 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -139,36 +139,35 @@ namespace Duplicati.Library.Main.Database throw new Exception("Connection is null"); using var cmd = dbnew.m_connection.CreateCommand(); - cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)").Await(); - cmd.ExecuteNonQueryAsync($@"CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" ON ""{dbnew.m_tempblocklist}"" (""BlockListHash"");").Await(); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)"); + await cmd.ExecuteNonQueryAsync($@"CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" ON ""{dbnew.m_tempblocklist}"" (""BlockListHash"");"); - cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)").Await(); - cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockSize"");").Await(); - cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");").Await(); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)"); + await cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockSize"");"); + await cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");"); - dbnew.m_insertFileCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (@PrefixId,@Path,@BlocksetId,@MetadataId); SELECT last_insert_rowid();"); - dbnew.m_insertFilesetEntryCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId,@FileId,@LastModified)"); - dbnew.m_insertMetadatasetCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); - dbnew.m_insertBlocksetCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length,@FullHash); SELECT last_insert_rowid();"); - dbnew.m_insertBlocklistHashCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId,@Index,@Hash)"); - dbnew.m_updateBlockVolumeCommand = dbnew.m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @VolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - dbnew.m_insertTempBlockListHash = dbnew.m_connection.CreateCommand($@"INSERT INTO ""{dbnew.m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (@BlocklistHash,@BlockHash,@Index) "); - dbnew.m_insertSmallBlockset = dbnew.m_connection.CreateCommand($@"INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (@FileHash,@BlockHash,@BlockSize) "); - dbnew.m_findBlocksetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash "); - dbnew.m_findMetadatasetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length "); - dbnew.m_findFilesetCommand = dbnew.m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId "); - dbnew.m_findTempBlockListHashCommand = dbnew.m_connection.CreateCommand($@"SELECT DISTINCT ""BlockListHash"" FROM ""{dbnew.m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash "); - dbnew.m_findHashBlockCommand = dbnew.m_connection.CreateCommand(@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); - dbnew.m_insertBlockCommand = dbnew.m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (@Hash,@Size,@VolumeId)"); - dbnew.m_insertDuplicateBlockCommand = dbnew.m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size), @VolumeId)"); + dbnew.m_insertFileCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (@PrefixId,@Path,@BlocksetId,@MetadataId); SELECT last_insert_rowid();"); + dbnew.m_insertFilesetEntryCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId,@FileId,@LastModified)"); + dbnew.m_insertMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); + dbnew.m_insertBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length,@FullHash); SELECT last_insert_rowid();"); + dbnew.m_insertBlocklistHashCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId,@Index,@Hash)"); + dbnew.m_updateBlockVolumeCommand = await dbnew.m_connection.CreateCommandAsync(@"UPDATE ""Block"" SET ""VolumeID"" = @VolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + dbnew.m_insertTempBlockListHash = await dbnew.m_connection.CreateCommandAsync($@"INSERT INTO ""{dbnew.m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (@BlocklistHash,@BlockHash,@Index) "); + dbnew.m_insertSmallBlockset = await dbnew.m_connection.CreateCommandAsync($@"INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (@FileHash,@BlockHash,@BlockSize) "); + dbnew.m_findBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash "); + dbnew.m_findMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length "); + dbnew.m_findFilesetCommand = await dbnew.m_connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId "); + dbnew.m_findTempBlockListHashCommand = await dbnew.m_connection.CreateCommandAsync($@"SELECT DISTINCT ""BlockListHash"" FROM ""{dbnew.m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash "); + dbnew.m_findHashBlockCommand = await dbnew.m_connection.CreateCommandAsync(@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); + dbnew.m_insertBlockCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (@Hash,@Size,@VolumeId)"); + dbnew.m_insertDuplicateBlockCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size), @VolumeId)"); return dbnew; } public async Task FindMissingBlocklistHashes(long hashsize, long blocksize, SqliteTransaction transaction) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; + using var cmd = m_connection.CreateCommand(m_rtr); //Update all small blocklists and matching blocks var selectSmallBlocks = $@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""; @@ -219,7 +218,7 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "; { Logging.Log.WriteErrorMessage(LOGTAG, "BlocksetInsertFailed", ex, "Blockset insert failed, committing temporary tables for trace purposes"); - using (var fixcmd = m_connection.CreateCommand()) + using (var fixcmd = m_connection.CreateCommand(m_rtr)) { await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" "); await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" "); @@ -247,8 +246,7 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "; /// and blocksize can vary from 0 to the configured block size for the backup public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, SqliteTransaction transaction, bool hashOnly = false) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; + using var cmd = m_connection.CreateCommand(m_rtr); var insertBlocksCommand = $@"INSERT INTO BLOCK (Hash, Size, VolumeID) SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM ( @@ -306,7 +304,7 @@ BE.BlocksetID IS NULL "; { Logging.Log.WriteErrorMessage(LOGTAG, "BlockOrBlocksetInsertFailed", ex, "Block or Blockset insert failed, committing temporary tables for trace purposes"); - using (var fixcmd = m_connection.CreateCommand()) + using (var fixcmd = m_connection.CreateCommand(m_rtr)) { await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" "); await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" "); @@ -333,28 +331,32 @@ BE.BlocksetID IS NULL "; private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, SqliteTransaction transaction) { - m_findFilesetCommand.Transaction = transaction; - m_findFilesetCommand.Parameters["@PrefixId"].Value = pathprefixid; - m_findFilesetCommand.Parameters["@Path"].Value = path; - m_findFilesetCommand.Parameters["@BlocksetId"].Value = blocksetid; - m_findFilesetCommand.Parameters["@MetadataId"].Value = metadataid; - var fileid = await m_findFilesetCommand.ExecuteScalarInt64Async(-1); + var fileid = await m_findFilesetCommand + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetid) + .SetParameterValue("@PrefixId", pathprefixid) + .SetParameterValue("@Path", path) + .SetParameterValue("@BlocksetId", blocksetid) + .SetParameterValue("@MetadataId", metadataid) + .ExecuteScalarInt64Async(-1); if (fileid < 0) { - m_insertFileCommand.Transaction = transaction; - m_insertFileCommand.Parameters["@PrefixId"].Value = pathprefixid; - m_insertFileCommand.Parameters["@Path"].Value = path; - m_insertFileCommand.Parameters["@BlocksetId"].Value = blocksetid; - m_insertFileCommand.Parameters["@MetadataId"].Value = metadataid; - fileid = await m_insertFileCommand.ExecuteScalarInt64Async(-1); + fileid = await m_insertFileCommand + .SetTransaction(m_rtr) + .SetParameterValue("@PrefixId", pathprefixid) + .SetParameterValue("@Path", path) + .SetParameterValue("@BlocksetId", blocksetid) + .SetParameterValue("@MetadataId", metadataid) + .ExecuteScalarInt64Async(-1); } - m_insertFilesetEntryCommand.Transaction = transaction; - m_insertFilesetEntryCommand.Parameters["@FilesetId"].Value = filesetid; - m_insertFilesetEntryCommand.Parameters["@FileId"].Value = fileid; - m_insertFilesetEntryCommand.Parameters["@LastModified"].Value = time.ToUniversalTime().Ticks; - await m_insertFilesetEntryCommand.ExecuteNonQueryAsync(); + await m_insertFilesetEntryCommand + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetid) + .SetParameterValue("@FileId", fileid) + .SetParameterValue("@LastModified", time.ToUniversalTime().Ticks) + .ExecuteNonQueryAsync(); } public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, SqliteTransaction transaction) @@ -363,44 +365,49 @@ BE.BlocksetID IS NULL "; if (metahash == null) return metadataid; - m_findMetadatasetCommand.Transaction = transaction; - m_findMetadatasetCommand.Parameters["@FullHash"].Value = metahash; - m_findMetadatasetCommand.Parameters["@Length"].Value = metahashsize; - metadataid = await m_findMetadatasetCommand.ExecuteScalarInt64Async(-1); + metadataid = await m_findMetadatasetCommand + .SetTransaction(m_rtr) + .SetParameterValue("@FullHash", metahash) + .SetParameterValue("@Length", metahashsize) + .ExecuteScalarInt64Async(-1); if (metadataid != -1) return metadataid; var blocksetid = await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction); - m_insertMetadatasetCommand.Transaction = transaction; - m_insertMetadatasetCommand.Parameters["@BlocksetId"].Value = blocksetid; - metadataid = await m_insertMetadatasetCommand.ExecuteScalarInt64Async(-1); + metadataid = await m_insertMetadatasetCommand + .SetTransaction(m_rtr) + .SetParameterValue("@BlocksetId", blocksetid) + .ExecuteScalarInt64Async(-1); return metadataid; } public async Task AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, SqliteTransaction transaction) { - m_findBlocksetCommand.Transaction = transaction; - m_findBlocksetCommand.Parameters["@Length"].Value = size; - m_findBlocksetCommand.Parameters["@FullHash"].Value = fullhash; - var blocksetid = await m_findBlocksetCommand.ExecuteScalarInt64Async(-1); + var blocksetid = await m_findBlocksetCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Length", size) + .SetParameterValue("@FullHash", fullhash) + .ExecuteScalarInt64Async(-1); if (blocksetid != -1) return blocksetid; - m_insertBlocksetCommand.Transaction = transaction; - m_insertBlocksetCommand.Parameters["@Length"].Value = size; - m_insertBlocksetCommand.Parameters["@FullHash"].Value = fullhash; - blocksetid = await m_insertBlocksetCommand.ExecuteScalarInt64Async(-1); + blocksetid = await m_insertBlocksetCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Length", size) + .SetParameterValue("@FullHash", fullhash) + .ExecuteScalarInt64Async(-1); long c = 0; if (blocklisthashes != null) { var index = 0L; - m_insertBlocklistHashCommand.Transaction = transaction; - m_insertBlocklistHashCommand.Parameters["@BlocksetId"].Value = blocksetid; + m_insertBlocklistHashCommand + .SetTransaction(m_rtr) + .SetParameterValue("@BlocksetId", blocksetid); foreach (var hash in blocklisthashes) { @@ -409,9 +416,10 @@ BE.BlocksetID IS NULL "; c++; if (c <= expectedblocklisthashes) { - m_insertBlocklistHashCommand.Parameters["@Index"].Value = index++; - m_insertBlocklistHashCommand.Parameters["@Hash"].Value = hash; - await m_insertBlocklistHashCommand.ExecuteNonQueryAsync(); + await m_insertBlocklistHashCommand + .SetParameterValue("@Index", index) + .SetParameterValue("@Hash", hash) + .ExecuteNonQueryAsync(); } } } @@ -426,10 +434,11 @@ BE.BlocksetID IS NULL "; public async Task<(bool, bool)> UpdateBlock(string hash, long size, long volumeID, SqliteTransaction transaction) { var anyChange = false; - m_findHashBlockCommand.Transaction = transaction; - m_findHashBlockCommand.Parameters["@Hash"].Value = hash; - m_findHashBlockCommand.Parameters["@Size"].Value = size; - var currentVolumeId = await m_findHashBlockCommand.ExecuteScalarInt64Async(-2); + var currentVolumeId = await m_findHashBlockCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteScalarInt64Async(-2); if (currentVolumeId == volumeID) return (anyChange, false); @@ -439,22 +448,24 @@ BE.BlocksetID IS NULL "; if (currentVolumeId == -2) { //Insert - m_insertBlockCommand.Transaction = transaction; - m_insertBlockCommand.Parameters["@Hash"].Value = hash; - m_insertBlockCommand.Parameters["@Size"].Value = size; - m_insertBlockCommand.Parameters["@VolumeId"].Value = volumeID; - await m_insertBlockCommand.ExecuteNonQueryAsync(); + await m_insertBlockCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .SetParameterValue("@VolumeId", volumeID) + .ExecuteNonQueryAsync(); return (anyChange, true); } else if (currentVolumeId == -1) { //Update - m_updateBlockVolumeCommand.Transaction = transaction; - m_updateBlockVolumeCommand.Parameters["@VolumeId"].Value = volumeID; - m_updateBlockVolumeCommand.Parameters["@Hash"].Value = hash; - m_updateBlockVolumeCommand.Parameters["@Size"].Value = size; - var c = await m_updateBlockVolumeCommand.ExecuteNonQueryAsync(); + var c = await m_updateBlockVolumeCommand + .SetTransaction(m_rtr) + .SetParameterValue("@VolumeId", volumeID) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(); if (c != 1) throw new Exception($"Failed to update table, found {c} entries for key {hash} with size {size}"); @@ -462,11 +473,12 @@ BE.BlocksetID IS NULL "; } else { - m_insertDuplicateBlockCommand.Transaction = transaction; - m_insertDuplicateBlockCommand.Parameters["@Hash"].Value = hash; - m_insertDuplicateBlockCommand.Parameters["@Size"].Value = size; - m_insertDuplicateBlockCommand.Parameters["@VolumeId"].Value = volumeID; - await m_insertDuplicateBlockCommand.ExecuteNonQueryAsync(); + await m_insertDuplicateBlockCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .SetParameterValue("@VolumeId", currentVolumeId) + .ExecuteNonQueryAsync(); return (anyChange, false); } @@ -474,32 +486,36 @@ BE.BlocksetID IS NULL "; public async Task AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, SqliteTransaction transaction) { - m_insertSmallBlockset.Transaction = transaction; - m_insertSmallBlockset.Parameters["@FileHash"].Value = filehash; - m_insertSmallBlockset.Parameters["@BlockHash"].Value = blockhash; - m_insertSmallBlockset.Parameters["@BlockSize"].Value = blocksize; - await m_insertSmallBlockset.ExecuteNonQueryAsync(); + await m_insertSmallBlockset + .SetTransaction(m_rtr) + .SetParameterValue("@FileHash", filehash) + .SetParameterValue("@BlockHash", blockhash) + .SetParameterValue("@BlockSize", blocksize) + .ExecuteNonQueryAsync(); } public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes, SqliteTransaction transaction) { - m_findTempBlockListHashCommand.Transaction = transaction; - m_findTempBlockListHashCommand.Parameters["@BlocklistHash"].Value = hash; - var r = await m_findTempBlockListHashCommand.ExecuteScalarAsync(); + var r = await m_findTempBlockListHashCommand + .SetTransaction(m_rtr) + .SetParameterValue("@BlocklistHash", hash) + .ExecuteScalarAsync(); if (r != null && r != DBNull.Value) return false; - m_insertTempBlockListHash.Transaction = transaction; - m_insertTempBlockListHash.Parameters["@BlocklistHash"].Value = hash; + m_insertTempBlockListHash + .SetTransaction(m_rtr) + .SetParameterValue("@BlocklistHash", hash); var index = 0L; foreach (var s in blocklisthashes) { - m_insertTempBlockListHash.Parameters["@BlockHash"].Value = s; - m_insertTempBlockListHash.Parameters["@Index"].Value = index++; - await m_insertTempBlockListHash.ExecuteNonQueryAsync(); + await m_insertTempBlockListHash + .SetParameterValue("@BlockHash", s) + .SetParameterValue("@Index", index++) + .ExecuteNonQueryAsync(); } return true; @@ -508,9 +524,10 @@ BE.BlocksetID IS NULL "; public async IAsyncEnumerable GetBlockLists(long volumeid) { - using var cmd = m_connection.CreateCommand(@"SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId"); + using var cmd = m_connection.CreateCommand(@"SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId") + .SetTransaction(m_rtr) + .SetParameterValue("@VolumeId", volumeid); - cmd.Parameters.AddWithValue("@VolumeId", volumeid); using var rd = await cmd.ExecuteReaderAsync(); while (rd.Read()) yield return rd.ConvertValueToString(0) ?? ""; @@ -538,7 +555,7 @@ BE.BlocksetID IS NULL "; { // On the first pass, we select all the volumes we know we need, // which may be an empty list - cmd.CommandText = $@"{selectCommand} WHERE ""ID"" IN ({missingBlocklistVolumes})"; + cmd.SetCommandAndParameters($@"{selectCommand} WHERE ""ID"" IN ({missingBlocklistVolumes})"); // Reset the list m_proccessedVolumes.Clear(); @@ -546,9 +563,10 @@ BE.BlocksetID IS NULL "; else { //On anything but the first pass, we check if we are done - cmd.CommandText = countMissingInformation; - cmd.Parameters.Clear(); - var r = await cmd.ExecuteScalarInt64Async(0); + var r = await cmd + .SetCommandAndParameters(countMissingInformation) + .ExecuteScalarInt64Async(0); + if (r == 0 && !forceBlockUse) yield break; @@ -559,9 +577,9 @@ BE.BlocksetID IS NULL "; var mentionedVolumes = @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; - cmd.CommandText = $@"{selectCommand} WHERE ""ID"" IN ({mentionedVolumes}) AND ""Type"" = @Type "; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@Type", RemoteVolumeType.Blocks.ToString()); + cmd + .SetCommandAndParameters($@"{selectCommand} WHERE ""ID"" IN ({mentionedVolumes}) AND ""Type"" = @Type ") + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); } @@ -569,9 +587,9 @@ BE.BlocksetID IS NULL "; { // On the final pass, we select all volumes // the filter will ensure that we do not download anything twice - cmd.CommandText = $@"{selectCommand} WHERE ""Type"" = @Type"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@Type", RemoteVolumeType.Blocks.ToString()); + cmd + .SetCommandAndParameters($@"{selectCommand} WHERE ""Type"" = @Type") + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); } } @@ -649,7 +667,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" var countsql = $@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' "; - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_rtr)) { var cnt = await cmd.ExecuteScalarInt64Async(countsql); if (cnt > 0) @@ -680,8 +698,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" var tmptablename = "DeletedBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(); + using var cmd = m_connection.CreateCommand(m_rtr); // 1. Select blocks not used by any file and not as a blocklist into temporary table await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tmptablename}"" AS SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""Block"" @@ -692,17 +709,23 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" // 3. Remove blocks from Block table await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{tmptablename}"")"); await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"""); - await cmd.Transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public override void Dispose() { - using (var cmd = m_connection.CreateCommand()) + DisposeAsync().Await(); + + } + + public async override Task DisposeAsync() + { + using (var cmd = m_connection.CreateCommand(m_rtr)) { if (m_tempblocklist != null) try { - cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""").Wait(); + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}"""); } catch (Exception ex) { @@ -716,7 +739,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempsmalllist != null) try { - cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""").Wait(); + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}"""); } catch (Exception ex) { From 4affe460ad1a6f8b0e8920d21ae48eca210023f5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 08:42:42 +0200 Subject: [PATCH 108/362] Restructered SQL queries in LocalRecreateDatabase --- .../Main/Database/LocalRecreateDatabase.cs | 882 ++++++++++++++---- 1 file changed, 690 insertions(+), 192 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 3b492aec1..9971201c1 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -86,45 +86,44 @@ namespace Duplicati.Library.Main.Database // {1} --> BlockHash-Size // {2} --> Temp-Table // {3} --> FullBlocklist-BlockCount [equals ({0} / {1}), if SQLite pays respect to ints] - private static string SELECT_BLOCKLIST_ENTRIES(long blocksize, long blockhashsize, string temptable, long fullBlockListBlockCount) => - $@" - SELECT - ""E"".""BlocksetID"", - ""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount}) AS ""FullIndex"", - ""F"".""BlockHash"", - MIN({blocksize}, ""E"".""Length"" - ((""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount})) * {blocksize})) AS ""BlockSize"", - ""E"".""Hash"", - ""E"".""BlocklistSize"", - ""E"".""BlocklistHash"" - FROM - ( - SELECT * FROM - ( - SELECT - ""A"".""BlocksetID"", - ""A"".""Index"" AS ""BlocklistIndex"", - MIN({fullBlockListBlockCount} * {blockhashsize}, (((""B"".""Length"" + {blocksize} - 1) / {blocksize}) - (""A"".""Index"" * ({fullBlockListBlockCount}))) * {blockhashsize}) AS ""BlocklistSize"", - ""A"".""Hash"" AS ""BlocklistHash"", - ""B"".""Length"" - FROM - ""BlocklistHash"" A, - ""Blockset"" B - WHERE - ""B"".""ID"" = ""A"".""BlocksetID"" - ) C, - ""Block"" D - WHERE - ""C"".""BlocklistHash"" = ""D"".""Hash"" - AND - ""C"".""BlocklistSize"" = ""D"".""Size"" - ) E, - ""{temptable}"" F - WHERE - ""F"".""BlocklistHash"" = ""E"".""Hash"" - ORDER BY - ""E"".""BlocksetID"", - ""FullIndex"" -"; + private static string SELECT_BLOCKLIST_ENTRIES(long blocksize, long blockhashsize, string temptable, long fullBlockListBlockCount) => $@" + SELECT + ""E"".""BlocksetID"", + ""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount}) AS ""FullIndex"", + ""F"".""BlockHash"", + MIN({blocksize}, ""E"".""Length"" - ((""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount})) * {blocksize})) AS ""BlockSize"", + ""E"".""Hash"", + ""E"".""BlocklistSize"", + ""E"".""BlocklistHash"" + FROM + ( + SELECT * + FROM + ( + SELECT + ""A"".""BlocksetID"", + ""A"".""Index"" AS ""BlocklistIndex"", + MIN({fullBlockListBlockCount} * {blockhashsize}, (((""B"".""Length"" + {blocksize} - 1) / {blocksize}) - (""A"".""Index"" * ({fullBlockListBlockCount}))) * {blockhashsize}) AS ""BlocklistSize"", + ""A"".""Hash"" AS ""BlocklistHash"", + ""B"".""Length"" + FROM + ""BlocklistHash"" A, + ""Blockset"" B + WHERE + ""B"".""ID"" = ""A"".""BlocksetID"" + ) C, + ""Block"" D + WHERE + ""C"".""BlocklistHash"" = ""D"".""Hash"" + AND ""C"".""BlocklistSize"" = ""D"".""Size"" + ) E, + ""{temptable}"" F + WHERE + ""F"".""BlocklistHash"" = ""E"".""Hash"" + ORDER BY + ""E"".""BlocksetID"", + ""FullIndex"" + "; public static async Task CreateAsync(LocalDatabase parentdb, Options options, LocalRecreateDatabase? dbnew = null) { @@ -139,28 +138,207 @@ namespace Duplicati.Library.Main.Database throw new Exception("Connection is null"); using var cmd = dbnew.m_connection.CreateCommand(); - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)"); - await cmd.ExecuteNonQueryAsync($@"CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" ON ""{dbnew.m_tempblocklist}"" (""BlockListHash"");"); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" ( + ""BlockListHash"" TEXT NOT NULL, + ""BlockHash"" TEXT NOT NULL, + ""Index"" INTEGER NOT NULL + ) + "); - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)"); - await cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockSize"");"); - await cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" ON ""{dbnew.m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");"); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" + ON ""{dbnew.m_tempblocklist}"" (""BlockListHash""); + "); - dbnew.m_insertFileCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (@PrefixId,@Path,@BlocksetId,@MetadataId); SELECT last_insert_rowid();"); - dbnew.m_insertFilesetEntryCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId,@FileId,@LastModified)"); - dbnew.m_insertMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); - dbnew.m_insertBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length,@FullHash); SELECT last_insert_rowid();"); - dbnew.m_insertBlocklistHashCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId,@Index,@Hash)"); - dbnew.m_updateBlockVolumeCommand = await dbnew.m_connection.CreateCommandAsync(@"UPDATE ""Block"" SET ""VolumeID"" = @VolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - dbnew.m_insertTempBlockListHash = await dbnew.m_connection.CreateCommandAsync($@"INSERT INTO ""{dbnew.m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (@BlocklistHash,@BlockHash,@Index) "); - dbnew.m_insertSmallBlockset = await dbnew.m_connection.CreateCommandAsync($@"INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (@FileHash,@BlockHash,@BlockSize) "); - dbnew.m_findBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash "); - dbnew.m_findMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length "); - dbnew.m_findFilesetCommand = await dbnew.m_connection.CreateCommandAsync(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId "); - dbnew.m_findTempBlockListHashCommand = await dbnew.m_connection.CreateCommandAsync($@"SELECT DISTINCT ""BlockListHash"" FROM ""{dbnew.m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash "); - dbnew.m_findHashBlockCommand = await dbnew.m_connection.CreateCommandAsync(@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); - dbnew.m_insertBlockCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (@Hash,@Size,@VolumeId)"); - dbnew.m_insertDuplicateBlockCommand = await dbnew.m_connection.CreateCommandAsync(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size), @VolumeId)"); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" ( + ""FileHash"" TEXT NOT NULL, + ""BlockHash"" TEXT NOT NULL, + ""BlockSize"" INTEGER NOT NULL + ) + "); + + await cmd.ExecuteNonQueryAsync($@" + CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" + ON ""{dbnew.m_tempsmalllist}"" ( + ""FileHash"", + ""BlockSize"" + ); + "); + + await cmd.ExecuteNonQueryAsync($@" + CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" + ON ""{dbnew.m_tempsmalllist}"" ( + ""BlockHash"", + ""BlockSize"" + ); + "); + + dbnew.m_insertFileCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""FileLookup"" ( + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + VALUES ( + @PrefixId, + @Path, + @BlocksetId, + @MetadataId + ); + SELECT last_insert_rowid(); + "); + + dbnew.m_insertFilesetEntryCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + VALUES ( + @FilesetId, + @FileId, + @LastModified + ) + "); + + dbnew.m_insertMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""Metadataset"" (""BlocksetID"") + VALUES (@BlocksetId); + SELECT last_insert_rowid(); + "); + + dbnew.m_insertBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""Blockset"" ( + ""Length"", + ""FullHash"" + ) + VALUES ( + @Length, + @FullHash + ); + SELECT last_insert_rowid(); + "); + + dbnew.m_insertBlocklistHashCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""BlocklistHash"" ( + ""BlocksetID"", + ""Index"", + ""Hash"" + ) + VALUES ( + @BlocksetId, + @Index, + @Hash + ) + "); + + dbnew.m_updateBlockVolumeCommand = await dbnew.m_connection.CreateCommandAsync(@" + UPDATE ""Block"" + SET ""VolumeID"" = @VolumeId + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + "); + + dbnew.m_insertTempBlockListHash = await dbnew.m_connection.CreateCommandAsync($@" + INSERT INTO ""{dbnew.m_tempblocklist}"" ( + ""BlocklistHash"", + ""BlockHash"", + ""Index"" + ) + VALUES ( + @BlocklistHash, + @BlockHash, + @Index + ) + "); + + dbnew.m_insertSmallBlockset = await dbnew.m_connection.CreateCommandAsync($@" + INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" ( + ""FileHash"", + ""BlockHash"", + ""BlockSize"" + ) + VALUES ( + @FileHash, + @BlockHash, + @BlockSize + ) + "); + + dbnew.m_findBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Blockset"" + WHERE + ""Length"" = @Length + AND ""FullHash"" = @FullHash + "); + + dbnew.m_findMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" + SELECT ""Metadataset"".""ID"" + FROM ""Metadataset"",""Blockset"" + WHERE + ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""Blockset"".""FullHash"" = @FullHash + AND ""Blockset"".""Length"" = @Length + "); + + dbnew.m_findFilesetCommand = await dbnew.m_connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + "); + + dbnew.m_findTempBlockListHashCommand = await dbnew.m_connection.CreateCommandAsync($@" + SELECT DISTINCT ""BlockListHash"" + FROM ""{dbnew.m_tempblocklist}"" + WHERE ""BlockListHash"" = @BlocklistHash + "); + + dbnew.m_findHashBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" + SELECT ""VolumeID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + "); + + dbnew.m_insertBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + VALUES ( + @Hash, + @Size, + @VolumeId + ) + "); + + dbnew.m_insertDuplicateBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT OR IGNORE INTO ""DuplicateBlock"" ( + ""BlockID"", + ""VolumeID"" + ) + VALUES ( + ( + SELECT ""ID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ), + @VolumeId + ) + "); return dbnew; } @@ -170,45 +348,146 @@ namespace Duplicati.Library.Main.Database using var cmd = m_connection.CreateCommand(m_rtr); //Update all small blocklists and matching blocks - var selectSmallBlocks = $@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""; + var selectSmallBlocks = $@" + SELECT + ""BlockHash"", + ""BlockSize"" + FROM ""{m_tempsmalllist}"" + "; - var selectBlockHashes = $@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} )"; + var selectBlockHashes = $@" + SELECT + ""BlockHash"" AS ""FullHash"", + ""BlockSize"" AS ""Length"" + FROM ( + {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} + ) + "; - var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; + var selectAllBlocks = @$" + SELECT DISTINCT + ""FullHash"", + ""Length"" + FROM ( + {selectBlockHashes} + UNION {selectSmallBlocks} + ) + "; - var selectNewBlocks = $@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" -FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({selectAllBlocks}) A -LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" ) -WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "; + var selectNewBlocks = $@" + SELECT + ""FullHash"" AS ""Hash"", + ""Length"" AS ""Size"", + -1 AS ""VolumeID"" + FROM ( + SELECT + ""A"".""FullHash"", + ""A"".""Length"", + CASE + WHEN ""B"".""Hash"" IS NULL + THEN '' + ELSE ""B"".""Hash"" + END AS ""Hash"", + CASE + WHEN ""B"".""Size"" is NULL + THEN -1 + ELSE ""B"".""Size"" + END AS ""Size"" + FROM ({selectAllBlocks}) A + LEFT OUTER JOIN ""Block"" B + ON ""B"".""Hash"" = ""A"".""FullHash"" + AND ""B"".""Size"" = ""A"".""Length"" + ) + WHERE + ""FullHash"" != ""Hash"" + AND ""Length"" != ""Size"" + "; - var insertBlocksCommand = - @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + - selectNewBlocks; + var insertBlocksCommand = @$" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + {selectNewBlocks} + "; // Insert all known blocks into block table with volumeid = -1 await cmd.ExecuteNonQueryAsync(insertBlocksCommand); - var selectBlocklistBlocksetEntries = - // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati - // this could be used instead - //@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" "; - $@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" " - ; + // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati + // this could be used instead + //@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" "; + var selectBlocklistBlocksetEntries = $@" + SELECT + ""E"".""BlocksetID"" AS ""BlocksetID"", + ""D"".""FullIndex"" AS ""Index"", + ""F"".""ID"" AS ""BlockID"" + FROM + ({SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) D, + ""BlocklistHash"" E, + ""Block"" F, + ""Block"" G + WHERE + ""D"".""BlocksetID"" = ""E"".""BlocksetID"" + AND ""D"".""BlocklistHash"" = ""E"".""Hash"" + AND ""D"".""BlocklistSize"" = ""G"".""Size"" + AND ""D"".""BlocklistHash"" = ""G"".""Hash"" + AND ""D"".""Blockhash"" = ""F"".""Hash"" + AND ""D"".""BlockSize"" = ""F"".""Size"" + "; - var selectBlocksetEntries = $@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{m_tempsmalllist}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {blocksize} "; + var selectBlocksetEntries = $@" + SELECT + ""Blockset"".""ID"" AS ""BlocksetID"", + 0 AS ""Index"", + ""Block"".""ID"" AS ""BlockID"" + FROM + ""Blockset"", + ""Block"", + ""{m_tempsmalllist}"" S + WHERE + ""Blockset"".""Fullhash"" = ""S"".""FileHash"" + AND ""S"".""BlockHash"" = ""Block"".""Hash"" + AND ""S"".""BlockSize"" = ""Block"".""Size"" + AND ""Blockset"".""Length"" = ""S"".""BlockSize"" + AND ""Blockset"".""Length"" <= {blocksize} + "; - var selectAllBlocksetEntries = - selectBlocklistBlocksetEntries + - @" UNION " + - selectBlocksetEntries; + var selectAllBlocksetEntries = @$" + {selectBlocklistBlocksetEntries} + UNION + {selectBlocksetEntries} + "; - var selectFiltered = - @"SELECT DISTINCT ""H"".""BlocksetID"", ""H"".""Index"", ""H"".""BlockID"" FROM (" + - selectAllBlocksetEntries + - @") H WHERE (""H"".""BlocksetID"" || ':' || ""H"".""Index"") NOT IN (SELECT (""ExistingBlocksetEntries"".""BlocksetID"" || ':' || ""ExistingBlocksetEntries"".""Index"") FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" )"; + var selectFiltered = @$" + SELECT DISTINCT + ""H"".""BlocksetID"", + ""H"".""Index"", + ""H"".""BlockID"" + FROM ({selectAllBlocksetEntries}) H + WHERE ( + ""H"".""BlocksetID"" + || ':' + || ""H"".""Index"" + ) NOT IN ( + SELECT ( + ""ExistingBlocksetEntries"".""BlocksetID"" + || ':' + || ""ExistingBlocksetEntries"".""Index"" + ) + FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" + ) + "; - var insertBlocksetEntriesCommand = - @"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered; + var insertBlocksetEntriesCommand = @$" + INSERT INTO ""BlocksetEntry"" ( + ""BlocksetID"", + ""Index"", + ""BlockID"" + ) + {selectFiltered} + "; try { @@ -220,8 +499,17 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "; using (var fixcmd = m_connection.CreateCommand(m_rtr)) { - await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" "); - await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" "); + await fixcmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{m_tempblocklist}-Failure"" AS + SELECT * + FROM ""{m_tempblocklist}"" + "); + + await fixcmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{m_tempsmalllist}-Failure"" AS + SELECT * + FROM ""{m_tempsmalllist}"" + "); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -247,51 +535,98 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "; public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, SqliteTransaction transaction, bool hashOnly = false) { using var cmd = m_connection.CreateCommand(m_rtr); - var insertBlocksCommand = $@"INSERT INTO BLOCK (Hash, Size, VolumeID) -SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM -( -SELECT NB.BlockHash, -MIN({blocksize}, BS.Length - ((NB.""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS BlockSize -FROM ( - SELECT TBL.BlockListHash, TBL.BlockHash, TBL.""Index"" FROM {m_tempblocklist} TBL - LEFT OUTER JOIN Block B ON (B.Hash = TBL.BlockHash) - WHERE B.Hash IS NULL - ) NB -JOIN BlocklistHash BH ON (BH.Hash = NB.BlocklistHash) -JOIN Blockset BS ON (BS.ID = BH.Blocksetid) "; - if (!hashOnly) - { - insertBlocksCommand += $@" UNION -SELECT TS.BlockHash, TS.BlockSize FROM -{m_tempsmalllist} TS -WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE - B.Hash = TS.BlockHash AND - B.Size = TS.BlockSize) -)"; - } + var extra = hashOnly ? "" : $@" + UNION + SELECT + TS.BlockHash, + TS.BlockSize + FROM {m_tempsmalllist} TS + WHERE NOT EXISTS ( + SELECT ""X"" + FROM Block AS B + WHERE + B.Hash = TS.BlockHash + AND B.Size = TS.BlockSize + ) + "; + var insertBlocksCommand = $@" + INSERT INTO BLOCK ( + Hash, + Size, + VolumeID + ) + SELECT DISTINCT + BlockHash AS Hash, + BlockSize AS Size, + -1 AS VolumeID + FROM ( + SELECT + NB.BlockHash, + MIN({blocksize}, BS.Length - ((NB.""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS BlockSize + FROM ( + SELECT + TBL.BlockListHash, + TBL.BlockHash, + TBL.""Index"" FROM {m_tempblocklist} TBL + LEFT OUTER JOIN Block B + ON (B.Hash = TBL.BlockHash) + WHERE B.Hash IS NULL + ) NB + JOIN BlocklistHash BH + ON (BH.Hash = NB.BlocklistHash) + JOIN Blockset BS + ON (BS.ID = BH.Blocksetid) + {extra} + ) + "; - var insertBlocksetEntriesCommand = $@"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) -SELECT DISTINCT BH.blocksetid, (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" as FullIndex, BK.ID AS BlockID -FROM {m_tempblocklist} TBL - JOIN blocklisthash BH ON (BH.hash = TBL.blocklisthash) - JOIN block BK ON (BK.Hash = TBL.BlockHash) - LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlockSetID = BH.BlocksetID AND BE.""Index"" = (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"") -WHERE -BE.BlockSetID IS NULL "; - if (!hashOnly) - { - insertBlocksetEntriesCommand += $@" UNION -SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID -FROM {m_tempsmalllist} TS - JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND - BS.Length = TS.BlockSize AND - BS.Length <= {blocksize}) - JOIN Block BL ON (BL.Hash = TS.BlockHash AND - BL.Size = TS.BlockSize) - LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) -WHERE -BE.BlocksetID IS NULL "; - } + extra = hashOnly ? "" : $@" + UNION + SELECT + BS.ID AS BlocksetID, + 0 AS ""Index"", + BL.ID AS BlockID + FROM {m_tempsmalllist} TS + JOIN Blockset BS + ON ( + BS.FullHash = TS.FileHash + AND BS.Length = TS.BlockSize + AND BS.Length <= {blocksize} + ) + JOIN Block BL + ON ( + BL.Hash = TS.BlockHash + AND BL.Size = TS.BlockSize + ) + LEFT OUTER JOIN BlocksetEntry BE + ON ( + BE.BlocksetID = BS.ID + AND BE.""Index"" = 0 + ) + WHERE BE.BlocksetID IS NULL + "; + var insertBlocksetEntriesCommand = $@" + INSERT INTO BlocksetEntry ( + BlocksetID, + ""Index"", + BlockID + ) + SELECT DISTINCT + BH.blocksetid, + (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" as FullIndex, + BK.ID AS BlockID + FROM {m_tempblocklist} TBL + JOIN blocklisthash BH + ON (BH.hash = TBL.blocklisthash) + JOIN block BK + ON (BK.Hash = TBL.BlockHash) + LEFT OUTER JOIN BlocksetEntry BE + ON ( + BE.BlockSetID = BH.BlocksetID + AND BE.""Index"" = (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" + ) + WHERE BE.BlockSetID IS NULL + {extra}"; try { @@ -306,8 +641,17 @@ BE.BlocksetID IS NULL "; using (var fixcmd = m_connection.CreateCommand(m_rtr)) { - await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" "); - await fixcmd.ExecuteNonQueryAsync($@"CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" "); + await fixcmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{m_tempblocklist}_Failure"" AS + SELECT * + FROM ""{m_tempblocklist}"" + "); + + await fixcmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{m_tempsmalllist}_Failure"" + AS SELECT * + FROM ""{m_tempsmalllist}"" + "); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -524,7 +868,15 @@ BE.BlocksetID IS NULL "; public async IAsyncEnumerable GetBlockLists(long volumeid) { - using var cmd = m_connection.CreateCommand(@"SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId") + using var cmd = m_connection.CreateCommand(@" + SELECT DISTINCT ""BlocklistHash"".""Hash"" + FROM + ""BlocklistHash"", + ""Block"" + WHERE + ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" + AND ""Block"".""VolumeID"" = @VolumeId + ") .SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); @@ -537,25 +889,59 @@ BE.BlocksetID IS NULL "; { using (var cmd = m_connection.CreateCommand()) { - var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume"""; + var selectCommand = @" + SELECT DISTINCT + ""RemoteVolume"".""Name"", + ""RemoteVolume"".""Hash"", + ""RemoteVolume"".""Size"", + ""RemoteVolume"".""ID"" + FROM ""RemoteVolume"" + "; - var missingBlocklistEntries = - $@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {blocksize / hashsize}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL"; + var missingBlocklistEntries = $@" + SELECT ""BlocklistHash"".""Hash"" + FROM ""BlocklistHash"" + LEFT OUTER JOIN ""BlocksetEntry"" + ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {blocksize / hashsize}) + AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" + WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL + "; - var missingBlockInfo = - @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 AND SIZE > 0"; + var missingBlockInfo = @" + SELECT ""VolumeID"" + FROM ""Block"" + WHERE + ""VolumeID"" < 0 + AND SIZE > 0 + "; - var missingBlocklistVolumes = - $@"SELECT ""VolumeID"" FROM ""Block"", ({missingBlocklistEntries}) A WHERE ""A"".""Hash"" = ""Block"".""Hash"" " - ; + var missingBlocklistVolumes = $@" + SELECT ""VolumeID"" + FROM + ""Block"", + ({missingBlocklistEntries}) A + WHERE ""A"".""Hash"" = ""Block"".""Hash"" + "; - var countMissingInformation = $@"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({missingBlockInfo} UNION {missingBlocklistVolumes}))"; + var countMissingInformation = $@" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT ""VolumeID"" + FROM ( + {missingBlockInfo} + UNION {missingBlocklistVolumes} + ) + ) + "; if (passNo == 0) { // On the first pass, we select all the volumes we know we need, // which may be an empty list - cmd.SetCommandAndParameters($@"{selectCommand} WHERE ""ID"" IN ({missingBlocklistVolumes})"); + cmd.SetCommandAndParameters($@" + {selectCommand} + WHERE ""ID"" IN ({missingBlocklistVolumes}) + "); // Reset the list m_proccessedVolumes.Clear(); @@ -574,11 +960,18 @@ BE.BlocksetID IS NULL "; { // On the second pass, we select all volumes that are not mentioned in the db - var mentionedVolumes = - @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; + var mentionedVolumes = @" + SELECT DISTINCT ""VolumeID"" + FROM ""Block"" + "; cmd - .SetCommandAndParameters($@"{selectCommand} WHERE ""ID"" IN ({mentionedVolumes}) AND ""Type"" = @Type ") + .SetCommandAndParameters($@" + {selectCommand} + WHERE + ""ID"" IN ({mentionedVolumes}) + AND ""Type"" = @Type + ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); @@ -588,7 +981,10 @@ BE.BlocksetID IS NULL "; // On the final pass, we select all volumes // the filter will ensure that we do not download anything twice cmd - .SetCommandAndParameters($@"{selectCommand} WHERE ""Type"" = @Type") + .SetCommandAndParameters($@" + {selectCommand} + WHERE ""Type"" = @Type + ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); } } @@ -622,50 +1018,118 @@ BE.BlocksetID IS NULL "; // The second part removes references to the non-present remote files, // and marks the index files that pointed to them, such that they will be removed later on var sql = $@" -CREATE TEMPORARY TABLE ""{tablename}"" AS -SELECT - ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" -FROM - (SELECT - ""Block"".""ID"", ""Block"".""VolumeID"", ""Remotevolume"".""State"" - FROM ""Block"", ""Remotevolume"" - WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Temporary}' - ) A, - (SELECT - ""DuplicateBlock"".""BlockID"", MIN(""DuplicateBlock"".""VolumeID"") AS ""VolumeID"", ""Remotevolume"".""State"" - FROM ""DuplicateBlock"", ""Remotevolume"" - WHERE ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Verified}' - GROUP BY ""DuplicateBlock"".""BlockID"", ""Remotevolume"".""State"" - ) B -WHERE ""A"".""ID"" = ""B"".""BlockID""; + CREATE TEMPORARY TABLE ""{tablename}"" AS + SELECT + ""A"".""ID"" AS ""BlockID"", + ""A"".""VolumeID"" AS ""SourceVolumeID"", + ""A"".""State"" AS ""SourceVolumeState"", + ""B"".""VolumeID"" AS ""TargetVolumeID"", + ""B"".""State"" AS ""TargetVolumeState"" + FROM + ( + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"", + ""Remotevolume"".""State"" + FROM + ""Block"", + ""Remotevolume"" + WHERE + ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Temporary}' + ) A, + ( + SELECT + ""DuplicateBlock"".""BlockID"", + MIN(""DuplicateBlock"".""VolumeID"") AS ""VolumeID"", + ""Remotevolume"".""State"" + FROM + ""DuplicateBlock"", + ""Remotevolume"" + WHERE + ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Verified}' + GROUP BY + ""DuplicateBlock"".""BlockID"", + ""Remotevolume"".""State"" + ) B + WHERE ""A"".""ID"" = ""B"".""BlockID""; -UPDATE ""Block"" - SET ""VolumeID"" = (SELECT - ""TargetVolumeID"" FROM ""{tablename}"" - WHERE ""Block"".""ID"" = ""{tablename}"".""BlockID"" - ) - WHERE ""Block"".""ID"" IN (SELECT ""BlockID"" FROM ""{tablename}""); + UPDATE ""Block"" + SET ""VolumeID"" = ( + SELECT ""TargetVolumeID"" + FROM ""{tablename}"" + WHERE ""Block"".""ID"" = ""{tablename}"".""BlockID"" + ) + WHERE ""Block"".""ID"" IN ( + SELECT ""BlockID"" + FROM ""{tablename}"" + ); -UPDATE ""DuplicateBlock"" - SET ""VolumeID"" = (SELECT - ""SourceVolumeID"" FROM ""{tablename}"" - WHERE ""DuplicateBlock"".""BlockID"" = ""{tablename}"".""BlockID"" - ) -WHERE - (""DuplicateBlock"".""BlockID"", ""DuplicateBlock"".""VolumeID"") - IN (SELECT ""BlockID"", ""TargetVolumeID"" FROM ""{tablename}""); + UPDATE ""DuplicateBlock"" + SET ""VolumeID"" = ( + SELECT ""SourceVolumeID"" + FROM ""{tablename}"" + WHERE ""DuplicateBlock"".""BlockID"" = ""{tablename}"".""BlockID"" + ) + WHERE ( + ""DuplicateBlock"".""BlockID"", + ""DuplicateBlock"".""VolumeID"" + ) IN ( + SELECT + ""BlockID"", + ""TargetVolumeID"" + FROM ""{tablename}"" + ); -DROP TABLE ""{tablename}""; + DROP TABLE ""{tablename}""; -DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); -DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); -DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""); -"; + DELETE FROM ""IndexBlockLink"" + WHERE ""BlockVolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = '{RemoteVolumeType.Blocks}' + AND ""State"" = '{RemoteVolumeState.Temporary}' + AND ""ID"" NOT IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Block"" + ) + ); + + DELETE FROM ""DuplicateBlock"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = '{RemoteVolumeType.Blocks}' + AND ""State"" = '{RemoteVolumeState.Temporary}' + AND ""ID"" NOT IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Block"" + ) + ); + + DELETE FROM ""RemoteVolume"" + WHERE + ""Type"" = '{RemoteVolumeType.Blocks}' + AND ""State"" = '{RemoteVolumeState.Temporary}' + AND ""ID"" NOT IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Block"" + ); + "; // We could delete these, but we don't have to, so we keep them around until the next compact is done // UPDATE ""RemoteVolume"" SET ""State"" = ""{3}"" WHERE ""Type"" = ""{5}"" AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink""); - var countsql = $@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' "; + var countsql = $@" + SELECT COUNT(*) + FROM ""RemoteVolume"" + WHERE + ""State"" = '{RemoteVolumeState.Temporary}' + AND ""Type"" = '{RemoteVolumeType.Blocks}' + "; using (var cmd = m_connection.CreateCommand(m_rtr)) { @@ -700,14 +1164,48 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" using var cmd = m_connection.CreateCommand(m_rtr); // 1. Select blocks not used by any file and not as a blocklist into temporary table - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tmptablename}"" - AS SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""Block"" - WHERE ""Block"".""ID"" NOT IN (SELECT ""BlocksetEntry"".""BlockID"" FROM ""BlocksetEntry"") - AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")"); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tmptablename}"" AS + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""Block"" + WHERE + ""Block"".""ID"" NOT IN ( + SELECT ""BlocksetEntry"".""BlockID"" + FROM ""BlocksetEntry"" + ) + AND ""Block"".""Hash"" NOT IN ( + SELECT ""BlocklistHash"".""Hash"" + FROM ""BlocklistHash"" + ) + "); + // 2. Insert blocks into DeletedBlock table - await cmd.ExecuteNonQueryAsync($@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{tmptablename}"""); + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""DeletedBlock"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""{tmptablename}"" + "); + // 3. Remove blocks from Block table - await cmd.ExecuteNonQueryAsync($@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{tmptablename}"")"); + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Block"" + WHERE ""ID"" IN ( + SELECT ""ID"" + FROM ""{tmptablename}"" + ) + "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"""); await m_rtr.CommitAsync(); } @@ -755,4 +1253,4 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" base.Dispose(); } } -} \ No newline at end of file +} From d45e977d53349136c5a5e3b22ec359a85b0a843f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 08:46:18 +0200 Subject: [PATCH 109/362] Found more factory construction inconsistencies --- .../Library/Main/Database/LocalPurgeDatabase.cs | 4 ++-- .../Library/Main/Database/LocalRecreateDatabase.cs | 2 +- .../Library/Main/Database/LocalRepairDatabase.cs | 8 ++++---- .../Library/Main/Database/LocalRestoreDatabase.cs | 14 +++++++------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 75afe24cb..1eb2759dd 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -31,7 +31,7 @@ namespace Duplicati.Library.Main.Database { internal class LocalPurgeDatabase : LocalDeleteDatabase { - public static new async Task CreateAsync(string path, long pagecachesize, LocalPurgeDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalPurgeDatabase? dbnew = null) { dbnew ??= new LocalPurgeDatabase(); @@ -42,7 +42,7 @@ namespace Duplicati.Library.Main.Database return dbnew; } - public static new async Task CreateAsync(LocalDatabase dbparent, LocalPurgeDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase dbparent, LocalPurgeDatabase? dbnew = null) { dbnew ??= new LocalPurgeDatabase(); diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 9971201c1..cd4169ae9 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -129,7 +129,7 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalRecreateDatabase(); - dbnew = (LocalRecreateDatabase)await CreateLocalDatabaseAsync(parentdb, dbnew); + dbnew = (LocalRecreateDatabase)await LocalRestoreDatabase.CreateAsync(parentdb, dbnew); dbnew.m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); dbnew.m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 675c938ee..ccfa756bc 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -41,13 +41,13 @@ namespace Duplicati.Library.Main.Database /// /// The path to the database /// The page cache size - public static async Task CreateRepairDatabase(string path, long pagecachesize) + public static async Task CreateRepairDatabase(string path, long pagecachesize, LocalRepairDatabase? dbnew = null) { - var db = new LocalRepairDatabase(); + dbnew ??= new LocalRepairDatabase(); - db = (LocalRepairDatabase)await CreateLocalDatabaseAsync(db, path, "Repair", false, pagecachesize); + dbnew = (LocalRepairDatabase)await CreateLocalDatabaseAsync(path, "Repair", false, pagecachesize, dbnew); - return db; + return dbnew; } /// diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 9901fda4a..784e89d88 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -59,19 +59,19 @@ namespace Duplicati.Library.Main.Database public DateTime RestoreTime { get { return m_restoreTime; } } - public static async Task CreateAsync(string path, long pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize, LocalRestoreDatabase? dbnew = null) { - var db = new LocalRestoreDatabase(); + dbnew ??= new LocalRestoreDatabase(); - db = (LocalRestoreDatabase)await CreateLocalDatabaseAsync(db, path, "Restore", false, pagecachesize); - db.ShouldCloseConnection = true; + dbnew = (LocalRestoreDatabase)await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew); + dbnew.ShouldCloseConnection = true; - return db; + return dbnew; } - public static async Task CreateAsync(LocalDatabase dbparent) + public static async Task CreateAsync(LocalDatabase dbparent, LocalRestoreDatabase? dbnew = null) { - var dbnew = new LocalRestoreDatabase(); + dbnew ??= new LocalRestoreDatabase(); return (LocalRestoreDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); } From 8c11cc9f71260d6202e3be035a2cc75c866f6a09 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 09:05:15 +0200 Subject: [PATCH 110/362] Removed whitespace --- .../Main/Database/LocalTestDatabase.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 6a66dbed8..ffe77917f 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From 60b8df6474d175bf32ff4180fe9100a0ad2392aa Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 09:05:34 +0200 Subject: [PATCH 111/362] Changed LocalTestDatabase to use factory construction --- .../Main/Database/LocalTestDatabase.cs | 84 +++++++++++++++---- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index ffe77917f..cc282fede 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -24,20 +24,28 @@ using System; using System.Data; using System.Linq; +using System.Threading.Tasks; using System.Collections.Generic; namespace Duplicati.Library.Main.Database { internal class LocalTestDatabase : LocalDatabase { - public LocalTestDatabase(string path, long pagecachesize) - : base(path, "Test", true, pagecachesize) + public static async Task CreateAsync(string path, long pagecachesize, LocalTestDatabase? dbnew = null) { + dbnew ??= new LocalTestDatabase(); + + dbnew = (LocalTestDatabase)await CreateLocalDatabaseAsync(path, "Test", true, pagecachesize, dbnew); + dbnew.ShouldCloseConnection = true; + + return dbnew; } - public LocalTestDatabase(LocalDatabase parent) - : base(parent) + public static async Task CreateAsync(LocalDatabase dbparent, LocalTestDatabase? dbnew = null) { + dbnew ??= new LocalTestDatabase(); + + return (LocalTestDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); } public void UpdateVerificationCount(string name, IDbTransaction? tr) @@ -70,7 +78,7 @@ namespace Duplicati.Library.Main.Database { var rnd = new Random(); - // First round is the new items + // First round is the new items var res = (from n in volumes where n.VerificationCount == 0 select n).ToList(); while (res.Count > samples) res.RemoveAt(rnd.Next(0, res.Count)); @@ -191,26 +199,36 @@ namespace Duplicati.Library.Main.Database private abstract class Basiclist : IDisposable { - protected readonly IDbConnection m_connection; - protected readonly string m_volumename; - protected string m_tablename; - protected ReusableTransaction m_rtr; - protected IDbCommand m_insertCommand; + protected IDbConnection m_connection = null!; + protected string m_volumename = null!; + protected string m_tablename = null!; + protected ReusableTransaction m_rtr = null!; + protected IDbCommand m_insertCommand = null!; + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] protected Basiclist(IDbConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) { - m_connection = connection; - m_volumename = volumename; - m_rtr = rtr; + throw new NotSupportedException("Use CreateAsync method instead."); + } + + protected Basiclist() { } + + protected static async Task CreateAsync(Basiclist bl, IDbConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) + { + bl.m_connection = connection; + bl.m_volumename = volumename; + bl.m_rtr = rtr; var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + using (var cmd = bl.m_connection.CreateCommand(bl.m_rtr.Transaction)) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}")); - m_tablename = tablename; + bl.m_tablename = tablename; } - m_insertCommand = m_connection.CreateCommand(m_rtr.Transaction, FormatInvariant($@"INSERT INTO ""{m_tablename}"" {insertCommand}")); + bl.m_insertCommand = bl.m_connection.CreateCommand(bl.m_rtr.Transaction, FormatInvariant($@"INSERT INTO ""{bl.m_tablename}"" {insertCommand}")); + + return bl; } public virtual void Dispose() @@ -239,9 +257,20 @@ namespace Duplicati.Library.Main.Database private const string TABLE_PREFIX = "Filelist"; private const string TABLE_FORMAT = @"(""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL)"; private const string INSERT_COMMAND = @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (@Path,@Size,@Hash,@Metasize,@Metahash)"; + + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Filelist(IDbConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { + throw new NotSupportedException("Use CreateAsync method instead."); + } + + private Filelist() { } + + public static async Task CreateAsync(IDbConnection connection, string volumename, ReusableTransaction rtr) + { + var bl = new Filelist(); + return (Filelist)await CreateAsync(bl, connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); } public void Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time) @@ -304,9 +333,19 @@ namespace Duplicati.Library.Main.Database private const string TABLE_FORMAT = @"(""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; private const string INSERT_COMMAND = @"(""Name"", ""Hash"", ""Size"") VALUES (@Name,@Hash,@Size)"; + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Indexlist(IDbConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { + throw new NotSupportedException("Use CreateAsync method instead."); + } + + private Indexlist() { } + + public static async Task CreateAsync(IDbConnection connection, string volumename, ReusableTransaction rtr) + { + var bl = new Indexlist(); + return (Indexlist)await CreateAsync(bl, connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); } public void AddBlockLink(string filename, string hash, long length) @@ -366,9 +405,20 @@ namespace Duplicati.Library.Main.Database private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Blocklist(IDbConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) - { } + { + throw new NotSupportedException("Use CreateAsync method instead."); + } + + private Blocklist() { } + + public static async Task CreateAsync(IDbConnection connection, string volumename, ReusableTransaction rtr) + { + var bl = new Blocklist(); + return (Blocklist)await CreateAsync(bl, connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + } public void AddBlock(string hash, long size) { From 602a30d8e24a49835078e6cdb7e9946e6043a196 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 09:11:25 +0200 Subject: [PATCH 112/362] Purged IDb from LocalTestDatabase --- .../Main/Database/LocalTestDatabase.cs | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index cc282fede..11a9273c3 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -22,10 +22,10 @@ #nullable enable using System; -using System.Data; using System.Linq; using System.Threading.Tasks; using System.Collections.Generic; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { @@ -48,9 +48,9 @@ namespace Duplicati.Library.Main.Database return (LocalTestDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); } - public void UpdateVerificationCount(string name, IDbTransaction? tr) + public void UpdateVerificationCount(string name) { - using (var cmd = m_connection.CreateCommand(tr)) + using (var cmd = m_connection.CreateCommand(m_rtr)) cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE WHEN ""VerificationCount"" <= 0 THEN (SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"") ELSE ""VerificationCount"" + 1 END) WHERE ""Name"" = @Name") .SetParameterValue("@Name", name) .ExecuteNonQuery(); @@ -64,7 +64,7 @@ namespace Duplicati.Library.Main.Database public string Hash { get; init; } public long VerificationCount { get; init; } - public RemoteVolume(IDataReader rd) + public RemoteVolume(SqliteDataReader rd) { ID = rd.ConvertValueToInt64(0); Name = rd.ConvertValueToString(1) ?? ""; @@ -126,12 +126,12 @@ namespace Duplicati.Library.Main.Database return res; } - public IEnumerable SelectTestTargets(long samples, Options options, IDbTransaction? tr) + public IEnumerable SelectTestTargets(long samples, Options options) { var tp = GetFilelistWhereClause(options.Time, options.Version); samples = Math.Max(1, samples); - using (var cmd = m_connection.CreateCommand(tr)) + using (var cmd = m_connection.CreateCommand(m_rtr)) { // Select any broken items cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) AND (""ArchiveTime"" = 0)") @@ -199,34 +199,32 @@ namespace Duplicati.Library.Main.Database private abstract class Basiclist : IDisposable { - protected IDbConnection m_connection = null!; + protected LocalDatabase m_db = null!; protected string m_volumename = null!; protected string m_tablename = null!; - protected ReusableTransaction m_rtr = null!; - protected IDbCommand m_insertCommand = null!; + protected SqliteCommand m_insertCommand = null!; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] - protected Basiclist(IDbConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) + protected Basiclist(SqliteConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) { throw new NotSupportedException("Use CreateAsync method instead."); } protected Basiclist() { } - protected static async Task CreateAsync(Basiclist bl, IDbConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) + protected static async Task CreateAsync(Basiclist bl, LocalDatabase db, string volumename, string tablePrefix, string tableFormat, string insertCommand) { - bl.m_connection = connection; + bl.m_db = db; bl.m_volumename = volumename; - bl.m_rtr = rtr; var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = bl.m_connection.CreateCommand(bl.m_rtr.Transaction)) + using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction)) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}")); bl.m_tablename = tablename; } - bl.m_insertCommand = bl.m_connection.CreateCommand(bl.m_rtr.Transaction, FormatInvariant($@"INSERT INTO ""{bl.m_tablename}"" {insertCommand}")); + bl.m_insertCommand = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction, FormatInvariant($@"INSERT INTO ""{bl.m_tablename}"" {insertCommand}")); return bl; } @@ -236,7 +234,7 @@ namespace Duplicati.Library.Main.Database if (m_tablename != null) try { - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction)) cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); } catch { } @@ -259,7 +257,7 @@ namespace Duplicati.Library.Main.Database private const string INSERT_COMMAND = @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (@Path,@Size,@Hash,@Metasize,@Metahash)"; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] - public Filelist(IDbConnection connection, string volumename, ReusableTransaction rtr) + public Filelist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { throw new NotSupportedException("Use CreateAsync method instead."); @@ -267,15 +265,15 @@ namespace Duplicati.Library.Main.Database private Filelist() { } - public static async Task CreateAsync(IDbConnection connection, string volumename, ReusableTransaction rtr) + public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Filelist(); - return (Filelist)await CreateAsync(bl, connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + return (Filelist)await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); } public void Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time) { - m_insertCommand.SetTransaction(m_rtr.Transaction) + m_insertCommand.SetTransaction(m_db.Transaction) .SetParameterValue("@Path", path) .SetParameterValue("@Size", hash == null ? -1 : size) .SetParameterValue("@Hash", hash) @@ -294,7 +292,7 @@ namespace Duplicati.Library.Main.Database var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"") "); var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { try { @@ -334,7 +332,7 @@ namespace Duplicati.Library.Main.Database private const string INSERT_COMMAND = @"(""Name"", ""Hash"", ""Size"") VALUES (@Name,@Hash,@Size)"; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] - public Indexlist(IDbConnection connection, string volumename, ReusableTransaction rtr) + public Indexlist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { throw new NotSupportedException("Use CreateAsync method instead."); @@ -342,15 +340,15 @@ namespace Duplicati.Library.Main.Database private Indexlist() { } - public static async Task CreateAsync(IDbConnection connection, string volumename, ReusableTransaction rtr) + public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Indexlist(); - return (Indexlist)await CreateAsync(bl, connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + return (Indexlist)await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); } public void AddBlockLink(string filename, string hash, long length) { - m_insertCommand.SetTransaction(m_rtr.Transaction) + m_insertCommand.SetTransaction(m_db.Transaction) .SetParameterValue("@Name", filename) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", length) @@ -366,7 +364,7 @@ namespace Duplicati.Library.Main.Database var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") "); var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { try { @@ -406,7 +404,7 @@ namespace Duplicati.Library.Main.Database private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] - public Blocklist(IDbConnection connection, string volumename, ReusableTransaction rtr) + public Blocklist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { throw new NotSupportedException("Use CreateAsync method instead."); @@ -414,15 +412,15 @@ namespace Duplicati.Library.Main.Database private Blocklist() { } - public static async Task CreateAsync(IDbConnection connection, string volumename, ReusableTransaction rtr) + public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Blocklist(); - return (Blocklist)await CreateAsync(bl, connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + return (Blocklist)await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); } public void AddBlock(string hash, long size) { - m_insertCommand.SetTransaction(m_rtr.Transaction) + m_insertCommand.SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .ExecuteNonQuery(); @@ -440,7 +438,7 @@ namespace Duplicati.Library.Main.Database var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "); var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { try { @@ -466,17 +464,17 @@ namespace Duplicati.Library.Main.Database } } - public IFilelist CreateFilelist(string name, ReusableTransaction rtr) + public IFilelist CreateFilelist(string name) { return new Filelist(m_connection, name, rtr); } - public IIndexlist CreateIndexlist(string name, ReusableTransaction rtr) + public IIndexlist CreateIndexlist(string name) { return new Indexlist(m_connection, name, rtr); } - public IBlocklist CreateBlocklist(string name, ReusableTransaction rtr) + public IBlocklist CreateBlocklist(string name) { return new Blocklist(m_connection, name, rtr); } From d6889569bbac78f8ea1cb98ab9fc26e84f9a5619 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 09:13:01 +0200 Subject: [PATCH 113/362] Removed FormatInvariant as it's now handled by the extension methods --- .../Main/Database/LocalTestDatabase.cs | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 11a9273c3..994a4ccc9 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -220,11 +220,11 @@ namespace Duplicati.Library.Main.Database using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction)) { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}")); + cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}"); bl.m_tablename = tablename; } - bl.m_insertCommand = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction, FormatInvariant($@"INSERT INTO ""{bl.m_tablename}"" {insertCommand}")); + bl.m_insertCommand = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction, $@"INSERT INTO ""{bl.m_tablename}"" {insertCommand}"); return bl; } @@ -235,7 +235,7 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction)) - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); + cmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{m_tablename}"""); } catch { } finally { m_tablename = null!; } @@ -286,11 +286,11 @@ namespace Duplicati.Library.Main.Database { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Path"" AS ""Path"", CASE WHEN ""B"".""Fullhash"" IS NULL THEN -1 ELSE ""B"".""Length"" END AS ""Size"", ""B"".""Fullhash"" AS ""Hash"", ""C"".""Length"" AS ""Metasize"", ""C"".""Fullhash"" AS ""Metahash"" FROM (SELECT ""File"".""Path"", ""File"".""BlocksetID"" AS ""FileBlocksetID"", ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" from ""Remotevolume"", ""Fileset"", ""FilesetEntry"", ""File"", ""Metadataset"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"") A LEFT OUTER JOIN ""Blockset"" B ON ""B"".""ID"" = ""A"".""FileBlocksetID"" LEFT OUTER JOIN ""Blockset"" C ON ""C"".""ID""=""A"".""MetadataBlocksetID"" "); - var extra = FormatInvariant($@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Path"" AS ""Path"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{cmpName}"" )"); - var missing = FormatInvariant($@"SELECT @TypeMissing AS ""Type"", ""Path"" AS ""Path"" FROM ""{cmpName}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tablename}"")"); - var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"") "); - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); + var create = $@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Path"" AS ""Path"", CASE WHEN ""B"".""Fullhash"" IS NULL THEN -1 ELSE ""B"".""Length"" END AS ""Size"", ""B"".""Fullhash"" AS ""Hash"", ""C"".""Length"" AS ""Metasize"", ""C"".""Fullhash"" AS ""Metahash"" FROM (SELECT ""File"".""Path"", ""File"".""BlocksetID"" AS ""FileBlocksetID"", ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" from ""Remotevolume"", ""Fileset"", ""FilesetEntry"", ""File"", ""Metadataset"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"") A LEFT OUTER JOIN ""Blockset"" B ON ""B"".""ID"" = ""A"".""FileBlocksetID"" LEFT OUTER JOIN ""Blockset"" C ON ""C"".""ID""=""A"".""MetadataBlocksetID"" "; + var extra = $@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Path"" AS ""Path"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{cmpName}"" )"; + var missing = $@"SELECT @TypeMissing AS ""Type"", ""Path"" AS ""Path"" FROM ""{cmpName}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tablename}"")"; + var modified = $@"SELECT @TypeModified AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"") "; + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { @@ -300,7 +300,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Name", m_volumename) .ExecuteNonQuery(); - cmd.SetCommandAndParameters(FormatInvariant($"{extra} UNION {missing} UNION {modified}")) + cmd.SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); @@ -358,11 +358,11 @@ namespace Duplicati.Library.Main.Database public IEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Name"", ""A"".""Hash"", ""A"".""Size"" FROM ""Remotevolume"" A, ""Remotevolume"" B, ""IndexBlockLink"" WHERE ""B"".""Name"" = @Name AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" "); - var extra = FormatInvariant($@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Name"" AS ""Name"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Name"" NOT IN ( SELECT ""Name"" FROM ""{cmpName}"" )"); - var missing = FormatInvariant($@"SELECT @TypeMissing AS ""Type"", ""Name"" AS ""Name"" FROM ""{cmpName}"" WHERE ""Name"" NOT IN (SELECT ""Name"" FROM ""{m_tablename}"")"); - var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") "); - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); + var create = $@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Name"", ""A"".""Hash"", ""A"".""Size"" FROM ""Remotevolume"" A, ""Remotevolume"" B, ""IndexBlockLink"" WHERE ""B"".""Name"" = @Name AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" "; + var extra = $@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Name"" AS ""Name"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Name"" NOT IN ( SELECT ""Name"" FROM ""{cmpName}"" )"; + var missing = $@"SELECT @TypeMissing AS ""Type"", ""Name"" AS ""Name"" FROM ""{cmpName}"" WHERE ""Name"" NOT IN (SELECT ""Name"" FROM ""{m_tablename}"")"; + var modified = $@"SELECT @TypeModified AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") "; + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { @@ -372,7 +372,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Name", m_volumename) .ExecuteNonQuery(); - cmd.SetCommandAndParameters(FormatInvariant($"{extra} UNION {missing} UNION {modified}")) + cmd.SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); @@ -432,11 +432,11 @@ namespace Duplicati.Library.Main.Database var curBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""Remotevolume"", ""Block"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"""; var duplBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""DuplicateBlock"", ""Block"" WHERE ""DuplicateBlock"".""VolumeID"" = (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name) AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"""; var delBlocks = @"SELECT ""DeletedBlock"".""Hash"" AS ""Hash"", ""DeletedBlock"".""Size"" AS ""Size"" FROM ""DeletedBlock"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = @Name AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"""; - var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT DISTINCT ""Hash"" AS ""Hash"", ""Size"" AS ""Size"" FROM ({curBlocks} UNION {delBlocks} UNION {duplBlocks})"); - var extra = FormatInvariant($@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{cmpName}"" )"); - var missing = FormatInvariant($@"SELECT @TypeMissing AS ""Type"", ""Hash"" AS ""Hash"" FROM ""{cmpName}"" WHERE ""Hash"" NOT IN (SELECT ""Hash"" FROM ""{m_tablename}"")"); - var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "); - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); + var create = $@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT DISTINCT ""Hash"" AS ""Hash"", ""Size"" AS ""Size"" FROM ({curBlocks} UNION {delBlocks} UNION {duplBlocks})"; + var extra = $@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{cmpName}"" )"; + var missing = $@"SELECT @TypeMissing AS ""Type"", ""Hash"" AS ""Hash"" FROM ""{cmpName}"" WHERE ""Hash"" NOT IN (SELECT ""Hash"" FROM ""{m_tablename}"")"; + var modified = $@"SELECT @TypeModified AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "; + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { @@ -446,7 +446,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Name", m_volumename) .ExecuteNonQuery(); - cmd.SetCommandAndParameters(FormatInvariant($"{extra} UNION {missing} UNION {modified}")) + cmd.SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); From ef2f856c61ecdec4a926f402d007d3373c005d85 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 09:22:29 +0200 Subject: [PATCH 114/362] Made everything in LocalTestDatabase async --- .../Main/Database/LocalTestDatabase.cs | 147 ++++++++++-------- 1 file changed, 85 insertions(+), 62 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 994a4ccc9..234ab0b09 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -26,6 +26,7 @@ using System.Linq; using System.Threading.Tasks; using System.Collections.Generic; using Microsoft.Data.Sqlite; +using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Database { @@ -74,7 +75,7 @@ namespace Duplicati.Library.Main.Database } } - private IEnumerable FilterByVerificationCount(IEnumerable volumes, long samples, long maxverification) + private static List FilterByVerificationCount(IEnumerable volumes, long samples, long maxverification) { var rnd = new Random(); @@ -126,9 +127,9 @@ namespace Duplicati.Library.Main.Database return res; } - public IEnumerable SelectTestTargets(long samples, Options options) + public async IAsyncEnumerable SelectTestTargets(long samples, Options options) { - var tp = GetFilelistWhereClause(options.Time, options.Version); + var tp = await GetFilelistWhereClause(options.Time, options.Version); samples = Math.Max(1, samples); using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -137,22 +138,22 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) AND (""ArchiveTime"" = 0)") .ExpandInClauseParameter("@States", [RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Uploaded.ToString()]); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new RemoteVolume(rd); //Grab the max value - var max = cmd.ExecuteScalarInt64(@"SELECT MAX(""VerificationCount"") FROM ""RemoteVolume""", 0); + var max = await cmd.ExecuteScalarInt64Async(@"SELECT MAX(""VerificationCount"") FROM ""RemoteVolume""", 0); //First we select some filesets var files = new List(); - var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); - using (var rd = cmd.SetCommandAndParameters(@"SELECT ""A"".""VolumeID"", ""A"".""Name"", ""A"".""Size"", ""A"".""Hash"", ""A"".""VerificationCount"" FROM (SELECT ""ID"" AS ""VolumeID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""ArchiveTime"" = 0 AND ""State"" IN (@State1, @State2)) A, ""Fileset"" " + whereClause + @" ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" ") + var whereClause = string.IsNullOrEmpty(tp.Query) ? " WHERE " : (" " + tp.Query + " AND "); + using (var rd = await cmd.SetCommandAndParameters(@"SELECT ""A"".""VolumeID"", ""A"".""Name"", ""A"".""Size"", ""A"".""Hash"", ""A"".""VerificationCount"" FROM (SELECT ""ID"" AS ""VolumeID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""ArchiveTime"" = 0 AND ""State"" IN (@State1, @State2)) A, ""Fileset"" " + whereClause + @" ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" ") .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .SetParameterValues(tp.Item2) - .ExecuteReader()) - while (rd.Read()) + .SetParameterValues(tp.Values) + .ExecuteReaderAsync()) + while (await rd.ReadAsync()) files.Add(new RemoteVolume(rd)); if (files.Count == 0) @@ -171,8 +172,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) files.Add(new RemoteVolume(rd)); foreach (var f in FilterByVerificationCount(files, samples, max)) @@ -188,8 +189,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) files.Add(new RemoteVolume(rd)); foreach (var f in FilterByVerificationCount(files, samples, max)) @@ -220,34 +221,39 @@ namespace Duplicati.Library.Main.Database using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction)) { - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}"); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}"); bl.m_tablename = tablename; } - bl.m_insertCommand = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction, $@"INSERT INTO ""{bl.m_tablename}"" {insertCommand}"); + bl.m_insertCommand = await bl.m_db.Connection.CreateCommandAsync($@"INSERT INTO ""{bl.m_tablename}"" {insertCommand}"); return bl; } - public virtual void Dispose() + public void Dispose() + { + DisposeAsync().Await(); + } + + public virtual async Task DisposeAsync() { if (m_tablename != null) try { using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction)) - cmd.ExecuteNonQuery($@"DROP TABLE IF EXISTS ""{m_tablename}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}"""); } catch { } finally { m_tablename = null!; } - m_insertCommand?.Dispose(); + await m_insertCommand.DisposeAsync(); } } public interface IFilelist : IDisposable { - void Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time); - IEnumerable> Compare(); + Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time); + IAsyncEnumerable> Compare(); } private class Filelist : Basiclist, IFilelist @@ -271,18 +277,19 @@ namespace Duplicati.Library.Main.Database return (Filelist)await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); } - public void Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time) + public async Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time) { - m_insertCommand.SetTransaction(m_db.Transaction) + await m_insertCommand + .SetTransaction(m_db.Transaction) .SetParameterValue("@Path", path) .SetParameterValue("@Size", hash == null ? -1 : size) .SetParameterValue("@Hash", hash) .SetParameterValue("@Metasize", metasize) .SetParameterValue("@Metahash", metahash) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); } - public IEnumerable> Compare() + public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -296,23 +303,28 @@ namespace Duplicati.Library.Main.Database { try { - cmd.SetCommandAndParameters(create) + await cmd + .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); - cmd.SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") + cmd + .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } finally { - try { cmd.ExecuteNonQuery(drop); } + try + { + await cmd.ExecuteNonQueryAsync(drop); + } catch { } } } @@ -321,8 +333,8 @@ namespace Duplicati.Library.Main.Database public interface IIndexlist : IDisposable { - void AddBlockLink(string filename, string hash, long length); - IEnumerable> Compare(); + Task AddBlockLink(string filename, string hash, long length); + IAsyncEnumerable> Compare(); } private class Indexlist : Basiclist, IIndexlist @@ -346,16 +358,16 @@ namespace Duplicati.Library.Main.Database return (Indexlist)await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); } - public void AddBlockLink(string filename, string hash, long length) + public async Task AddBlockLink(string filename, string hash, long length) { - m_insertCommand.SetTransaction(m_db.Transaction) + await m_insertCommand.SetTransaction(m_db.Transaction) .SetParameterValue("@Name", filename) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", length) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); } - public IEnumerable> Compare() + public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var create = $@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Name"", ""A"".""Hash"", ""A"".""Size"" FROM ""Remotevolume"" A, ""Remotevolume"" B, ""IndexBlockLink"" WHERE ""B"".""Name"" = @Name AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" "; @@ -368,23 +380,28 @@ namespace Duplicati.Library.Main.Database { try { - cmd.SetCommandAndParameters(create) + await cmd + .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); - cmd.SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") + cmd + .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } finally { - try { cmd.ExecuteNonQuery(drop); } + try + { + await cmd.ExecuteNonQueryAsync(drop); + } catch { } } } @@ -393,8 +410,8 @@ namespace Duplicati.Library.Main.Database public interface IBlocklist : IDisposable { - void AddBlock(string key, long value); - IEnumerable> Compare(); + Task AddBlock(string key, long value); + IAsyncEnumerable> Compare(); } private class Blocklist : Basiclist, IBlocklist @@ -418,15 +435,15 @@ namespace Duplicati.Library.Main.Database return (Blocklist)await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); } - public void AddBlock(string hash, long size) + public async Task AddBlock(string hash, long size) { - m_insertCommand.SetTransaction(m_db.Transaction) + await m_insertCommand.SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); } - public IEnumerable> Compare() + public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var curBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""Remotevolume"", ""Block"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"""; @@ -442,41 +459,47 @@ namespace Duplicati.Library.Main.Database { try { - cmd.SetCommandAndParameters(create) + await cmd + .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); - cmd.SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") + cmd + .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } finally { - try { cmd.ExecuteNonQuery(drop); } + try + { + await cmd.ExecuteNonQueryAsync(drop); + } catch { } } } } } - public IFilelist CreateFilelist(string name) + public async Task CreateFilelist(string name) { - return new Filelist(m_connection, name, rtr); + return await Filelist.CreateAsync(this, name); } - public IIndexlist CreateIndexlist(string name) + public async Task CreateIndexlist(string name) { - return new Indexlist(m_connection, name, rtr); + return await Indexlist.CreateAsync(this, name); } - public IBlocklist CreateBlocklist(string name) + public async Task CreateBlocklist(string name) { - return new Blocklist(m_connection, name, rtr); + return await Blocklist.CreateAsync(this, name); } } } From e4cffd573599e5f5c3ffb2aa3de7bb4d2d58583e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 09:40:32 +0200 Subject: [PATCH 115/362] Restructured SQL queries in LocalTestDatabase --- .../Main/Database/LocalTestDatabase.cs | 406 ++++++++++++++++-- 1 file changed, 372 insertions(+), 34 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 234ab0b09..a1898be54 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -52,7 +52,20 @@ namespace Duplicati.Library.Main.Database public void UpdateVerificationCount(string name) { using (var cmd = m_connection.CreateCommand(m_rtr)) - cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE WHEN ""VerificationCount"" <= 0 THEN (SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"") ELSE ""VerificationCount"" + 1 END) WHERE ""Name"" = @Name") + cmd.SetCommandAndParameters(@" + UPDATE ""RemoteVolume"" + SET ""VerificationCount"" = MAX(1, + CASE + WHEN ""VerificationCount"" <= 0 + THEN ( + SELECT MAX(""VerificationCount"") + FROM ""RemoteVolume"" + ) + ELSE ""VerificationCount"" + 1 + END + ) + WHERE ""Name"" = @Name + ") .SetParameterValue("@Name", name) .ExecuteNonQuery(); } @@ -135,7 +148,19 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand(m_rtr)) { // Select any broken items - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) AND (""ArchiveTime"" = 0)") + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + (""State"" IN (@States)) + AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) + AND (""ArchiveTime"" = 0) + ") .ExpandInClauseParameter("@States", [RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Uploaded.ToString()]); using (var rd = await cmd.ExecuteReaderAsync()) @@ -143,16 +168,43 @@ namespace Duplicati.Library.Main.Database yield return new RemoteVolume(rd); //Grab the max value - var max = await cmd.ExecuteScalarInt64Async(@"SELECT MAX(""VerificationCount"") FROM ""RemoteVolume""", 0); + var max = await cmd.ExecuteScalarInt64Async(@" + SELECT MAX(""VerificationCount"") + FROM ""RemoteVolume"" + ", 0); //First we select some filesets var files = new List(); var whereClause = string.IsNullOrEmpty(tp.Query) ? " WHERE " : (" " + tp.Query + " AND "); - using (var rd = await cmd.SetCommandAndParameters(@"SELECT ""A"".""VolumeID"", ""A"".""Name"", ""A"".""Size"", ""A"".""Hash"", ""A"".""VerificationCount"" FROM (SELECT ""ID"" AS ""VolumeID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""ArchiveTime"" = 0 AND ""State"" IN (@State1, @State2)) A, ""Fileset"" " + whereClause + @" ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" ") - .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .SetParameterValues(tp.Values) - .ExecuteReaderAsync()) + using (var rd = await cmd.SetCommandAndParameters(@$" + SELECT + ""A"".""VolumeID"", + ""A"".""Name"", + ""A"".""Size"", + ""A"".""Hash"", + ""A"".""VerificationCount"" + FROM + ( + SELECT + ""ID"" AS ""VolumeID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + ""ArchiveTime"" = 0 + AND ""State"" IN (@State1, @State2) + ) A, + ""Fileset"" + {whereClause} + ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" + ORDER BY ""Fileset"".""Timestamp"" + ") + .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) + .SetParameterValues(tp.Values) + .ExecuteReaderAsync()) while (await rd.ReadAsync()) files.Add(new RemoteVolume(rd)); @@ -168,7 +220,19 @@ namespace Duplicati.Library.Main.Database //Then we select some index files files.Clear(); - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ArchiveTime"" = 0") + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + ""Type"" = @Type + AND ""State"" IN (@States) + AND ""ArchiveTime"" = 0 + ") .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); @@ -185,7 +249,19 @@ namespace Duplicati.Library.Main.Database //And finally some block files files.Clear(); - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ArchiveTime"" = 0") + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + ""Type"" = @Type + AND ""State"" IN (@States) + AND ""ArchiveTime"" = 0 + ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); @@ -221,11 +297,17 @@ namespace Duplicati.Library.Main.Database using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction)) { - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}"); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tablename}"" + {tableFormat} + "); bl.m_tablename = tablename; } - bl.m_insertCommand = await bl.m_db.Connection.CreateCommandAsync($@"INSERT INTO ""{bl.m_tablename}"" {insertCommand}"); + bl.m_insertCommand = await bl.m_db.Connection.CreateCommandAsync($@" + INSERT INTO ""{bl.m_tablename}"" + {insertCommand} + "); return bl; } @@ -259,8 +341,31 @@ namespace Duplicati.Library.Main.Database private class Filelist : Basiclist, IFilelist { private const string TABLE_PREFIX = "Filelist"; - private const string TABLE_FORMAT = @"(""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL)"; - private const string INSERT_COMMAND = @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (@Path,@Size,@Hash,@Metasize,@Metahash)"; + private const string TABLE_FORMAT = @" + ( + ""Path"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Hash"" TEXT NULL, + ""Metasize"" INTEGER NOT NULL, + ""Metahash"" TEXT NOT NULL + ) + "; + private const string INSERT_COMMAND = @" + ( + ""Path"", + ""Size"", + ""Hash"", + ""Metasize"", + ""Metahash"" + ) + VALUES ( + @Path, + @Size, + @Hash, + @Metasize, + @Metahash + ) + "; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Filelist(SqliteConnection connection, string volumename, ReusableTransaction rtr) @@ -293,10 +398,79 @@ namespace Duplicati.Library.Main.Database { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = $@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Path"" AS ""Path"", CASE WHEN ""B"".""Fullhash"" IS NULL THEN -1 ELSE ""B"".""Length"" END AS ""Size"", ""B"".""Fullhash"" AS ""Hash"", ""C"".""Length"" AS ""Metasize"", ""C"".""Fullhash"" AS ""Metahash"" FROM (SELECT ""File"".""Path"", ""File"".""BlocksetID"" AS ""FileBlocksetID"", ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" from ""Remotevolume"", ""Fileset"", ""FilesetEntry"", ""File"", ""Metadataset"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"") A LEFT OUTER JOIN ""Blockset"" B ON ""B"".""ID"" = ""A"".""FileBlocksetID"" LEFT OUTER JOIN ""Blockset"" C ON ""C"".""ID""=""A"".""MetadataBlocksetID"" "; - var extra = $@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Path"" AS ""Path"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{cmpName}"" )"; - var missing = $@"SELECT @TypeMissing AS ""Type"", ""Path"" AS ""Path"" FROM ""{cmpName}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tablename}"")"; - var modified = $@"SELECT @TypeModified AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"") "; + var create = $@" + CREATE TEMPORARY TABLE ""{cmpName}"" AS + SELECT + ""A"".""Path"" AS ""Path"", + CASE + WHEN ""B"".""Fullhash"" IS NULL + THEN -1 + ELSE ""B"".""Length"" END AS ""Size"", + ""B"".""Fullhash"" AS ""Hash"", + ""C"".""Length"" AS ""Metasize"", + ""C"".""Fullhash"" AS ""Metahash"" + FROM ( + SELECT + ""File"".""Path"", + ""File"".""BlocksetID"" AS ""FileBlocksetID"", + ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" + FROM + ""Remotevolume"", + ""Fileset"", + ""FilesetEntry"", + ""File"", + ""Metadataset"" + WHERE + ""Remotevolume"".""Name"" = @Name + AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" + ) A + LEFT OUTER JOIN ""Blockset"" B + ON ""B"".""ID"" = ""A"".""FileBlocksetID"" + LEFT OUTER JOIN ""Blockset"" C + ON ""C"".""ID""=""A"".""MetadataBlocksetID"" + "; + + var extra = $@" + SELECT + @TypeExtra AS ""Type"", + ""{m_tablename}"".""Path"" AS ""Path"" + FROM ""{m_tablename}"" + WHERE ""{m_tablename}"".""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{cmpName}"" + )"; + + var missing = $@" + SELECT + @TypeMissing AS ""Type"", + ""Path"" AS ""Path"" + FROM ""{cmpName}"" + WHERE ""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_tablename}"" + ) + "; + + var modified = $@" + SELECT + @TypeModified AS ""Type"", + ""E"".""Path"" AS ""Path"" + FROM + ""{m_tablename}"" E, + ""{cmpName}"" D + WHERE + ""D"".""Path"" = ""E"".""Path"" + AND ( + ""D"".""Size"" != ""E"".""Size"" + OR ""D"".""Hash"" != ""E"".""Hash"" + OR ""D"".""Metasize"" != ""E"".""Metasize"" + OR ""D"".""Metahash"" != ""E"".""Metahash"" + ) + "; + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) @@ -316,7 +490,10 @@ namespace Duplicati.Library.Main.Database using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) - yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); + yield return new KeyValuePair( + (Interface.TestEntryStatus)rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1) ?? "" + ); } finally @@ -340,8 +517,26 @@ namespace Duplicati.Library.Main.Database private class Indexlist : Basiclist, IIndexlist { private const string TABLE_PREFIX = "Indexlist"; - private const string TABLE_FORMAT = @"(""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; - private const string INSERT_COMMAND = @"(""Name"", ""Hash"", ""Size"") VALUES (@Name,@Hash,@Size)"; + private const string TABLE_FORMAT = @" + ( + ""Name"" TEXT NOT NULL, + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + ) + "; + + private const string INSERT_COMMAND = @" + ( + ""Name"", + ""Hash"", + ""Size"" + ) + VALUES ( + @Name, + @Hash, + @Size + ) + "; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Indexlist(SqliteConnection connection, string volumename, ReusableTransaction rtr) @@ -370,10 +565,59 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = $@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Name"", ""A"".""Hash"", ""A"".""Size"" FROM ""Remotevolume"" A, ""Remotevolume"" B, ""IndexBlockLink"" WHERE ""B"".""Name"" = @Name AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" "; - var extra = $@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Name"" AS ""Name"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Name"" NOT IN ( SELECT ""Name"" FROM ""{cmpName}"" )"; - var missing = $@"SELECT @TypeMissing AS ""Type"", ""Name"" AS ""Name"" FROM ""{cmpName}"" WHERE ""Name"" NOT IN (SELECT ""Name"" FROM ""{m_tablename}"")"; - var modified = $@"SELECT @TypeModified AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") "; + var create = $@" + CREATE TEMPORARY TABLE ""{cmpName}"" AS + SELECT + ""A"".""Name"", + ""A"".""Hash"", + ""A"".""Size"" + FROM + ""Remotevolume"" A, + ""Remotevolume"" B, + ""IndexBlockLink"" + WHERE + ""B"".""Name"" = @Name + AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" + AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" + "; + + var extra = $@" + SELECT + @TypeExtra AS ""Type"", + ""{m_tablename}"".""Name"" AS ""Name"" + FROM ""{m_tablename}"" + WHERE ""{m_tablename}"".""Name"" NOT IN ( + SELECT ""Name"" + FROM ""{cmpName}"" + ) + "; + + var missing = $@" + SELECT + @TypeMissing AS ""Type"", + ""Name"" AS ""Name"" + FROM ""{cmpName}"" + WHERE ""Name"" NOT IN ( + SELECT ""Name"" + FROM ""{m_tablename}"" + ) + "; + + var modified = $@" + SELECT + @TypeModified AS ""Type"", + ""E"".""Name"" AS ""Name"" + FROM + ""{m_tablename}"" E, + ""{cmpName}"" D + WHERE + ""D"".""Name"" = ""E"".""Name"" + AND ( + ""D"".""Hash"" != ""E"".""Hash"" + OR ""D"".""Size"" != ""E"".""Size"" + ) + "; + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) @@ -417,8 +661,23 @@ namespace Duplicati.Library.Main.Database private class Blocklist : Basiclist, IBlocklist { private const string TABLE_PREFIX = "Blocklist"; - private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; - private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; + private const string TABLE_FORMAT = @" + ( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + ) + "; + + private const string INSERT_COMMAND = @" + ( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size + ) + "; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Blocklist(SqliteConnection connection, string volumename, ReusableTransaction rtr) @@ -446,13 +705,92 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var curBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""Remotevolume"", ""Block"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"""; - var duplBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""DuplicateBlock"", ""Block"" WHERE ""DuplicateBlock"".""VolumeID"" = (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name) AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"""; - var delBlocks = @"SELECT ""DeletedBlock"".""Hash"" AS ""Hash"", ""DeletedBlock"".""Size"" AS ""Size"" FROM ""DeletedBlock"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = @Name AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"""; - var create = $@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT DISTINCT ""Hash"" AS ""Hash"", ""Size"" AS ""Size"" FROM ({curBlocks} UNION {delBlocks} UNION {duplBlocks})"; - var extra = $@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{cmpName}"" )"; - var missing = $@"SELECT @TypeMissing AS ""Type"", ""Hash"" AS ""Hash"" FROM ""{cmpName}"" WHERE ""Hash"" NOT IN (SELECT ""Hash"" FROM ""{m_tablename}"")"; - var modified = $@"SELECT @TypeModified AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "; + var curBlocks = @" + SELECT + ""Block"".""Hash"" AS ""Hash"", + ""Block"".""Size"" AS ""Size"" + FROM + ""Remotevolume"", + ""Block"" + WHERE + ""Remotevolume"".""Name"" = @Name + AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"" + "; + + var duplBlocks = @" + SELECT + ""Block"".""Hash"" AS ""Hash"", + ""Block"".""Size"" AS ""Size"" + FROM + ""DuplicateBlock"", + ""Block"" + WHERE + ""DuplicateBlock"".""VolumeID"" = ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" = @Name + ) + AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"" + "; + + var delBlocks = @" + SELECT + ""DeletedBlock"".""Hash"" AS ""Hash"", + ""DeletedBlock"".""Size"" AS ""Size"" + FROM + ""DeletedBlock"", + ""RemoteVolume"" + WHERE + ""RemoteVolume"".""Name"" = @Name + AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"" + "; + + var create = $@" + CREATE TEMPORARY TABLE ""{cmpName}"" AS + SELECT DISTINCT + ""Hash"" AS ""Hash"", + ""Size"" AS ""Size"" + FROM ( + {curBlocks} + UNION {delBlocks} + UNION {duplBlocks} + ) + "; + + var extra = $@" + SELECT + @TypeExtra AS ""Type"", + ""{m_tablename}"".""Hash"" AS ""Hash"" + FROM ""{m_tablename}"" + WHERE ""{m_tablename}"".""Hash"" NOT IN ( + SELECT ""Hash"" + FROM ""{cmpName}"" + ) + "; + + var missing = $@" + SELECT + @TypeMissing AS ""Type"", + ""Hash"" AS ""Hash"" + FROM ""{cmpName}"" + WHERE ""Hash"" NOT IN ( + SELECT ""Hash"" + FROM ""{m_tablename}"" + ) + "; + + var modified = $@" + SELECT + @TypeModified AS ""Type"", + ""E"".""Hash"" AS ""Hash"" + FROM + ""{m_tablename}"" E, + ""{cmpName}"" D + WHERE + ""D"".""Hash"" = ""E"".""Hash"" + AND ""D"".""Size"" != ""E"".""Size"" + "; + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) From edf1d93f564a002fda212cb3a57dd7cd384e3a2f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 09:58:24 +0200 Subject: [PATCH 116/362] Removed transactions as arguments as part of the transition to implicit transactions --- .../Library/Main/Database/LocalDatabase.cs | 105 +++++++++--------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 1209511fc..abd0a3f93 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -533,7 +533,7 @@ namespace Duplicati.Library.Main.Database .ExecuteScalarInt64Async(-1); } - public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files, SqliteTransaction transaction) + public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files) { using var cmd = await m_connection.CreateCommandAsync(@" SELECT @@ -542,7 +542,7 @@ namespace Duplicati.Library.Main.Database FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name) "); - cmd.SetTransaction(transaction); + cmd.SetTransaction(m_rtr); using var tmptable = await TemporaryDbValueList.CreateAsync(this, files); await cmd.ExpandInClauseParameterAsync("@Name", tmptable); @@ -573,9 +573,9 @@ namespace Duplicati.Library.Main.Database return RemoteVolumeEntry.Empty; } - public async IAsyncEnumerable> DuplicateRemoteVolumes(SqliteTransaction transaction) + public async IAsyncEnumerable> DuplicateRemoteVolumes() { - m_selectduplicateRemoteVolumesCommand.SetTransaction(transaction); + m_selectduplicateRemoteVolumesCommand.SetTransaction(m_rtr); await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync()) { @@ -586,9 +586,9 @@ namespace Duplicati.Library.Main.Database } } - public async IAsyncEnumerable GetRemoteVolumes(SqliteTransaction transaction) + public async IAsyncEnumerable GetRemoteVolumes() { - m_selectremotevolumesCommand.SetTransaction(transaction); + m_selectremotevolumesCommand.SetTransaction(m_rtr); using var rd = await m_selectremotevolumesCommand.ExecuteReaderAsync(); while (await rd.ReadAsync()) { @@ -611,10 +611,10 @@ namespace Duplicati.Library.Main.Database /// The operation performed /// The path involved /// Any data relating to the operation - public async Task LogRemoteOperation(string operation, string path, string? data, SqliteTransaction transaction) + public async Task LogRemoteOperation(string operation, string path, string? data) { await m_insertremotelogCommand - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@OperationID", m_operationid) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) .SetParameterValue("@Operation", operation) @@ -641,7 +641,7 @@ namespace Duplicati.Library.Main.Database .ExecuteNonQueryAsync(); } - public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state, SqliteTransaction transaction) + public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state) { using var cmd = await m_connection.CreateCommandAsync(@" DELETE FROM ""RemoteVolume"" @@ -649,7 +649,7 @@ namespace Duplicati.Library.Main.Database ""Name"" = @Name AND ""State"" = @State "); - var c = await cmd.SetTransaction(transaction) + var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@Name", name) .SetParameterValue("@State", state.ToString()) .ExecuteNonQueryAsync(); @@ -657,7 +657,7 @@ namespace Duplicati.Library.Main.Database if (c != 1) throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public async Task RemoveRemoteVolume(string name) @@ -1049,7 +1049,7 @@ namespace Duplicati.Library.Main.Database return res; } - public async Task IsFilesetFullBackup(DateTime filesetTime, SqliteTransaction transaction) + public async Task IsFilesetFullBackup(DateTime filesetTime) { using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters($@" @@ -1057,7 +1057,7 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" WHERE ""Timestamp"" = @Timestamp ") - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)); using var rd = await cmd.ExecuteReaderAsync(); @@ -1067,7 +1067,7 @@ namespace Duplicati.Library.Main.Database return isFullBackup == BackupType.FULL_BACKUP; } - private async IAsyncEnumerable> GetDbOptionList(SqliteTransaction transaction) + private async IAsyncEnumerable> GetDbOptionList() { using var cmd = m_connection.CreateCommand(@" SELECT @@ -1075,19 +1075,19 @@ namespace Duplicati.Library.Main.Database ""Value"" FROM ""Configuration"" ") - .SetTransaction(transaction); + .SetTransaction(m_rtr); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? ""); } - public async Task> GetDbOptions(SqliteTransaction transaction) + public async Task> GetDbOptions() { - var res = await GetDbOptionList(transaction) + var res = await GetDbOptionList() .ToDictionaryAsync(x => x.Key, x => x.Value); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); return res; } @@ -1098,22 +1098,21 @@ namespace Duplicati.Library.Main.Database /// The value to set private async Task UpdateDbOption(string key, bool value) { - var transaction = m_connection.BeginTransaction(); - var opts = await GetDbOptions(transaction); + var opts = await GetDbOptions(); if (value) opts[key] = "true"; else opts.Remove(key); - await SetDbOptions(opts, transaction); - await transaction.CommitAsync(); + await SetDbOptions(opts); + await m_rtr.CommitAsync(); } /// /// Flag indicating if a repair is in progress /// - public async Task RepairInProgress(SqliteTransaction transaction, bool? value = null) + public async Task RepairInProgress(bool? value = null) { if (value is bool v) { @@ -1121,14 +1120,14 @@ namespace Duplicati.Library.Main.Database return v; } - var opts = await GetDbOptions(transaction); + var opts = await GetDbOptions(); return opts.ContainsKey("repair-in-progress"); } /// /// Flag indicating if a repair is in progress /// - public async Task PartiallyRecreated(SqliteTransaction transaction, bool? value = null) + public async Task PartiallyRecreated(bool? value = null) { if (value is bool v) { @@ -1136,14 +1135,14 @@ namespace Duplicati.Library.Main.Database return v; } - var opts = await GetDbOptions(transaction); + var opts = await GetDbOptions(); return opts.ContainsKey("partially-recreated"); } /// /// Flag indicating if the database can contain partial uploads /// - public async Task TerminatedWithActiveUploads(SqliteTransaction transaction, bool? value = null) + public async Task TerminatedWithActiveUploads(bool? value = null) { if (value is bool v) { @@ -1151,7 +1150,7 @@ namespace Duplicati.Library.Main.Database return v; } - var opts = await GetDbOptions(transaction); + var opts = await GetDbOptions(); return opts.ContainsKey("terminated-with-active-uploads"); } @@ -1160,7 +1159,7 @@ namespace Duplicati.Library.Main.Database /// /// The options to set /// An optional transaction - public async Task SetDbOptions(IDictionary options, SqliteTransaction transaction) + public async Task SetDbOptions(IDictionary options) { using var cmd = m_connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@" @@ -1184,7 +1183,7 @@ namespace Duplicati.Library.Main.Database .ExecuteNonQueryAsync(); } - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public async Task GetBlocksLargerThan(long fhblocksize) @@ -1206,8 +1205,8 @@ namespace Duplicati.Library.Main.Database /// The hash size in byts /// Also verify filelists (can be slow) /// The transaction to run in - public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists, SqliteTransaction transaction) - => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false, transaction); + public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists) + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false); /// /// Verifies the consistency of the database prior to repair @@ -1216,8 +1215,8 @@ namespace Duplicati.Library.Main.Database /// The hash size in byts /// Also verify filelists (can be slow) /// The transaction to run in - public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists, SqliteTransaction transaction) - => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true, transaction); + public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists) + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true); /// /// Verifies the consistency of the database @@ -1227,10 +1226,10 @@ namespace Duplicati.Library.Main.Database /// Also verify filelists (can be slow) /// Disable verify for errors that will be fixed by repair /// The transaction to run in - private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair, SqliteTransaction transaction) + private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair) { - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); // Calculate the lengths for each blockset var combinedLengths = @" SELECT @@ -1504,7 +1503,7 @@ namespace Duplicati.Library.Main.Database var anyError = new List(); using (var cmd2 = m_connection.CreateCommand()) { - cmd2.SetTransaction(transaction); + cmd2.SetTransaction(m_rtr); cmd.SetCommandAndParameters(@" SELECT ""ID"" FROM ""Fileset"" @@ -1566,7 +1565,7 @@ namespace Duplicati.Library.Main.Database public long Size { get; private set; } = size; } - public async IAsyncEnumerable GetBlocks(long volumeid, SqliteTransaction transaction) + public async IAsyncEnumerable GetBlocks(long volumeid) { using var cmd = await m_connection.CreateCommandAsync(@" SELECT DISTINCT @@ -1575,7 +1574,7 @@ namespace Duplicati.Library.Main.Database FROM ""Block"" WHERE ""VolumeID"" = @VolumeId "); - cmd.SetTransaction(transaction) + cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) @@ -1799,11 +1798,11 @@ namespace Duplicati.Library.Main.Database ""H"".""Index"" "; - public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, SqliteTransaction transaction) + public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId) { using var cmd = m_connection.CreateCommand() .SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS) - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); @@ -1871,14 +1870,14 @@ namespace Duplicati.Library.Main.Database } } - public async Task LinkFilesetToVolume(long filesetid, long volumeid, SqliteTransaction transaction) + public async Task LinkFilesetToVolume(long filesetid, long volumeid) { using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""Fileset"" SET ""VolumeID"" = @VolumeId WHERE ""ID"" = @FilesetId "); - var c = await cmd.SetTransaction(transaction) + var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@FilesetId", filesetid) .ExecuteNonQueryAsync(); @@ -1887,7 +1886,7 @@ namespace Duplicati.Library.Main.Database throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); } - public async Task PushTimestampChangesToPreviousVersion(long filesetId, SqliteTransaction transaction) + public async Task PushTimestampChangesToPreviousVersion(long filesetId) { var query = @" UPDATE FilesetEntry AS oldVersion @@ -1906,7 +1905,7 @@ namespace Duplicati.Library.Main.Database "; using var cmd = await m_connection.CreateCommandAsync(query); - await cmd.SetTransaction(transaction) + await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQueryAsync(); } @@ -1926,7 +1925,7 @@ namespace Duplicati.Library.Main.Database } [Obsolete("Calling this constructor will throw an exception. Use the Create method instead.")] - public FilteredFilenameTable(SqliteConnection connection, IFilter filter, SqliteTransaction transaction) + public FilteredFilenameTable(SqliteConnection connection, IFilter filter) { throw new NotImplementedException("Use the Create method instead."); } @@ -2128,14 +2127,14 @@ namespace Duplicati.Library.Main.Database /// The ID of the index volume. /// The ID of the block volume. /// An optional transaction. - public async Task AddIndexBlockLink(long indexVolumeID, long blockVolumeID, SqliteTransaction transaction) + public async Task AddIndexBlockLink(long indexVolumeID, long blockVolumeID) { if (indexVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(indexVolumeID), "Index volume ID must be greater than 0."); if (blockVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0."); - await m_insertIndexBlockLink.SetTransaction(transaction) + await m_insertIndexBlockLink.SetTransaction(m_rtr) .SetParameterValue("@IndexVolumeId", indexVolumeID) .SetParameterValue("@BlockVolumeId", blockVolumeID) .ExecuteNonQueryAsync(); @@ -2149,7 +2148,7 @@ namespace Duplicati.Library.Main.Database /// The size of the hash /// An optional external transaction /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data - public async IAsyncEnumerable> GetBlocklists(long volumeid, long blocksize, int hashsize, SqliteTransaction transaction) + public async IAsyncEnumerable> GetBlocklists(long volumeid, long blocksize, int hashsize) { // Group subquery by hash to ensure that each blocklist hash appears only once in the result using var cmd = await m_connection.CreateCommandAsync($@" @@ -2187,7 +2186,7 @@ namespace Duplicati.Library.Main.Database var count = 0; var buffer = new byte[blocksize]; - cmd.SetTransaction(transaction) + cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) @@ -2236,13 +2235,13 @@ namespace Duplicati.Library.Main.Database /// /// The fileset ID to clear /// The transaction to use - public async Task ClearFilesetEntries(long filesetId, SqliteTransaction transaction) + public async Task ClearFilesetEntries(long filesetId) { using var cmd = await m_connection.CreateCommandAsync(@" DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId "); - await cmd.SetTransaction(transaction) + await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQueryAsync(); } From cc2f9740aed97a40b69e02b9d07d3c2092e7b115 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 14:51:22 +0200 Subject: [PATCH 117/362] Reintroduced the helper method for constructing a LocalDeleteDatabase.BlockQuery --- Duplicati/Library/Main/Database/LocalDeleteDatabase.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 64f11506d..4c1391cce 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -647,14 +647,13 @@ namespace Duplicati.Library.Main.Database } } - // TODO signature has been changed. /// /// Builds a lookup table to enable faster response to block queries /// - //public IBlockQuery CreateBlockQueryHelper(IDbTransaction transaction) - //{ - // return new BlockQuery(m_connection, transaction); - //} + public async Task CreateBlockQueryHelper() + { + return await BlockQuery.CreateAsync(this); + } /// /// Registers a block as moved to a new volume From 5ffff0e32e014602057794f03a83fc327cc85151 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 14:51:47 +0200 Subject: [PATCH 118/362] Fixed a method being synchronous in LocalTestDatabase --- Duplicati/Library/Main/Database/LocalTestDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index a1898be54..27a954dd0 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -49,10 +49,10 @@ namespace Duplicati.Library.Main.Database return (LocalTestDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); } - public void UpdateVerificationCount(string name) + public async Task UpdateVerificationCount(string name) { using (var cmd = m_connection.CreateCommand(m_rtr)) - cmd.SetCommandAndParameters(@" + await cmd.SetCommandAndParameters(@" UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE @@ -67,7 +67,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name ") .SetParameterValue("@Name", name) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); } private record RemoteVolume : IRemoteVolume From 96449bcc6bc1e22324c6b95dbfae0809c5efe26a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 14:52:10 +0200 Subject: [PATCH 119/362] Removed whitespace --- .../Main/Operation/Backup/BackupDatabase.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index ccaa96d5e..319cbc26e 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From c7f7b2cefeb192a009ea51b593389ee740a5f3aa Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:10:27 +0200 Subject: [PATCH 120/362] Updated BackupDatabase async interface to call the new async database methods --- .../Main/Operation/Backup/BackupDatabase.cs | 131 ++++++++---------- .../Operation/Backup/MetadataPreProcess.cs | 32 ++--- 2 files changed, 75 insertions(+), 88 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index 319cbc26e..e28dcec6f 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -29,7 +29,6 @@ using Duplicati.Library.Main.Volumes; using System.Linq; using Duplicati.Library.Interface; - namespace Duplicati.Library.Main.Operation.Backup { /// @@ -57,120 +56,108 @@ namespace Duplicati.Library.Main.Operation.Backup public Task FindBlockIDAsync(string hash, long size) { - return RunOnMain(() => m_database.FindBlockID(hash, size, GetTransaction())); + return RunOnMain(async () => await m_database.FindBlockID(hash, size)); } public Task AddBlockAsync(string hash, long size, long volumeid) { - return RunOnMain(() => m_database.AddBlock(hash, size, volumeid, GetTransaction())); + return RunOnMain(async () => await m_database.AddBlock(hash, size, volumeid)); } public Task GetFileHashAsync(long fileid) { - return RunOnMain(() => m_database.GetFileHash(fileid, GetTransaction())); + return RunOnMain(async () => await m_database.GetFileHash(fileid)); } - public Task> AddMetadatasetAsync(string hash, long size, long blocksetid) + public Task<(bool, long)> AddMetadatasetAsync(string hash, long size, long blocksetid) { - return RunOnMain(() => - { - long metadataid; - var n = m_database.AddMetadataset(hash, size, blocksetid, out metadataid, GetTransaction()); - return new Tuple(n, metadataid); - }); + return RunOnMain(async () => await m_database.AddMetadataset(hash, size, blocksetid)); } - public Task> GetMetadataIDAsync(string hash, long size) + public Task<(bool, long)> GetMetadataIDAsync(string hash, long size) { - return RunOnMain(() => - { - long metadataid; - var r = m_database.GetMetadatasetID(hash, size, out metadataid, GetTransaction()); - return new Tuple(r, metadataid); - }); + return RunOnMain(async () => await m_database.GetMetadatasetID(hash, size)); } public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified) { - return RunOnMain(() => m_database.AddDirectoryEntry(filename, metadataid, lastModified, GetTransaction())); + return RunOnMain(async () => await m_database.AddDirectoryEntry(filename, metadataid, lastModified)); } public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified) { - return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, GetTransaction())); + return RunOnMain(async () => await m_database.AddSymlinkEntry(filename, metadataid, lastModified)); } public Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFileAsync(long fileid) { - return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, GetTransaction())); + return RunOnMain(async () => await m_database.GetMetadataHashAndSizeForFile(fileid)); } - public Task> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength) + public Task<(long, DateTime, long)> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength) { - return RunOnMain(() => - { - var id = m_database.GetFileLastModified(prefixid, path, lastfilesetid, includeLength, out var lastModified, out var length, GetTransaction()); - return new Tuple(id, lastModified, length); - }); + return RunOnMain(async () => await m_database.GetFileLastModified(prefixid, path, lastfilesetid, includeLength)); } public Task GetFileEntryAsync(long prefixid, string path, long lastfilesetid) { - return RunOnMain(() => + return RunOnMain(async () => { - DateTime oldModified; - long lastFileSize; - string oldMetahash; - long oldMetasize; - - var id = m_database.GetFileEntry(prefixid, path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize, GetTransaction()); + var (id, oldModified, lastFileSize, oldMetahash, oldMetasize) = + await m_database.GetFileEntry(prefixid, path, lastfilesetid); return id < 0 ? null : - new FileEntryData() { id = id, modified = oldModified, filesize = lastFileSize, metahash = oldMetahash, metasize = oldMetasize }; + new FileEntryData() + { + id = id, + modified = oldModified, + filesize = lastFileSize, + metahash = oldMetahash, + metasize = oldMetasize + }; }); } public Task AddBlocksetAsync(string filehash, long size, int blocksize, IEnumerable hashlist, IEnumerable blocklisthashes) { - return RunOnMain(() => + return RunOnMain(async () => { - long blocksetid; + var (_, blocksetid) = await m_database.AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes); - m_database.AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes, out blocksetid, GetTransaction()); return blocksetid; }); } public Task GetOrCreatePathPrefix(string prefix) { - return RunOnMain(() => m_database.GetOrCreatePathPrefix(prefix, GetTransaction())); + return RunOnMain(async () => await m_database.GetOrCreatePathPrefix(prefix)); } public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid) { - return RunOnMain(() => m_database.AddFile(prefixid, filename, lastmodified, blocksetid, metadataid, GetTransaction())); + return RunOnMain(async () => await m_database.AddFile(prefixid, filename, lastmodified, blocksetid, metadataid)); } public Task AddUnmodifiedAsync(long fileid, DateTime lastModified) { - return RunOnMain(() => m_database.AddKnownFile(fileid, lastModified, GetTransaction())); + return RunOnMain(async () => await m_database.AddKnownFile(fileid, lastModified)); } public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid) { - return RunOnMain(() => m_database.MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid, GetTransaction())); + return RunOnMain(async () => await m_database.MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid)); } public Task SafeDeleteRemoteVolumeAsync(string remotename) { - return RunOnMain(() => m_database.SafeDeleteRemoteVolume(remotename, GetTransaction())); + return RunOnMain(async () => await m_database.SafeDeleteRemoteVolume(remotename)); } public Task GetBlocklistHashesAsync(string remotename) { - return RunOnMain(() => Task.FromResult(m_database.GetBlocklistHashes(remotename, GetTransaction()))); + return RunOnMain(async () => await m_database.GetBlocklistHashes(remotename)); } public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume) @@ -178,47 +165,47 @@ namespace Duplicati.Library.Main.Operation.Backup if (indexvolume != null) return Task.FromResult(true); - return RunOnMain(() => + return RunOnMain(async () => { - m_database.AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID, GetTransaction()); + await m_database.AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID); indexvolume.StartVolume(blockvolume.RemoteFilename); - foreach (var b in m_database.GetBlocks(blockvolume.VolumeID, GetTransaction())) + await foreach (var b in m_database.GetBlocks(blockvolume.VolumeID)) indexvolume.AddBlock(b.Hash, b.Size); - m_database.UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, GetTransaction()); + await m_database.UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); }); } public Task AppendFilesFromPreviousSetAsync() { - return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction())); + return RunOnMain(async () => await m_database.AppendFilesFromPreviousSet()); } public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction(), deletedFilelist)); + return RunOnMain(async () => await m_database.AppendFilesFromPreviousSet(deletedFilelist)); } public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction(), deletedFilelist, filesetid, prevId, timestamp)); + return RunOnMain(async () => await m_database.AppendFilesFromPreviousSet(deletedFilelist, filesetid, prevId, timestamp)); } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// /// Optional exclusion predicate (true = exclude file) public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSetWithPredicate(GetTransaction(), exclusionPredicate)); + return RunOnMain(async () => await m_database.AppendFilesFromPreviousSetWithPredicate(exclusionPredicate)); } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// /// Optional exclusion predicate (true = exclude file) @@ -227,83 +214,83 @@ namespace Duplicati.Library.Main.Operation.Backup /// If prevFileSetId == -1, used to locate previous fileset public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSetWithPredicate(GetTransaction(), exclusionPredicate, fileSetId, prevFileSetId, timestamp)); + return RunOnMain(async () => await m_database.AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, fileSetId, prevFileSetId, timestamp)); } public Task[]> GetIncompleteFilesetsAsync() { - return RunOnMain(() => m_database.GetIncompleteFilesets(GetTransaction()).OrderBy(x => x.Value).ToArray()); + return RunOnMain(async () => await m_database.GetIncompleteFilesets().OrderBy(x => x.Value).ToArrayAsync()); } public Task[]> GetFilesetTimesAsync() { - return RunOnMain(() => m_database.FilesetTimes.ToArray()); + return RunOnMain(async () => await m_database.FilesetTimes().ToArrayAsync()); } public Task CreateFilesetAsync(long volumeID, DateTime fileTime) { - return RunOnMain(() => m_database.CreateFileset(volumeID, fileTime, GetTransaction())); + return RunOnMain(async () => await m_database.CreateFileset(volumeID, fileTime)); } public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid) { - return RunOnMain(() => m_database.LinkFilesetToVolume(filesetid, volumeid, GetTransaction())); + return RunOnMain(async () => await m_database.LinkFilesetToVolume(filesetid, volumeid)); } public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid) { - return RunOnMain(() => m_database.WriteFileset(fsw, filesetid, GetTransaction())); + return RunOnMain(async () => await m_database.WriteFileset(fsw, filesetid)); } public Task PushTimestampChangesToPreviousVersionAsync(long filesetid) { - return RunOnMain(() => m_database.PushTimestampChangesToPreviousVersion(filesetid, GetTransaction())); + return RunOnMain(async () => await m_database.PushTimestampChangesToPreviousVersion(filesetid)); } public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid) { - return RunOnMain(() => m_database.UpdateFullBackupStateInFileset(filesetid, true, GetTransaction())); + return RunOnMain(async () => await m_database.UpdateFullBackupStateInFileset(filesetid, true)); } // TODO: Make IAsyncEnumerable or get rid of the SingleRunner setup public Task GetMissingIndexFilesAsync() { - return RunOnMain(() => m_database.GetMissingIndexFiles(GetTransaction()).ToArray()); + return RunOnMain(async () => await m_database.GetMissingIndexFiles().ToArrayAsync()); } public Task UpdateChangeStatisticsAsync(BackupResults result) { - return RunOnMain(() => m_database.UpdateChangeStatistics(result, GetTransaction())); + return RunOnMain(async () => await m_database.UpdateChangeStatistics(result)); } public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists) { - return RunOnMain(() => m_database.VerifyConsistency(blocksize, blockhashSize, verifyfilelists, GetTransaction())); + return RunOnMain(async () => await m_database.VerifyConsistency(blocksize, blockhashSize, verifyfilelists)); } public Task RemoveRemoteVolumeAsync(string remoteFilename) { - return RunOnMain(() => m_database.RemoveRemoteVolume(remoteFilename, GetTransaction())); + return RunOnMain(async () => await m_database.RemoveRemoteVolume(remoteFilename)); } public Task GetRemoteVolumeFromFilesetIDAsync(long filesetID) { - return RunOnMain(() => m_database.GetRemoteVolumeFromFilesetID(filesetID, GetTransaction())); + return RunOnMain(async () => await m_database.GetRemoteVolumeFromFilesetID(filesetID)); } public Task CreateChangeJournalDataAsync(IEnumerable journalData) { - return RunOnMain(() => m_database.CreateChangeJournalData(journalData, GetTransaction())); + return RunOnMain(async () => await m_database.CreateChangeJournalData(journalData)); } public Task UpdateChangeJournalDataAsync(IEnumerable journalData, long lastfilesetid) { - return RunOnMain(() => m_database.UpdateChangeJournalData(journalData, lastfilesetid, GetTransaction())); + return RunOnMain(async () => await m_database.UpdateChangeJournalData(journalData, lastfilesetid)); } public Task IsBlocklistHashKnownAsync(string hash) { - return RunOnMain(() => m_database.IsBlocklistHashKnown(hash, GetTransaction())); + return RunOnMain(async () => await m_database.IsBlocklistHashKnown(hash)); } } diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index 2ae10faa1..f1e1a1743 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -258,7 +258,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The metadata entry. /// The database connection. /// The channel to write streams to. - internal static async Task> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) + internal static async Task<(bool, long)> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) { StreamProcessResult res; using (var ms = new MemoryStream(meta.Blob)) From 6e26d685ea2f9fd46399431ee69cddc24ebc8443 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:20:52 +0200 Subject: [PATCH 121/362] Removed whitespace --- .../Main/Operation/Common/DatabaseCommon.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs index 89b4de75c..eec46880f 100644 --- a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs +++ b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 8d00d24ed6a3f1476530d50bd06a5c78185c205c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:21:18 +0200 Subject: [PATCH 122/362] Updated DatabaseCommon async interface to use the new async database methods --- .../Main/Operation/Common/DatabaseCommon.cs | 66 +++++-------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs index eec46880f..2b499fc1e 100644 --- a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs +++ b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs @@ -40,7 +40,6 @@ namespace Duplicati.Library.Main.Operation.Common private static readonly string LOGTAG = Logging.Log.LogTagFromType(); protected readonly LocalDatabase m_db; - private System.Data.IDbTransaction m_transaction; protected readonly Options m_options; public DatabaseCommon(LocalDatabase db, Options options) @@ -48,26 +47,16 @@ namespace Duplicati.Library.Main.Operation.Common { m_db = db; m_options = options; - m_transaction = db.BeginTransaction(); - } - - protected System.Data.IDbTransaction GetTransaction() - { - m_workerSource.Token.ThrowIfCancellationRequested(); - - if (m_transaction == null) - m_transaction = m_db.BeginTransaction(); - return m_transaction; } public Task RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state) { - return RunOnMain(() => m_db.RegisterRemoteVolume(name, type, state, GetTransaction())); + return RunOnMain(async () => await m_db.RegisterRemoteVolume(name, type, state)); } public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan), bool? setArchived = null) { - return RunOnMain(() => m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived, GetTransaction())); + return RunOnMain(async () => await m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived)); } public async Task FlushBackendMessagesAndCommitAsync(IBackendManager backendManager) @@ -77,83 +66,66 @@ namespace Duplicati.Library.Main.Operation.Common } private Task FlushPendingBackendMessagesAsync(IBackendManager backendManager) - => RunOnMain(() => backendManager.FlushPendingMessagesAsync(m_db, GetTransaction(), CancellationToken.None).ConfigureAwait(false)); + => RunOnMain(async () => await backendManager.FlushPendingMessagesAsync(m_db, CancellationToken.None).ConfigureAwait(false)); public Task CommitTransactionAsync(string message, bool restart = true) { - return RunOnMain(() => + return RunOnMain(async () => { if (m_options.Dryrun) { - if (!restart) - { - if (m_transaction != null) - m_transaction.Rollback(); - m_transaction = null; - } - return; + await m_db.Transaction.RollBackAsync(); } - - if (m_transaction != null) + else { using (new Logging.Timer(LOGTAG, "CommitTransactionAsync", message)) - m_transaction.Commit(); - m_transaction = null; + await m_db.Transaction.CommitAsync(message, restart); } }); } public Task RollbackTransactionAsync() { - return RunOnMain(() => - { - if (m_transaction != null) - { - m_transaction.Rollback(); - m_transaction = null; - } - }); + return RunOnMain(async () => await m_db.Transaction.RollBackAsync()); } - - public Task RenameRemoteFileAsync(string oldname, string newname) { - return RunOnMain(() => m_db.RenameRemoteFile(oldname, newname, GetTransaction())); + return RunOnMain(async () => await m_db.RenameRemoteFile(oldname, newname)); } public Task LogRemoteOperationAsync(string operation, string path, string data) { - return RunOnMain(() => m_db.LogRemoteOperation(operation, path, data, GetTransaction())); + return RunOnMain(async () => await m_db.LogRemoteOperation(operation, path, data)); } public Task GetBlocksAsync(long volumeid) { // TODO: Figure out how to return the enumerable, while keeping the lock // and not creating the entire result in memory - return RunOnMain(() => m_db.GetBlocks(volumeid, GetTransaction()).ToArray()); + return RunOnMain(async () => await m_db.GetBlocks(volumeid).ToArrayAsync()); } public Task GetVolumeInfoAsync(string remotename) { - return RunOnMain(() => m_db.GetRemoteVolume(remotename, GetTransaction())); + return RunOnMain(async () => await m_db.GetRemoteVolume(remotename)); } public Task[]> GetBlocklistsAsync(long volumeid, int blocksize, int hashsize) { // TODO: Figure out how to return the enumerable, while keeping the lock // and not creating the entire result in memory - return RunOnMain(() => m_db.GetBlocklists(volumeid, blocksize, hashsize, GetTransaction()).ToArray()); + return RunOnMain(async () => await m_db.GetBlocklists(volumeid, blocksize, hashsize).ToArrayAsync()); } public Task GetRemoteVolumeIDAsync(string remotename) { - return RunOnMain(() => m_db.GetRemoteVolumeID(remotename, GetTransaction())); + return RunOnMain(async () => await m_db.GetRemoteVolumeID(remotename)); } public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID) { - return RunOnMain(() => m_db.AddIndexBlockLink(indexVolumeID, blockVolumeID, GetTransaction())); + return RunOnMain(async () => await m_db.AddIndexBlockLink(indexVolumeID, blockVolumeID)); } @@ -162,13 +134,11 @@ namespace Duplicati.Library.Main.Operation.Common if (m_workerSource.IsCancellationRequested) return; - RunOnMain(() => + RunOnMain(async () => { base.Dispose(isDisposing); - - var tr = Interlocked.Exchange(ref m_transaction, null); - tr?.Commit(); - tr?.Dispose(); + await m_db.Transaction.CommitAsync(restart: false); + await m_db.Transaction.DisposeAsync(); }).Await(); } From 85c645c7f1efa63b2b8a0b85510ab3979b3ce47d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:29:13 +0200 Subject: [PATCH 123/362] Updated the internal processes of the reworked restore flow to use the new async database methods --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 2 +- Duplicati/Library/Main/Operation/Restore/FileLister.cs | 6 ++++-- .../Library/Main/Operation/Restore/FileProcessor.cs | 9 +++++++-- .../Library/Main/Operation/Restore/VolumeDownloader.cs | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index af8a34d0a..d6501975e 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -172,7 +172,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_set_wake_get = options.InternalProfiling ? new() : null; sw_set_wake_set = options.InternalProfiling ? new() : null; - foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata)) + foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata).ToEnumerable()) { var bc = m_blockcount.TryGetValue(block_id, out var c); m_blockcount[block_id] = bc ? c + 1 : 1; diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index b115d6b8b..0c0f13d0f 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -66,7 +66,8 @@ namespace Duplicati.Library.Main.Operation.Restore try { sw_get_files?.Start(); - var files = db.GetFilesAndSymlinksToRestore().OrderByDescending(x => x.Length).ToArray(); // Get started on big files first + // The enumerables are cast to arrays to force the query to be executed and release the database lock. + var files = await db.GetFilesAndSymlinksToRestore().OrderByDescending(x => x.Length).ToArrayAsync(); // Get started on big files first result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); sw_get_files?.Stop(); @@ -78,7 +79,8 @@ namespace Duplicati.Library.Main.Operation.Restore if (!options.SkipMetadata) { sw_get_folders?.Start(); - var folders = db.GetFolderMetadataToRestore().ToArray(); + // The enumerables are cast to arrays to force the query to be executed and release the database lock. + var folders = await db.GetFolderMetadataToRestore().ToArrayAsync(); sw_get_folders?.Stop(); sw_write_folder?.Start(); diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 1ebd2504c..e26215fc0 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -122,7 +122,7 @@ namespace Duplicati.Library.Main.Operation.Restore // Get information about the blocks for the file // TODO rather than keeping all of the blocks in memory, we could do a single pass over the blocks using a cursor, only keeping the relevant block requests in memory. Maybe even only a single block request at a time. sw_block?.Start(); - var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray(); + var blocks = await db.GetBlocksFromFile(file.BlocksetID).ToArrayAsync(); sw_block?.Stop(); sw_work_verify_target?.Start(); @@ -577,12 +577,17 @@ namespace Duplicati.Library.Main.Operation.Restore private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp) { sw_meta?.Start(); + // Since each FileProcessor should have its own connection, it's ok + // to keep the lock associated with the read transaction for a long + // time, as the other processes should still be able to read. + // Therefore, we don't need to read the entire result set into + // memory. var blocks = db.GetMetadataBlocksFromFile(file.ID); sw_meta?.Stop(); using var ms = new MemoryStream(); - foreach (var block in blocks) + await foreach (var block in blocks) { sw_work?.Stop(); sw_req?.Start(); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index b615c4d8e..dccbd2828 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -75,7 +75,7 @@ namespace Duplicati.Library.Main.Operation.Restore // Trigger the download. sw_wait?.Start(); TempFile f; - var (volume_name, size, hash) = db.GetVolumeInfo(volume_id).First(); + var (volume_name, size, hash) = await db.GetVolumeInfo(volume_id).FirstAsync(); try { f = await backend.GetDirectAsync(volume_name, hash, size, CancellationToken.None).ConfigureAwait(false); From 6dc40fc5a7e3b6611872881edfd087fa4856863e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:29:55 +0200 Subject: [PATCH 124/362] Removed whitespace --- .../Library/Main/Operation/BackupHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 5d3bc96eb..b176f450a 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From d02608f925ac80ed9011afdf1159210f320b99ba Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:30:54 +0200 Subject: [PATCH 125/362] Updated the BackupHandler to use the new async database methods --- .../Library/Main/Operation/BackupHandler.cs | 129 +++++++++--------- 1 file changed, 66 insertions(+), 63 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index b176f450a..546188de2 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -208,7 +208,13 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException("Multiple USN journal services are not supported", "MultipleUSNJournalServices"); var fileProvider = providers.First().Provider as LocalFileSource; - var journalData = m_database.GetChangeJournalData(lastfilesetid); + // TODO because of the new SQLite backend, this method returns an + // async enumerable. The proper way to handle this would be to + // change the UsnJournalService to use factory construction (see how + // all of the local database classes handle this), as the constructor + // is not async and shouldn't be executing "heavy" code, such as + // database queries. + var journalData = m_database.GetChangeJournalData(lastfilesetid).ToEnumerable(); var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, journalData, cancellationTokenSource.Token); @@ -294,12 +300,12 @@ namespace Duplicati.Library.Main.Operation { try { - database = new LocalBackupDatabase(options.Dbpath, options); - var lastFilestTime = database.FilesetTimes + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options); + var lastFilestTime = await database.FilesetTimes() .Select(x => x.Value) .Append(DateTime.UnixEpoch) .Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds) - .Max(); + .MaxAsync(); var currentStamp = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(database.OperationTimestamp); if (lastFilestTime > currentStamp) @@ -311,7 +317,7 @@ namespace Duplicati.Library.Main.Operation Log.WriteVerboseMessage(LOGTAG, "DatabaseTimestampError", "The database has a timestamp of {0}, but the last fileset has a timestamp of {1}. Sleeping for {2} seconds to allow the clock to catch up.", database.OperationTimestamp.ToLocalTime(), Library.Utility.Utility.EPOCH.AddSeconds(lastFilestTime).ToLocalTime(), sleepSeconds); Thread.Sleep(TimeSpan.FromSeconds(sleepSeconds)); database.Dispose(); - database = new LocalBackupDatabase(options.Dbpath, options); + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options); } result.Dryrun = options.Dryrun; @@ -320,34 +326,34 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(database, options); Utility.VerifyOptionsAndUpdateDatabase(database, options); - var probe_path = database.GetFirstPath(); + var probe_path = await database.GetFirstPath(); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); - if (database.PartiallyRecreated) + if (await database.PartiallyRecreated()) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (database.RepairInProgress) + if (await database.RepairInProgress()) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress"); // Make sure the database is sane - database.VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks, null); + await database.VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks); // Guard the last filelist if (!options.DisableSyntheticFilelist) - lastTempFilelist = database.GetLastIncompleteFilesetVolume(null); + lastTempFilelist = await database.GetLastIncompleteFilesetVolume(); try { if (options.NoBackendverification) { - await FilelistProcessor.VerifyLocalList(backendManager, database, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + await FilelistProcessor.VerifyLocalList(backendManager, database, result.TaskControl.ProgressToken).ConfigureAwait(false); await UpdateStorageStatsFromDatabase(result, database, options, backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); } else { - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, null, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); - var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : database.GetRemoteVolume(lastTempFilelist.Name, null); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); + var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name); if (!string.IsNullOrWhiteSpace(updatedLastTemp.Name) && updatedLastTemp.State == RemoteVolumeState.Deleting) { // The last temporary filelist was emptied, and scheduled for deletion, so we need to delete it @@ -360,9 +366,9 @@ namespace Duplicati.Library.Main.Operation Log.WriteMessage(LogMessageType.Information, LOGTAG, "BackendDeleteFailed", ex, "Backend delete failed: {0}", ex.Message); } - await backendManager.WaitForEmptyAsync(database, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, result.TaskControl.ProgressToken).ConfigureAwait(false); // In case the backend delete failed, manually set the state to deleted - database.UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null, null); + await database.UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null); // The last temporary filelist is no more lastTempFilelist = default; } @@ -382,17 +388,17 @@ namespace Duplicati.Library.Main.Operation .ConfigureAwait(false); // Re-open the database and backend manager - database = new LocalBackupDatabase(options.Dbpath, options); + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, null, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); } else throw; } // Return the updated version of the fileset, in case it was actually complete - return new PreBackupVerifyResult(database, database.GetLastIncompleteFilesetVolume(null)); + return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume()); } catch { @@ -502,7 +508,7 @@ namespace Duplicati.Library.Main.Operation } } - private async Task CompactIfRequired(IBackendManager backendManager, ReusableTransaction transaction, long lastVolumeSize) + private async Task CompactIfRequired(IBackendManager backendManager, long lastVolumeSize) { var currentIsSmall = lastVolumeSize != -1 && lastVolumeSize <= m_options.SmallFileSize; @@ -510,9 +516,9 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); - using (var db = new LocalDeleteDatabase(m_database)) + using (var db = await LocalDeleteDatabase.CreateAsync(m_database)) await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) - .DoRunAsync(db, transaction, true, currentIsSmall, backendManager) + .DoRunAsync(db, true, currentIsSmall, backendManager) .ConfigureAwait(false); } @@ -520,26 +526,26 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); - using (var db = new LocalDeleteDatabase(m_database)) + using (var db = await LocalDeleteDatabase.CreateAsync(m_database)) await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) - .DoCompactAsync(db, true, transaction, backendManager) + .DoCompactAsync(db, true, backendManager) .ConfigureAwait(false); } } - private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager, ReusableTransaction rtr) + private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, rtr.Transaction, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(m_database, rtr.Transaction, m_taskReader.ProgressToken); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(m_database, m_taskReader.ProgressToken); // Calculate the number of samples to test, using the largest number of file of a given type - long remoteVolumeCount = m_database.GetRemoteVolumes() + long remoteVolumeCount = await await m_database.GetRemoteVolumes() .Where(x => x.State == RemoteVolumeState.Verified) .GroupBy(x => x.Type) - .Select(x => x.LongCount()) - .Max(); + .Select(async x => await x.LongCountAsync()) + .MaxAsync(); long samplesToTest = Math.Max(m_options.BackupTestSampleCount, (long)Math.Round(remoteVolumeCount * (m_options.BackupTestPercentage / 100m), MidpointRounding.AwayFromZero)); if (samplesToTest > 0 && remoteVolumeCount > 0) @@ -547,9 +553,9 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest); m_result.TestResults = new TestResults(m_result); - using (var testdb = new LocalTestDatabase(m_database)) + using (var testdb = await LocalTestDatabase.CreateAsync(m_database)) await new TestHandler(m_options, (TestResults)m_result.TestResults) - .DoRunAsync(samplesToTest, testdb, rtr, backendManager) + .DoRunAsync(samplesToTest, testdb, backendManager) .ConfigureAwait(false); } } @@ -561,15 +567,15 @@ namespace Duplicati.Library.Main.Operation { if (result.BackendWriter != null) { - result.BackendWriter.KnownFileCount = database.GetRemoteVolumes().Count(); - result.BackendWriter.KnownFilesets = database.GetRemoteVolumes().Count(x => x.Type == RemoteVolumeType.Files); - result.BackendWriter.KnownFileSize = database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).Sum(); + result.BackendWriter.KnownFileCount = await database.GetRemoteVolumes().CountAsync(); + result.BackendWriter.KnownFilesets = await database.GetRemoteVolumes().CountAsync(x => x.Type == RemoteVolumeType.Files); + result.BackendWriter.KnownFileSize = await database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).SumAsync(); result.BackendWriter.UnknownFileCount = 0; result.BackendWriter.UnknownFileSize = 0; - result.BackendWriter.BackupListCount = database.FilesetTimes.Count(); - result.BackendWriter.LastBackupDate = database.FilesetTimes.FirstOrDefault().Value.ToLocalTime(); + result.BackendWriter.BackupListCount = await database.FilesetTimes().CountAsync(); + result.BackendWriter.LastBackupDate = (await database.FilesetTimes().FirstOrDefaultAsync()).Value.ToLocalTime(); if (!options.QuotaDisable) { @@ -651,7 +657,7 @@ namespace Duplicati.Library.Main.Operation { // If we crash now, there is a chance we leave behind partial remote files if (!m_options.Dryrun) - database.TerminatedWithActiveUploads = true; + await database.TerminatedWithActiveUploads(true); using (m_database = database) using (var db = new Backup.BackupDatabase(m_database, m_options)) @@ -669,7 +675,7 @@ namespace Duplicati.Library.Main.Operation await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist); // Grab the previous backup ID, if any - var prevfileset = m_database.FilesetTimes.FirstOrDefault(); + var prevfileset = await m_database.FilesetTimes().FirstOrDefaultAsync(); if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime()) throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime())); @@ -696,7 +702,7 @@ namespace Duplicati.Library.Main.Operation // Start parallel scan, or use the database if (m_options.DisableFileScanner) { - var d = m_database.GetLastBackupFileCountAndSize(); + var d = await m_database.GetLastBackupFileCountAndSize(); m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true); } else @@ -734,37 +740,34 @@ namespace Duplicati.Library.Main.Operation await FlushBackend(db, m_result, backendManager).ConfigureAwait(false); if (!m_options.Dryrun) - database.TerminatedWithActiveUploads = false; + await database.TerminatedWithActiveUploads(false); // Make sure we have the database up-to-date await db.CommitTransactionAsync("CommitAfterUpload", false); - using (var rtr = new ReusableTransaction(m_database)) + // If this throws, we should roll back the transaction + if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - // If this throws, we should roll back the transaction + var lastVolumeSize = await m_database.GetLastWrittenDBlockVolumeSize(); + await CompactIfRequired(backendManager, lastVolumeSize); + } + + if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database); + } + + if (!m_options.Dryrun) + { + await database.Transaction.CommitAsync("CommitFinalizingBackup"); if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - var lastVolumeSize = m_database.GetLastWrittenDBlockVolumeSize(rtr.Transaction); - await CompactIfRequired(backendManager, rtr, lastVolumeSize); - } - - if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, rtr.Transaction); - } - - if (!m_options.Dryrun) - { - rtr.Commit("CommitFinalizingBackup"); - if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - if (m_options.NoBackendverification) - await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); - else - await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager, rtr).ConfigureAwait(false); - } - rtr.Commit("CommitAfterVerification", restart: false); + if (m_options.NoBackendverification) + await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); + else + await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager).ConfigureAwait(false); } + await database.Transaction.CommitAsync("CommitAfterVerification", restart: false); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); From 99dddd483903550fa71900c53a4e1f90bb71fa54 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:31:04 +0200 Subject: [PATCH 126/362] Removed whitespace --- .../Library/Main/Operation/CompactHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 95c4f88e8..fb4cd241f 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Main.Database; From f4981ec0e4e7bd183bf6b7efc7b3328a0bfec418 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:31:23 +0200 Subject: [PATCH 127/362] Updated CompactHandler to use the new async database methods --- .../Library/Main/Operation/CompactHandler.cs | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index fb4cd241f..6cdf9869e 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -52,23 +52,22 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - using (var db = new LocalDeleteDatabase(m_options.Dbpath, "Compact", m_options.SqlitePageCache)) - using (var tr = new ReusableTransaction(db)) + using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache)) { Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - var changed = await DoCompactAsync(db, false, tr, backendManager).ConfigureAwait(false); + var changed = await DoCompactAsync(db, false, backendManager).ConfigureAwait(false); if (changed && m_options.UploadVerificationFile) - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, null); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db); if (!m_options.Dryrun) { - tr.Commit("CommitCompact", restart: false); + await db.Transaction.CommitAsync("CommitCompact", restart: false); if (changed) { - db.WriteResults(m_result); + await db.WriteResults(m_result); if (m_options.AutoVacuum) { m_result.VacuumResults = new VacuumResults(m_result); @@ -79,31 +78,31 @@ namespace Duplicati.Library.Main.Operation } } - internal async Task DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, ReusableTransaction rtr, IBackendManager backendManager) + internal async Task DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, IBackendManager backendManager) { - var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, rtr.Transaction); + var report = await db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount); report.ReportCompactData(); if (report.ShouldReclaim || report.ShouldCompact) { // Workaround where we allow a running backendmanager to be used if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, rtr.Transaction).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); var newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, rtr.Transaction); + newvol.VolumeID = await db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); IndexVolumeWriter newvolindex = null; if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); - db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, rtr.Transaction); + newvolindex.VolumeID = await db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); + await db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID); } var blocksInVolume = 0L; var buffer = new byte[m_options.Blocksize]; - var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray(); + var remoteList = await db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArrayAsync(); //These are for bookkeeping var uploadedVolumes = new List>(); @@ -121,7 +120,7 @@ namespace Duplicati.Library.Main.Operation .Cast() .ToList(); } - await foreach (var d in DoDelete(db, backendManager, fullyDeleteable, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var d in DoDelete(db, backendManager, fullyDeleteable, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) deletedVolumes.Add(d); // This list is used to pick up unused volumes, @@ -133,7 +132,7 @@ namespace Duplicati.Library.Main.Operation { // If we crash now, we may leave partial files if (!m_options.Dryrun) - db.TerminatedWithActiveUploads = true; + await db.TerminatedWithActiveUploads(true); newvolindex?.StartVolume(newvol.RemoteFilename); List volumesToDownload = []; @@ -147,7 +146,7 @@ namespace Duplicati.Library.Main.Operation .ToList(); } - using (var q = db.CreateBlockQueryHelper(rtr.Transaction)) + using (var q = await db.CreateBlockQueryHelper()) { await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { @@ -156,18 +155,18 @@ namespace Duplicati.Library.Main.Operation var entry = new RemoteVolume(name, hash, size); if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); return false; } downloadedVolumes.Add(new KeyValuePair(entry.Name, entry.Size)); - var volumeid = db.GetRemoteVolumeID(entry.Name, rtr.Transaction); + var volumeid = await db.GetRemoteVolumeID(entry.Name); var inst = VolumeBase.ParseFilename(entry.Name); using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options)) { foreach (var e in f.Blocks) { - if (q.UseBlock(e.Key, e.Value, volumeid, rtr.Transaction)) + if (await q.UseBlock(e.Key, e.Value, volumeid)) { //TODO: How do we get the compression hint? Reverse query for filename in db? var s = f.ReadBlock(e.Key, buffer); @@ -178,32 +177,32 @@ namespace Duplicati.Library.Main.Operation if (newvolindex != null) newvolindex.AddBlock(e.Key, e.Value); - db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, rtr.Transaction); + await db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID); blocksInVolume++; if (newvol.Filesize > (m_options.VolumeSize - m_options.Blocksize)) { - await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes, rtr); + await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes); newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, rtr.Transaction); + newvol.VolumeID = await db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); - db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, rtr.Transaction); + newvolindex.VolumeID = await db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); + await db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID); newvolindex.StartVolume(newvol.RemoteFilename); } blocksInVolume = 0; // Wait for the backend to catch up - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Commit as we have uploaded a volume if (!m_options.Dryrun) - rtr.Commit("CommitCompact"); + await db.Transaction.CommitAsync("CommitCompact"); } } } @@ -215,14 +214,14 @@ namespace Duplicati.Library.Main.Operation if (blocksInVolume > 0) { - await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes, rtr).ConfigureAwait(false); + await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes).ConfigureAwait(false); } else { - db.RemoveRemoteVolume(newvol.RemoteFilename, rtr.Transaction); + await db.RemoveRemoteVolume(newvol.RemoteFilename); if (newvolindex != null) { - db.RemoveRemoteVolume(newvolindex.RemoteFilename, rtr.Transaction); + await db.RemoveRemoteVolume(newvolindex.RemoteFilename); newvolindex.FinishVolume(null, 0); } } @@ -230,7 +229,7 @@ namespace Duplicati.Library.Main.Operation // The remainder of the operation cannot leave partial files if (!m_options.Dryrun) - db.TerminatedWithActiveUploads = false; + await db.TerminatedWithActiveUploads(false); } else { @@ -238,7 +237,7 @@ namespace Duplicati.Library.Main.Operation newvol.Dispose(); } - await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) deletedVolumes.Add(d); var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); @@ -283,7 +282,7 @@ namespace Duplicati.Library.Main.Operation Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)); } - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return (m_result.DeletedFileCount + m_result.UploadedFileCount) > 0; @@ -295,41 +294,42 @@ namespace Duplicati.Library.Main.Operation } } - private async IAsyncEnumerable> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable deleteableVolumes, ReusableTransaction rtr, [EnumeratorCancellation] CancellationToken cancellationToken) + private async IAsyncEnumerable> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable deleteableVolumes, [EnumeratorCancellation] CancellationToken cancellationToken) { // Find volumes that can be deleted - var remoteFilesToRemove = db.ReOrderDeleteableVolumes(deleteableVolumes, rtr.Transaction).ToList(); + var remoteFilesToRemove = await db.ReOrderDeleteableVolumes(deleteableVolumes).ToListAsync(); // Make sure we do not re-assign blocks to any of the volumes we are about to delete - var toRemoveVolumeIds = db.GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), rtr.Transaction) + var toRemoveVolumeIds = await db + .GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name)) .Select(x => x.Value) .Distinct() - .ToList(); + .ToListAsync(); // Mark all volumes and relevant index files as disposable foreach (var f in remoteFilesToRemove) { - db.PrepareForDelete(f.Name, toRemoveVolumeIds, rtr.Transaction); - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, rtr.Transaction); + await db.PrepareForDelete(f.Name, toRemoveVolumeIds); + await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash); } // Before we commit the current state, make sure the backend has caught up - await backend.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backend.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitDelete"); + await db.Transaction.CommitAsync("CommitDelete"); await foreach (var d in PerformDelete(backend, remoteFilesToRemove, cancellationToken).ConfigureAwait(false)) yield return d; } - private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes, ReusableTransaction rtr) + private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes) { Action indexVolumeFinished = null; if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - indexVolumeFinished = () => + indexVolumeFinished = async () => { - foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) + await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); }; @@ -343,7 +343,7 @@ namespace Duplicati.Library.Main.Operation // Once fixed, we can perhaps let he backend manager simply call the database directly if (!m_options.Dryrun) { - rtr.Commit("CommitUpload"); + await db.Transaction.CommitAsync("CommitUpload"); await backendManager.PutAsync(newvol, newvolindex, indexVolumeFinished, false, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } else From b7beea429b47a37d0fd01eee1a83b6608d92cf1b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:31:35 +0200 Subject: [PATCH 128/362] Removed whitespace --- .../Main/Operation/CreateBugReportHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index f46ec3701..2fbcd86ff 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From e9bb7189de7b50141460f5c88f93eae06a9b2746 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:31:56 +0200 Subject: [PATCH 129/362] Updated CreateBugReportHandler to use the new async database methods --- Duplicati/Library/Main/Operation/CreateBugReportHandler.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index 2fbcd86ff..bf5df7633 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -68,11 +68,11 @@ namespace Duplicati.Library.Main.Operation using (var tmp = new TempFile()) { File.Copy(m_options.Dbpath, tmp, true); - using (var db = new LocalBugReportDatabase(tmp, m_options.SqlitePageCache)) + using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache)) { - db.Fix(); + await db.Fix(); if (m_options.AutoVacuum) - db.Vacuum(); + await db.Vacuum(); } // Apply zip64 option if not already set From 300d80f2510f116b535bb0316d1ee910912de8bf Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:32:56 +0200 Subject: [PATCH 130/362] Removed whitespace --- .../Library/Main/Operation/DeleteHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 6e291fc4e..d699703a4 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 43e5e8f9955fae57ef2be3a561fad15f67a834af Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:33:09 +0200 Subject: [PATCH 131/362] Updated DeleteHandler to use the new async database methods --- .../Library/Main/Operation/DeleteHandler.cs | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index d699703a4..fb866d7f6 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -51,25 +51,26 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - using (var db = new LocalDeleteDatabase(m_options.Dbpath, "Delete", m_options.SqlitePageCache)) - using (var tr = new ReusableTransaction(db)) + using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache)) { Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - await DoRunAsync(db, tr, false, false, backendManager).ConfigureAwait(false); + await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); if (!m_options.Dryrun) - tr.Commit("ComitDelete", restart: false); + await db.Transaction.CommitAsync("ComitDelete", restart: false); } } - public async Task DoRunAsync(LocalDeleteDatabase db, ReusableTransaction rtr, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) + public async Task DoRunAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) { if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, rtr.Transaction).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); - var filesets = db.FilesetsWithBackupVersion.ToArray(); + // We collapse the async enumerablo into a array to avoid multiple + // enumerations (and thus multiple database queries) + var filesets = await db.FilesetsWithBackupVersion().ToArrayAsync(); List versionsToDelete = [ .. new SpecificVersionsRemover(m_options).GetFilesetsToDelete(filesets), @@ -97,18 +98,18 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray(); + var lst = await db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray()).ToArrayAsync(); foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction); + await db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null); if (!m_options.Dryrun) - rtr.Commit("CommitBeforeDelete"); + await db.Transaction.CommitAsync("CommitBeforeDelete"); foreach (var f in lst) { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); return; } @@ -118,7 +119,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); } - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var count = lst.Length; if (!m_options.Dryrun) @@ -145,7 +146,7 @@ namespace Duplicati.Library.Main.Operation { m_result.CompactResults = new CompactResults(m_result); await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) - .DoCompactAsync(db, true, rtr, backendManager).ConfigureAwait(false); + .DoCompactAsync(db, true, backendManager).ConfigureAwait(false); } m_result.SetResults(versionsToDelete.Select(v => new Tuple(v.Version, v.Time)), m_options.Dryrun); From 63141d4645610fcabd5327691aecc8b4eab25df3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:33:19 +0200 Subject: [PATCH 132/362] Removed whitespace --- .../Main/Operation/FilelistProcessor.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 1f5239faf..952272da2 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 996f2b16d833f3626328765997900a29f33343ab Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:34:00 +0200 Subject: [PATCH 133/362] Updated the FilelistProcessor operation to use the new async database methods --- .../Main/Operation/FilelistProcessor.cs | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 952272da2..c4d4d07d6 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -66,10 +66,10 @@ namespace Duplicati.Library.Main.Operation /// Throws an error if there are issues with the remote storage /// /// The database to compare with - public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, IDbTransaction transaction, CancellationToken cancellationToken) + public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) { - var locallist = database.GetRemoteVolumes(transaction); - foreach (var i in locallist) + var locallist = database.GetRemoteVolumes(); + await foreach (var i in locallist) { switch (i.State) { @@ -98,7 +98,7 @@ namespace Duplicati.Library.Main.Operation break; } - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); } } @@ -112,15 +112,14 @@ namespace Duplicati.Library.Main.Operation /// The backend writer to use /// True if only the latest volumes should be verified /// The mode to use for verification - /// An optional transaction object /// An awaitable task - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, IDbTransaction transaction) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode) { if (!options.NoBackendverification) { - LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options); - IEnumerable protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction); - await VerifyRemoteList(backend, options, database, transaction, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); + LocalBackupDatabase backupDatabase = await LocalBackupDatabase.CreateAsync(database, options); + IEnumerable protectedFiles = await backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly); + await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); } } @@ -137,9 +136,9 @@ namespace Duplicati.Library.Main.Operation /// The mode to use for verification /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless /// An awaitable task - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IDbTransaction transaction, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) { - var tp = await RemoteListAnalysis(backend, options, database, transaction, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); + var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); long extraCount = 0; long missingCount = 0; @@ -208,7 +207,7 @@ namespace Duplicati.Library.Main.Operation public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream) { var s = new Newtonsoft.Json.JsonSerializer(); - s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).Cast().ToArray()); + s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).ToEnumerable().Cast().ToArray()); } /// @@ -217,8 +216,7 @@ namespace Duplicati.Library.Main.Operation /// The backend url /// The options to use /// The attached database - /// An optional transaction object - public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, IDbTransaction transaction) + public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db) { using (var tempfile = new Library.Utility.TempFile()) { @@ -233,7 +231,7 @@ namespace Duplicati.Library.Main.Operation else { await backendManager.PutVerificationFileAsync(remotename, tempfile, CancellationToken.None).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(db, transaction, CancellationToken.None).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, CancellationToken.None).ConfigureAwait(false); } } } @@ -249,10 +247,10 @@ namespace Duplicati.Library.Main.Operation /// Filenames that should be exempted from strict verification /// The mode to use for verification /// The result of the analysis - public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IDbTransaction transaction, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) + public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) { // If the last operation completed, no cleanup should be required - if (verifyMode == VerifyMode.VerifyAndClean && !database.TerminatedWithActiveUploads) + if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads()) verifyMode = VerifyMode.VerifyStrict; // Force cleanup should set the mode to cleanup @@ -290,7 +288,7 @@ namespace Duplicati.Library.Main.Operation log.KnownFileSize = knownFileSize; log.UnknownFileCount = unknownlist.Count; log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum(); - log.BackupListCount = database.FilesetTimes.Count(); + log.BackupListCount = await database.FilesetTimes().CountAsync(); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); await CheckQuota(backendManager, options, log, knownFileSize).ConfigureAwait(false); @@ -304,16 +302,16 @@ namespace Duplicati.Library.Main.Operation var missingUploadingVolumes = new HashSet(); var protectedVolumes = new HashSet(); - foreach (var e in database.DuplicateRemoteVolumes(transaction)) + await foreach (var e in database.DuplicateRemoteVolumes()) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) - database.UnlinkRemoteVolume(e.Key, e.Value, transaction); + await database.UnlinkRemoteVolume(e.Key, e.Value); else throw new RemoteListVerificationException(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()), "AmbiguousStateRemoteFiles"); } - var locallist = database.GetRemoteVolumes(transaction); - foreach (var i in locallist) + var locallist = database.GetRemoteVolumes(); + await foreach (var i in locallist) { var remoteFound = lookup.TryGetValue(i.Name, out var r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); @@ -321,9 +319,9 @@ namespace Duplicati.Library.Main.Operation if (archived && r.File.Size == 0) correctSize = true; if (archived && i.ArchiveTime <= DateTime.UnixEpoch) - database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true, transaction); + await database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true); else if (!archived && i.ArchiveTime > DateTime.UnixEpoch) - database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false, transaction); + await database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false); lookup.Remove(i.Name); @@ -378,7 +376,7 @@ namespace Duplicati.Library.Main.Operation if (remoteFound && correctSize && r.File.Size >= 0) { Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, transaction); + await database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); } else if (!remoteFound) { @@ -386,7 +384,7 @@ namespace Duplicati.Library.Main.Operation { protectedVolumes.Add(i.Name); Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, transaction); + await database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null); } else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) { @@ -397,7 +395,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); missingUploadingVolumes.Add(i.Name); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, transaction); + await database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null); } } else @@ -435,7 +433,7 @@ namespace Duplicati.Library.Main.Operation if (!remoteFound) missing.Add(i); else if (correctSize) - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, transaction); + await database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); else missingHash.Add(new Tuple(r.File.Size, i)); @@ -455,19 +453,19 @@ namespace Duplicati.Library.Main.Operation } } - await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, CancellationToken.None).ConfigureAwait(false); // Batch cleanup deleted volumes in DB if (verifyMode == VerifyMode.VerifyStrict && missingUploadingVolumes.Count > 0) throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); else if (verifyMode == VerifyMode.VerifyAndClean || verifyMode == VerifyMode.VerifyStrict) { - database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), transaction); + await database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes)); // Clear the flag after we have cleaned up if (!options.Dryrun && protectedVolumes.Count == 0) - database.TerminatedWithActiveUploads = false; + await database.TerminatedWithActiveUploads(false); } - else if (verifyMode == VerifyMode.VerifyOnly && database.TerminatedWithActiveUploads) + else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads()) { Logging.Log.WriteWarningMessage(LOGTAG, "ActiveUploadsDetected", null, "Active uploads detected, but no cleanup was performed. Run the \"repair\" command to clean up the incomplete files"); } From 1f8c1747e8f0c92162b121588cba41ca70afe7f5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:34:10 +0200 Subject: [PATCH 134/362] Removed whitespace --- .../Library/Main/Operation/TestHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index de15b58e0..01d0e73a8 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 40bc75b585140d61fa86fb9ca15ca2f242e15dde Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:34:40 +0200 Subject: [PATCH 135/362] Updated TestHandler to use the new async database methods --- .../Library/Main/Operation/TestHandler.cs | 63 +++++++++---------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 01d0e73a8..98fa75aee 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -50,21 +50,20 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new LocalTestDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var rtr = new ReusableTransaction(db)) + using (var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) { - Utility.UpdateOptionsFromDb(db, m_options, rtr.Transaction); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options, rtr.Transaction); - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, rtr.Transaction); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, rtr.Transaction).ConfigureAwait(false); - await DoRunAsync(samples, db, rtr, backendManager).ConfigureAwait(false); - rtr.Commit("TestHandlerComplete"); + Utility.UpdateOptionsFromDb(db, m_options); + Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); + await db.Transaction.CommitAsync("TestHandlerComplete"); } } - public async Task DoRunAsync(long samples, LocalTestDatabase db, ReusableTransaction rtr, IBackendManager backend) + public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend) { - var files = db.SelectTestTargets(samples, m_options, rtr.Transaction).ToList(); + var files = await db.SelectTestTargets(samples, m_options).ToListAsync(); m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); m_results.OperationProgressUpdater.UpdateProgress(0); @@ -79,7 +78,7 @@ namespace Duplicati.Library.Main.Operation { if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backend.WaitForEmptyAsync(db, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await backend.WaitForEmptyAsync(db, m_results.TaskControl.ProgressToken).ConfigureAwait(false); m_results.EndTime = DateTime.UtcNow; return; } @@ -89,14 +88,14 @@ namespace Duplicati.Library.Main.Operation KeyValuePair>> res; using (tf) - res = TestVolumeInternals(db, rtr, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2); + res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2); m_results.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0) { if (res.Value == null || !res.Value.Any()) { - var rv = db.GetRemoteVolume(vol.Name, rtr.Transaction); + var rv = await db.GetRemoteVolume(vol.Name); if (rv.ID < 0) { @@ -109,14 +108,14 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", "Successfully captured hash and size for {0}, updating database", vol.Name); - db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, rtr.Transaction); + await db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash); } } } } } - db.UpdateVerificationCount(vol.Name, rtr.Transaction); + await db.UpdateVerificationCount(vol.Name); } catch (Exception ex) { @@ -153,7 +152,7 @@ namespace Duplicati.Library.Main.Operation (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false); using (tf) - res = TestVolumeInternals(db, rtr, f, tf, m_options, 1); + res = await TestVolumeInternals(db, f, tf, m_options, 1); m_results.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(hash) && size > 0) @@ -167,7 +166,7 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", "Successfully captured hash and size for {0}, updating database", f.Name); - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); + await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash); } } } @@ -178,7 +177,7 @@ namespace Duplicati.Library.Main.Operation { } } - db.UpdateVerificationCount(f.Name, rtr.Transaction); + await db.UpdateVerificationCount(f.Name); m_results.AddResult(f.Name, []); } catch (Exception ex) @@ -214,7 +213,7 @@ namespace Duplicati.Library.Main.Operation /// The remote volume being examined /// The path to the downloaded copy of the file /// A value between 0 and 1 that indicates how many blocks are tested in a dblock file - public static KeyValuePair>> TestVolumeInternals(LocalTestDatabase db, ReusableTransaction rtr, IRemoteVolume vol, string tf, Options options, double sample_percent) + public static async Task>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent) { var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm); var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name); @@ -223,15 +222,15 @@ namespace Duplicati.Library.Main.Operation switch (parsedInfo.FileType) { case RemoteVolumeType.Files: - //Compare with db and see if all files are accounted for + //Compare with db and see if all files are accounted for // with correct file hashes and blocklist hashes - using (var fl = db.CreateFilelist(vol.Name, rtr)) + using (var fl = await db.CreateFilelist(vol.Name)) { using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options)) foreach (var f in rd.Files) - fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time); + await fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time); - return new KeyValuePair>>(vol.Name, fl.Compare().ToList()); + return new KeyValuePair>>(vol.Name, await fl.Compare().ToListAsync()); } case RemoteVolumeType.Index: @@ -243,32 +242,32 @@ namespace Duplicati.Library.Main.Operation foreach (var v in rd.Volumes) { blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); - using (var bl = db.CreateBlocklist(v.Filename, rtr)) + using (var bl = await db.CreateBlocklist(v.Filename)) { foreach (var h in v.Blocks) - bl.AddBlock(h.Key, h.Value); + await bl.AddBlock(h.Key, h.Value); - combined = combined.Union(bl.Compare().ToArray()); + combined = combined.Union(await bl.Compare().ToArrayAsync()); } } - using (var il = db.CreateIndexlist(vol.Name, rtr)) + using (var il = await db.CreateIndexlist(vol.Name)) { foreach (var t in blocklinks) - il.AddBlockLink(t.Item1, t.Item2, t.Item3); + await il.AddBlockLink(t.Item1, t.Item2, t.Item3); - combined = combined.Union(il.Compare()).ToList(); + combined = combined.Union(await il.Compare().ToArrayAsync()).ToList(); } return new KeyValuePair>>(vol.Name, combined.ToList()); case RemoteVolumeType.Blocks: using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var bl = db.CreateBlocklist(vol.Name, rtr)) + using (var bl = await db.CreateBlocklist(vol.Name)) using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options)) { //Verify that all blocks are in the file foreach (var b in rd.Blocks) - bl.AddBlock(b.Key, b.Value); + await bl.AddBlock(b.Key, b.Value); //Select random blocks and verify their hashes match the filename and size var hashsamples = new List>(rd.Blocks); @@ -293,7 +292,7 @@ namespace Duplicati.Library.Main.Operation } } - return new KeyValuePair>>(vol.Name, changes.Union(bl.Compare().ToList())); + return new KeyValuePair>>(vol.Name, changes.Union(await bl.Compare().ToListAsync())); } } From 9d6078021c50a3bd3fe73a8bf70b127673528974 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:35:16 +0200 Subject: [PATCH 136/362] Removed whitespace --- .../Library/Main/Operation/ListAffected.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 77baf1ca9..ea1d21214 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 42dabccd0fae32908c912ed08e7cd9de666823a0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:37:31 +0200 Subject: [PATCH 137/362] Updated the ListAffected operation to use the new async database methods --- .../Library/Main/Operation/ListAffected.cs | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index ea1d21214..1586e3df5 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -39,36 +39,34 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public Task RunAsync(List args, Action callback = null) + public async Task RunAsync(List args, Action callback = null) { if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new Database.LocalListAffectedDatabase(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) { if (callback == null) { m_result.SetResult( - db.GetFilesets(args).OrderByDescending(x => x.Time).ToArray(), - db.GetFiles(args).ToArray(), - db.GetLogLines(args).ToArray(), - db.GetVolumes(args).ToArray() + await db.GetFilesets(args).OrderByDescending(x => x.Time).ToArrayAsync(), + await db.GetFiles(args).ToArrayAsync(), + await db.GetLogLines(args).ToArrayAsync(), + await db.GetVolumes(args).ToArrayAsync() ); } else { m_result.SetResult( - db.GetFilesets(args).OrderByDescending(x => x.Time), - db.GetFiles(args), - db.GetLogLines(args), - db.GetVolumes(args) + await db.GetFilesets(args).OrderByDescending(x => x.Time).ToArrayAsync(), + await db.GetFiles(args).ToArrayAsync(), + await db.GetLogLines(args).ToArrayAsync(), + await db.GetVolumes(args).ToArrayAsync() ); callback(m_result); } } - - return Task.CompletedTask; } } } From b0890d3e023222e783317d9dcc55b0a29d81e6b6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:37:46 +0200 Subject: [PATCH 138/362] Removed whitespace --- .../Main/Operation/ListBrokenFilesHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index dcca28a80..bac324e84 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 1d08e0359bbe4d7bbb0b1742b5b712f010759acf Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:43:08 +0200 Subject: [PATCH 139/362] Updated ListBrokenFilesHandler to use the new async database methods --- .../Main/Operation/ListBrokenFilesHandler.cs | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index bac324e84..56f31d5bd 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -50,24 +50,23 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var tr = db.BeginTransaction()) - await DoRunAsync(backendManager, db, tr, filter, callbackhandler).ConfigureAwait(false); + using (var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) + await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); } - public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[], List Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options) + public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[], List Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, Options options) { List missing = null; - var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); + var brokensets = await db.GetBrokenFilesets(options.Time, options.Version).ToArrayAsync(); if (brokensets.Length == 0) { - if (db.RepairInProgress) + if (await db.RepairInProgress()) throw new UserInformationException("Cannot continue because the database is marked as being under repair, but does not have broken files.", "CannotListOnDatabaseInRepair"); Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, transaction, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); if (!remotestate.ParsedVolumes.Any()) throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); @@ -80,32 +79,32 @@ namespace Duplicati.Library.Main.Operation // Mark all volumes as disposable foreach (var f in missing) - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); + await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash); Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count); // Drop all content from tables - db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction); + await db.RemoveMissingBlocks(missing.Select(x => x.Name)); // Mark all orphaned index files as disposable after removing the missing block files - foreach (var f in db.GetOrphanedIndexFiles(transaction).ToList()) - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); + await foreach (var f in db.GetOrphanedIndexFiles()) + await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash); - brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); + brokensets = await db.GetBrokenFilesets(options.Time, options.Version).ToArrayAsync(); } return (brokensets, missing); } - private async Task DoRunAsync(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, IFilter filter, Func callbackhandler) + private async Task DoRunAsync(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, IFilter filter, Func callbackhandler) { if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles"); - if (db.PartiallyRecreated) + if (await db.PartiallyRecreated()) throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase"); - (var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, transaction, m_options).ConfigureAwait(false); + (var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); if (brokensets == null) return; @@ -123,15 +122,15 @@ namespace Duplicati.Library.Main.Operation return; } - var fstimes = db.FilesetTimes.ToList(); + var fstimes = await db.FilesetTimes().ToListAsync(); var brokenfilesets = brokensets.Select(x => new { - Version = fstimes.FindIndex(y => y.Key == x.Item2), - Timestamp = x.Item1, - FilesetID = x.Item2, - BrokenCount = x.Item3 + Version = fstimes.FindIndex(y => y.Key == x.FilesetID), + Timestamp = x.FilesetTime, + FilesetID = x.FilesetID, + BrokenCount = x.RemoveCount } ) .ToArray(); @@ -143,7 +142,7 @@ namespace Duplicati.Library.Main.Operation x.Version, x.Timestamp, callbackhandler == null && !m_options.ListSetsOnly - ? db.GetBrokenFilenames(x.FilesetID, transaction).ToArray().AsEnumerable() + ? db.GetBrokenFilenames(x.FilesetID).ToEnumerable() : new MockList>((int)x.BrokenCount) )) .ToArray(); @@ -151,7 +150,7 @@ namespace Duplicati.Library.Main.Operation if (callbackhandler != null) foreach (var bs in brokenfilesets) - foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, transaction)) + await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID)) if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2)) break; } From 0eba962fb968b77e9018a8f46ed963cfe66ec19f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:45:18 +0200 Subject: [PATCH 140/362] Reintroduced helper function for constructing a LocalListChangesDatabase.StorageHelper --- Duplicati/Library/Main/Database/LocalListChangesDatabase.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index ff499fbb8..204ce8545 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -618,6 +618,11 @@ namespace Duplicati.Library.Main.Database } } } + + public async Task CreateStorageHelper() + { + return await StorageHelper.CreateAsync(this); + } } } From 5d89e5ff1b7f7a86d395581300f0cfbee642eaff Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:48:39 +0200 Subject: [PATCH 141/362] Removed whitespace --- .../Main/Operation/ListChangesHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index 603a6f331..f1d7062b5 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From f700d8d540d6c1858c368106bd934555a7612c92 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:48:53 +0200 Subject: [PATCH 142/362] Updated ListChangesHandler to use the new async database methods --- .../Main/Operation/ListChangesHandler.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index f1d7062b5..f056c3e76 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -80,12 +80,12 @@ namespace Duplicati.Library.Main.Operation DateTime compareVersionTime; using (var tmpdb = useLocalDb ? null : new TempFile()) - using (var db = new Database.LocalListChangesDatabase(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache)) - using (var storageKeeper = db.CreateStorageHelper()) + using (var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache)) + using (var storageKeeper = await db.CreateStorageHelper()) { if (useLocalDb) { - var dbtimes = db.FilesetTimes.ToList(); + var dbtimes = await db.FilesetTimes().ToListAsync(); if (dbtimes.Count < 2) throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", dbtimes.Count), "NeedTwoBackupsToStartDiff"); @@ -97,8 +97,8 @@ namespace Duplicati.Library.Main.Operation times.Remove(bt); SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId); - storageKeeper.AddFromDb(baseVersionId, false, filter); - storageKeeper.AddFromDb(compareVersionId, true, filter); + await storageKeeper.AddFromDb(baseVersionId, false, filter); + await storageKeeper.AddFromDb(compareVersionId, true, filter); } else { @@ -143,7 +143,7 @@ namespace Duplicati.Library.Main.Operation using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options)) foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) - storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false); + await storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false); if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -152,11 +152,11 @@ namespace Duplicati.Library.Main.Operation using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options)) foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) - storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true); + await storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true); } - var changes = storageKeeper.CreateChangeCountReport(); - var sizes = storageKeeper.CreateChangeSizeReport(); + var changes = await storageKeeper.CreateChangeCountReport(); + var sizes = await storageKeeper.CreateChangeSizeReport(); var lst = (m_options.FullResult || callback != null) ? (from n in storageKeeper.CreateChangedFileReport() @@ -168,11 +168,11 @@ namespace Duplicati.Library.Main.Operation changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles, changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles, sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize, - (lst == null || callback == null) ? null : lst.ToArray() + (lst == null || callback == null) ? null : await lst.ToArrayAsync() ); if (callback != null) - callback(m_result, lst); + callback(m_result, lst.ToEnumerable()); return; } From f501d39fafa0db73ea0f0358d6cb4fb1f33e9c54 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:49:16 +0200 Subject: [PATCH 143/362] Removed whitespace --- .../Main/Operation/ListControlFilesHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index 02c5bf8da..0bbddc1c5 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 7ac37c8ba43c3beb9160ff2525643d95c21c662b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:50:17 +0200 Subject: [PATCH 144/362] Updated ListControlFilesHandler to use the new async database methods --- Duplicati/Library/Main/Operation/ListControlFilesHandler.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index 0bbddc1c5..f43e75d94 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.Operation { var cancellationToken = CancellationToken.None; using (var tmpdb = new TempFile()) - using (var db = new LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache)) + using (var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache)) { var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); @@ -62,7 +62,7 @@ namespace Duplicati.Library.Main.Operation return; var file = fileversion.Value.File; - var entry = db.GetRemoteVolume(file.Name); + var entry = await db.GetRemoteVolume(file.Name); var files = new List(); using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false)) @@ -85,7 +85,7 @@ namespace Duplicati.Library.Main.Operation } finally { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); } } } From 724d29f9fef263e319d3eada9b558a5ec1440c18 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:51:32 +0200 Subject: [PATCH 145/362] Removed whitespace --- .../Main/Operation/ListFilesetsHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs index 81d7c311f..bd55cf33e 100644 --- a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From a79cd659beb2df53527c5d0fc6923e641258c4e6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:51:49 +0200 Subject: [PATCH 146/362] Updated ListFilesetsHandler to use the new async database methods --- Duplicati/Library/Main/Operation/ListFilesetsHandler.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs index bd55cf33e..d5e74b1bc 100644 --- a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs @@ -46,11 +46,11 @@ internal static class ListFilesetsHandler /// A task representing the asynchronous operation public static async Task RunAsync(Options options, ListFilesetResults result, IBackendManager backendManager) { - //Use a speedy local query + // Use a speedy local query if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb) { - using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache); - result.Filesets = db.ListFilesetsExtended().ToArray(); + using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache); + result.Filesets = await db.ListFilesetsExtended().ToArrayAsync(); return; } From 8020b0a30c1cb03bc7eecdd263c8a9509b25f73b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:54:01 +0200 Subject: [PATCH 147/362] Removed whitespace --- .../Main/Operation/ListFilesHandler.cs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index ec68ca2b9..88664be75 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -1,23 +1,24 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System.Collections.Generic; using System.Linq; using System.Threading; From 61f5990e3d7bcff8c1ff870adbc71e937b8789cc Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:54:17 +0200 Subject: [PATCH 148/362] Updated ListFilesHandler to use the new async database methods --- .../Main/Operation/ListFilesHandler.cs | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 88664be75..46409229b 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -55,26 +55,26 @@ namespace Duplicati.Library.Main.Operation //Use a speedy local query if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) - using (var db = new Database.LocalListDatabase(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) { - using (var filesets = db.SelectFileSets(m_options.Time, m_options.Version)) + using (var filesets = await db.SelectFileSets(m_options.Time, m_options.Version)) { if (!filter.Empty) { if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) { - filesets.TakeFirst(); + await filesets.TakeFirst(); } } IEnumerable files; if (m_options.ListFolderContents) { - files = filesets.SelectFolderContents(filter); + files = filesets.SelectFolderContents(filter).ToEnumerable(); } else if (m_options.ListPrefixOnly) { - files = filesets.GetLargestPrefix(filter); + files = filesets.GetLargestPrefix(filter).ToEnumerable(); } else if (filter.Empty) { @@ -82,21 +82,26 @@ namespace Duplicati.Library.Main.Operation } else { - files = filesets.SelectFiles(filter); + files = filesets.SelectFiles(filter).ToEnumerable(); } if (m_options.ListSetsOnly) { m_result.SetResult( - filesets.QuickSets.Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)).ToArray(), + await filesets + .QuickSets() + .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) + .ToArrayAsync(), null ); } else { m_result.SetResult( - filesets.Sets.Select(x => - new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)).ToArray(), + await filesets + .Sets() + .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) + .ToArrayAsync(), files == null ? null : (from n in files @@ -120,7 +125,7 @@ namespace Duplicati.Library.Main.Operation // Otherwise, grab info from remote location using (var tmpdb = new TempFile()) - using (var db = new LocalDatabase(tmpdb, "List", true, m_options.SqlitePageCache)) + using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache)) { var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) From 58fc35e9fd6abf34bd12355c451dc2b596404002 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:56:59 +0200 Subject: [PATCH 149/362] Changed LocalDatabase.GetFilesetIDs to return an async enumerable, rather than a task containing an enumerable. --- Duplicati/Library/Main/Database/LocalDatabase.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index abd0a3f93..7c5d5aa30 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -987,7 +987,7 @@ namespace Duplicati.Library.Main.Database return r; } - public async Task> GetFilesetIDs(DateTime restoretime, long[] versions, bool singleTimeMatch = false) + public async IAsyncEnumerable GetFilesetIDs(DateTime restoretime, long[] versions, bool singleTimeMatch = false) { if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); @@ -1023,7 +1023,8 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteWarningMessage(LOGTAG, "RestoreTimeNoMatch", null, "Restore time or version did not match any existing backups, selecting newest backup"); } - return res; + foreach (var el in res) + yield return el; } public async Task> FindMatchingFilesets(DateTime restoretime, long[] versions) From 51bbc8c3a6cb1ba1800ab02b878965d58e6f5693 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:57:49 +0200 Subject: [PATCH 150/362] Removed whitespace --- .../Main/Operation/ListFileVersionsHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs index 95e53370b..47c101f09 100644 --- a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From b40981aad7fd04170cb1308d0083b9959749c879 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:58:04 +0200 Subject: [PATCH 151/362] Updated ListFileVersionsHandler to use the new async database methods --- .../Library/Main/Operation/ListFileVersionsHandler.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs index 47c101f09..af107ad7b 100644 --- a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs @@ -42,16 +42,16 @@ internal static class ListFileVersionsHandler /// The offset to start listing from /// The maximum number of results to return /// A task representing the asynchronous operation - public static Task RunAsync(Options options, ListFileVersionsResults result, string[] paths, long offset, long limit) + public async static Task RunAsync(Options options, ListFileVersionsResults result, string[] paths, long offset, long limit) { if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache); + using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache); long[]? filesetIds = null; if (!options.AllVersions) { - filesetIds = db.GetFilesetIDs(options.Time, options.Version).ToArray(); + filesetIds = await db.GetFilesetIDs(options.Time, options.Version).ToArrayAsync(); if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); } @@ -60,8 +60,6 @@ internal static class ListFileVersionsHandler throw new UserInformationException("No path specified", "NoPathSpecified"); paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); - result.FileVersions = db.ListFileVersions(paths, filesetIds, offset, limit); - - return Task.CompletedTask; + result.FileVersions = await db.ListFileVersions(paths, filesetIds, offset, limit); } } From e1f570c0f8d700d59ea1e0fc3650179cb32c87a8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 15:59:50 +0200 Subject: [PATCH 152/362] Removed whitespace --- .../Main/Operation/ListFolderHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index cc844d4cb..ef6589303 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From ecbc844647c74ad2634af279cebe772d55062037 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 16:00:08 +0200 Subject: [PATCH 153/362] Updated ListFolderHandler to use the new async database methods --- Duplicati/Library/Main/Operation/ListFolderHandler.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index ef6589303..7d90dfec3 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -41,13 +41,13 @@ internal static class ListFolderHandler /// The offset to start listing from /// The maximum number of entries to list /// A task representing the asynchronous operation - public static Task RunAsync(Options options, ListFolderResults result, string[] folders, long offset, long limit) + public static async Task RunAsync(Options options, ListFolderResults result, string[] folders, long offset, long limit) { if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache); - var filesetIds = db.GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true).ToArray(); + using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache); + var filesetIds = await db.GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true).ToArrayAsync(); if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); if (filesetIds.Length > 1) @@ -57,12 +57,11 @@ internal static class ListFolderHandler { if (folders != null && folders.Length > 1) throw new UserInformationException("When no folder is specified, only one folder can be listed", "MultipleFoldersFound"); - result.Entries = db.ListFilesetEntries(db.GetRootPrefixes(filesetIds[0]), filesetIds[0], offset, limit); + result.Entries = await db.ListFilesetEntries(db.GetRootPrefixes(filesetIds[0]).ToEnumerable(), filesetIds[0], offset, limit); } else { - result.Entries = db.ListFolder(db.GetPrefixIds(folders), filesetIds[0], offset, limit); + result.Entries = await db.ListFolder(db.GetPrefixIds(folders).ToEnumerable(), filesetIds[0], offset, limit); } - return Task.CompletedTask; } } From 0744767dbb3c7811db4041c3ca08d10d57ba5cee Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 21 May 2025 16:00:29 +0200 Subject: [PATCH 154/362] Removed whitespace --- .../Main/Operation/PurgeBrokenFilesHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 491958fee..5cd67c57f 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 5c2b172b78f2c496f3d060ac2f5b1b1b9c17987e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 05:46:42 +0200 Subject: [PATCH 155/362] Updated PurgeBrokenFilesHandler to use the new assync database methods --- .../Main/Operation/PurgeBrokenFilesHandler.cs | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 5cd67c57f..fdb64db82 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -25,6 +25,7 @@ using System.Linq; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; +using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation { @@ -51,13 +52,12 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - using (var db = new LocalListBrokenFilesDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var tr = db.BeginTransaction()) + using (var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) { - if (db.PartiallyRecreated) + if (await db.PartiallyRecreated()) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - (var sets, var missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, tr, m_options).ConfigureAwait(false); + var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); if (sets == null) return; @@ -77,22 +77,24 @@ namespace Duplicati.Library.Main.Operation var pgoffset = 0.0f; var pgspan = 0.95f / sets.Length; - var filesets = db.FilesetTimes.ToList(); + var filesets = await db.FilesetTimes().ToListAsync(); var compare_list = sets.Select(x => new { - FilesetID = x.Item2, - Timestamp = x.Item1, - RemoveCount = x.Item3, - Version = filesets.FindIndex(y => y.Key == x.Item2), - SetCount = db.GetFilesetFileCount(x.Item2, tr) + FilesetID = x.FilesetID, + Timestamp = x.FilesetTime, + RemoveCount = x.RemoveCount, + Version = filesets.FindIndex(y => y.Key == x.FilesetID), + // TODO if we await this properly, we'll have to await + // it again when using it. + SetCount = db.GetFilesetFileCount(x.FilesetID).Await() }).ToArray(); var emptyBlocksetId = -1L; if (!m_options.DisableReplaceMissingMetadata) { var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); - emptyBlocksetId = db.GetEmptyMetadataBlocksetId(missing.Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length, null); + emptyBlocksetId = await db.GetEmptyMetadataBlocksetId(missing.Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length); if (emptyBlocksetId < 0) throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); } @@ -100,11 +102,11 @@ namespace Duplicati.Library.Main.Operation var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); - if (fully_emptied.Length == db.FilesetTimes.Count()) + if (fully_emptied.Length == await db.FilesetTimes().CountAsync()) throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); if (!m_options.Dryrun) - tr.Commit(); + await db.Transaction.CommitAsync(); if (fully_emptied.Length != 0) @@ -115,8 +117,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); m_result.DeleteResults = new DeleteResults(m_result); - using (var rmdb = new LocalDeleteDatabase(db)) - using (var deltr = new ReusableTransaction(rmdb)) + using (var rmdb = await LocalDeleteDatabase.CreateAsync(db)) { var opts = new Options(new Dictionary(m_options.RawOptions)); opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); @@ -124,10 +125,10 @@ namespace Duplicati.Library.Main.Operation opts.RawOptions["no-auto-compact"] = "true"; await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) - .DoRunAsync(rmdb, deltr, true, false, backendManager).ConfigureAwait(false); + .DoRunAsync(rmdb, true, false, backendManager).ConfigureAwait(false); if (!m_options.Dryrun) - deltr.Commit("CommitDelete", restart: false); + await rmdb.Transaction.CommitAsync("CommitDelete", restart: false); } pgoffset += (pgspan * fully_emptied.Length); @@ -143,10 +144,10 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); var opts = new Options(new Dictionary(m_options.RawOptions)); - using (var pgdb = new Database.LocalPurgeDatabase(db)) + using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db)) { // Recompute the version number after we deleted the versions before - filesets = pgdb.FilesetTimes.ToList(); + filesets = await pgdb.FilesetTimes().ToListAsync(); var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); if (thisversion < 0) throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); @@ -155,7 +156,7 @@ namespace Duplicati.Library.Main.Operation opts.RawOptions.Remove("time"); opts.RawOptions["no-auto-compact"] = "true"; - await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) => + await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, async (cmd, filesetid, tablename) => { if (filesetid != bs.FilesetID) throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); @@ -163,9 +164,9 @@ namespace Duplicati.Library.Main.Operation // Update entries that would be removed because of missing metadata var updatedEntries = 0; if (!m_options.DisableReplaceMissingMetadata) - updatedEntries = db.ReplaceMetadata(filesetid, emptyBlocksetId, cmd.Transaction); + updatedEntries = await db.ReplaceMetadata(filesetid, emptyBlocksetId); - db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction); + await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID"); return updatedEntries; }).ConfigureAwait(false); } @@ -178,16 +179,16 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0.95f); - if (!m_options.Dryrun && db.RepairInProgress) + if (!m_options.Dryrun && await db.RepairInProgress()) { Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); - db.RepairInProgress = false; + await db.RepairInProgress(false); } else { - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); } m_result.OperationProgressUpdater.UpdateProgress(1.0f); From 46bdfc54f0d871bcb6a92f33ce7874b874b07ad9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 05:59:01 +0200 Subject: [PATCH 156/362] Removed whitespace --- .../Main/Operation/PurgeFilesHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 99cf71b8e..258956fa4 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From e0fed2f03d57f93a658a25dfe9acf64c0762d529 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 05:59:18 +0200 Subject: [PATCH 157/362] Updated PurgeFilesHandler to use the new async database methods --- .../Main/Operation/PurgeFilesHandler.cs | 187 +++++++++--------- 1 file changed, 95 insertions(+), 92 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 258956fa4..0b6517a03 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -25,6 +25,7 @@ using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Operation { @@ -51,34 +52,37 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new Database.LocalPurgeDatabase(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) { await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); } } - public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func filtercommand) + public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func> filtercommand) => DoRunAsync(backendManager, db, null, filtercommand, pgoffset, pgspan); - private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func filtercommand, float pgoffset, float pgspan) + private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func> filtercommand, float pgoffset, float pgspan) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin); Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation"); var doCompactStep = !m_options.NoAutoCompact && filtercommand == null; - if (db.PartiallyRecreated) + if (await db.PartiallyRecreated()) throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase"); - if (db.RepairInProgress && filtercommand == null) + if (await db.RepairInProgress() && filtercommand == null) throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase"); - var versions = db.GetFilesetIDs(m_options.Time, m_options.Version).OrderByDescending(x => x).ToArray(); + var versions = await db + .GetFilesetIDs(m_options.Time, m_options.Version) + .OrderByDescending(x => x) + .ToArrayAsync(); if (versions.Length <= 0) throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion"); - var orphans = db.CountOrphanFiles(null); + var orphans = await db.CountOrphanFiles(); if (orphans != 0) throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans"); @@ -87,15 +91,18 @@ namespace Duplicati.Library.Main.Operation if (filtercommand == null) { - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false); if (m_options.NoBackendverification) - await FilelistProcessor.VerifyLocalList(backendManager, db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, null, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); } - var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); + var filesets = await db + .FilesetTimes() + .OrderByDescending(x => x.Value) + .ToArrayAsync(); var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan; var currentprogress = pgoffset; @@ -106,7 +113,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, it is possible that the remote storage contains partial files if (!m_options.Dryrun) - db.TerminatedWithActiveUploads = true; + await db.TerminatedWithActiveUploads(true); // Reverse makes sure we re-write the old versions first foreach (var versionid in versions.Reverse()) @@ -114,93 +121,90 @@ namespace Duplicati.Library.Main.Operation progress++; Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, versions.Length); - using (var tr = db.BeginTransaction()) + (var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid); + if (ix < 0 || tsOriginal.Ticks == 0) + throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); + + var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal); + var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key); + + if (ix != 0 && filesets[ix - 1].Value <= ts) + throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); + + using (var tempset = await db.CreateTemporaryFileset(versionid)) { - (var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid); - if (ix < 0 || tsOriginal.Ticks == 0) - throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); + if (filtercommand == null) + await tempset.ApplyFilter(filter); + else + await tempset.ApplyFilter(filtercommand); - var ts = FilesetVolumeWriter.ProbeUnusedFilenameName(db, tr, m_options, tsOriginal); - var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr); - - if (ix != 0 && filesets[ix - 1].Value <= ts) - throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); - - using (var tempset = db.CreateTemporaryFileset(versionid, tr)) + if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) { - if (filtercommand == null) - tempset.ApplyFilter(filter); - else - tempset.ApplyFilter(filtercommand); + Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); + currentprogress += versionprogress; + await db.Transaction.RollBackAsync(); + continue; + } + else + { + using (var tf = new TempFile()) + using (var vol = new FilesetVolumeWriter(m_options, ts)) + { + var isOriginalFilesetFullBackup = await db.IsFilesetFullBackup(tsOriginal); + var newids = await tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); + vol.VolumeID = newids.Item1; + vol.CreateFilesetFile(isOriginalFilesetFullBackup); - if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); - currentprogress += versionprogress; - tr.Rollback(); - continue; - } - else - { - using (var tf = new TempFile()) - using (var vol = new FilesetVolumeWriter(m_options, ts)) + Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); + + await db.WriteFileset(vol, newids.Item2); + + m_result.RemovedFileSize += tempset.RemovedFileSize; + m_result.RemovedFileCount += tempset.RemovedFileCount; + m_result.UpdatedFileCount += tempset.UpdatedFileCount; + m_result.RewrittenFileLists++; + + currentprogress += (versionprogress / 2); + m_result.OperationProgressUpdater.UpdateProgress(currentprogress); + + if (m_options.Dryrun || m_options.FullResult) { - var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal, tr); - var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); - vol.VolumeID = newids.Item1; - vol.CreateFilesetFile(isOriginalFilesetFullBackup); - - Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); - - db.WriteFileset(vol, newids.Item2, tr); - - m_result.RemovedFileSize += tempset.RemovedFileSize; - m_result.RemovedFileCount += tempset.RemovedFileCount; - m_result.UpdatedFileCount += tempset.UpdatedFileCount; - m_result.RewrittenFileLists++; - - currentprogress += (versionprogress / 2); - m_result.OperationProgressUpdater.UpdateProgress(currentprogress); - - if (m_options.Dryrun || m_options.FullResult) + await foreach (var fe in tempset.ListAllDeletedFiles()) { - foreach (var fe in tempset.ListAllDeletedFiles()) - { - var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); + var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); - Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg); - Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg); - - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg); - } + Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg); + Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg); if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage"); - - Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage"); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg); } if (m_options.Dryrun) - { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount); - tr.Rollback(); - } - else - { - var lst = db.DropFilesetsFromTable(new[] { tsOriginal }, tr).ToArray(); - foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage"); - tr.Commit(); + Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage"); + } - await backendManager.PutAsync(vol, null, null, true, async () => - { - await backendManager.FlushPendingMessagesAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - } + if (m_options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount); + await db.Transaction.RollBackAsync(); + } + else + { + var lst = await db.DropFilesetsFromTable(new[] { tsOriginal }).ToArrayAsync(); + foreach (var f in lst) + await db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null); + + await db.Transaction.CommitAsync(); + + await backendManager.PutAsync(vol, null, null, true, async () => + { + await backendManager.FlushPendingMessagesAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } @@ -211,7 +215,7 @@ namespace Duplicati.Library.Main.Operation } if (!m_options.Dryrun) - db.TerminatedWithActiveUploads = false; + await db.TerminatedWithActiveUploads(false); if (doCompactStep) { @@ -224,21 +228,20 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); m_result.CompactResults = new CompactResults(m_result); - using (var cdb = new Database.LocalDeleteDatabase(db)) - using (var ctr = new Database.ReusableTransaction(cdb)) + using (var cdb = await Database.LocalDeleteDatabase.CreateAsync(db)) { await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) - .DoCompactAsync(cdb, true, ctr, backendManager) + .DoCompactAsync(cdb, true, backendManager) .ConfigureAwait(false); - ctr.Commit("PostCompact", restart: false); + await cdb.Transaction.CommitAsync("PostCompact", restart: false); } } m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete); - await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } From 67c17871f906324668bffa2aa5b6e3010d6ef9a8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:00:01 +0200 Subject: [PATCH 158/362] Reintroduced CreateTemporaryFileset helper function for instantiating inner class of LocalPurgeDatabase --- Duplicati/Library/Main/Database/LocalPurgeDatabase.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 1eb2759dd..61a5d9e05 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -51,6 +51,11 @@ namespace Duplicati.Library.Main.Database return dbnew; } + public async Task CreateTemporaryFileset(long parentid) + { + return await TemporaryFileset.CreateAsync(parentid, this); + } + public async Task GetRemoteVolumeNameForFileset(long id) { using var cmd = m_connection.CreateCommand(@" From 1f77ce7d84076dfe5b60e9f67fd5d20a866b968d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:00:18 +0200 Subject: [PATCH 159/362] Removed whitespace --- .../Main/Volumes/FilesetVolumeWriter.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 79f384494..be834e266 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -111,7 +111,7 @@ namespace Duplicati.Library.Main.Volumes if (blocklisthashes != null) { - //Slightly awkward, but we avoid writing if there are no entries + //Slightly awkward, but we avoid writing if there are no entries using (var en = blocklisthashes.GetEnumerator()) { if (en.MoveNext() && !string.IsNullOrEmpty(en.Current)) From 4d8d1a80d131e31257981be715e877afc3fec5f4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:01:22 +0200 Subject: [PATCH 160/362] Updated an inner call of FilesetVolumeWriter to use the new async database methods --- Duplicati/Library/Main/Database/LocalPurgeDatabase.cs | 6 +++--- Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 61a5d9e05..514c75609 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -103,7 +103,7 @@ namespace Duplicati.Library.Main.Database long UpdatedFileCount { get; } Task ApplyFilter(Library.Utility.IFilter filter); - Task ApplyFilter(Func filtercommand); + Task ApplyFilter(Func> filtercommand); Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); IAsyncEnumerable> ListAllDeletedFiles(); } @@ -145,11 +145,11 @@ namespace Duplicati.Library.Main.Database return tempf; } - public async Task ApplyFilter(Func filtercommand) + public async Task ApplyFilter(Func> filtercommand) { int updated; using (var cmd = m_db.Connection.CreateCommand()) - updated = filtercommand(cmd, ParentID, m_tablename); + updated = await filtercommand(cmd, ParentID, m_tablename); await PostFilterChecks(updated); } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index be834e266..7347a0ad8 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -250,8 +250,8 @@ namespace Duplicati.Library.Main.Volumes /// The time to increment by each probe /// The maximum number of tries to probe /// The first unused filename - internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, IDbTransaction transaction, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) - => ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name, transaction)), options, start, increment, maxTries).Await(); + internal static async Task ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) + => await ProbeUnusedFilenameName(database.GetRemoteVolumeID, options, start, increment, maxTries); /// /// Probes for an unused filename, using the current time as a starting point From ad2c7b804521dc5a125dded7df4212c812bcf2ac Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:01:59 +0200 Subject: [PATCH 161/362] Removed transactions as parameters as part of the process towards implicit transactions --- Duplicati/Library/Main/IBackendManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/IBackendManager.cs b/Duplicati/Library/Main/IBackendManager.cs index ada2c758e..e44d69e49 100644 --- a/Duplicati/Library/Main/IBackendManager.cs +++ b/Duplicati/Library/Main/IBackendManager.cs @@ -53,7 +53,7 @@ internal interface IBackendManager : IDisposable /// The transaction to use, if any /// The cancellation token /// An awaitable task - Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken); + Task WaitForEmptyAsync(LocalDatabase database, CancellationToken cancellationToken); /// /// Lists the files on the backend @@ -133,7 +133,7 @@ internal interface IBackendManager : IDisposable /// The transaction to use /// The cancellation token /// - Task FlushPendingMessagesAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken); + Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken); /// /// Updates the throttle values for upload and download From 7a6bc2ba26aad9e3d5e8a6818a888772f104095e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:14:16 +0200 Subject: [PATCH 162/362] Removed transactions as parameter in the methods of LocalRecreateDatabase as part of the move towards implicit transactions --- .../Main/Database/LocalRecreateDatabase.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index cd4169ae9..90974792f 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -343,7 +343,7 @@ namespace Duplicati.Library.Main.Database return dbnew; } - public async Task FindMissingBlocklistHashes(long hashsize, long blocksize, SqliteTransaction transaction) + public async Task FindMissingBlocklistHashes(long hashsize, long blocksize) { using var cmd = m_connection.CreateCommand(m_rtr); //Update all small blocklists and matching blocks @@ -532,7 +532,7 @@ namespace Duplicati.Library.Main.Database /// temp small list structure: filehash, blockhash, blocksize: as the small files are defined /// by the fact that they are contained in a single block, blockhash is the same as the filehash, /// and blocksize can vary from 0 to the configured block size for the backup - public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, SqliteTransaction transaction, bool hashOnly = false) + public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, bool hashOnly = false) { using var cmd = m_connection.CreateCommand(m_rtr); var extra = hashOnly ? "" : $@" @@ -658,22 +658,22 @@ namespace Duplicati.Library.Main.Database } } - public async Task AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, SqliteTransaction transaction) + public async Task AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid) { - await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid); } - public async Task AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, SqliteTransaction transaction) + public async Task AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid) { - await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid); } - public async Task AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, SqliteTransaction transaction) + public async Task AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid) { - await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid); } - private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, SqliteTransaction transaction) + private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid) { var fileid = await m_findFilesetCommand .SetTransaction(m_rtr) @@ -703,7 +703,7 @@ namespace Duplicati.Library.Main.Database .ExecuteNonQueryAsync(); } - public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, SqliteTransaction transaction) + public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes) { var metadataid = -1L; if (metahash == null) @@ -718,7 +718,7 @@ namespace Duplicati.Library.Main.Database if (metadataid != -1) return metadataid; - var blocksetid = await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction); + var blocksetid = await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes); metadataid = await m_insertMetadatasetCommand .SetTransaction(m_rtr) @@ -728,7 +728,7 @@ namespace Duplicati.Library.Main.Database return metadataid; } - public async Task AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, SqliteTransaction transaction) + public async Task AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes) { var blocksetid = await m_findBlocksetCommand .SetTransaction(m_rtr) @@ -775,7 +775,7 @@ namespace Duplicati.Library.Main.Database return blocksetid; } - public async Task<(bool, bool)> UpdateBlock(string hash, long size, long volumeID, SqliteTransaction transaction) + public async Task<(bool, bool)> UpdateBlock(string hash, long size, long volumeID) { var anyChange = false; var currentVolumeId = await m_findHashBlockCommand @@ -828,7 +828,7 @@ namespace Duplicati.Library.Main.Database } } - public async Task AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, SqliteTransaction transaction) + public async Task AddSmallBlocksetLink(string filehash, string blockhash, long blocksize) { await m_insertSmallBlockset .SetTransaction(m_rtr) @@ -838,7 +838,7 @@ namespace Duplicati.Library.Main.Database .ExecuteNonQueryAsync(); } - public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes, SqliteTransaction transaction) + public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes) { var r = await m_findTempBlockListHashCommand .SetTransaction(m_rtr) @@ -1155,7 +1155,7 @@ namespace Duplicati.Library.Main.Database /// Move blocks that are not referenced by any files to DeletedBlock table. /// /// Needs to be called after the last FindMissingBlocklistHashes, otherwise the tables are not up to date. - public async Task CleanupDeletedBlocks(SqliteTransaction transaction) + public async Task CleanupDeletedBlocks() { // Find out which blocks are deleted and move them into DeletedBlock, so that compact notices these blocks are empty // Deleted blocks do not appear in the BlocksetEntry and not in the BlocklistHash table From 3fe3e113a7ba722cc25e93957ba02765d045ef80 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:14:51 +0200 Subject: [PATCH 163/362] Updated some of the database calls in RecreateDatabaseHandler to match the new signatures --- .../Main/Operation/RecreateDatabaseHandler.cs | 279 +++++++++--------- 1 file changed, 136 insertions(+), 143 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 42e66abeb..15cf5012a 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -64,12 +64,12 @@ namespace Duplicati.Library.Main.Operation if (File.Exists(path)) throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); - using var db = new LocalDatabase(path, "Recreate", true, m_options.SqlitePageCache); + using var db = await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache); await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); // Ensure database is consistent after the recreate - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); } /// @@ -83,7 +83,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.RepairOnlyPaths) throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); - using var db = new LocalDatabase(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache); + using var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache); if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version)).Any()) { if (m_options.IgnoreUpdateIfVersionExists) @@ -96,7 +96,7 @@ namespace Duplicati.Library.Main.Operation } // Mark as incomplete - db.PartiallyRecreated = true; + await db.PartiallyRecreated(true); var preexistingOptionsInDatabase = Utility.ContainsOptionsForVerification(db); Utility.UpdateOptionsFromDb(db, m_options, null); @@ -121,8 +121,8 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running); //We build a local database in steps. - using var restoredb = new LocalRecreateDatabase(dbparent, m_options); - restoredb.RepairInProgress = true; + using var restoredb = await LocalRecreateDatabase.CreateAsync(dbparent, m_options); + await restoredb.RepairInProgress(true); var expRecreateDb = false; // experimental recreate db code flag var volumeIds = new Dictionary(); @@ -194,101 +194,97 @@ namespace Duplicati.Library.Main.Operation var hasUpdatedOptions = false; //Record all blocksets and files needed - using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) + var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); + Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count); + + var progress = 0; + + // Register the files we are working with, if not already updated + if (updating) { - var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); - Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count); - - var progress = 0; - - // Register the files we are working with, if not already updated - if (updating) - { - foreach (var n in filelists) - if (volumeIds[n.File.Name] == -1) - volumeIds[n.File.Name] = await restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), tr); - } - - var isFirstFilelist = true; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - { - var entry = new RemoteVolume(name, hash, size); - try - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - m_result.EndTime = DateTime.UtcNow; - return; - } - - progress++; - if (filelistWork.Count == 1 && m_options.RepairOnlyPaths) - { - m_result.OperationProgressUpdater.UpdateProgress(0.5f); - } - else - { - m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f)); - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count); - } - - using (tmpfile) - { - isFirstFilelist = false; - - if (!string.IsNullOrWhiteSpace(hash) && size > 0) - await restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, tr); - - var parsed = VolumeBase.ParseFilename(entry.Name); - - using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); - using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - - if (!hasUpdatedOptions) - { - VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options); - hasUpdatedOptions = true; - } - - // Create timestamped operations based on the file timestamp - var filesetid = await restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); - - await RecreateFilesetFromRemoteList(restoredb, tr, compressor, filesetid, m_options, filter); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FileProcessingFailed", ex, "Failed to process file: {0}", entry.Name); - if (ex.IsAbortException()) - { - m_result.EndTime = DateTime.UtcNow; - throw; - } - - if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException) - { - m_result.EndTime = DateTime.UtcNow; - throw; - } - - if (m_options.UnittestMode) - throw; - - } - } - - //Make sure we write the config if it has been read from a manifest - if (hasUpdatedOptions) - Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, tr); - - using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) - if (tr.Connection != null) - await tr.CommitAsync(); + foreach (var n in filelists) + if (volumeIds[n.File.Name] == -1) + volumeIds[n.File.Name] = await restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0)); } + var isFirstFilelist = true; + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + var entry = new RemoteVolume(name, hash, size); + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + m_result.EndTime = DateTime.UtcNow; + return; + } + + progress++; + if (filelistWork.Count == 1 && m_options.RepairOnlyPaths) + { + m_result.OperationProgressUpdater.UpdateProgress(0.5f); + } + else + { + m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f)); + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count); + } + + using (tmpfile) + { + isFirstFilelist = false; + + if (!string.IsNullOrWhiteSpace(hash) && size > 0) + await restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash); + + var parsed = VolumeBase.ParseFilename(entry.Name); + + using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + + if (!hasUpdatedOptions) + { + VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options); + hasUpdatedOptions = true; + } + + // Create timestamped operations based on the file timestamp + var filesetid = await restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time); + + await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FileProcessingFailed", ex, "Failed to process file: {0}", entry.Name); + if (ex.IsAbortException()) + { + m_result.EndTime = DateTime.UtcNow; + throw; + } + + if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException) + { + m_result.EndTime = DateTime.UtcNow; + throw; + } + + if (m_options.UnittestMode) + throw; + + } + } + + //Make sure we write the config if it has been read from a manifest + if (hasUpdatedOptions) + Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options); + + using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) + await restoredb.Transaction.CommitAsync(); + // do we stop after just handling the dlist files ? // (if yes, we never will be able to do a backup !) if (!m_options.RepairOnlyPaths) @@ -297,7 +293,6 @@ namespace Duplicati.Library.Main.Operation //Grab all index files, and update the block table using (var hashalg = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) { hashsize = hashalg.HashSize / 8; @@ -308,7 +303,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "FilelistsRestored", "Filelists restored, downloading {0} index files", indexfiles.Count); - var progress = 0; + progress = 0; await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { @@ -316,7 +311,7 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -328,14 +323,14 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) { if (!string.IsNullOrWhiteSpace(hash) && size > 0) - await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, tr); + await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash); using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) { foreach (var a in svr.Volumes) { var filename = a.Filename; - var volumeID = await restoredb.GetRemoteVolumeID(filename, tr); + var volumeID = await restoredb.GetRemoteVolumeID(filename); // No such file if (volumeID < 0) @@ -350,18 +345,18 @@ namespace Duplicati.Library.Main.Operation throw new Exception(string.Format("Unable to parse filename: {0}", filename)); Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); missing = true; - volumeID = await restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, tr); + volumeID = await restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary); } bool anyChange = false; //Add all block/volume mappings foreach (var b in a.Blocks) { - anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr)).Item1; + anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID)).Item1; } - await restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, tr); - await restoredb.AddIndexBlockLink(await restoredb.GetRemoteVolumeID(name, tr), volumeID, tr); + await restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash); + await restoredb.AddIndexBlockLink(await restoredb.GetRemoteVolumeID(name), volumeID); } //If there are blocklists in the index file, add them to the temp blocklist hashes table @@ -375,7 +370,7 @@ namespace Duplicati.Library.Main.Operation // We need to instantiate the list to ensure the verification is // done before we add it to the database, since we do not have nested transactions var list = b.Blocklist.ToList(); - await restoredb.AddTempBlockListHash(b.Hash, list, tr); + await restoredb.AddTempBlockListHash(b.Hash, list); } catch (System.IO.InvalidDataException e) { @@ -406,7 +401,7 @@ namespace Duplicati.Library.Main.Operation } using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) - tr.Commit(); + await restoredb.Transaction.CommitAsync(); // TODO: In some cases, we can avoid downloading all index files, // if we are lucky and pick the right ones @@ -418,9 +413,9 @@ namespace Duplicati.Library.Main.Operation if (expRecreateDb) // add missing blocks and blocksetentry data (at this point // we have not yet anything in the blocksetentry table) - await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, null); + await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize); else - await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, null); + await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize); // We have now grabbed as much information as possible, // if we are still missing data, we must now fetch block files @@ -449,18 +444,17 @@ namespace Duplicati.Library.Main.Operation } } - var progress = 0; + progress = 0; await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { try { using (tmpfile) using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - using (var tr = restoredb.Connection.BeginTransaction(deferred: true)) { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -471,13 +465,13 @@ namespace Duplicati.Library.Main.Operation var volumeid = await restoredb.GetRemoteVolumeID(name); - await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, tr); + await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash); bool anyChange = false; // Update the block table so we know about the block/volume map foreach (var h in rd.Blocks) { - anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr)).Item1; + anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid)).Item1; } // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes @@ -485,7 +479,7 @@ namespace Duplicati.Library.Main.Operation // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) { - if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), tr)) + if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize))) { anyChange = true; } @@ -503,13 +497,13 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name); } if (expRecreateDb) - await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, tr, false); + await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false); else - await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, tr); + await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize); } using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) - tr.Commit(); + await restoredb.Transaction.CommitAsync(); //At this point we can patch files with data from the block volume if (blockprocessor != null) @@ -526,14 +520,14 @@ namespace Duplicati.Library.Main.Operation } } - await backendManager.WaitForEmptyAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (!m_options.RepairOnlyPaths) { // All blocks are collected and added into the Block table // Find out which blocks are deleted and move them into DeletedBlock, // so that compact can calculate the unused space - await restoredb.CleanupDeletedBlocks(null); + await restoredb.CleanupDeletedBlocks(); } await restoredb.CleanupMissingVolumes(); @@ -551,24 +545,24 @@ namespace Duplicati.Library.Main.Operation // except to continue a backup m_result.EndTime = DateTime.UtcNow; - using (var lbfdb = new LocalListBrokenFilesDatabase(restoredb)) + using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb)) { - var broken = lbfdb.GetBrokenFilesets(new DateTime(0), null, null).Count(); + var broken = await lbfdb.GetBrokenFilesets(new DateTime(0), null).CountAsync(); if (broken != 0) throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"), "DatabaseIsBrokenConsiderPurge"); } - await restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); + await restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); - restoredb.RepairInProgress = false; + await restoredb.RepairInProgress(false); } m_result.EndTime = DateTime.UtcNow; } - public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, SqliteTransaction transaction, ICompression compressor, long filesetid, Options options, IFilter filter) + public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter) { var blocksize = options.Blocksize; var hashes_pr_block = blocksize / options.BlockhashSize; @@ -577,11 +571,10 @@ namespace Duplicati.Library.Main.Operation var filesetData = VolumeReaderBase.GetFilesetData(compressor, options); // update fileset using filesetData - await restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, transaction); - transaction = restoredb.Connection.BeginTransaction(deferred: true); + await restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup); // clear any existing fileset entries - await restoredb.ClearFilesetEntries(filesetid, transaction); + await restoredb.ClearFilesetEntries(filesetid); using (var filelistreader = new FilesetVolumeReader(compressor, options)) foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) @@ -594,37 +587,37 @@ namespace Duplicati.Library.Main.Operation var metadataid = long.MinValue; var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path); - var prefixid = await restoredb.GetOrCreatePathPrefix(split.Key, transaction); + var prefixid = await restoredb.GetOrCreatePathPrefix(split.Key); switch (fe.Type) { case FilelistEntryType.Folder: - metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - await restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); + metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes); + await restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid); break; case FilelistEntryType.File: var expectedblocks = (fe.Size + blocksize - 1) / blocksize; var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block; if (expectedblocks <= 1) expectedblocklisthashes = 0; - var blocksetid = await restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, transaction); - metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - await restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, transaction); + var blocksetid = await restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes); + metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes); + await restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid); if (fe.Size <= blocksize) { if (!string.IsNullOrWhiteSpace(fe.Blockhash)) - await restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, transaction); + await restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) - await restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, transaction); + await restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size); else if (fe.Size > 0) Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); } break; case FilelistEntryType.Symlink: - metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - await restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); + metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes); + await restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid); break; default: Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path); @@ -634,9 +627,9 @@ namespace Duplicati.Library.Main.Operation if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink)) { if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) - await restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, transaction); + await restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) - await restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, transaction); + await restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize); else Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path); } @@ -647,7 +640,7 @@ namespace Duplicati.Library.Main.Operation } } - await transaction.CommitAsync(); + await restoredb.Transaction.CommitAsync(); } /// From c2a73613d711e2fbb055578bcb86b8f623881c3d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:35:34 +0200 Subject: [PATCH 164/362] Updated RepairHandler to match the new method signatures and removed transactions as parameters towards implicit transactions --- .../Library/Main/Operation/RepairHandler.cs | 232 +++++++++--------- 1 file changed, 113 insertions(+), 119 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 49c4d9f7e..9defbc63d 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -65,7 +65,7 @@ namespace Duplicati.Library.Main.Operation long knownRemotes = -1; try { - using var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); + using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); knownRemotes = await db.GetRemoteVolumes().CountAsync(); } catch (Exception ex) @@ -128,32 +128,31 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); - using var rtr = new ReusableTransaction(db); + using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - if (db.PartiallyRecreated) + if (await db.PartiallyRecreated()) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (db.RepairInProgress) + if (await db.RepairInProgress()) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); // Ensure the database is consistent before we start fixing the remote - await db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, rtr.Transaction); + await db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true); // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist - var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(rtr.Transaction); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, rtr.Transaction, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); - await rtr.CommitAsync("CommitRemoteListAnalysisTransaction"); + var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + await db.Transaction.CommitAsync("CommitRemoteListAnalysisTransaction"); var buffer = new byte[m_options.Blocksize]; var hashsize = m_options.BlockhashSize; - var missingRemoteFilesets = await db.MissingRemoteFilesets(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); - var missingLocalFilesets = await db.MissingLocalFilesets(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); - var emptyIndexFiles = await db.EmptyIndexFiles(rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + var missingRemoteFilesets = await db.MissingRemoteFilesets().ToListAsync(cancellationToken: cancellationToken); + var missingLocalFilesets = await db.MissingLocalFilesets().ToListAsync(cancellationToken: cancellationToken); + var emptyIndexFiles = await db.EmptyIndexFiles().ToListAsync(cancellationToken: cancellationToken); var progress = 0; var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; @@ -189,16 +188,16 @@ namespace Duplicati.Library.Main.Operation { if (tp.VerificationRequiredVolumes.Any()) { - using var testdb = new LocalTestDatabase(db); + using var testdb = await LocalTestDatabase.CreateAsync(db); foreach (var n in tp.VerificationRequiredVolumes) try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(testdb, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(testdb, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await rtr.CommitAsync("CommitEarlyExit", false); + await testdb.Transaction.CommitAsync("CommitEarlyExit", false); return; } @@ -208,7 +207,7 @@ namespace Duplicati.Library.Main.Operation KeyValuePair>> res; var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); using (tf) - res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1); + res = await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1); if (res.Value.Any()) throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); @@ -216,7 +215,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); - await db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); + await db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash); } } @@ -227,7 +226,7 @@ namespace Duplicati.Library.Main.Operation throw; } - await rtr.CommitAsync("CommitVerificationTransaction"); + await db.Transaction.CommitAsync("CommitVerificationTransaction"); } // TODO: It is actually possible to use the extra files if we parse them @@ -236,9 +235,9 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await rtr.CommitAsync("CommitEarlyExit", false); + await db.Transaction.CommitAsync("CommitEarlyExit", false); return; } @@ -267,31 +266,31 @@ namespace Duplicati.Library.Main.Operation if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); - await db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, rtr.Transaction); + await db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks); } var blocksize = m_options.Blocksize; foreach (var ixb in ifr.BlockLists) - await db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, rtr.Transaction); + await db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize); // Register the new index file and link it to the block files - var selfid = await db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), rtr.Transaction); + var selfid = await db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0)); foreach (var rv in ifr.Volumes) { // Guard against unknown block files - long id = await db.GetRemoteVolumeID(rv.Filename, rtr.Transaction); + long id = await db.GetRemoteVolumeID(rv.Filename); if (id == -1) Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); else - await db.AddIndexBlockLink(selfid, id, rtr.Transaction); + await db.AddIndexBlockLink(selfid, id); } if (!m_options.Dryrun) - await rtr.CommitAsync("CommitIndexFileTransaction"); + await db.Transaction.CommitAsync("CommitIndexFileTransaction"); } // All checks fine, we accept the new index file Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); - await db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); + await db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash); continue; } catch (Exception rex) @@ -324,15 +323,15 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await rtr.CommitAsync("CommitEarlyExit", false); + await db.Transaction.CommitAsync("CommitEarlyExit", false); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); + var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); var fsw = new FilesetVolumeWriter(m_options, fileTime); Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); @@ -342,12 +341,12 @@ namespace Duplicati.Library.Main.Operation fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); fsw.CreateFilesetFile(isfull); - await db.WriteFileset(fsw, filesetId, null); + await db.WriteFileset(fsw, filesetId); fsw.Close(); fsw.VolumeID = await db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); - await db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); + await db.LinkFilesetToVolume(filesetId, fsw.VolumeID); if (m_options.Dryrun) { fsw.Dispose(); @@ -359,10 +358,10 @@ namespace Duplicati.Library.Main.Operation if (anyDlistUploads) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await rtr.CommitAsync("CommitFilesetTransaction"); + await db.Transaction.CommitAsync("CommitFilesetTransaction"); } foreach (var volumename in missingLocalFilesets) @@ -373,22 +372,22 @@ namespace Duplicati.Library.Main.Operation var parsed = VolumeBase.ParseFilename(remoteVolume.Name); using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) - using (var recreatedb = new LocalRecreateDatabase(db, m_options)) + using (var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options)) { if (compressor == null) throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - var filesetid = await db.CreateFileset(remoteVolume.ID, parsed.Time, rtr.Transaction); + var filesetid = await db.CreateFileset(remoteVolume.ID, parsed.Time); - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()); if (!m_options.Dryrun) - await rtr.CommitAsync("CommitRecreateFilesetTransaction"); + await recreatedb.Transaction.CommitAsync("CommitRecreateFilesetTransaction"); } } } if (!m_options.Dryrun && tp.MissingVolumes.Any()) - db.TerminatedWithActiveUploads = true; + await db.TerminatedWithActiveUploads(true); if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); @@ -403,7 +402,7 @@ namespace Duplicati.Library.Main.Operation } // Blocks are recreated with the entire list of missing files to handle cases where partial recreation is needed - await RunRepairDblocks(backendManager, db, rtr, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); + await RunRepairDblocks(backendManager, db, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); // Filesets are recreated one at a time foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Files)) @@ -414,10 +413,10 @@ namespace Duplicati.Library.Main.Operation try { var timestamp = VolumeBase.ParseFilename(n.Name).Time; - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); + var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); - await RunRepairDlist(backendManager, db, rtr, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); + await RunRepairDlist(backendManager, db, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -441,12 +440,12 @@ namespace Duplicati.Library.Main.Operation try { // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = (await db.GetRemoteVolume(n.Name, rtr.Transaction)).State; + var currentState = (await db.GetRemoteVolume(n.Name)).State; if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) continue; var w = newEntry = new IndexVolumeWriter(m_options); - await RunRepairDindex(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); + await RunRepairDindex(backendManager, db, w, n, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -467,7 +466,7 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); return; } @@ -486,7 +485,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); - await backendManager.FlushPendingMessagesAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); } } } @@ -505,11 +504,11 @@ namespace Duplicati.Library.Main.Operation } m_result.OperationProgressUpdater.UpdateProgress(1); - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) { - db.TerminatedWithActiveUploads = false; - await rtr.CommitAsync("CommitRepairTransaction", false); + await db.TerminatedWithActiveUploads(false); + await db.Transaction.CommitAsync("CommitRepairTransaction", false); } } @@ -524,40 +523,40 @@ namespace Duplicati.Library.Main.Operation /// The time of the new fileset to create /// Cancellation token to cancel the operation /// A task representing the asynchronous operation - private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) + private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) { - volumeWriter.VolumeID = await db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, rtr.Transaction); - (var prevFilesetId, var _, var isPrevFull) = await db.GetFilesetFromRemotename(originalVolume.Name, rtr.Transaction); + volumeWriter.VolumeID = await db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero); + (var prevFilesetId, var _, var isPrevFull) = await db.GetFilesetFromRemotename(originalVolume.Name); if (!string.IsNullOrEmpty(m_options.ControlFiles)) foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) volumeWriter.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); volumeWriter.CreateFilesetFile(isPrevFull); - var newFilesetID = await db.CreateFileset(volumeWriter.VolumeID, filesetTime, rtr.Transaction); - await db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull, rtr.Transaction); + var newFilesetID = await db.CreateFileset(volumeWriter.VolumeID, filesetTime); + await db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull); - await db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, rtr.Transaction); - await db.MoveFilesFromFileset(newFilesetID, prevFilesetId, rtr.Transaction); + await db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID); + await db.MoveFilesFromFileset(newFilesetID, prevFilesetId); - await db.WriteFileset(volumeWriter, newFilesetID, rtr.Transaction); + await db.WriteFileset(volumeWriter, newFilesetID); volumeWriter.Close(); - await db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); - await db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - await rtr.CommitAsync("CommitPriorToFilesetUpload"); + await db.Transaction.CommitAsync("CommitPriorToFilesetUpload"); await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await rtr.CommitAsync("CommitFilesetTransaction"); + await db.Transaction.CommitAsync("CommitFilesetTransaction"); } /// @@ -572,25 +571,25 @@ namespace Duplicati.Library.Main.Operation /// Cancellation token to cancel the operation /// A task representing the asynchronous operation - private async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IndexVolumeWriter indexWriter, RemoteVolumeEntry originalVolume, CancellationToken cancellationToken) + private async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, IndexVolumeWriter indexWriter, RemoteVolumeEntry originalVolume, CancellationToken cancellationToken) { - indexWriter.VolumeID = await db.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); + indexWriter.VolumeID = await db.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero); var blockvolumeids = new List(); using var h = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); - await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name, rtr.Transaction)) + await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name)) { indexWriter.StartVolume(blockvolume.Name); - var volumeid = await db.GetRemoteVolumeID(blockvolume.Name, rtr.Transaction); + var volumeid = await db.GetRemoteVolumeID(blockvolume.Name); - await foreach (var b in db.GetBlocks(volumeid, rtr.Transaction)) + await foreach (var b in db.GetBlocks(volumeid)) indexWriter.AddBlock(b.Hash, b.Size); indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size); blockvolumeids.Add(volumeid); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - await foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, m_options.BlockhashSize, rtr.Transaction)) + await foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, m_options.BlockhashSize)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); if (bh != b.Item1) @@ -601,7 +600,7 @@ namespace Duplicati.Library.Main.Operation } foreach (var blockvolumeid in blockvolumeids) - await db.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, rtr.Transaction); + await db.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid); indexWriter.Close(); @@ -609,14 +608,14 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(indexWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - await db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await rtr.CommitAsync("CommitRepairTransaction"); + await db.Transaction.CommitAsync("CommitRepairTransaction"); } /// @@ -629,10 +628,6 @@ namespace Duplicati.Library.Main.Operation /// private readonly LocalDatabase m_database; /// - /// Transaction to use for the operation - /// - private readonly ReusableTransaction m_transaction; - /// /// List of volumes to delete after the upload is complete /// private readonly List m_toDelete = new List(); @@ -663,14 +658,14 @@ namespace Duplicati.Library.Main.Operation /// The options for the current operation /// The database to use for registering a volume in progress /// The transaction to use for the operation - public InProgressDblockVolumes(Options options, LocalDatabase db, ReusableTransaction transaction) + public InProgressDblockVolumes(Options options, LocalDatabase db) { m_database = db; - m_transaction = transaction; m_options = options; m_maxVolumeSize = options.VolumeSize - m_options.Blocksize; m_activeWriter = new BlockVolumeWriter(options); - m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction).Await(); + // TODO this constructor should be converted to a factory method + m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero).Await(); } /// @@ -726,7 +721,7 @@ namespace Duplicati.Library.Main.Operation m_completedWriters.Add(m_activeWriter); m_activeWriter = new BlockVolumeWriter(m_options); - m_activeWriter.VolumeID = await m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); + m_activeWriter.VolumeID = await m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero); m_anyData = false; } } @@ -741,9 +736,9 @@ namespace Duplicati.Library.Main.Operation /// /// Cancellation token to cancel the operation /// >A task representing the asynchronous operation - private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) + private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) { - var currentVolume = new InProgressDblockVolumes(m_options, db, rtr); + var currentVolume = new InProgressDblockVolumes(m_options, db); foreach (var n in missingDblockFiles) { @@ -751,17 +746,17 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await rtr.CommitAsync("CommitEarlyExit", false); + await db.Transaction.CommitAsync("CommitEarlyExit", false); return; } incrementProgress(); - await RunRepairDblock(backendManager, db, rtr, n, currentVolume, cancellationToken).ConfigureAwait(false); + await RunRepairDblock(backendManager, db, n, currentVolume, cancellationToken).ConfigureAwait(false); if (currentVolume.CompletedWriters.Count > 0) - await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); + await UploadCompletedBlockVolumes(backendManager, currentVolume, db, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -775,10 +770,10 @@ namespace Duplicati.Library.Main.Operation if (currentVolume.AnyData) await currentVolume.StartNewVolume(); // Complete any pending uploads - await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); + await UploadCompletedBlockVolumes(backendManager, currentVolume, db, cancellationToken).ConfigureAwait(false); // After reset, we need to remove the temporary volume from the database - await db.RemoveRemoteVolume(currentVolume.RemoteFilename, rtr.Transaction); + await db.RemoveRemoteVolume(currentVolume.RemoteFilename); } /// @@ -790,11 +785,11 @@ namespace Duplicati.Library.Main.Operation /// >The transaction to use for the upload /// Cancellation token to cancel the operation /// >A task representing the asynchronous operation - async Task UploadCompletedBlockVolumes(IBackendManager backendManager, InProgressDblockVolumes activeVolume, LocalRepairDatabase db, ReusableTransaction rtr, CancellationToken cancellationToken) + async Task UploadCompletedBlockVolumes(IBackendManager backendManager, InProgressDblockVolumes activeVolume, LocalRepairDatabase db, CancellationToken cancellationToken) { foreach (var completedVolume in activeVolume.CompletedWriters) { - await db.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + await db.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); // Create a new index file that points to the new volume IndexVolumeWriter newvolindex = null; @@ -802,41 +797,41 @@ namespace Duplicati.Library.Main.Operation if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = await db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); + newvolindex.VolumeID = await db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); newvolindex.StartVolume(completedVolume.RemoteFilename); - await foreach (var b in db.GetBlocks(completedVolume.VolumeID, rtr.Transaction)) + await foreach (var b in db.GetBlocks(completedVolume.VolumeID)) newvolindex.AddBlock(b.Hash, b.Size); - await db.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, rtr.Transaction); + await db.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { - await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize, rtr.Transaction)) + await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); }; } // All information is in place, we can now upload the new volume - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReplaceBlockFile", "would upload new block file {0}", completedVolume.RemoteFilename); else { - await rtr.CommitAsync("PostRepairPreUploadBlockVolume"); + await db.Transaction.CommitAsync("PostRepairPreUploadBlockVolume"); await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); } } // Flush the database as we have new volumes uploaded - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); // Prepare for deleting the old stuff foreach (var vol in activeVolume.VolumesToDelete) - await db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null); // Persist desired state prior to deleting the old files if (!m_options.Dryrun) - await rtr.CommitAsync("PostRepairBlockVolume"); + await db.Transaction.CommitAsync("PostRepairBlockVolume"); // Delete the old files foreach (var vol in activeVolume.VolumesToDelete) @@ -846,9 +841,9 @@ namespace Duplicati.Library.Main.Operation await backendManager.DeleteAsync(vol.Name, vol.Size, false, cancellationToken).ConfigureAwait(false); // All done, the new dblocks are now in place - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await rtr.CommitAsync("PostRepairBlockVolume"); + await db.Transaction.CommitAsync("PostRepairBlockVolume"); activeVolume.CompletedWriters.Clear(); activeVolume.VolumesToDelete.Clear(); @@ -864,12 +859,12 @@ namespace Duplicati.Library.Main.Operation /// The in-progress dblock volume to use for the recreate /// Cancellation token to cancel the operation /// >A task representing the asynchronous operation - private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, RemoteVolumeEntry originalVolume, InProgressDblockVolumes pendingVolume, CancellationToken cancellationToken) + private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, RemoteVolumeEntry originalVolume, InProgressDblockVolumes pendingVolume, CancellationToken cancellationToken) { // The dblock files are the most complex to recreate // as data can be either file contents, metadata or blocklist hashes // We attempt to recover all three source parts in the steps below - using var mbl = db.CreateBlockList(originalVolume.Name, rtr); + using var mbl = await db.CreateBlockList(originalVolume.Name); var originalMissingBlockCount = await mbl.GetMissingBlockCount(); var recoveredSourceBlocks = 0L; @@ -1071,15 +1066,15 @@ namespace Duplicati.Library.Main.Operation if (missingBlocks == 0) { pendingVolume.VolumesToDelete.Add(originalVolume); - var oldIndexFiles = await db.GetIndexFilesReferencingBlockFile(originalVolume.ID, rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + var oldIndexFiles = await db.GetIndexFilesReferencingBlockFile(originalVolume.ID).ToListAsync(cancellationToken: cancellationToken); // Find all index files that point to the old volume foreach (var oldIndexFile in oldIndexFiles) { - var oldVolume = await db.GetRemoteVolume(oldIndexFile, rtr.Transaction); + var oldVolume = await db.GetRemoteVolume(oldIndexFile); if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) { - var blockVolumesReferenced = await db.GetBlockVolumesFromIndexName(oldIndexFile, rtr.Transaction).ToListAsync(cancellationToken: cancellationToken); + var blockVolumesReferenced = await db.GetBlockVolumesFromIndexName(oldIndexFile).ToListAsync(cancellationToken: cancellationToken); if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); else @@ -1099,10 +1094,9 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); - using var tr = new ReusableTransaction(db); + using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); - var sets = await db.GetFilesetsWithMissingFiles(null).ToListAsync(); + var sets = await db.GetFilesetsWithMissingFiles().ToListAsync(); if (sets.Count == 0) return; @@ -1112,7 +1106,7 @@ namespace Duplicati.Library.Main.Operation { ix++; Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); - var volume = await db.GetRemoteVolumeFromFilesetID(entry.Key, tr.Transaction); + var volume = await db.GetRemoteVolumeFromFilesetID(entry.Key); var parsed = VolumeBase.ParseFilename(volume.Name); using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); @@ -1121,13 +1115,13 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); // Clear out the old fileset - await db.DeleteFilesetEntries(entry.Key, tr.Transaction); - using (var rdb = new LocalRecreateDatabase(db, m_options)) + await db.DeleteFilesetEntries(entry.Key); + using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options)) { - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, tr.Transaction, compressor, entry.Key, m_options, new FilterExpression()); + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression()); } - await tr.CommitAsync("PostRepairFileset"); + await db.Transaction.CommitAsync("PostRepairFileset"); } } @@ -1138,11 +1132,11 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); + using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); Utility.UpdateOptionsFromDb(db, m_options); - if (db.RepairInProgress || db.PartiallyRecreated) + if (await db.RepairInProgress() || await db.PartiallyRecreated()) Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); await db.FixDuplicateMetahash(); From 443e478ea91fbde447fde05051cb7cb2bf2bcd4d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:36:28 +0200 Subject: [PATCH 165/362] Removed whitespace --- .../Operation/RestoreControlFilesHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index e481c0f9c..0699589c6 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; From 7903045e369c20e55480d1b056a94a06cfbbf620 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:36:47 +0200 Subject: [PATCH 166/362] Updated RestoreControlFilesHandler to use the new async database methods --- .../Library/Main/Operation/RestoreControlFilesHandler.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index 0699589c6..9600239cf 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Operation Directory.CreateDirectory(m_options.Restorepath); using (var tmpdb = new TempFile()) - using (var db = new Database.LocalDatabase(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache)) + using (var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache)) { var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); @@ -63,12 +63,12 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); return; } var file = fileversion.Value.File; - var entry = db.GetRemoteVolume(file.Name); + var entry = await db.GetRemoteVolume(file.Name); var res = new List(); using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) @@ -99,7 +99,7 @@ namespace Duplicati.Library.Main.Operation } finally { - await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } From 6a0b836a7939a01de20ec4e1c5b3a0e2cca2e287 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:47:16 +0200 Subject: [PATCH 167/362] Updated RestoreHandler to use the new async database methods --- .../Library/Main/Operation/RestoreHandler.cs | 130 +++++++++--------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index b4746e3a9..1cd7293bc 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -115,14 +115,14 @@ namespace Duplicati.Library.Main.Operation { if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) { - db = new LocalRestoreDatabase(m_options.Dbpath, m_options.SqlitePageCache); + db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache); } else { Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); tmpdb = new TempFile(); RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); - db = new LocalRestoreDatabase(tmpdb, m_options.SqlitePageCache); + db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) await new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) @@ -150,17 +150,17 @@ namespace Duplicati.Library.Main.Operation } } - private static void PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage) + private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage) { var blocksize = options.Blocksize; var updateCounter = 0L; var fullblockverification = options.FullBlockVerification; using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var blockmarker = database.CreateBlockMarker()) - using (var volumekeeper = database.GetMissingBlockData(blocks, options.Blocksize)) + using (var blockmarker = await database.CreateBlockMarkerAsync()) + using (var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize)) { - foreach (var restorelist in volumekeeper.FilesWithMissingBlocks) + await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks()) { var targetpath = restorelist.Path; @@ -184,7 +184,7 @@ namespace Duplicati.Library.Main.Operation // TODO: Much faster if we iterate the volume and checks what blocks are used, // because the compressors usually like sequential reading using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - foreach (var targetblock in restorelist.Blocks) + await foreach (var targetblock in restorelist.Blocks()) { file.Position = targetblock.Offset; var size = blocks.ReadBlock(targetblock.Key, blockbuffer); @@ -203,7 +203,7 @@ namespace Duplicati.Library.Main.Operation if (valid) { file.Write(blockbuffer, 0, size); - blockmarker.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false); + await blockmarker.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false); } } else @@ -213,7 +213,7 @@ namespace Duplicati.Library.Main.Operation } if ((++updateCounter) % 20 == 0) - blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker.UpdateProcessed(result.OperationProgressUpdater); } catch (Exception ex) { @@ -226,7 +226,7 @@ namespace Duplicati.Library.Main.Operation if (!options.SkipMetadata) { - foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks) + await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks()) { var targetpath = restoremetadata.Path; Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); @@ -236,14 +236,14 @@ namespace Duplicati.Library.Main.Operation // TODO: When we support multi-block metadata this needs to deal with it using (var ms = new System.IO.MemoryStream()) { - foreach (var targetblock in restoremetadata.Blocks) + await foreach (var targetblock in restoremetadata.Blocks()) { ms.Position = targetblock.Offset; var size = blocks.ReadBlock(targetblock.Key, blockbuffer); if (targetblock.Size == size) { ms.Write(blockbuffer, 0, size); - blockmarker.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true); + await blockmarker.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true); } } @@ -260,8 +260,8 @@ namespace Duplicati.Library.Main.Operation } } } - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); + await blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker.CommitAsync(); } } @@ -315,13 +315,13 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); } // Prepare the block and file list and create the directory structure m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - PrepareBlockAndFileList(database, m_options, filter, m_result); + await PrepareBlockAndFileList(database, m_options, filter, m_result); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); @@ -400,8 +400,8 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - database.DropRestoreTable(); - await backendManager.WaitForEmptyAsync(database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await database.DropRestoreTable(); + await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Report that the restore is complete m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); @@ -428,13 +428,13 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); } //Figure out what files are to be patched, and what blocks are needed m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - PrepareBlockAndFileList(database, m_options, filter, m_result); + await PrepareBlockAndFileList(database, m_options, filter, m_result); //Make the entire output setup m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); @@ -461,7 +461,7 @@ namespace Duplicati.Library.Main.Operation if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); return; } @@ -476,14 +476,14 @@ namespace Duplicati.Library.Main.Operation if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); return; } // Fill BLOCKS with remote sources List volumes; using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes")) - volumes = database.GetMissingVolumes().ToList(); + volumes = await database.GetMissingVolumes().ToListAsync(); if (volumes.Count > 0) { @@ -500,13 +500,13 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); return; } using (tmpfile) using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) - PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); + await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); } catch (Exception ex) { @@ -520,7 +520,7 @@ namespace Duplicati.Library.Main.Operation var fileErrors = 0L; // Restore empty files. They might not have any blocks so don't appear in any volume. - foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0)) + await foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0)) { Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path); @@ -559,13 +559,13 @@ namespace Duplicati.Library.Main.Operation // After all blocks in the files are restored, verify the file hash using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification")) - foreach (var file in database.GetFilesToRestore(true)) + await foreach (var file in database.GetFilesToRestore(true)) { try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); return; } @@ -602,8 +602,8 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - database.DropRestoreTable(); - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await database.DropRestoreTable(); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); @@ -682,10 +682,10 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) { // Fill BLOCKS with data from known local source files - using (var blockmarker = database.CreateBlockMarker()) + using (var blockmarker = await database.CreateBlockMarkerAsync()) { var updateCount = 0L; - foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize)) + await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize)) { var targetpath = entry.TargetPath; var targetfileid = entry.TargetFileID; @@ -709,7 +709,7 @@ namespace Duplicati.Library.Main.Operation { using (var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath)) { - foreach (var block in entry.Blocks) + await foreach (var block in entry.Blocks()) { if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -733,7 +733,7 @@ namespace Duplicati.Library.Main.Operation targetstream.Write(blockbuffer, 0, size); } - blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false); + await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false); } } } @@ -750,7 +750,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) { - blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker.UpdateProcessed(result.OperationProgressUpdater); if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; } @@ -779,18 +779,18 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath); } - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); + await blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker.CommitAsync(); } } private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) { // Fill BLOCKS with data from known local source files - using (var blockmarker = database.CreateBlockMarker()) + using (var blockmarker = await database.CreateBlockMarkerAsync()) { var updateCount = 0L; - foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize)) + await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize)) { var targetpath = restorelist.TargetPath; var targetfileid = restorelist.TargetFileID; @@ -808,9 +808,9 @@ namespace Duplicati.Library.Main.Operation } using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - foreach (var targetblock in restorelist.Blocks) + await foreach (var targetblock in restorelist.Blocks()) { - foreach (var source in targetblock.Blocksources) + await foreach (var source in targetblock.BlockSources()) { try { @@ -848,7 +848,7 @@ namespace Duplicati.Library.Main.Operation } } - blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false); + await blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false); patched = true; break; } @@ -867,7 +867,7 @@ namespace Duplicati.Library.Main.Operation } if ((++updateCount) % 20 == 0) - blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker.UpdateProcessed(result.OperationProgressUpdater); } catch (Exception ex) { @@ -885,18 +885,18 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath)); } - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); + await blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker.CommitAsync(); } } - private static void PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) + private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) { // Create a temporary table FILES by selecting the files from fileset that matches a specific operation id // Delete all entries from the temp table that are excluded by the filter(s) using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList")) { - var c = database.PrepareRestoreFilelist(options.Time, options.Version, filter); + var c = await database.PrepareRestoreFilelist(options.Time, options.Version, filter); result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); } @@ -904,25 +904,25 @@ namespace Duplicati.Library.Main.Operation if (!string.IsNullOrEmpty(options.Restorepath)) { // Find the largest common prefix - var largest_prefix = options.DontCompressRestorePaths ? "" : database.GetLargestPrefix(); + var largest_prefix = options.DontCompressRestorePaths ? "" : await database.GetLargestPrefix(); Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath)); // Set the target paths, special care with C:\ and / - database.SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)); + await database.SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)); } else { - database.SetTargetPaths("", ""); + await database.SetTargetPaths("", ""); } // Create a temporary table BLOCKS that lists all blocks that needs to be recovered using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks")) - database.FindMissingBlocks(options.SkipMetadata); + await database.FindMissingBlocks(options.SkipMetadata); // Create temporary tables and triggers that automatically track progress using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker")) - database.CreateProgressTracker(false); + await database.CreateProgressTracker(false); } @@ -940,7 +940,7 @@ namespace Duplicati.Library.Main.Operation SystemIO.IO_OS.DirectoryCreate(options.Restorepath); } - foreach (var folder in database.GetTargetFolders()) + await foreach (var folder in database.GetTargetFolders()) { try { @@ -971,10 +971,10 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) { // Scan existing files for existing BLOCKS - using (var blockmarker = database.CreateBlockMarker()) + using (var blockmarker = await database.CreateBlockMarkerAsync()) { var updateCount = 0L; - foreach (var restorelist in database.GetExistingFilesWithBlocks()) + await foreach (var restorelist in database.GetExistingFilesWithBlocks()) { var rename = !options.Overwrite; var targetpath = restorelist.TargetPath; @@ -1027,7 +1027,7 @@ namespace Duplicati.Library.Main.Operation using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) using (var block = new Blockprocessor(file, blockbuffer)) - foreach (var targetblock in restorelist.Blocks) + await foreach (var targetblock in restorelist.Blocks()) { var size = block.Readblock(); if (size <= 0) @@ -1052,7 +1052,7 @@ namespace Duplicati.Library.Main.Operation if (key == targetblock.Hash) { - blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, size, false); + await blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, size, false); blockhashmatch = true; } } @@ -1087,7 +1087,7 @@ namespace Duplicati.Library.Main.Operation if (fullfilehashmatch) { //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. - blockmarker.SetFileDataVerified(targetfileid); + await blockmarker.SetFileDataVerified(targetfileid); Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); rename = false; } @@ -1095,13 +1095,13 @@ namespace Duplicati.Library.Main.Operation { // The new file will have none of the correct blocks, // even if the scanned file had some - blockmarker.SetAllBlocksMissing(targetfileid); + await blockmarker.SetAllBlocksMissing(targetfileid); } } if ((++updateCount) % 20 == 0) { - blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker.UpdateProcessed(result.OperationProgressUpdater); if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; } @@ -1148,8 +1148,8 @@ namespace Duplicati.Library.Main.Operation { //TODO: Also needs metadata check to make correct decision. // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. - blockmarker.SetAllBlocksRestored(targetfileid, false); - blockmarker.SetFileDataVerified(targetfileid); + await blockmarker.SetAllBlocksRestored(targetfileid, false); + await blockmarker.SetFileDataVerified(targetfileid); break; } } @@ -1165,13 +1165,13 @@ namespace Duplicati.Library.Main.Operation newname = tr; Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); - database.UpdateTargetPath(targetfileid, newname); + await database.UpdateTargetPath(targetfileid, newname); } } - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); + await blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker.CommitAsync(); } } } From 9693840611d1bde203e2021d6d2c92e37e17f75a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 06:47:38 +0200 Subject: [PATCH 168/362] Updated one call that went from Task to IAsyncEnumerable --- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 784e89d88..2f32a3542 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -335,7 +335,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); - var filesetIds = (await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions)).ToList(); + var filesetIds = await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToListAsync(); while (filesetIds.Count > 0) { var filesetId = filesetIds[0]; From 26db3a8f262f855ced330ca37ff2bcc3263d8d5f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 07:55:01 +0200 Subject: [PATCH 169/362] Removed whitespace --- .../Main/Operation/SearchEntriesHandler.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs index bae0f7408..502403093 100644 --- a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs +++ b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable From a9681db613f574a81df45d6a236b8814fc82aa84 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 07:55:14 +0200 Subject: [PATCH 170/362] Updated SearchEntriesHandler to use the new async database methods --- Duplicati/Library/Main/Operation/SearchEntriesHandler.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs index 502403093..76817b01e 100644 --- a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs +++ b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs @@ -44,16 +44,16 @@ internal static class SearchEntriesHandler /// The offset to start searching from /// The maximum number of results to return /// A task representing the asynchronous operation - public static Task RunAsync(Options options, SearchFilesResults result, string[] paths, IFilter filter, long offset, long limit) + public static async Task RunAsync(Options options, SearchFilesResults result, string[] paths, IFilter filter, long offset, long limit) { if (!System.IO.File.Exists(options.Dbpath)) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache); + using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache); long[]? filesetIds = null; if (!options.AllVersions) { - filesetIds = db.GetFilesetIDs(options.Time, options.Version).ToArray(); + filesetIds = await db.GetFilesetIDs(options.Time, options.Version).ToArrayAsync(); if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); } @@ -61,7 +61,6 @@ internal static class SearchEntriesHandler if (paths != null) paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); - result.FileVersions = db.SearchEntries(paths, filter, filesetIds, offset, limit); - return Task.CompletedTask; + result.FileVersions = await db.SearchEntries(paths, filter, filesetIds, offset, limit); } } From 9444aa63377bcc8dc33f9008ee2920d7fd4c5ace Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 07:56:42 +0200 Subject: [PATCH 171/362] Removed whitespace --- .../Library/Main/Operation/VacuumHandler.cs | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index 2c376e716..4da37f471 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -1,27 +1,25 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation From 97bfa11611e997ee46db5d52e64ade9d0031f83a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 07:56:55 +0200 Subject: [PATCH 172/362] Updated VacuumHandler to use the new async database methods --- .../Library/Main/Operation/VacuumHandler.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index 4da37f471..e531e5cd6 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -35,15 +35,12 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public virtual Task RunAsync() - => Task.Run(() => - { - using (var db = new Database.LocalDatabase(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache)) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); - db.Vacuum(); - m_result.EndTime = DateTime.UtcNow; - } - }); + public virtual async Task RunAsync() + { + using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); + await db.Vacuum(); + m_result.EndTime = DateTime.UtcNow; + } } } From 2c408d795f74faff90cd888090f0b5fa5e25c906 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:01:16 +0200 Subject: [PATCH 173/362] LocalDatabase always starts a new ReusableTransaction --- Duplicati/Library/Main/Database/LocalDatabase.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 7c5d5aa30..4c7157eb2 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -90,7 +90,10 @@ namespace Duplicati.Library.Main.Database public bool ShouldCloseConnection { get; set; } // Constructor is private to force use of CreateLocalDatabaseAsync - protected LocalDatabase() { } + protected LocalDatabase() + { + m_rtr = new ReusableTransaction(this); + } // Arguments are not used, but are required to match the constructor signatures [Obsolete("Calling the constructor will throw an exception. Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead")] From 9c0f2ca980a9c6de6eac22dcbf7ef1a1253940d7 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:01:45 +0200 Subject: [PATCH 174/362] LocalDatabase.Dispose() wasn't calling the database correctly with the ReusableTransaction --- Duplicati/Library/Main/Database/LocalDatabase.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 4c7157eb2..b35286608 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2400,16 +2400,16 @@ namespace Duplicati.Library.Main.Database { if (m_connection.State == ConnectionState.Open && !m_hasExecutedVacuum) { - using (var transaction = m_connection.BeginTransaction()) using (var command = m_connection.CreateCommand()) { - command.Transaction = transaction; // SQLite recommends that PRAGMA optimize is run just before closing each database connection. - await command.ExecuteNonQueryAsync("PRAGMA optimize"); + await command + .SetTransaction(m_rtr) + .ExecuteNonQueryAsync("PRAGMA optimize"); try { - await transaction.CommitAsync(); + await m_rtr.CommitAsync(restart: false); } catch (Exception ex) { From f8f0fd39afa6fb0f46243d424e43a88aa4556de6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:03:32 +0200 Subject: [PATCH 175/362] Removed comments --- .../Library/Main/Database/LocalDatabase.cs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index b35286608..5bed17361 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -57,7 +57,6 @@ namespace Duplicati.Library.Main.Database protected long m_operationid = -1; protected long m_pagecachesize; private bool m_hasExecutedVacuum; - // TODO use this rather than passing down transactions. // TODO dispose should check this, and report if it was not disposed prior to calling dispose. Maybe the rtr itself should check this during dispose? And just throw a warning during runtime. protected ReusableTransaction m_rtr; public ReusableTransaction Transaction { get { return m_rtr; } } @@ -129,27 +128,6 @@ namespace Duplicati.Library.Main.Database return c; } - // TODO Create these once the signature is stable. - // public static LocalDatabase CreateLocalDatabase(string path, string operation, bool shouldclose, long pagecachesize) - // { - // return CreateLocalDatabaseAsync(path, operation, shouldclose, pagecachesize).Await(); - // } - - // public static LocalDatabase CreateLocalDatabase(LocalDatabase db) - // { - // return CreateLocalDatabaseAsync(db).Await(); - // } - - // public static LocalDatabase CreateLocalDatabase(SqliteConnection connection, string operation) - // { - // return CreateLocalDatabaseAsync(connection, operation).Await(); - // } - - // private static LocalDatabase CreateLocalDatabase(SqliteConnection connection) - // { - // return CreateLocalDatabaseAsync(connection).Await(); - // } - public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize, LocalDatabase? db = null) { db ??= new LocalDatabase(); @@ -1938,7 +1916,6 @@ namespace Duplicati.Library.Main.Database { var ftt = new FilteredFilenameTable(db); - // TODO factory var type = FilterType.Regexp; if (filter is FilterExpression expression) type = expression.Type; From 46f394d6d567b987dc9c74cea4cc099942ed93a6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:07:23 +0200 Subject: [PATCH 176/362] Updated the BackendManager implementation to not pass down transactions (as they should be implicitly handled by the databases) and to use the new async database methods --- .../Backend/BackendManager.DatabaseCollector.cs | 15 ++++++++------- Duplicati/Library/Main/Backend/BackendManager.cs | 15 +++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs index 74bcc3327..ed66c61e6 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Duplicati.Library.Logging; using Duplicati.Library.Main.Database; @@ -17,7 +18,7 @@ partial class BackendManager /// Since the backend communication is done in parallel with the actual operations, /// we store the performed operations in a queue and flush them to the database when the database is available. /// The main operations are responsible for flushing the messages when comitting a transaction. - /// If the operation fails, the logged messages here should still be flushed to the database, + /// If the operation fails, the logged messages here should still be flushed to the database, /// as they have already been performed on the remote destination. /// private class DatabaseCollector @@ -116,7 +117,7 @@ partial class BackendManager /// The database to write pending messages to /// The transaction to use, if any /// Whether any messages were flushed - public bool FlushPendingMessages(LocalDatabase db, System.Data.IDbTransaction? transaction) + public async Task FlushPendingMessages(LocalDatabase db) { List entries; lock (m_dbqueuelock) @@ -134,22 +135,22 @@ partial class BackendManager // As we replace the list, we can now freely access the elements without locking foreach (var e in entries) if (e is RemoteOperationLogEntry operation) - db.LogRemoteOperation(operation.Action, operation.File, operation.Result, transaction); + await db.LogRemoteOperation(operation.Action, operation.File, operation.Result); else if (e is RemoteVolumeUpdate update && update.State == RemoteVolumeState.Deleted) { - db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null, transaction); + await db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null); volsRemoved.Add(update.Remotename); } else if (e is RemoteVolumeUpdate dbUpdate) - db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, transaction); + await db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash); else if (e is RenameRemoteVolume rename) - db.RenameRemoteFile(rename.Oldname, rename.Newname, transaction); + await db.RenameRemoteFile(rename.Oldname, rename.Newname); else if (e != null) Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e); // Finally remove volumes from DB. if (volsRemoved.Count > 0) - db.RemoveRemoteVolumes(volsRemoved); + await db.RemoveRemoteVolumes(volsRemoved); return true; } diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index f888cf6f2..a04f5a1a9 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -295,10 +295,9 @@ internal partial class BackendManager : IBackendManager /// The transaction to use /// The cancellation token /// - public Task FlushPendingMessagesAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken) + public async Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken) { - context.Database.FlushPendingMessages(database, transaction); - return Task.CompletedTask; + await context.Database.FlushPendingMessages(database); } /// @@ -320,11 +319,11 @@ internal partial class BackendManager : IBackendManager /// The transaction to use /// The cancellation token /// An awaitable task - public async Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken) + public async Task WaitForEmptyAsync(LocalDatabase database, CancellationToken cancellationToken) { - await FlushPendingMessagesAsync(database, transaction, cancellationToken).ConfigureAwait(false); + await FlushPendingMessagesAsync(database, cancellationToken).ConfigureAwait(false); await WaitForEmptyAsync(cancellationToken).ConfigureAwait(false); - await FlushPendingMessagesAsync(database, transaction, cancellationToken).ConfigureAwait(false); + await FlushPendingMessagesAsync(database, cancellationToken).ConfigureAwait(false); } /// @@ -332,10 +331,10 @@ internal partial class BackendManager : IBackendManager /// /// The database to write pending messages to /// The transaction to use, if any - public async Task StopRunnerAndFlushMessages(LocalDatabase database, IDbTransaction? transaction) + public async Task StopRunnerAndFlushMessages(LocalDatabase database) { await requestChannel.RetireAsync().ConfigureAwait(false); - await FlushPendingMessagesAsync(database, transaction, CancellationToken.None).ConfigureAwait(false); + await FlushPendingMessagesAsync(database, CancellationToken.None).ConfigureAwait(false); if (queueRunner.IsFaulted) Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed"); From 3e19db371d9254bb2542efb3cbbb5118ba8bebb4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:17:53 +0200 Subject: [PATCH 177/362] Removed whitespace --- Duplicati/Library/Main/Controller.cs | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 3e242612d..9c5074d04 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 8999dc4baa9d99971d9e480fe0a68fb61094d6c5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:18:10 +0200 Subject: [PATCH 178/362] Updated Controller to use the new async database methods --- Duplicati/Library/Main/Controller.cs | 34 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 9c5074d04..429c001c6 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -253,7 +253,7 @@ namespace Duplicati.Library.Main return RunAction(new ListRemoteResults(), async (result, backendManager) => { using (var tf = File.Exists(m_options.Dbpath) ? null : new TempFile()) - using (var db = new LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache)) + using (var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache)) result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); }); } @@ -277,10 +277,10 @@ namespace Duplicati.Library.Main // and uses the same prefix (see issues #2678, #3845, and #4244). if (File.Exists(m_options.Dbpath)) { - using (LocalDatabase db = new LocalDatabase(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache)) + using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache)) { var dbRemoteVolumes = db.GetRemoteVolumes(); - var dbRemoteFiles = dbRemoteVolumes.Select(x => x.Name).ToHashSet(); + var dbRemoteFiles = await dbRemoteVolumes.Select(x => x.Name).ToHashSetAsync(); list = list.Where(x => dbRemoteFiles.Contains(x.File.Name)).ToList(); } } @@ -517,10 +517,10 @@ namespace Duplicati.Library.Main // This would also allow us to control the unclean shutdown flag, // by toggling this on start and completion of transfers in the manager, // instead of relying on the operations to correctly toggle the flag - if (LocalDatabase.Exists(m_options.Dbpath)) + if (File.Exists(m_options.Dbpath)) { - using (var db = new LocalDatabase(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache)) - backend.StopRunnerAndFlushMessages(db, null).Await(); + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache).Await()) + backend.StopRunnerAndFlushMessages(db).Await(); } else { @@ -532,13 +532,13 @@ namespace Duplicati.Library.Main resultSetter.EndTime = DateTime.UtcNow; resultSetter.Interrupted = false; - if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun) + if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) { - using (var db = new LocalDatabase(m_options.Dbpath, null, true, m_options.SqlitePageCache)) + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache).Await()) { - db.WriteResults(result); - db.PurgeLogData(m_options.LogRetention); - db.PurgeDeletedVolumes(DateTime.UtcNow); + db.WriteResults(result).Await(); + db.PurgeLogData(m_options.LogRetention).Await(); + db.PurgeDeletedVolumes(DateTime.UtcNow).Await(); // Vacuum is done AFTER the results are written to the database // This means that the information about the vacuum is not stored in the database, @@ -577,9 +577,9 @@ namespace Duplicati.Library.Main try { // No operation was started in database, so write logs to new operation - if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun) - using (var db = new LocalDatabase(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache)) - db.WriteResults(result); + if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache).Await()) + db.WriteResults(result).Await(); } catch (Exception we) { @@ -602,9 +602,9 @@ namespace Duplicati.Library.Main resultSetter.Fatal = true; // Write logs to previous operation if database exists - if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun) - using (var db = new LocalDatabase(m_options.Dbpath, null, true, m_options.SqlitePageCache)) - db.WriteResults(result); + if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache).Await()) + db.WriteResults(result).Await(); } catch (Exception we) { From a2e3eeb727dd19f4e123cd9a1f2dcca57a8d77d1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:18:21 +0200 Subject: [PATCH 179/362] Removed whitespace --- Duplicati/Library/Main/ResultClasses.cs | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 2ab7aa2ca..0d0958736 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 0df878e455758661c1b748f1aa03ca525c8cb7f4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:18:33 +0200 Subject: [PATCH 180/362] Reordered usings --- Duplicati/Library/Main/ResultClasses.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 0d0958736..a6c679691 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -20,14 +20,16 @@ // DEALINGS IN THE SOFTWARE. using System; -using Duplicati.Library.Interface; using System.Collections.Generic; -using Duplicati.Library.Main.Database; -using Duplicati.Library.Logging; using System.Linq; -using Newtonsoft.Json; using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Logging; +using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Operation.Common; +using Newtonsoft.Json; namespace Duplicati.Library.Main { From 28fcff68ba65f3c096bfc019c309be6918aaa0ed Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:19:11 +0200 Subject: [PATCH 181/362] Changed BasicResults.FlushLog to be async --- Duplicati/Library/Main/ResultClasses.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index a6c679691..aae7c8697 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -260,10 +260,10 @@ namespace Duplicati.Library.Main } } - public void FlushLog(LocalDatabase db) + public async Task FlushLog(LocalDatabase db) { if (m_parent != null) - m_parent.FlushLog(db); + await m_parent.FlushLog(db); else { lock (m_lock) @@ -271,7 +271,7 @@ namespace Duplicati.Library.Main while (m_dbqueue.Count > 0) { var el = m_dbqueue.Dequeue(); - db.LogMessage(el.Type, el.Message, el.Exception, null); + await db.LogMessage(el.Type, el.Message, el.Exception); } } } From 2b5dfbbb1599e1ec30749eb91d8a0dfa55dd7714 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:19:56 +0200 Subject: [PATCH 182/362] Changed the lock in BasicResults from a regular object lock to a semaphore in order to be able to lock asynchronously --- Duplicati/Library/Main/ResultClasses.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index aae7c8697..0073acf96 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -181,7 +181,7 @@ namespace Duplicati.Library.Main protected readonly BasicResults m_parent; protected System.Threading.Thread m_callerThread; - protected readonly object m_lock = new object(); + protected readonly SemaphoreSlim m_lock = new(1, 1); protected readonly Queue m_dbqueue; public virtual ParsedResultType ParsedResult @@ -266,7 +266,8 @@ namespace Duplicati.Library.Main await m_parent.FlushLog(db); else { - lock (m_lock) + await m_lock.WaitAsync(); + try { while (m_dbqueue.Count > 0) { @@ -274,6 +275,10 @@ namespace Duplicati.Library.Main await db.LogMessage(el.Type, el.Message, el.Exception); } } + finally + { + m_lock.Release(); + } } } From b9f9b0360b0f7f8ac635e38945c7deccd552d1aa Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:24:57 +0200 Subject: [PATCH 183/362] Removed whitespace --- Duplicati/Library/Main/Utility.cs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index 844d28b56..e84ec3ecc 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 30ca4f927ed462c083311ea1f70def926a07b35a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:25:08 +0200 Subject: [PATCH 184/362] Reordered usings --- Duplicati/Library/Main/Utility.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index e84ec3ecc..ab23eb21a 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -22,11 +22,12 @@ using System; using System.Collections.Generic; using System.IO; -using Newtonsoft.Json; +using System.Linq; using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; -using System.Linq; namespace Duplicati.Library.Main { From cdba7ef6e90249b6e76a87a2e90222cc746dde9b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:25:25 +0200 Subject: [PATCH 185/362] Updated Utility to use the new async database methods --- .../Library/Main/Operation/BackupHandler.cs | 4 ++-- .../Library/Main/Operation/CompactHandler.cs | 4 ++-- .../Library/Main/Operation/DeleteHandler.cs | 4 ++-- .../Main/Operation/PurgeFilesHandler.cs | 4 ++-- .../Main/Operation/RecreateDatabaseHandler.cs | 8 ++++---- .../Library/Main/Operation/RepairHandler.cs | 6 +++--- .../Library/Main/Operation/RestoreHandler.cs | 8 ++++---- .../Library/Main/Operation/TestHandler.cs | 4 ++-- Duplicati/Library/Main/Utility.cs | 18 +++++++++--------- 9 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 546188de2..12424efd5 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -323,8 +323,8 @@ namespace Duplicati.Library.Main.Operation result.Dryrun = options.Dryrun; // Check the database integrity - Utility.UpdateOptionsFromDb(database, options); - Utility.VerifyOptionsAndUpdateDatabase(database, options); + await Utility.UpdateOptionsFromDb(database, options); + await Utility.VerifyOptionsAndUpdateDatabase(database, options); var probe_path = await database.GetFirstPath(); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 6cdf9869e..f49ffe9dd 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -54,8 +54,8 @@ namespace Duplicati.Library.Main.Operation using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache)) { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); var changed = await DoCompactAsync(db, false, backendManager).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index fb866d7f6..89bc89df1 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -53,8 +53,8 @@ namespace Duplicati.Library.Main.Operation using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache)) { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 0b6517a03..d0f722668 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -86,8 +86,8 @@ namespace Duplicati.Library.Main.Operation if (orphans != 0) throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans"); - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); if (filtercommand == null) { diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 15cf5012a..d2c458653 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -98,12 +98,12 @@ namespace Duplicati.Library.Main.Operation // Mark as incomplete await db.PartiallyRecreated(true); - var preexistingOptionsInDatabase = Utility.ContainsOptionsForVerification(db); - Utility.UpdateOptionsFromDb(db, m_options, null); + var preexistingOptionsInDatabase = await Utility.ContainsOptionsForVerification(db); + await Utility.UpdateOptionsFromDb(db, m_options); // Make sure the options have not changed between calls, unless there are no previous options if (preexistingOptionsInDatabase) - Utility.VerifyOptionsAndUpdateDatabase(db, m_options, null); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); } @@ -280,7 +280,7 @@ namespace Duplicati.Library.Main.Operation //Make sure we write the config if it has been read from a manifest if (hasUpdatedOptions) - Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options); using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) await restoredb.Transaction.CommitAsync(); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 9defbc63d..1bf49cfa7 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -130,8 +130,8 @@ namespace Duplicati.Library.Main.Operation using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); if (await db.PartiallyRecreated()) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); @@ -1134,7 +1134,7 @@ namespace Duplicati.Library.Main.Operation using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); - Utility.UpdateOptionsFromDb(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options); if (await db.RepairInProgress() || await db.PartiallyRecreated()) Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 1cd7293bc..4f24d817f 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -308,8 +308,8 @@ namespace Duplicati.Library.Main.Operation private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { // Perform initial setup - Utility.UpdateOptionsFromDb(database, m_options); - Utility.VerifyOptionsAndUpdateDatabase(database, m_options); + await Utility.UpdateOptionsFromDb(database, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options); // Verify the backend if necessary if (!m_options.NoBackendverification) @@ -421,8 +421,8 @@ namespace Duplicati.Library.Main.Operation //using (var database = new LocalRestoreDatabase(dbparent)) using (var metadatastorage = new RestoreHandlerMetadataStorage()) { - Utility.UpdateOptionsFromDb(database, m_options); - Utility.VerifyOptionsAndUpdateDatabase(database, m_options); + await Utility.UpdateOptionsFromDb(database, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options); m_blockbuffer = new byte[m_options.Blocksize]; if (!m_options.NoBackendverification) diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 98fa75aee..d8d70cc16 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -52,8 +52,8 @@ namespace Duplicati.Library.Main.Operation using (var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks); await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index ab23eb21a..28efaa721 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -104,10 +104,10 @@ namespace Duplicati.Library.Main /// The database to read from /// The options to update /// The transaction to use, if any - internal static void UpdateOptionsFromDb(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null) + internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options) { string n = null; - var opts = db.GetDbOptions(transaction); + var opts = await db.GetDbOptions(); if (opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n))) options.RawOptions["blocksize"] = opts["blocksize"] + "b"; @@ -122,9 +122,9 @@ namespace Duplicati.Library.Main /// /// The database to check /// true if the database contains options that need to be verified; false otherwise - internal static bool ContainsOptionsForVerification(LocalDatabase db) + internal static async Task ContainsOptionsForVerification(LocalDatabase db) { - var opts = db.GetDbOptions(); + var opts = await db.GetDbOptions(); return new[] { "blocksize", "blockhash", "filehash", "passphrase" }.Any(opts.ContainsKey); } @@ -134,7 +134,7 @@ namespace Duplicati.Library.Main /// The database to check /// The options to verify /// The transaction to use, if any - internal static void VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null) + internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options) { var newDict = new Dictionary { @@ -142,7 +142,7 @@ namespace Duplicati.Library.Main { "blockhash", options.BlockHashAlgorithm }, { "filehash", options.FileHashAlgorithm } }; - var opts = db.GetDbOptions(transaction); + var opts = await db.GetDbOptions(); if (options.NoEncryption) { @@ -163,7 +163,7 @@ namespace Duplicati.Library.Main newDict["passphrase-salt"] = salt; - // We avoid storing the passphrase directly, + // We avoid storing the passphrase directly, // instead we salt and rehash repeatedly newDict.Add("passphrase", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(options.Passphrase, salt, 1200))); } @@ -192,7 +192,7 @@ namespace Duplicati.Library.Main } //Extra sanity check - if (db.GetBlocksLargerThan(options.Blocksize) > 0) + if (await db.GetBlocksLargerThan(options.Blocksize) > 0) throw new Duplicati.Library.Interface.UserInformationException("You have attempted to change the block-size on an existing backup, which is not supported. Please configure a new clean backup if you want to change the block-size.", "BlockSizeChangeNotSupported"); if (needsUpdate) @@ -202,7 +202,7 @@ namespace Duplicati.Library.Main if (!newDict.ContainsKey(k.Key)) newDict[k.Key] = k.Value; - db.SetDbOptions(newDict, transaction); + await db.SetDbOptions(newDict); } } } From afddb93e5d5bb0bdf8e52c2108c5a1b65c2b63ff Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:26:32 +0200 Subject: [PATCH 186/362] Async method wasn't awaited --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 5bed17361..c968b1742 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2459,7 +2459,7 @@ namespace Duplicati.Library.Main.Database { if (result is BasicResults basicResults) { - basicResults.FlushLog(this); + await basicResults.FlushLog(this); if (basicResults.EndTime.Ticks == 0) basicResults.EndTime = DateTime.UtcNow; } From e643c2dfce2f392353d95fcf8e1ccf9c04d8e1c3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 08:26:43 +0200 Subject: [PATCH 187/362] Method signature changed --- Duplicati/Library/Main/Database/LocalRepairDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index ccfa756bc..a3a23e559 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1471,7 +1471,7 @@ namespace Duplicati.Library.Main.Database try { - await VerifyConsistency(blocksize, hashsize, true, m_rtr.Transaction); + await VerifyConsistency(blocksize, hashsize, true); } catch (Exception ex) { From 21a9f0036c83fa1dd27381c60191bd7ca2dcc355 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:23:27 +0200 Subject: [PATCH 188/362] Reintroduced legacy database methods for non-main operation calls to be able to function. --- .../Library/Main/Database/ExtensionMethods.cs | 896 +++++++++--------- 1 file changed, 452 insertions(+), 444 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 99b40f17c..7d3d780fd 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -383,8 +383,6 @@ public static partial class ExtensionMethods [GeneratedRegex(@"@\w+")] private static partial Regex MyRegex(); - // TODO commennt from here on - /// /// Adds a parameter to the command with the given value. /// @@ -403,60 +401,72 @@ public static partial class ExtensionMethods return self; } - // /// - // /// Sets the parameter values for the command, the parameters must already be added. - // /// - // /// The type of the command - // /// The command to set the parameter values for - // /// The values to set the parameters to - // /// The command with the parameter values set - // public static T SetParameterValues(this T self, Dictionary values) - // where T : IDbCommand - // { - // foreach (var kvp in values) - // ((IDataParameter)self.Parameters[kvp.Key]!).Value = kvp.Value; - // return self; - // } + // TODO commennt from here on - // /// - // /// Sets the parameter value for the command at the given index. - // /// - // /// The type of the command - // /// The command to set the parameter value for - // /// The name of the parameter to set the value for - // /// The value to set the parameter to - // /// The command with the parameter value set - // public static T SetParameterValue(this T self, string name, object? value) - // where T : IDbCommand - // { - // #if DEBUG - // if (value is not null && value is System.Collections.IEnumerable && value is not string) - // throw new ArgumentException($"Cannot set parameter '{name}' to an array or enumerable type, as the SQLite bindings does not support it.", nameof(value)); - // #endif - // ((IDataParameter)self.Parameters[name]!).Value = value; - // return self; - // } + public static IDbCommand AddNamedParameter(this IDbCommand self, string name, object? value = null) + { + var p = self.CreateParameter(); + p.ParameterName = name; + if (value != null) + p.Value = value; + self.Parameters.Add(p); + return self; + } - // /// - // /// Sets the transaction for the command. - // /// - // /// The type of the command - // /// The command to set the transaction for - // /// The transaction to set for the command - // /// >The command with the transaction set - // public static T SetTransaction(this T self, IDbTransaction? transaction) - // where T : IDbCommand - // { - // self.Transaction = transaction; - // return self; - // } + /// + /// Sets the parameter values for the command, the parameters must already be added. + /// + /// The type of the command + /// The command to set the parameter values for + /// The values to set the parameters to + /// The command with the parameter values set + public static T SetParameterValues(this T self, Dictionary values) + where T : IDbCommand + { + foreach (var kvp in values) + ((IDataParameter)self.Parameters[kvp.Key]!).Value = kvp.Value; + return self; + } + + /// + /// Sets the parameter value for the command at the given index. + /// + /// The type of the command + /// The command to set the parameter value for + /// The name of the parameter to set the value for + /// The value to set the parameter to + /// The command with the parameter value set + public static T SetParameterValue(this T self, string name, object? value) + where T : IDbCommand + { +#if DEBUG + if (value is not null && value is System.Collections.IEnumerable && value is not string) + throw new ArgumentException($"Cannot set parameter '{name}' to an array or enumerable type, as the SQLite bindings does not support it.", nameof(value)); +#endif + ((IDataParameter)self.Parameters[name]!).Value = value; + return self; + } + + /// + /// Sets the transaction for the command. + /// + /// The type of the command + /// The command to set the transaction for + /// The transaction to set for the command + /// >The command with the transaction set + public static T SetTransaction(this T self, IDbTransaction? transaction) + where T : IDbCommand + { + self.Transaction = transaction; + return self; + } /// /// Gets the printable command text for the given command. /// The command to get the printable command text for /// The printable command text - public static string GetPrintableCommandText(this SqliteCommand self) + public static string GetPrintableCommandText(this IDbCommand self) { var txt = self.CommandText; // TODO not adjusted to named parameters - it's using positional parameters. @@ -481,450 +491,448 @@ public static partial class ExtensionMethods return txt; } - // /// - // /// Executes the command and returns the number of rows affected. - // /// - // /// The command instance to execute on - // /// Whether to write a log entry - // /// The number of rows affected - // public static int ExecuteNonQuery(this IDbCommand self, bool writeLog) - // { - // return ExecuteNonQuery(self, writeLog, null); - // } + /// + /// Executes the command and returns the number of rows affected. + /// + /// The command instance to execute on + /// Whether to write a log entry + /// The number of rows affected + public static int ExecuteNonQuery(this IDbCommand self, bool writeLog) + { + return ExecuteNonQuery(self, writeLog, null); + } - // /// - // /// Executes the command and returns the number of rows affected. - // /// - // /// The command instance to execute on - // /// The command string to execute - // /// The values to use as parameters. The parameters must already be added. - // /// The number of rows affected - // public static int ExecuteNonQuery(this IDbCommand self, string cmd) - // { - // return ExecuteNonQuery(self, true, cmd); - // } + /// + /// Executes the command and returns the number of rows affected. + /// + /// The command instance to execute on + /// The command string to execute + /// The values to use as parameters. The parameters must already be added. + /// The number of rows affected + public static int ExecuteNonQuery(this IDbCommand self, string cmd) + { + return ExecuteNonQuery(self, true, cmd); + } - // /// - // /// Executes the command and returns the number of rows affected. - // /// - // /// The command instance to execute on - // /// The command string to execute - // /// The values to use as parameters. The parameters must already be added. - // /// The number of rows affected - // public static int ExecuteNonQuery(this IDbCommand self, string cmd, Dictionary values) - // { - // return ExecuteNonQuery(self, true, cmd, values); - // } + /// + /// Executes the command and returns the number of rows affected. + /// + /// The command instance to execute on + /// The command string to execute + /// The values to use as parameters. The parameters must already be added. + /// The number of rows affected + public static int ExecuteNonQuery(this IDbCommand self, string cmd, Dictionary values) + { + return ExecuteNonQuery(self, true, cmd, values); + } - // /// - // /// Executes the command and returns the number of rows affected. - // /// - // /// The command instance to execute on - // /// Whether to write a log entry - // /// The command string to execute - // /// The values to use as parameters. The parameters must already be added. - // /// The number of rows affected - // public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd, Dictionary values) - // { - // if (cmd != null) - // self.SetCommandAndParameters(cmd); + /// + /// Executes the command and returns the number of rows affected. + /// + /// The command instance to execute on + /// Whether to write a log entry + /// The command string to execute + /// The values to use as parameters. The parameters must already be added. + /// The number of rows affected + public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd, Dictionary values) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); - // if (values != null && values.Count > 0) - // self.SetParameterValues(values); + if (values != null && values.Count > 0) + self.SetParameterValues(values); - // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) - // return self.ExecuteNonQuery(); - // } + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) + return self.ExecuteNonQuery(); + } - // /// - // /// Executes the command and returns the number of rows affected. - // /// - // /// The command instance to execute on - // /// Whether to write a log entry - // /// The command string to execute - // /// The number of rows affected - // public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd) - // { - // if (cmd != null) - // self.SetCommandAndParameters(cmd); + /// + /// Executes the command and returns the number of rows affected. + /// + /// The command instance to execute on + /// Whether to write a log entry + /// The command string to execute + /// The number of rows affected + public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); - // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) - // return self.ExecuteNonQuery(); - // } + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null) + return self.ExecuteNonQuery(); + } - // /// - // /// Executes the command and returns the number of rows affected. - // /// - // /// The command instance to execute on - // /// The transaction to use for the command - // /// The number of rows affected - // public static int ExecuteNonQuery(this IDbCommand self, IDbTransaction? transaction) - // { - // self.Transaction = transaction; - // return self.ExecuteNonQuery(); - // } + /// + /// Executes the command and returns the number of rows affected. + /// + /// The command instance to execute on + /// The transaction to use for the command + /// The number of rows affected + public static int ExecuteNonQuery(this IDbCommand self, IDbTransaction? transaction) + { + self.Transaction = transaction; + return self.ExecuteNonQuery(); + } - // /// - // /// Executes the command and returns the scalar value of the first row. - // /// - // /// The command instance to execute on - // /// The command string to execute - // /// The scalar value of the first row - // public static object? ExecuteScalar(this IDbCommand self, string cmd) - // { - // return ExecuteScalar(self, true, cmd); - // } + /// + /// Executes the command and returns the scalar value of the first row. + /// + /// The command instance to execute on + /// The command string to execute + /// The scalar value of the first row + public static object? ExecuteScalar(this IDbCommand self, string cmd) + { + return ExecuteScalar(self, true, cmd); + } - // /// - // /// Executes the command and returns the scalar value of the first row. - // /// - // /// The command instance to execute on - // /// Whether to write a log entry - // /// The command string to execute - // /// The scalar value of the first row - // public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string? cmd) - // { - // if (cmd != null) - // self.SetCommandAndParameters(cmd); + /// + /// Executes the command and returns the scalar value of the first row. + /// + /// The command instance to execute on + /// Whether to write a log entry + /// The command string to execute + /// The scalar value of the first row + public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string? cmd) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); - // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalar", string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())) : null) - // return self.ExecuteScalar(); - // } + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalar", string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())) : null) + return self.ExecuteScalar(); + } - // /// - // /// Executes the command and returns the scalar int64 value of the first row as a string. - // /// - // /// The command instance to execute on - // /// Whether to write a log entry - // /// The default value to return if no value is found - // /// The scalar int64 value of the first row as a string - // public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, long defaultvalue = -1) - // { - // return ExecuteScalarInt64(self, writeLog, null, defaultvalue); - // } + /// + /// Executes the command and returns the scalar int64 value of the first row as a string. + /// + /// The command instance to execute on + /// Whether to write a log entry + /// The default value to return if no value is found + /// The scalar int64 value of the first row as a string + public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, long defaultvalue = -1) + { + return ExecuteScalarInt64(self, writeLog, null, defaultvalue); + } - // /// - // /// Executes the command and returns the scalar int64 value of the first row as a string. - // /// - // /// The command instance to execute on - // /// The default value to return if no value is found - // /// The scalar int64 value of the first row as a string - // public static long ExecuteScalarInt64(this IDbCommand self, long defaultvalue = -1) - // { - // return ExecuteScalarInt64(self, true, null, defaultvalue); - // } + /// + /// Executes the command and returns the scalar int64 value of the first row as a string. + /// + /// The command instance to execute on + /// The default value to return if no value is found + /// The scalar int64 value of the first row as a string + public static long ExecuteScalarInt64(this IDbCommand self, long defaultvalue = -1) + { + return ExecuteScalarInt64(self, true, null, defaultvalue); + } - // /// - // /// Executes the command and returns the scalar int64 value of the first row as a string. - // /// - // /// The command instance to execute on - // /// The transaction to use for the command - // /// The default value to return if no value is found - // /// The scalar int64 value of the first row as a string - // public static long ExecuteScalarInt64(this IDbCommand self, IDbTransaction? transaction, long defaultvalue = -1) - // { - // self.Transaction = transaction; - // return ExecuteScalarInt64(self, true, null, defaultvalue); - // } + /// + /// Executes the command and returns the scalar int64 value of the first row as a string. + /// + /// The command instance to execute on + /// The transaction to use for the command + /// The default value to return if no value is found + /// The scalar int64 value of the first row as a string + public static long ExecuteScalarInt64(this IDbCommand self, IDbTransaction? transaction, long defaultvalue = -1) + { + self.Transaction = transaction; + return ExecuteScalarInt64(self, true, null, defaultvalue); + } - // /// - // /// Executes the command and returns the scalar int64 value of the first row as a string. - // /// - // /// The command instance to execute on - // /// The command string to execute - // /// The default value to return if no value is found - // /// The scalar int64 value of the first row as a string - // public static long ExecuteScalarInt64(this IDbCommand self, string? cmd, long defaultvalue = -1) - // { - // return ExecuteScalarInt64(self, true, cmd, defaultvalue); - // } + /// + /// Executes the command and returns the scalar int64 value of the first row as a string. + /// + /// The command instance to execute on + /// The command string to execute + /// The default value to return if no value is found + /// The scalar int64 value of the first row as a string + public static long ExecuteScalarInt64(this IDbCommand self, string? cmd, long defaultvalue = -1) + { + return ExecuteScalarInt64(self, true, cmd, defaultvalue); + } - // /// - // /// Executes the command and returns the scalar int64 value of the first row as a string. - // /// - // /// The command instance to execute on - // /// Whether to write a log entry - // /// The command string to execute - // /// The default value to return if no value is found - // /// The scalar int64 value of the first row as a string - // public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, string? cmd, long defaultvalue) - // { - // if (cmd != null) - // self.SetCommandAndParameters(cmd); + /// + /// Executes the command and returns the scalar int64 value of the first row as a string. + /// + /// The command instance to execute on + /// Whether to write a log entry + /// The command string to execute + /// The default value to return if no value is found + /// The scalar int64 value of the first row as a string + public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, string? cmd, long defaultvalue) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); - // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())) : null) - // using (var rd = self.ExecuteReader()) - // if (rd.Read()) - // return ConvertValueToInt64(rd, 0, defaultvalue); + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())) : null) + using (var rd = self.ExecuteReader()) + if (rd.Read()) + return ConvertValueToInt64(rd, 0, defaultvalue); - // return defaultvalue; - // } + return defaultvalue; + } - // /// - // /// Executes the command and returns a data reader. - // /// - // /// The command instance to execute on - // /// The command string to execute - // /// The values to use as parameters. The parameters must already be added. - // /// A instance - // public static IDataReader ExecuteReader(this IDbCommand self, string cmd, Dictionary? values) - // { - // return ExecuteReader(self, true, cmd, values); - // } + /// + /// Executes the command and returns a data reader. + /// + /// The command instance to execute on + /// The command string to execute + /// The values to use as parameters. The parameters must already be added. + /// A instance + public static IDataReader ExecuteReader(this IDbCommand self, string cmd, Dictionary? values) + { + return ExecuteReader(self, true, cmd, values); + } - // /// - // /// Executes the command and returns a data reader. - // /// - // /// The command instance to execute on - // /// The command string to execute - // /// A instance - // public static IDataReader ExecuteReader(this IDbCommand self, string cmd) - // { - // return ExecuteReader(self, true, cmd); - // } + /// + /// Executes the command and returns a data reader. + /// + /// The command instance to execute on + /// The command string to execute + /// A instance + public static IDataReader ExecuteReader(this IDbCommand self, string cmd) + { + return ExecuteReader(self, true, cmd); + } - // /// - // /// Executes the command and returns a data reader. - // /// - // /// The command instance to execute on - // /// Whether to write a log entry - // /// The command string to execute - // /// A instance - // public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd) - // { - // if (cmd != null) - // self.SetCommandAndParameters(cmd); + /// + /// Executes the command and returns a data reader. + /// + /// The command instance to execute on + /// Whether to write a log entry + /// The command string to execute + /// A instance + public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); - // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) - // return self.ExecuteReader(); - // } + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) + return self.ExecuteReader(); + } - // /// - // /// Executes the command and returns a data reader. - // /// - // /// The command instance to execute on - // /// Whether to write a log entry - // /// The command string to execute - // /// The values to use as parameters. The parameters must already be added. - // /// A instance - // public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd, Dictionary? values) - // { - // if (cmd != null) - // self.SetCommandAndParameters(cmd); + /// + /// Executes the command and returns a data reader. + /// + /// The command instance to execute on + /// Whether to write a log entry + /// The command string to execute + /// The values to use as parameters. The parameters must already be added. + /// A instance + public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd, Dictionary? values) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); - // if (values != null && values.Count > 0) - // self.SetParameterValues(values); + if (values != null && values.Count > 0) + self.SetParameterValues(values); - // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) - // return self.ExecuteReader(); - // } + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) + return self.ExecuteReader(); + } - // /// - // /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. - // /// - // /// The database command to execute on. - // /// The command string to execute. - // /// - // public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self, string cmd) - // { - // using var rd = ExecuteReader(self, cmd); - // while (rd.Read()) - // yield return rd; - // } + /// + /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. + /// + /// The database command to execute on. + /// The command string to execute. + /// + public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self, string cmd) + { + using var rd = ExecuteReader(self, cmd); + while (rd.Read()) + yield return rd; + } - // /// - // /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. - // /// - // /// The database command to execute on. - // /// The command string to execute. - // /// - // public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self) - // { - // using var rd = self.ExecuteReader(); - // while (rd.Read()) - // yield return rd; - // } + /// + /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. + /// + /// The database command to execute on. + /// The command string to execute. + /// + public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self) + { + using var rd = self.ExecuteReader(); + while (rd.Read()) + yield return rd; + } - // /// - // /// Converts the value at the given index of the given data reader to a string. - // /// - // /// The data reader to convert the value from. - // /// The index of the value to convert. - // /// The value at the given index as a string. - // public static string? ConvertValueToString(this IDataReader reader, int index) - // { - // var v = reader.GetValue(index); - // if (v == null || v == DBNull.Value) - // return null; + /// + /// Converts the value at the given index of the given data reader to a string. + /// + /// The data reader to convert the value from. + /// The index of the value to convert. + /// The value at the given index as a string. + public static string? ConvertValueToString(this IDataReader reader, int index) + { + var v = reader.GetValue(index); + if (v == null || v == DBNull.Value) + return null; - // return v.ToString(); - // } + return v.ToString(); + } - // /// - // /// Converts the value at the given index of the given data reader to a long. - // /// - // /// The data reader to convert the value from. - // /// The index of the value to convert. - // /// The default value to return if the value is null or cannot be converted. - // /// The value at the given index as a long. - // public static long ConvertValueToInt64(this IDataReader reader, int index, long defaultvalue = -1) - // { - // try - // { - // if (!reader.IsDBNull(index)) - // return reader.GetInt64(index); - // } - // catch - // { - // } + /// + /// Converts the value at the given index of the given data reader to a long. + /// + /// The data reader to convert the value from. + /// The index of the value to convert. + /// The default value to return if the value is null or cannot be converted. + /// The value at the given index as a long. + public static long ConvertValueToInt64(this IDataReader reader, int index, long defaultvalue = -1) + { + try + { + if (!reader.IsDBNull(index)) + return reader.GetInt64(index); + } + catch + { + } - // return defaultvalue; - // } + return defaultvalue; + } - // /// - // /// Creates a command with the given transaction. - // /// - // /// The connection to create the command on. - // /// The transaction to use for the command. - // /// The command string to create the command with. - // /// A new command with the given transaction. - // public static IDbCommand CreateCommand(this IDbConnection self, IDbTransaction? transaction, string? cmdtext = null) - // { - // var cmd = self.CreateCommand(); - // cmd.Transaction = transaction; - // if (!string.IsNullOrEmpty(cmdtext)) - // cmd.SetCommandAndParameters(cmdtext); - // return cmd; - // } + /// + /// Creates a command with the given transaction. + /// + /// The connection to create the command on. + /// The transaction to use for the command. + /// The command string to create the command with. + /// A new command with the given transaction. + public static IDbCommand CreateCommand(this IDbConnection self, IDbTransaction? transaction, string? cmdtext = null) + { + var cmd = self.CreateCommand(); + cmd.Transaction = transaction; + if (!string.IsNullOrEmpty(cmdtext)) + cmd.SetCommandAndParameters(cmdtext); + return cmd; + } - // /// - // /// Sets the command text and adds parameters to the command. - // /// - // /// The command to set the command text and add parameters to. - // /// The transaction to use for the command. - // /// The command text to set. - // /// The command with the command text set and parameters added. - // public static IDbCommand SetCommandAndParameters(this IDbCommand cmd, IDbTransaction transaction, string cmdtext) - // { - // cmd.Transaction = transaction; - // return cmd.SetCommandAndParameters(cmdtext); - // } + /// + /// Sets the command text and adds parameters to the command. + /// + /// The command to set the command text and add parameters to. + /// The transaction to use for the command. + /// The command text to set. + /// The command with the command text set and parameters added. + public static IDbCommand SetCommandAndParameters(this IDbCommand cmd, IDbTransaction transaction, string cmdtext) + { + cmd.Transaction = transaction; + return cmd.SetCommandAndParameters(cmdtext); + } - // /// - // /// Sets the command text and adds parameters to the command. - // /// - // /// The command to set the command text and add parameters to. - // /// The command text to set. - // /// The command with the command text set and parameters added. - // public static IDbCommand SetCommandAndParameters(this IDbCommand cmd, string cmdtext) - // { - // cmd.CommandText = cmdtext; - // cmd.Parameters.Clear(); + /// + /// Sets the command text and adds parameters to the command. + /// + /// The command to set the command text and add parameters to. + /// The command text to set. + /// The command with the command text set and parameters added. + public static IDbCommand SetCommandAndParameters(this IDbCommand cmd, string cmdtext) + { + cmd.CommandText = cmdtext; + cmd.Parameters.Clear(); - // #if DEBUG - // if (cmd.CommandText.Contains("?", StringComparison.OrdinalIgnoreCase)) - // throw new ArgumentException("Command text cannot contain '?' as a parameter placeholder, use '@' instead.", nameof(cmdtext)); - // #endif +#if DEBUG + if (cmd.CommandText.Contains("?", StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("Command text cannot contain '?' as a parameter placeholder, use '@' instead.", nameof(cmdtext)); +#endif - // var parameters = Regex.Matches(cmdtext, @"@\w+"); - // var found = new HashSet(StringComparer.OrdinalIgnoreCase); - // foreach (Match match in parameters) - // { - // if (found.Contains(match.Value)) - // continue; - // found.Add(match.Value); - // cmd.AddNamedParameter(match.Value); - // } + var parameters = Regex.Matches(cmdtext, @"@\w+"); + var found = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in parameters) + { + if (found.Contains(match.Value)) + continue; + found.Add(match.Value); + cmd.AddNamedParameter(match.Value); + } - // return cmd; - // } + return cmd; + } - // /// - // /// Creates the transaction for the given connection and executes a "BEGIN IMMEDIATE" command to ensure that the transaction can be comitted or rolled back. - // /// This works around a quirk in SQLite where the transaction is not initialized until the first command is executed. - // /// This means that if the transaction is rolled back or committed with no commands executed, it will fail with an exception. - // /// - // /// The connection to create the transaction on - // /// The transaction - // //public static IDbTransaction BeginTransactionSafe(this IDbConnection self) - // //{ - // // var transaction = self.BeginTransaction(); - // // // using (var cmd = self.CreateCommand(transaction, "BEGIN IMMEDIATE;")) - // // // try { cmd.ExecuteNonQuery(); } - // // // catch { - // // // ignore - // // } - // // - // // return transaction; - // //} + /// + /// Creates the transaction for the given connection and executes a "BEGIN IMMEDIATE" command to ensure that the transaction can be comitted or rolled back. + /// This works around a quirk in SQLite where the transaction is not initialized until the first command is executed. + /// This means that if the transaction is rolled back or committed with no commands executed, it will fail with an exception. + /// + /// The connection to create the transaction on + /// The transaction + //public static IDbTransaction BeginTransactionSafe(this IDbConnection self) + //{ + // var transaction = self.BeginTransaction(); + // // using (var cmd = self.CreateCommand(transaction, "BEGIN IMMEDIATE;")) + // // try { cmd.ExecuteNonQuery(); } + // // catch { + // // ignore + // } + // + // return transaction; + //} - // /// - // /// Creates a command with the given command string, and adds parameters to fit the input - // /// - // /// The connection to create the command on - // /// The command string to create the command with - // /// The command with the parameters added - // public static IDbCommand CreateCommand(this IDbConnection self, string cmdtext) - // => CreateCommand(self, null, cmdtext); + /// + /// Creates a command with the given command string, and adds parameters to fit the input + /// + /// The connection to create the command on + /// The command string to create the command with + /// The command with the parameters added + public static IDbCommand CreateCommand(this IDbConnection self, string cmdtext) + => CreateCommand(self, null, cmdtext); - // /// - // /// Expands the given parameter name to a list of parameters for an IN clause. - // /// - // /// The type of the command - // /// The type of the values - // /// The command to expand the parameter for - // /// The original parameter name to expand - // /// The values to expand the parameter for - // public static TCommand ExpandInClauseParameter(this TCommand cmd, string originalParamName, IEnumerable values) - // where TCommand : IDbCommand - // { - // if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) - // throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); + /// + /// Expands the given parameter name to a list of parameters for an IN clause. + /// + /// The type of the command + /// The type of the values + /// The command to expand the parameter for + /// The original parameter name to expand + /// The values to expand the parameter for + public static IDbCommand ExpandInClauseParameter(this IDbCommand cmd, string originalParamName, IEnumerable values) + { + if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) + throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); - // foreach (var p in cmd.Parameters) - // if (p is IDataParameter parameter && parameter.ParameterName.Equals(originalParamName, StringComparison.OrdinalIgnoreCase)) - // { - // cmd.Parameters.Remove(parameter); - // break; - // } + foreach (var p in cmd.Parameters) + if (p is IDataParameter parameter && parameter.ParameterName.Equals(originalParamName, StringComparison.OrdinalIgnoreCase)) + { + cmd.Parameters.Remove(parameter); + break; + } - // foreach ((var value, var index) in values.Select((value, index) => (value, index))) - // cmd.AddNamedParameter($"{originalParamName}{index}", value); + foreach ((var value, var index) in values.Select((value, index) => (value, index))) + cmd.AddNamedParameter($"{originalParamName}{index}", value); - // var inClause = string.Join(", ", values.Select((_, index) => $"{originalParamName}{index}")); - // if (string.IsNullOrWhiteSpace(inClause) && values.Any()) - // throw new ArgumentException("IN clause cannot be empty", nameof(values)); + var inClause = string.Join(", ", values.Select((_, index) => $"{originalParamName}{index}")); + if (string.IsNullOrWhiteSpace(inClause) && values.Any()) + throw new ArgumentException("IN clause cannot be empty", nameof(values)); - // #if DEBUG - // if (!cmd.CommandText.Contains(originalParamName, StringComparison.OrdinalIgnoreCase)) - // throw new ArgumentException($"Command text does not contain parameter '{originalParamName}'", nameof(originalParamName)); - // #endif - // cmd.CommandText = cmd.CommandText.Replace(originalParamName, inClause, StringComparison.OrdinalIgnoreCase); +#if DEBUG + if (!cmd.CommandText.Contains(originalParamName, StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException($"Command text does not contain parameter '{originalParamName}'", nameof(originalParamName)); +#endif + cmd.CommandText = cmd.CommandText.Replace(originalParamName, inClause, StringComparison.OrdinalIgnoreCase); - // return cmd; - // } + return cmd; + } - // /// - // /// Expands the given parameter name to a list of parameters for an IN clause. - // /// - // /// The type of the command - // /// The command to expand the parameter for - // /// The original parameter name to expand - // /// The values to expand the parameter for - // public static T ExpandInClauseParameter(this T cmd, string originalParamName, TemporaryDbValueList values) - // where T : IDbCommand - // { - // if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) - // throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); + /// + /// Expands the given parameter name to a list of parameters for an IN clause. + /// + /// The type of the command + /// The command to expand the parameter for + /// The original parameter name to expand + /// The values to expand the parameter for + internal static IDbCommand ExpandInClauseParameter(this IDbCommand cmd, string originalParamName, TemporaryDbValueList values) + { + if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) + throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); - // if (!values.IsTableCreated) - // return ExpandInClauseParameter(cmd, originalParamName, values.Values); + if (!values.IsTableCreated) + return ExpandInClauseParameter(cmd, originalParamName, values.Values); - // // We have a temporary table, so we need to replace the parameter with the table name - // cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(), StringComparison.OrdinalIgnoreCase); - // return cmd; - // } - // // TODO comment to here + // We have a temporary table, so we need to replace the parameter with the table name + cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause().Await(), StringComparison.OrdinalIgnoreCase); + return cmd; + } + // TODO comment to here } \ No newline at end of file From 2cd169253272d7865d8260dc5251a99e93e957c6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:24:00 +0200 Subject: [PATCH 189/362] Temporary fix to distinguish between new and legacy extension method --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 5 +++-- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 4 ++-- Duplicati/Library/Main/Database/LocalDatabase.cs | 4 ++-- Duplicati/Library/Main/Database/LocalDeleteDatabase.cs | 4 ++-- .../Library/Main/Database/LocalListAffectedDatabase.cs | 2 +- .../Library/Main/Database/LocalListBrokenFilesDatabase.cs | 6 +++--- Duplicati/Library/Main/Database/LocalListDatabase.cs | 4 ++-- Duplicati/Library/Main/Database/LocalRepairDatabase.cs | 6 +++--- 8 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 7d3d780fd..4a6fdbdc1 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -263,7 +263,8 @@ public static partial class ExtensionMethods return await ExecuteScalarInt64Async(self, defaultvalue).ConfigureAwait(false); } - public static SqliteCommand ExpandInClauseParameter(this SqliteCommand cmd, string originalParamName, IEnumerable values) + [Obsolete("Technically this method isn't async, but it is used in an async context and to differntiate this with the IDb version. ")] + public static SqliteCommand ExpandInClauseParameterAsync(this SqliteCommand cmd, string originalParamName, IEnumerable values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); @@ -297,7 +298,7 @@ public static partial class ExtensionMethods throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); if (!values.IsTableCreated) - return ExpandInClauseParameter(cmd, originalParamName, values.Values); + return ExpandInClauseParameterAsync(cmd, originalParamName, values.Values); // We have a temporary table, so we need to replace the parameter with the table name cmd.CommandText = cmd.CommandText.Replace(originalParamName, await values.GetInClause(), StringComparison.OrdinalIgnoreCase); diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index f3da98fa1..e5fae16d1 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -320,7 +320,7 @@ namespace Duplicati.Library.Main.Database ""Hash"", ""Size"" ") - .ExpandInClauseParameter("@States", [ + .ExpandInClauseParameterAsync("@States", [ RemoteVolumeState.Deleted, RemoteVolumeState.Deleting ]) @@ -1412,7 +1412,7 @@ namespace Duplicati.Library.Main.Database AND ""State"" IN (@States) ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameter("@States", [ + .ExpandInClauseParameterAsync("@States", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() ]); diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index c968b1742..79136d41f 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -668,7 +668,7 @@ namespace Duplicati.Library.Main.Database FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames) ") - .ExpandInClauseParameter("@VolumeNames", [.. names]) + .ExpandInClauseParameterAsync("@VolumeNames", [.. names]) .ExecuteNonQueryAsync(); var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" "; @@ -867,7 +867,7 @@ namespace Duplicati.Library.Main.Database AND ""State"" IN (@AllowedStates) ") .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) - .ExpandInClauseParameter("@AllowedStates", [ + .ExpandInClauseParameterAsync("@AllowedStates", [ RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 4c1391cce..d01aa4765 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -204,7 +204,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) .SetParameterValue("@CurrentType", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@AllowedStates", [ + .ExpandInClauseParameterAsync("@AllowedStates", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString(), @@ -427,7 +427,7 @@ namespace Duplicati.Library.Main.Database { await cmd.SetCommandAndParameters(createtable) .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameter("@AllowedStates", [ + .ExpandInClauseParameterAsync("@AllowedStates", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() ]) diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 5ba622a94..2b317e945 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -301,7 +301,7 @@ namespace Duplicati.Library.Main.Database ) "; - using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", items.ToArray())) + using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterAsync("@Names", items.ToArray())) using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) yield return new ListResultRemoteVolume() diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 962fb28e0..88d64ce9a 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -275,7 +275,7 @@ namespace Duplicati.Library.Main.Database ) ") .SetTransaction(m_rtr) - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterAsync("@BlockVolumeIds", blockVolumeIds.Cast()) .SetParameterValue("@EmptyHash", emptyHash) .SetParameterValue("@EmptyHashSize", emptyHashSize); @@ -298,7 +298,7 @@ namespace Duplicati.Library.Main.Database AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ) ") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterAsync("@BlockVolumeIds", blockVolumeIds.Cast()) .SetParameterValue("@EmptyHashSize", 0) .ExecuteScalarInt64Async(-1); @@ -319,7 +319,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Blockset"".""Length"" ASC LIMIT 1 ") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterAsync("@BlockVolumeIds", blockVolumeIds.Cast()) .ExecuteScalarInt64Async(-1); return res; diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index ee0eac7d8..ef081349c 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -835,7 +835,7 @@ namespace Duplicati.Library.Main.Database fe.FilesetID = @filesetid AND fl.PrefixID IN (@PrefixIds) ") - .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterAsync("@PrefixIds", prefixIds.Cast()) .SetParameterValue("@filesetid", filesetid) .ExecuteScalarInt64Async(0); } @@ -938,7 +938,7 @@ namespace Duplicati.Library.Main.Database AND fl.""PrefixId"" IN (@PrefixIds) ") .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterAsync("@PrefixIds", prefixIds.Cast()) .ExecuteScalarInt64Async(0); } diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index a3a23e559..9917a0fd8 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1616,7 +1616,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [ + .ExpandInClauseParameterAsync("@States", [ RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString() ]); @@ -1643,7 +1643,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [ + .ExpandInClauseParameterAsync("@States", [ RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString() ]); @@ -1674,7 +1674,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) - .ExpandInClauseParameter("@States", [ + .ExpandInClauseParameterAsync("@States", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() ]); From 37039f467ee1cb340285bee3db6970cf39e217f4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:26:05 +0200 Subject: [PATCH 190/362] Removed whitespace --- .../Library/RestAPI/Database/Connection.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index 421031e14..bbb31834b 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -108,7 +108,7 @@ namespace Duplicati.Server.Database public void ReWriteAllFieldsIfEncryptionChanged() { - // The token is automatically decrypted when the settings are loaded + // The token is automatically decrypted when the settings are loaded // In case the password has changed, this will fail and return the encrypted // hex-string, but will crash before reaching this point if (this.ApplicationSettings.EncryptedFields != m_encryptSensitiveFields) From b399adf5341f863537de16fb17778c6e5768c7ed Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:27:36 +0200 Subject: [PATCH 191/362] BeginTransactionSafe() was a legacy helper method that called BeginTransaction() directly --- .../Library/RestAPI/Database/Connection.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index bbb31834b..070bae6b3 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -246,7 +246,7 @@ namespace Duplicati.Server.Database internal void SetMetadata(IDictionary values, long id, IDbTransaction? transaction) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + using (var tr = transaction == null ? m_connection.BeginTransaction() : null) { OverwriteAndUpdateDb( tr, @@ -282,7 +282,7 @@ namespace Duplicati.Server.Database internal void SetFilters(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + using (var tr = transaction == null ? m_connection.BeginTransaction() : null) { OverwriteAndUpdateDb( tr, @@ -320,7 +320,7 @@ namespace Duplicati.Server.Database internal void SetSettings(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + using (var tr = transaction == null ? m_connection.BeginTransaction() : null) { if (m_encryptSensitiveFields) values = values.Select(x => new Setting @@ -366,7 +366,7 @@ namespace Duplicati.Server.Database internal void SetSources(IEnumerable values, long id, IDbTransaction transaction) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + using (var tr = transaction == null ? m_connection.BeginTransaction() : null) { OverwriteAndUpdateDb( tr, @@ -624,7 +624,7 @@ namespace Duplicati.Server.Database { lock (m_lock) { - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { using (var cmd = m_connection.CreateCommand(tr, @"UPDATE ""Backup"" SET ""DBPath""= @Dbpath WHERE ""ID""= @Id")) { @@ -665,7 +665,7 @@ namespace Duplicati.Server.Database throw new Exception("Unable to generate a unique database file name"); } - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { OverwriteAndUpdateDb( tr, @@ -759,7 +759,7 @@ namespace Duplicati.Server.Database internal void AddOrUpdateSchedule(ISchedule item) { lock (m_lock) - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { AddOrUpdateSchedule(item, tr); tr.Commit(); @@ -809,7 +809,7 @@ namespace Duplicati.Server.Database lock (m_lock) { - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { var existing = GetScheduleIDsFromTags(new string[] { "ID=" + ID.ToString() }); if (existing.Any()) @@ -995,7 +995,7 @@ namespace Duplicati.Server.Database //Workaround to clean up the database after invalid settings update public void FixInvalidBackupId() { - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) using (var cmd = m_connection.CreateCommand(tr)) { cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @BackupId") @@ -1046,7 +1046,7 @@ namespace Duplicati.Server.Database public void SetUISettings(string scheme, IDictionary values, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + using (var tr = transaction == null ? m_connection.BeginTransaction() : null) { OverwriteAndUpdateDb( tr, @@ -1070,7 +1070,7 @@ namespace Duplicati.Server.Database public void UpdateUISettings(string scheme, IDictionary values, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + using (var tr = transaction == null ? m_connection.BeginTransaction() : null) { OverwriteAndUpdateDb( tr, @@ -1123,7 +1123,7 @@ namespace Duplicati.Server.Database { var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(purgeDate); - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) using (var cmd = m_connection.CreateCommand(tr)) { cmd.SetCommandAndParameters(@"DELETE FROM ""ErrorLog"" WHERE ""Timestamp"" < @Time") @@ -1202,7 +1202,7 @@ namespace Duplicati.Server.Database { if (transaction == null) { - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { var r = DeleteFromDb(tablename, id, tr); tr.Commit(); From eef156161b7b1593d0f2b8cb6b4c6bdbf55978f1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:29:34 +0200 Subject: [PATCH 192/362] Reintroduced the old interface for TemporaryDbValueList and ReusableTransaction, as some of the unit tests construct these without a database instance. --- .../Main/Database/ReusableTransaction.cs | 14 ++++++----- .../Main/Database/TemporaryDbValueList.cs | 24 ++++++++++++++----- Duplicati/UnitTest/SQLiteTests.cs | 5 ++-- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index dc0253089..1564cf8b0 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.Database; /// /// The database to use. /// The transaction to use. If null, a new transaction is created. -internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) : IDisposable +internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? transaction = null) : IDisposable { /// /// The tag used for logging. @@ -46,20 +46,22 @@ internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transact /// /// The database to use. /// - private readonly LocalDatabase m_db = db; + private readonly SqliteConnection m_con = con; /// /// The current transaction. /// - private SqliteTransaction m_transaction = transaction ?? db.Connection.BeginTransaction(); + private SqliteTransaction m_transaction = transaction ?? con.BeginTransaction(deferred: true); /// /// True if the transaction is disposed. /// private bool m_disposed = false; + public ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) : this(db.Connection, transaction) { } + /// /// The current transaction. /// - public SqliteTransaction Transaction => m_disposed ? m_transaction : throw new InvalidOperationException("Transaction is disposed"); + public SqliteTransaction Transaction => m_disposed ? throw new InvalidOperationException("Transaction is disposed") : m_transaction; /// /// Commits the current transaction and optionally restarts it. @@ -92,7 +94,7 @@ internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transact await m_transaction.DisposeAsync(); if (restart) - m_transaction = m_db.Connection.BeginTransaction(); + m_transaction = m_con.BeginTransaction(); else m_disposed = true; } @@ -130,7 +132,7 @@ internal class ReusableTransaction(LocalDatabase db, SqliteTransaction? transact if (restart) { - m_transaction = m_db.Connection.BeginTransaction(); + m_transaction = m_con.BeginTransaction(); } else { diff --git a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs index a442b1773..9f51192c2 100644 --- a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs +++ b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs @@ -44,7 +44,8 @@ internal class TemporaryDbValueList : IDisposable /// The command to use /// private SqliteCommand? _cmd; - private readonly LocalDatabase _db; + private SqliteConnection _con; + private ReusableTransaction _rtr; /// /// The table to use /// @@ -73,9 +74,10 @@ internal class TemporaryDbValueList : IDisposable /// The command to use /// The connection to use /// The values to use - private TemporaryDbValueList(LocalDatabase db, IEnumerable values, string valuesType) + private TemporaryDbValueList(SqliteConnection con, ReusableTransaction rtr, IEnumerable values, string valuesType) { - _db = db; + _con = con; + _rtr = rtr; _valuesType = valuesType; _values = values; @@ -84,12 +86,22 @@ internal class TemporaryDbValueList : IDisposable internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) { - return await DoCreateAsync(new TemporaryDbValueList(db, values.Cast(), "INTEGER")); + return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "INTEGER")); } internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) { - return await DoCreateAsync(new TemporaryDbValueList(db, values.Cast(), "TEXT")); + return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "TEXT")); + } + + internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values) + { + return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "INTEGER")); + } + + internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values) + { + return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "TEXT")); } private static async Task DoCreateAsync(TemporaryDbValueList valueList) @@ -138,7 +150,7 @@ internal class TemporaryDbValueList : IDisposable if (_isTable) return; - _cmd = _db.Connection.CreateCommand(_db.Transaction); + _cmd = _con.CreateCommand(_rtr); await _cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType}) "); diff --git a/Duplicati/UnitTest/SQLiteTests.cs b/Duplicati/UnitTest/SQLiteTests.cs index 95b78b0ba..789ec9f26 100755 --- a/Duplicati/UnitTest/SQLiteTests.cs +++ b/Duplicati/UnitTest/SQLiteTests.cs @@ -67,6 +67,7 @@ namespace Duplicati.UnitTest { using var tf = new TempFile(); using var connection = CreateDummyDatabase(tf); + using var rtr = new ReusableTransaction(connection); using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { @@ -98,7 +99,7 @@ namespace Duplicati.UnitTest using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { - using var tmplist = new TemporaryDbValueList(connection, null, list); + using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list); command.ExpandInClauseParameter("@List", list); Assert.AreEqual(3, command.ExecuteScalarInt64()); } @@ -108,7 +109,7 @@ namespace Duplicati.UnitTest list.Remove(1); list.Remove(2); - using var tmplist = new TemporaryDbValueList(connection, null, list); + using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list); command.ExpandInClauseParameter("@List", list); Assert.AreEqual(1, command.ExecuteScalarInt64()); } From 9a08c7b12ece167f94d2ab913ec19593c6f76d8c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:30:02 +0200 Subject: [PATCH 193/362] Updated DirectListHandlerTests to use the new async database methods --- Duplicati/UnitTest/DirectListHandlerTests.cs | 25 ++++++++++---------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Duplicati/UnitTest/DirectListHandlerTests.cs b/Duplicati/UnitTest/DirectListHandlerTests.cs index 953c2332c..125bb4340 100644 --- a/Duplicati/UnitTest/DirectListHandlerTests.cs +++ b/Duplicati/UnitTest/DirectListHandlerTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; @@ -321,11 +322,11 @@ namespace Duplicati.UnitTest } [Test] - public void GetRootPrefixes_ShouldReturnCorrectLinuxPrefixes() + public async Task GetRootPrefixes_ShouldReturnCorrectLinuxPrefixes() { // Arrange using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, 1); SeedTestData(db, [ "/folder1/", // 1 "/folder1/sub1/", @@ -335,7 +336,7 @@ namespace Duplicati.UnitTest ]); // Act - var result = db.GetRootPrefixes(1).ToList(); + var result = await db.GetRootPrefixes(1).ToListAsync(); // Assert Assert.That(result, Does.Contain(1L)); @@ -347,11 +348,11 @@ namespace Duplicati.UnitTest } [Test] - public void GetRootPrefixes_ShouldReturnCorrectWindowsDrivePrefixes() + public async Task GetRootPrefixes_ShouldReturnCorrectWindowsDrivePrefixes() { // Arrange using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, 1); SeedTestData(db, [ "C:\\folder1\\", "C:\\folder1\\sub1\\", @@ -360,7 +361,7 @@ namespace Duplicati.UnitTest ]); // Act - var result = db.GetRootPrefixes(1).ToList(); + var result = await db.GetRootPrefixes(1).ToListAsync(); // Assert Assert.That(result, Does.Contain(1L)); @@ -371,11 +372,11 @@ namespace Duplicati.UnitTest } [Test] - public void GetRootPrefixes_ShouldReturnCorrectWindowsUncPrefixes() + public async Task GetRootPrefixes_ShouldReturnCorrectWindowsUncPrefixes() { // Arrange using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, 1); SeedTestData(db, [ "\\\\server\\share\\folder\\", "\\\\server\\share\\folder\\subfolder\\", @@ -383,7 +384,7 @@ namespace Duplicati.UnitTest ]); // Act - var result = db.GetRootPrefixes(1).ToList(); + var result = await db.GetRootPrefixes(1).ToListAsync(); // Assert Assert.That(result, Does.Contain(1L)); @@ -393,11 +394,11 @@ namespace Duplicati.UnitTest } [Test] - public void GetRootPrefixes_ShouldHandleMixedWindowsDriveAndUncPaths() + public async Task GetRootPrefixes_ShouldHandleMixedWindowsDriveAndUncPaths() { // Arrange using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, 1); SeedTestData(db, [ "C:\\data\\", // 1 "C:\\data\\sub1\\", @@ -409,7 +410,7 @@ namespace Duplicati.UnitTest ]); // Act - var result = db.GetRootPrefixes(1).ToList(); + var result = await db.GetRootPrefixes(1).ToListAsync(); // Assert Assert.That(result, Does.Contain(1L)); From 2363e3a26df1eea7ee932c400b792a78dccae108 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:30:52 +0200 Subject: [PATCH 194/362] Removed whitespace --- Duplicati/UnitTest/SQLiteTests.cs | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Duplicati/UnitTest/SQLiteTests.cs b/Duplicati/UnitTest/SQLiteTests.cs index 789ec9f26..de62a308b 100755 --- a/Duplicati/UnitTest/SQLiteTests.cs +++ b/Duplicati/UnitTest/SQLiteTests.cs @@ -1,23 +1,24 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System.Collections.Generic; using System.Data; using Duplicati.Library.Main.Database; @@ -58,7 +59,7 @@ namespace Duplicati.UnitTest t1.Commit(); // No exception should be thrown using var t2 = connection.BeginTransaction(); - t2.Rollback(); // No exception should be thrown + t2.Rollback(); // No exception should be thrown } [Test] From 33f8a87b672c02ff24418c74397b8689b12a33e4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:32:22 +0200 Subject: [PATCH 195/362] Updated SQLiteTests to be async --- Duplicati/UnitTest/SQLiteTests.cs | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Duplicati/UnitTest/SQLiteTests.cs b/Duplicati/UnitTest/SQLiteTests.cs index de62a308b..7eb5d5ae3 100755 --- a/Duplicati/UnitTest/SQLiteTests.cs +++ b/Duplicati/UnitTest/SQLiteTests.cs @@ -20,10 +20,11 @@ // DEALINGS IN THE SOFTWARE. using System.Collections.Generic; -using System.Data; +using System.Threading.Tasks; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; using NUnit.Framework; namespace Duplicati.UnitTest @@ -31,17 +32,17 @@ namespace Duplicati.UnitTest [TestFixture] public class SQLiteTests : BasicSetupHelper { - private static IDbConnection CreateDummyDatabase(string path) + private static async Task CreateDummyDatabase(string path) { var connection = SQLiteLoader.LoadConnection(path, 0); using (var command = connection.CreateCommand()) { command.CommandText = "CREATE TABLE TestTable (ID INTEGER PRIMARY KEY, Name TEXT)"; - command.ExecuteNonQuery(); + await command.ExecuteNonQueryAsync(); command.CommandText = "INSERT INTO TestTable (ID, Name) VALUES (1, 'Test1'), (2, 'Test2'), (3, 'Test3')"; - command.ExecuteNonQuery(); + await command.ExecuteNonQueryAsync(); } return connection; @@ -50,48 +51,48 @@ namespace Duplicati.UnitTest [Test] [Category("SQLite")] - public void TestEmptyTransaction() + public async Task TestEmptyTransaction() { using var tf = new TempFile(); - using var connection = CreateDummyDatabase(tf); + using var connection = await CreateDummyDatabase(tf); using var t1 = connection.BeginTransaction(); - t1.Commit(); // No exception should be thrown + await t1.CommitAsync(); // No exception should be thrown using var t2 = connection.BeginTransaction(); - t2.Rollback(); // No exception should be thrown + await t2.RollbackAsync(); // No exception should be thrown } [Test] [Category("SQLite")] - public void TestListExpansion() + public async Task TestListExpansion() { using var tf = new TempFile(); - using var connection = CreateDummyDatabase(tf); + using var connection = await CreateDummyDatabase(tf); using var rtr = new ReusableTransaction(connection); using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1, 2, 3 }); - Assert.AreEqual(3, command.ExecuteScalarInt64()); + Assert.AreEqual(3, await command.ExecuteScalarInt64Async()); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1, 2 }); - Assert.AreEqual(2, command.ExecuteScalarInt64()); + Assert.AreEqual(2, await command.ExecuteScalarInt64Async()); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1 }); - Assert.AreEqual(1, command.ExecuteScalarInt64()); + Assert.AreEqual(1, await command.ExecuteScalarInt64Async()); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[0]); - Assert.AreEqual(0, command.ExecuteScalarInt64()); + Assert.AreEqual(0, await command.ExecuteScalarInt64Async()); } var list = new List(); @@ -102,7 +103,7 @@ namespace Duplicati.UnitTest { using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list); command.ExpandInClauseParameter("@List", list); - Assert.AreEqual(3, command.ExecuteScalarInt64()); + Assert.AreEqual(3, await command.ExecuteScalarInt64Async()); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) @@ -112,7 +113,7 @@ namespace Duplicati.UnitTest using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list); command.ExpandInClauseParameter("@List", list); - Assert.AreEqual(1, command.ExecuteScalarInt64()); + Assert.AreEqual(1, await command.ExecuteScalarInt64Async()); } } } From c5248e007f5a91fd47ff99bbfe72d10c33827984 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:33:10 +0200 Subject: [PATCH 196/362] Construction chain of databases wrongfully tried to start multiple transactions --- Duplicati/Library/Main/Database/LocalDatabase.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 79136d41f..76291969d 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -89,10 +89,7 @@ namespace Duplicati.Library.Main.Database public bool ShouldCloseConnection { get; set; } // Constructor is private to force use of CreateLocalDatabaseAsync - protected LocalDatabase() - { - m_rtr = new ReusableTransaction(this); - } + protected LocalDatabase() { } // Arguments are not used, but are required to match the constructor signatures [Obsolete("Calling the constructor will throw an exception. Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead")] @@ -145,10 +142,11 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalDatabase(); + dbnew.m_connection = dbparent.m_connection; + dbnew.m_rtr = dbparent.m_rtr; dbnew = await CreateLocalDatabaseAsync(dbparent.m_connection, dbnew); dbnew.OperationTimestamp = dbparent.OperationTimestamp; - dbnew.m_connection = dbparent.m_connection; dbnew.m_operationid = dbparent.m_operationid; dbnew.m_pagecachesize = dbparent.m_pagecachesize; @@ -165,9 +163,8 @@ namespace Duplicati.Library.Main.Database if (dbnew.m_connection.State != ConnectionState.Open) await dbnew.m_connection.OpenAsync(); - using var cmd = dbnew.m_connection.CreateCommand(); - using var transaction = dbnew.m_connection.BeginTransaction(); - cmd.Transaction = transaction; + using var cmd = dbnew.m_connection.CreateCommand() + .SetTransaction(dbnew.m_rtr); if (operation != null) { dbnew.m_operationid = await cmd.SetCommandAndParameters(@" @@ -211,6 +208,7 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalDatabase(); dbnew.m_connection = connection; + dbnew.m_rtr ??= new ReusableTransaction(connection); var selectremotevolumes_sql = @" SELECT From 22550be73c76d31ca1d89f8fe3d47bc654cd30a5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 13:34:29 +0200 Subject: [PATCH 197/362] SQLite parameters are case sensitive. --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 2 +- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index e5fae16d1..4c8e000cf 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -107,7 +107,7 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""Block"" WHERE - ""Hash"" = @hash + ""Hash"" = @Hash AND ""Size"" = @Size "); diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 76291969d..91a5f9590 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -951,7 +951,7 @@ namespace Duplicati.Library.Main.Database { var r = await m_createremotevolumeCommand .SetTransaction(m_rtr) - .SetParameterValue("@OperationId", m_operationid) + .SetParameterValue("@OperationID", m_operationid) .SetParameterValue("@Name", name) .SetParameterValue("@Type", type.ToString()) .SetParameterValue("@State", state.ToString()) From 6c7e67dd282cba443d239ded7d033d55c1a8e688 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 14:35:15 +0200 Subject: [PATCH 198/362] Post master merge fixes --- .../Library/Main/Database/LocalDatabase.cs | 90 +++++++++++-------- .../Library/Main/Operation/CompactHandler.cs | 8 +- .../Main/Operation/PurgeFilesHandler.cs | 2 +- .../Main/Operation/RecreateDatabaseHandler.cs | 2 +- .../Library/Main/Operation/RepairHandler.cs | 7 +- 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index a8a9729e4..68b675e7b 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -58,7 +58,7 @@ namespace Duplicati.Library.Main.Database protected long m_pagecachesize; private bool m_hasExecutedVacuum; // TODO dispose should check this, and report if it was not disposed prior to calling dispose. Maybe the rtr itself should check this during dispose? And just throw a warning during runtime. - protected ReusableTransaction m_rtr; + protected ReusableTransaction m_rtr = null!; public ReusableTransaction Transaction { get { return m_rtr; } } private SqliteCommand m_updateremotevolumeCommand = null!; @@ -1958,7 +1958,7 @@ namespace Duplicati.Library.Main.Database while (await rd.ReadAsync()) { var p = rd.ConvertValueToString(0); - if (FilterExpression.Matches(filter, p)) + if (p != null && FilterExpression.Matches(filter, p)) { await cmd.SetParameterValue("@Path", p) .ExecuteNonQueryAsync(); @@ -2060,7 +2060,7 @@ namespace Duplicati.Library.Main.Database //Create a fake new entry with the old name and mark as deleting // as this ensures we will remove it, if it shows up in some later listing - RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, tr.Parent); + await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting); await m_rtr.CommitAsync(); } @@ -2127,45 +2127,63 @@ namespace Duplicati.Library.Main.Database /// The size of the hash /// An optional external transaction /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data - public IEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize, IDbTransaction? transaction = null) + public async IAsyncEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize) { - using (var cmd = m_connection.CreateCommand(transaction)) + using (var cmd = m_connection.CreateCommand(m_rtr)) { // Group subquery by hash to ensure that each blocklist hash appears only once in the result // The AllBlocks CTE is used to map both active and duplicate blocks from the volume, // because the new volume is initially registered with only duplicate blocks. var sql = @" -WITH ""AllBlocksInVolume"" AS ( - SELECT DISTINCT ""ID"", ""Hash"", ""Size"" FROM ( - SELECT ""ID"", ""Hash"", ""Size"" - FROM ""Block"" - WHERE ""VolumeID"" = @VolumeId + WITH ""AllBlocksInVolume"" AS ( + SELECT DISTINCT + ""ID"", + ""Hash"", + ""Size"" + FROM ( + SELECT + ""ID"", + ""Hash"", + ""Size"" + FROM ""Block"" + WHERE ""VolumeID"" = @VolumeId + UNION + SELECT + ""DuplicateBlock"".""BlockID"" AS ""ID"", + ""Block"".""Hash"" AS ""Hash"", + ""Block"".""Size"" AS ""Size"" + FROM ""DuplicateBlock"" + INNER JOIN ""Block"" + ON ""DuplicateBlock"".""BlockID"" = ""Block"".""ID"" + WHERE ""DuplicateBlock"".""VolumeID"" = @VolumeId + ) + ) - UNION + SELECT + ""A"".""Hash"" AS ""BlockHash"", + ""C"".""Hash"" AS ""ItemHash"" + FROM + ( + SELECT + ""BlocklistHash"".""BlocksetID"", + ""AllBlocksInVolume"".""Hash"", + ""BlocklistHash"".""Index"" + FROM ""BlocklistHash"",""AllBlocksInVolume"" + WHERE ""BlocklistHash"".""Hash"" = ""AllBlocksInVolume"".""Hash"" + GROUP BY + ""AllBlocksInVolume"".""Hash"", + ""AllBlocksInVolume"".""Size"" + ) A, + ""BlocksetEntry"" B, + ""Block"" C - SELECT ""DuplicateBlock"".""BlockID"" AS ""ID"", ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" - FROM ""DuplicateBlock"" INNER JOIN ""Block"" - ON ""DuplicateBlock"".""BlockID"" = ""Block"".""ID"" - WHERE ""DuplicateBlock"".""VolumeID"" = @VolumeId - ) -) - -SELECT ""A"".""Hash"" AS ""BlockHash"", ""C"".""Hash"" AS ""ItemHash"" -FROM ( - SELECT ""BlocklistHash"".""BlocksetID"", ""AllBlocksInVolume"".""Hash"", ""BlocklistHash"".""Index"" - FROM ""BlocklistHash"",""AllBlocksInVolume"" - WHERE ""BlocklistHash"".""Hash"" = ""AllBlocksInVolume"".""Hash"" GROUP BY ""AllBlocksInVolume"".""Hash"", ""AllBlocksInVolume"".""Size"" -) A, -""BlocksetEntry"" B, -""Block"" C - -WHERE - ""B"".""BlocksetID"" = ""A"".""BlocksetID"" - AND ""B"".""Index"" >= (""A"".""Index"" * @HashesPerBlock) - AND ""B"".""Index"" < ((""A"".""Index"" + 1) * @HashesPerBlock) - AND ""C"".""ID"" = ""B"".""BlockID"" -ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" -"; + WHERE + ""B"".""BlocksetID"" = ""A"".""BlocksetID"" + AND ""B"".""Index"" >= (""A"".""Index"" * @HashesPerBlock) + AND ""B"".""Index"" < ((""A"".""Index"" + 1) * @HashesPerBlock) + AND ""C"".""ID"" = ""B"".""BlockID"" + ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" + "; string? curHash = null; var count = 0; @@ -2175,8 +2193,8 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@HashesPerBlock", blocksize / hashsize); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) { var blockhash = rd.ConvertValueToString(0); if (curHash != null && (blockhash != curHash || count + hashsize > buffer.Length)) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index f618c68e7..de17447ff 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -207,7 +207,7 @@ namespace Duplicati.Library.Main.Operation if (deleteableVolumes.Any()) { // Preserve space by deleting the old volume - await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) deletedVolumes.Add(d); deleteableVolumes.Clear(); } @@ -337,7 +337,7 @@ namespace Duplicati.Library.Main.Operation if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { - await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize, rtr.Transaction)) + await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) newvolindex.WriteBlocklist(blocklist.Hash, blocklist.Buffer, 0, blocklist.Size); }; @@ -347,8 +347,8 @@ namespace Duplicati.Library.Main.Operation // We can handle at most one in-flight upload at a time, // because the transaction is not thread-safe, and shared with the upload - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - db.UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await db.UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null); // TODO: The upload here does not flush the database messages, // and this can leave the database in a state where it does not know of the remote file diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 5fee8a32b..3064765f5 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -193,7 +193,7 @@ namespace Duplicati.Library.Main.Operation } else { - db.UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, tr); + await db.UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null); var lst = await db.DropFilesetsFromTable(new[] { tsOriginal }).ToArrayAsync(); foreach (var f in lst) await db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null); diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index b52bf3a5b..3d8a737c8 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -423,7 +423,7 @@ namespace Duplicati.Library.Main.Operation for (var i = 0; i < 3; i++) { // Grab the list matching the pass type - var lst = restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse).ToList(); + var lst = await restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse).ToListAsync(); if (lst.Count == 0) continue; diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index b5f04831f..7bb49e5ef 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -158,11 +158,12 @@ namespace Duplicati.Library.Main.Operation var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; // Find the most recent timestamp from either a fileset or a remote volume - var mostRecentLocal = db.GetRemoteVolumes(rtr.Transaction) + var mostRecentLocal = await db.GetRemoteVolumes() .Where(x => x.Type == RemoteVolumeType.Files) .Select(x => VolumeBase.ParseFilename(x.Name).Time.ToLocalTime()) - .Concat(db.FilesetTimes.Select(x => x.Value.ToLocalTime())) - .Append(DateTime.MinValue).Max(); + .Concat(db.FilesetTimes().Select(x => x.Value.ToLocalTime())) + .Append(DateTime.MinValue) + .MaxAsync(); var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); if (mostRecentLocal < DateTime.UnixEpoch) From 344d3dcb5b421c0fa056442b438bbddf67da90d2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 15:28:14 +0200 Subject: [PATCH 199/362] The new backend requires a value to be set --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 4a6fdbdc1..5b9dd5dde 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -341,7 +341,7 @@ public static partial class ExtensionMethods if (value is not null && value is System.Collections.IEnumerable && value is not string) throw new ArgumentException($"Cannot set parameter '{name}' to an array or enumerable type, as the SQLite bindings does not support it.", nameof(value)); #endif - self.Parameters[name].Value = value; + self.Parameters[name].Value = value ?? DBNull.Value; return self; } From dc4404e7512db0be03cad7e468d6253bda405db7 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 15:28:35 +0200 Subject: [PATCH 200/362] Added missing await --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 4c8e000cf..bf1ffb75d 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -1447,7 +1447,7 @@ namespace Duplicati.Library.Main.Database public async Task SafeDeleteRemoteVolume(string name) { - var volumeid = GetRemoteVolumeID(name); + var volumeid = await GetRemoteVolumeID(name); using (var cmd = m_connection.CreateCommand(m_rtr)) { From afa468377b9162787a4d36c5451813387a9eb719 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 15:28:46 +0200 Subject: [PATCH 201/362] Fixed typo --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 68b675e7b..79ca27659 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -706,7 +706,7 @@ namespace Duplicati.Library.Main.Database // Create a temp table to associate metadata that is being deleted to a fileset var metadataFilesetQuery = $@" - SELECT Metadataset.ID, FilesetEntry.FilesetD + SELECT Metadataset.ID, FilesetEntry.FilesetID FROM Metadataset INNER JOIN FileLookup ON FileLookup.MetadataID = Metadataset.ID From 4ffbc36abf699452153558bbcd6e724dca092ee9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 15:29:10 +0200 Subject: [PATCH 202/362] IAsyncEnumerable was being chained incorrectly --- Duplicati/Library/Main/Operation/BackupHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 12424efd5..565fb5a28 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -541,10 +541,10 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(m_database, m_taskReader.ProgressToken); // Calculate the number of samples to test, using the largest number of file of a given type - long remoteVolumeCount = await await m_database.GetRemoteVolumes() + long remoteVolumeCount = await m_database.GetRemoteVolumes() .Where(x => x.State == RemoteVolumeState.Verified) .GroupBy(x => x.Type) - .Select(async x => await x.LongCountAsync()) + .SelectAwait(async x => await x.LongCountAsync()) .MaxAsync(); long samplesToTest = Math.Max(m_options.BackupTestSampleCount, (long)Math.Round(remoteVolumeCount * (m_options.BackupTestPercentage / 100m), MidpointRounding.AwayFromZero)); From dc4ef34cbd3bb32404b2fe907b27959269a86af2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 22 May 2025 15:29:48 +0200 Subject: [PATCH 203/362] Since transactions are now shared across the lifespan of a database object, the transaction shouldn't be disposed halfway through. --- Duplicati/Library/Main/Operation/BackupHandler.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 565fb5a28..419398052 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -741,8 +741,9 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await database.TerminatedWithActiveUploads(false); + // Make sure we have the database up-to-date - await db.CommitTransactionAsync("CommitAfterUpload", false); + await db.CommitTransactionAsync("CommitAfterUpload"); // If this throws, we should roll back the transaction if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) From ad2d66ce2bb6e8309e4cf59eaf05f1a1aec4b446 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 08:15:51 +0200 Subject: [PATCH 204/362] Made ReusableTransaction automatically try to commit when disposing --- .../Main/Database/ReusableTransaction.cs | 17 +++++++++++++++-- .../Main/Operation/Common/DatabaseCommon.cs | 1 - 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 1564cf8b0..593bd7b6e 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -116,8 +116,21 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran { if (!m_disposed) { - m_disposed = true; - await m_transaction.DisposeAsync(); + try + { + using (var timer = new Logging.Timer(LOGTAG, "Dispose", "Commit during transaction dispose")) + await m_transaction.CommitAsync(); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "ReusableTransaction dispose", ex, "Transaction disposed with error: {0}", ex.Message); + throw; + } + finally + { + m_disposed = true; + await m_transaction.DisposeAsync(); + } } } diff --git a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs index 59b217ade..3bbe57136 100644 --- a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs +++ b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs @@ -137,7 +137,6 @@ namespace Duplicati.Library.Main.Operation.Common RunOnMain(async () => { base.Dispose(isDisposing); - await m_db.Transaction.CommitAsync(restart: false); await m_db.Transaction.DisposeAsync(); }).Await(); } From abc7c79841d08ddb9512305bf15ae459e4eea68b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 08:20:00 +0200 Subject: [PATCH 205/362] Dispose made infinite recursion between different class derivations --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 4 ++-- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index bf1ffb75d..2a72cf58a 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -1050,7 +1050,7 @@ namespace Duplicati.Library.Main.Database public override void Dispose() { - DisposeAsync().Await(); + this.DisposeAsync().Await(); } public override async Task DisposeAsync() @@ -1066,7 +1066,7 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteWarningMessage(LOGTAG, "DropTempTableFailed", ex, "Failed to drop temporary table {0}: {1}", m_tempDeletedBlockTable, ex.Message); } - base.Dispose(); + await base.DisposeAsync(); } public async Task GetLastWrittenDBlockVolumeSize() diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 79ca27659..8978f2fd1 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2388,7 +2388,7 @@ namespace Duplicati.Library.Main.Database public virtual void Dispose() { - DisposeAsync().Await(); + this.DisposeAsync().Await(); } public virtual async Task DisposeAsync() diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 90974792f..7f4c0c081 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -1250,7 +1250,7 @@ namespace Duplicati.Library.Main.Database } - base.Dispose(); + await base.DisposeAsync(); } } } From d27dec0a2f2f7079e3f29fe608f4720a72cd0bb4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 08:26:06 +0200 Subject: [PATCH 206/362] Updated methods to use the instance ReusableTransacition instead of starting a new one --- Duplicati/Library/Main/Database/LocalDatabase.cs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 8978f2fd1..bc0e391e1 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2352,9 +2352,7 @@ namespace Duplicati.Library.Main.Database public async Task PurgeLogData(DateTime threshold) { - using var transaction = m_connection.BeginTransaction(); - using var cmd = m_connection.CreateCommand(); - cmd.Transaction = transaction; + using var cmd = m_connection.CreateCommand(m_rtr); var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold); await cmd.SetCommandAndParameters(@" @@ -2371,19 +2369,17 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Timestamp", t) .ExecuteNonQueryAsync(); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public async Task PurgeDeletedVolumes(DateTime threshold) { - using var transaction = m_connection.BeginTransaction(); - await m_removedeletedremotevolumeCommand - .SetTransaction(transaction) + .SetTransaction(m_rtr) .SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)) .ExecuteNonQueryAsync(); - await transaction.CommitAsync(); + await m_rtr.CommitAsync(); } public virtual void Dispose() From d0df1386cca7860ada9835c4a06f949f46d8e00e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 11:16:17 +0200 Subject: [PATCH 207/362] Removed line that's also been removed on master --- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 7f4c0c081..87e7e8a0e 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -677,7 +677,6 @@ namespace Duplicati.Library.Main.Database { var fileid = await m_findFilesetCommand .SetTransaction(m_rtr) - .SetParameterValue("@FilesetId", filesetid) .SetParameterValue("@PrefixId", pathprefixid) .SetParameterValue("@Path", path) .SetParameterValue("@BlocksetId", blocksetid) From 006efa875d951ab3ea4f3b28c3b9e0e669091750 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 11:16:30 +0200 Subject: [PATCH 208/362] Fixed missing increment --- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 87e7e8a0e..1b7013715 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -760,7 +760,7 @@ namespace Duplicati.Library.Main.Database if (c <= expectedblocklisthashes) { await m_insertBlocklistHashCommand - .SetParameterValue("@Index", index) + .SetParameterValue("@Index", index++) .SetParameterValue("@Hash", hash) .ExecuteNonQueryAsync(); } From 9419f225c0c1f85063f9a5dc2f74f270d4763619 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 11:17:30 +0200 Subject: [PATCH 209/362] Creating a new LocalRepairDatabase should close the database --- Duplicati/Library/Main/Database/LocalRepairDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 9917a0fd8..71a8295c9 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -45,7 +45,7 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalRepairDatabase(); - dbnew = (LocalRepairDatabase)await CreateLocalDatabaseAsync(path, "Repair", false, pagecachesize, dbnew); + dbnew = (LocalRepairDatabase)await CreateLocalDatabaseAsync(path, "Repair", true, pagecachesize, dbnew); return dbnew; } From 41ebeee4f06a30387f0192c918bb74ad0d16fa71 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 11:18:03 +0200 Subject: [PATCH 210/362] All transactions should be deferred --- Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index d32420dba..d3801170a 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -147,7 +147,7 @@ namespace Duplicati.Library.SQLiteHelper return con; using (var cmd = con.CreateCommand()) - using (var transaction = con.BeginTransaction()) + using (var transaction = con.BeginTransaction(deferred: true)) { cmd.Transaction = transaction; foreach (var opt in opts.Split([';'], StringSplitOptions.RemoveEmptyEntries)) From da487182ae83b8c48415b06ffabbffbaa2ea25a0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 15:46:30 +0200 Subject: [PATCH 211/362] Connection pool in RestoreDatabase now has a transaction associated with each connection --- .../Main/Database/LocalRestoreDatabase.cs | 112 ++++++++++-------- .../Library/Main/Operation/RestoreHandler.cs | 7 ++ 2 files changed, 72 insertions(+), 47 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 2f32a3542..4b07194e7 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -49,7 +49,7 @@ namespace Duplicati.Library.Main.Database /// protected string? m_tempfiletable; protected string? m_tempblocktable; - protected ConcurrentBag m_connection_pool = []; + protected ConcurrentBag<(SqliteConnection, ReusableTransaction)> m_connection_pool = []; protected string? m_latestblocktable; protected string? m_fileprogtable; protected string? m_totalprogtable; @@ -1450,18 +1450,20 @@ namespace Duplicati.Library.Main.Database /// Returns a connection from the connection pool. /// /// A connection from the connection pool. - public async Task GetConnectionFromPool() + public async Task<(SqliteConnection, ReusableTransaction)> GetConnectionFromPool() { - if (!m_connection_pool.TryTake(out var connection)) + if (!m_connection_pool.TryTake(out var entry)) { - connection = await SQLiteLoader.LoadConnectionAsync(); + var connection = await SQLiteLoader.LoadConnectionAsync(); connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared"; - connection.Open(); - + await connection.OpenAsync(); await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize); + var transaction = new ReusableTransaction(connection); + + return (connection, transaction); } - return connection; + return entry; } private class FileToRestore : IFileToRestore @@ -1538,7 +1540,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable GetFolderMetadataToRestore() { using var cmd = m_connection.CreateCommand(); - cmd.Transaction = m_connection.BeginTransaction(deferred: true); + cmd.SetTransaction(m_rtr); using var rd = await cmd.ExecuteReaderAsync($@" SELECT F.ID, @@ -1563,8 +1565,6 @@ namespace Duplicati.Library.Main.Database rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5) ); - - await m_rtr.CommitAsync(); } /// @@ -1616,8 +1616,11 @@ namespace Duplicati.Library.Main.Database /// A list of needed to restore the given file. public async IAsyncEnumerable GetBlocksFromFile(long blocksetID) { - var connection = await GetConnectionFromPool(); - using var cmd = connection.CreateCommand(@$" + var (connection, transaction) = await GetConnectionFromPool(); + try + { + + using var cmd = connection.CreateCommand(@$" SELECT ""Block"".""ID"", ""Block"".""Hash"", @@ -1628,24 +1631,25 @@ namespace Duplicati.Library.Main.Database ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID ") - .SetTransaction(m_rtr) - .SetParameterValue("@BlocksetID", blocksetID); + .SetTransaction(transaction) + .SetParameterValue("@BlocksetID", blocksetID); - using var reader = await cmd.ExecuteReaderAsync(); - for (long i = 0; await reader.ReadAsync(); i++) - yield return new BlockRequest( - reader.ConvertValueToInt64(0), - i, - reader.ConvertValueToString(1), - reader.ConvertValueToInt64(2), - reader.ConvertValueToInt64(3), - false - ); - - await m_rtr.CommitAsync(); - - // Return the connection to the pool - m_connection_pool.Add(connection); + using var reader = await cmd.ExecuteReaderAsync(); + for (long i = 0; await reader.ReadAsync(); i++) + yield return new BlockRequest( + reader.ConvertValueToInt64(0), + i, + reader.ConvertValueToString(1), + reader.ConvertValueToInt64(2), + reader.ConvertValueToInt64(3), + false + ); + } + finally + { + // Return the connection to the pool + m_connection_pool.Add((connection, transaction)); + } } /// @@ -1655,8 +1659,10 @@ namespace Duplicati.Library.Main.Database /// A list of needed to restore the metadata of the given file. public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID) { - var connection = await GetConnectionFromPool(); - using var cmd = connection.CreateCommand($@" + var (connection, transaction) = await GetConnectionFromPool(); + try + { + using var cmd = connection.CreateCommand($@" SELECT ""Block"".""ID"", ""Block"".""Hash"", @@ -1671,19 +1677,20 @@ namespace Duplicati.Library.Main.Database ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" WHERE ""File"".""ID"" = @FileID ") - .SetTransaction(m_rtr) - .SetParameterValue("@FileID", fileID); + .SetTransaction(transaction) + .SetParameterValue("@FileID", fileID); - using var reader = await cmd.ExecuteReaderAsync(); - for (long i = 0; await reader.ReadAsync(); i++) - { - yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); + using var reader = await cmd.ExecuteReaderAsync(); + for (long i = 0; await reader.ReadAsync(); i++) + { + yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); + } + } + finally + { + // Return the connection to the pool + m_connection_pool.Add((connection, transaction)); } - - await m_rtr.CommitAsync(); - - // Return the connection to the pool - m_connection_pool.Add(connection); } /// @@ -1968,14 +1975,25 @@ namespace Duplicati.Library.Main.Database public override void Dispose() { - foreach (var connection in m_connection_pool) + DisposeAsync().Await(); + } + + public override async Task DisposeAsync() + { + await DisposePoolAsync(); + await DropRestoreTable(); + await base.DisposeAsync(); + } + + public async Task DisposePoolAsync() + { + foreach (var (connection, transaction) in m_connection_pool) { - connection.Close(); - connection.Dispose(); + await transaction.DisposeAsync(); + await connection.CloseAsync(); + await connection.DisposeAsync(); } m_connection_pool.Clear(); - DropRestoreTable().Await(); - base.Dispose(); } public async IAsyncEnumerable GetTargetFolders() diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 4f24d817f..13ba8da44 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -326,6 +326,9 @@ namespace Duplicati.Library.Main.Operation using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); + // At this point, there should be no more writes to the database, so we have to unlock the database: + await database.Transaction.CommitAsync("CommitBeforeRestore", restart: true).ConfigureAwait(false); + using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); // Create the channels between BlockManager and FileProcessor Restore.Channels.BufferSize = m_options.RestoreChannelBufferSize; @@ -374,6 +377,10 @@ namespace Duplicati.Library.Main.Operation kill_updater.Cancel(); } + await database.Transaction.CommitAsync("CommitAfterRestore", restart: true).ConfigureAwait(false); + + await database.DisposePoolAsync().ConfigureAwait(false); + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; From e57768f6045b3ca0f2e995beb7c773105b506656 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 15:46:58 +0200 Subject: [PATCH 212/362] All transactions should be deferred --- Duplicati/Library/Main/Database/ReusableTransaction.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 593bd7b6e..1d2461f55 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -94,7 +94,7 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran await m_transaction.DisposeAsync(); if (restart) - m_transaction = m_con.BeginTransaction(); + m_transaction = m_con.BeginTransaction(deferred: true); else m_disposed = true; } From 806d3b512cd12ee6ef2f555adcd2ecd4b7b9a175 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 15:48:24 +0200 Subject: [PATCH 213/362] Fixed livelock in the new restore flow by changing from a spinlock to an awaitable task. --- .../Main/Operation/Restore/FileProcessor.cs | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index e26215fc0..52bd07367 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -53,6 +53,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// The number of file processors that are still restoring files. It is set by the . /// public static int file_processors_restoring_files; + public static object file_processor_continue_lock = new object(); + public static TaskCompletionSource file_processor_continue = new(); /// /// Runs the file processor process that restores the files that need to be restored. @@ -110,7 +112,10 @@ namespace Duplicati.Library.Main.Operation.Restore // Check if there are other FileProcessor's still restoring files if (!decremented) { - await RendezvousBeforeProcessingFolderMetadata(); + if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted) + file_processor_continue.SetResult(); + else + await RendezvousBeforeProcessingFolderMetadata(); decremented = true; } @@ -459,7 +464,12 @@ namespace Duplicati.Library.Main.Operation.Restore finally { if (!decremented) - Interlocked.Decrement(ref file_processors_restoring_files); + lock (file_processor_continue_lock) + { + file_processors_restoring_files--; + if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted) + file_processor_continue.SetResult(); + } block_request.Retire(); block_response.Retire(); @@ -554,11 +564,18 @@ namespace Duplicati.Library.Main.Operation.Restore /// An awaitable task that completes once all of the FileProcessor's have rendezvoused. private static async Task RendezvousBeforeProcessingFolderMetadata() { - Interlocked.Decrement(ref file_processors_restoring_files); - - while (file_processors_restoring_files > 0) + var should_wait = false; + lock (file_processor_continue_lock) { - await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + file_processors_restoring_files--; + if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted) + file_processor_continue.SetResult(); + should_wait = file_processors_restoring_files > 0; + } + + if (should_wait) + { + await file_processor_continue.Task.ConfigureAwait(false); } } From 9a34ace37db64262c02130bc32ddd393d5e20268 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 23 May 2025 15:49:10 +0200 Subject: [PATCH 214/362] Microsoft.Data.Sqlite doesn't have OpenAsync(), only Open() --- Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index d3801170a..afa2df576 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -276,7 +276,7 @@ namespace Duplicati.Library.SQLiteHelper private static async Task OpenSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con, string path) { con.ConnectionString = "Data Source=" + path; - await con.OpenAsync(); + con.Open(); // TODO legacy configuration? //if (con is Microsoft.Data.Sqlite.SqliteConnection sqlitecon && !OperatingSystem.IsMacOS()) //{ From 2f1cf9f0108c11bc48af088724cf493c2df8ed3c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 26 May 2025 09:02:08 +0200 Subject: [PATCH 215/362] The databases themselves handles disposing the transactions --- Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs index 3bbe57136..364b4bd00 100644 --- a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs +++ b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs @@ -134,11 +134,7 @@ namespace Duplicati.Library.Main.Operation.Common if (m_workerSource.IsCancellationRequested) return; - RunOnMain(async () => - { - base.Dispose(isDisposing); - await m_db.Transaction.DisposeAsync(); - }).Await(); + base.Dispose(isDisposing); } } From 26e738e8d6af98067a2fa89fce5d494c5602ac8f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 26 May 2025 09:02:52 +0200 Subject: [PATCH 216/362] The databases may still need to use the transaction when disposing for running PRAGMA optimize --- Duplicati/Library/Main/Operation/BackupHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 419398052..abce13574 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -768,7 +768,7 @@ namespace Duplicati.Library.Main.Operation else await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager).ConfigureAwait(false); } - await database.Transaction.CommitAsync("CommitAfterVerification", restart: false); + await database.Transaction.CommitAsync("CommitAfterVerification"); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); From 5408570643ef0d745d534bfdde7daf82ed0ef46d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 26 May 2025 09:03:21 +0200 Subject: [PATCH 217/362] All SQL queries should be performed inside of a transaction --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 2a72cf58a..5abd9a0cb 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -1058,8 +1058,9 @@ namespace Duplicati.Library.Main.Database if (!string.IsNullOrWhiteSpace(m_tempDeletedBlockTable)) try { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_rtr)) await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}"""); + await m_rtr.CommitAsync(); } catch (Exception ex) { From e11d6377acf253d91dcb31a2a9a7953dc6737659 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 26 May 2025 09:06:42 +0200 Subject: [PATCH 218/362] Construction chain was incorrect for LocalBackupDatabase --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 5abd9a0cb..fcd3c5e79 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -90,9 +90,10 @@ namespace Duplicati.Library.Main.Database dbnew ??= new LocalBackupDatabase(); dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(path, "Backup", false, options.SqlitePageCache, dbnew); + dbnew = await CreateAsync(dbnew, options); dbnew.ShouldCloseConnection = true; - return await CreateAsync(dbnew, options); + return dbnew; } public static async Task CreateAsync(LocalDatabase dbparent, Options options, LocalBackupDatabase? dbnew = null) From 115bf269d37d8f1309b571255722d9432f8401f0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 05:39:31 +0200 Subject: [PATCH 219/362] Pooling ensures that a connection (and thus also filehandle) may persist even after all of the database connections have been closed, which is undesirable. --- Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index afa2df576..acd9e7026 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -275,7 +275,7 @@ namespace Duplicati.Library.SQLiteHelper /// Path to the file to open, which may not exist. private static async Task OpenSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con, string path) { - con.ConnectionString = "Data Source=" + path; + con.ConnectionString = $"Data Source={path};Pooling=false"; con.Open(); // TODO legacy configuration? //if (con is Microsoft.Data.Sqlite.SqliteConnection sqlitecon && !OperatingSystem.IsMacOS()) From 7fa181a7598370c9542e7c535fb6ba8096ee3e4f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 05:39:51 +0200 Subject: [PATCH 220/362] Commit to ensure that changes are visible --- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 1b7013715..2f2ed0e73 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -514,6 +514,8 @@ namespace Duplicati.Library.Main.Database throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); } + + await m_rtr.CommitAsync(); } /// From b54bcba624b2619dcc6f158727b3c951c85377cd Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 05:40:07 +0200 Subject: [PATCH 221/362] All commands should run in a transaction --- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 2f2ed0e73..04b566833 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -888,7 +888,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_rtr)) { var selectCommand = @" SELECT DISTINCT From 5463cdcf24c4ccfbff4ebf8a3c0e5af01520c6db Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 05:40:52 +0200 Subject: [PATCH 222/362] NOT keyword was accidentally removed from an SQL query. --- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 04b566833..dad1a582e 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -970,7 +970,7 @@ namespace Duplicati.Library.Main.Database .SetCommandAndParameters($@" {selectCommand} WHERE - ""ID"" IN ({mentionedVolumes}) + ""ID"" NOT IN ({mentionedVolumes}) AND ""Type"" = @Type ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); From a4e608d7c423b78f35bd3b8adf3d54878734f1d3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 06:23:49 +0200 Subject: [PATCH 223/362] Updated transaction propagation logic, as all commands must have a transaction --- .../Library/RestAPI/Database/Connection.cs | 154 ++++++++++-------- 1 file changed, 83 insertions(+), 71 deletions(-) diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index 070bae6b3..0b464ed81 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -246,22 +246,25 @@ namespace Duplicati.Server.Database internal void SetMetadata(IDictionary values, long id, IDbTransaction? transaction) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransaction() : null) - { - OverwriteAndUpdateDb( - tr, - cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Metadata"" WHERE ""BackupID"" = @Id") - .SetParameterValue("@Id", id), - values ?? new Dictionary(), - cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Metadata"" (""BackupID"", ""Name"", ""Value"") VALUES (@BackupId, @Name, @Value)"), - (cmd, f) => cmd.SetParameterValue("@BackupId", id) - .SetParameterValue("@Name", f.Key) - .SetParameterValue("@Value", f.Value) - ); + { + var tr = transaction ?? m_connection.BeginTransaction(); + OverwriteAndUpdateDb( + tr, + cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Metadata"" WHERE ""BackupID"" = @Id") + .SetParameterValue("@Id", id), + values ?? new Dictionary(), + cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Metadata"" (""BackupID"", ""Name"", ""Value"") VALUES (@BackupId, @Name, @Value)"), + (cmd, f) => cmd.SetParameterValue("@BackupId", id) + .SetParameterValue("@Name", f.Key) + .SetParameterValue("@Value", f.Value) + ); - if (tr != null) - tr.Commit(); + if (transaction == null) + { + tr.Commit(); + tr.Dispose(); } + } } internal IFilter[] GetFilters(long id) @@ -282,23 +285,26 @@ namespace Duplicati.Server.Database internal void SetFilters(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransaction() : null) - { - OverwriteAndUpdateDb( - tr, - cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Filter"" WHERE ""BackupID"" = @Id") - .SetParameterValue("@Id", id), - values, - cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Filter"" (""BackupID"", ""Order"", ""Include"", ""Expression"") VALUES (@Id, @Order, @Include, @Expression)"), - (cmd, f) => cmd.SetParameterValue("@Id", id) - .SetParameterValue("@Order", f.Order) - .SetParameterValue("@Include", f.Include) - .SetParameterValue("@Expression", f.Expression) - ); + { + var tr = transaction ?? m_connection.BeginTransaction(); + OverwriteAndUpdateDb( + tr, + cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Filter"" WHERE ""BackupID"" = @Id") + .SetParameterValue("@Id", id), + values, + cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Filter"" (""BackupID"", ""Order"", ""Include"", ""Expression"") VALUES (@Id, @Order, @Include, @Expression)"), + (cmd, f) => cmd.SetParameterValue("@Id", id) + .SetParameterValue("@Order", f.Order) + .SetParameterValue("@Include", f.Include) + .SetParameterValue("@Expression", f.Expression) + ); - if (tr != null) - tr.Commit(); + if (transaction == null) + { + tr.Commit(); + tr.Dispose(); } + } } public ISetting[] GetSettings(long id) @@ -320,37 +326,40 @@ namespace Duplicati.Server.Database internal void SetSettings(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransaction() : null) + { + var tr = transaction ?? m_connection.BeginTransaction(); + if (m_encryptSensitiveFields) + values = values.Select(x => new Setting + { + Filter = x.Filter, + Name = x.Name, + Value = EncryptSensitiveFields(x.Name, x.Value, m_key) + }).ToList(); + + OverwriteAndUpdateDb( + tr, + cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @Id") + .SetParameterValue("@Id", id), + values, + cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"), + (cmd, f) => + { + if (PASSWORD_PLACEHOLDER.Equals(f.Value)) + throw new Exception("Attempted to save a property with the placeholder password"); + + cmd.SetParameterValue("@BackupId", id) + .SetParameterValue("@Filter", f.Filter ?? "") + .SetParameterValue("@Name", f.Name ?? "") + .SetParameterValue("@Value", f.Value ?? ""); + } + ); + + if (transaction == null) { - if (m_encryptSensitiveFields) - values = values.Select(x => new Setting - { - Filter = x.Filter, - Name = x.Name, - Value = EncryptSensitiveFields(x.Name, x.Value, m_key) - }).ToList(); - - OverwriteAndUpdateDb( - tr, - cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @Id") - .SetParameterValue("@Id", id), - values, - cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"), - (cmd, f) => - { - if (PASSWORD_PLACEHOLDER.Equals(f.Value)) - throw new Exception("Attempted to save a property with the placeholder password"); - - cmd.SetParameterValue("@BackupId", id) - .SetParameterValue("@Filter", f.Filter ?? "") - .SetParameterValue("@Name", f.Name ?? "") - .SetParameterValue("@Value", f.Value ?? ""); - } - ); - - if (tr != null) - tr.Commit(); + tr.Commit(); + tr.Dispose(); } + } } internal string?[] GetSources(long id) @@ -366,21 +375,24 @@ namespace Duplicati.Server.Database internal void SetSources(IEnumerable values, long id, IDbTransaction transaction) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransaction() : null) - { - OverwriteAndUpdateDb( - tr, - cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Source"" WHERE ""BackupID"" = @Id") - .SetParameterValue("@Id", id), - values, - cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Source"" (""BackupID"", ""Path"") VALUES (@BackupId, @Path)"), - (cmd, f) => cmd.SetParameterValue("@BackupId", id) - .SetParameterValue("@Path", f) - ); + { + var tr = transaction ?? m_connection.BeginTransaction(); + OverwriteAndUpdateDb( + tr, + cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Source"" WHERE ""BackupID"" = @Id") + .SetParameterValue("@Id", id), + values, + cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Source"" (""BackupID"", ""Path"") VALUES (@BackupId, @Path)"), + (cmd, f) => cmd.SetParameterValue("@BackupId", id) + .SetParameterValue("@Path", f) + ); - if (tr != null) - tr.Commit(); + if (transaction == null) + { + tr.Commit(); + tr.Dispose(); } + } } internal long[] GetBackupIDsForTags(string[] tags) From 2dafca03d4a54ca05fbff9789aabc63bbb43d3a5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 06:37:41 +0200 Subject: [PATCH 224/362] Found statements missing await --- Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 2c2fd133b..6a67f8299 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -54,8 +54,8 @@ namespace Duplicati.Library.Main.Operation using (var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); } } From 26736daa7db93f8cc684943a8b2c804894ada35c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 06:41:35 +0200 Subject: [PATCH 225/362] IEnumerable resulted in late expansion of the query, which led to the transaction being disposed when the results were enumerated. --- .../Library/Main/Operation/ListBrokenFilesHandler.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 6a67f8299..33981faa2 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -143,15 +143,14 @@ namespace Duplicati.Library.Main.Operation m_result.BrokenFiles = - brokenfilesets.Select( - x => new Tuple>>( + await Task.WhenAll(brokenfilesets.Select( + async x => new Tuple>>( x.Version, x.Timestamp, callbackhandler == null && !m_options.ListSetsOnly - ? db.GetBrokenFilenames(x.FilesetID).ToEnumerable() + ? await db.GetBrokenFilenames(x.FilesetID).ToArrayAsync() : new MockList>((int)x.BrokenCount) - )) - .ToArray(); + ))); if (callbackhandler != null) From 4032e8b07a4c71f6dbbcf0e7f171edd9f0abe493 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 07:07:38 +0200 Subject: [PATCH 226/362] Disposing a database calls PRAGMA optimize, and thus needs a transaction. This means that the database themselves handles disposing and not restarting the final transaction --- Duplicati/Library/Main/Operation/CompactHandler.cs | 2 +- Duplicati/Library/Main/Operation/DeleteHandler.cs | 2 +- Duplicati/Library/Main/Operation/RepairHandler.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index de17447ff..39a313a54 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -64,7 +64,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { - await db.Transaction.CommitAsync("CommitCompact", restart: false); + await db.Transaction.CommitAsync("CommitCompact"); if (changed) { await db.WriteResults(m_result); diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 89bc89df1..6516fbd69 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -59,7 +59,7 @@ namespace Duplicati.Library.Main.Operation await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("ComitDelete", restart: false); + await db.Transaction.CommitAsync("ComitDelete"); } } diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 7bb49e5ef..faac8044f 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -515,7 +515,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { await db.TerminatedWithActiveUploads(false); - await db.Transaction.CommitAsync("CommitRepairTransaction", false); + await db.Transaction.CommitAsync("CommitRepairTransaction"); } } From 0ebcad0b85df159da1062b070ea6a08ff12244c0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 09:22:08 +0200 Subject: [PATCH 227/362] Removed unnecessary (and incorrect) unpacking of an SQL query --- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 4b07194e7..731b8ea8b 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1213,10 +1213,8 @@ namespace Duplicati.Library.Main.Database // One could also use like the average block number in a volume, that needs to be measured. using var rd = await cmd.ExecuteReaderAsync(); - object[] r = new object[3]; while (await rd.ReadAsync()) { - rd.GetValues(r); yield return new RemoteVolume( rd.ConvertValueToString(0), rd.ConvertValueToString(1), From 09490b835b53b8453dc691124115ead7bb91d962 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 09:22:30 +0200 Subject: [PATCH 228/362] Accidentally committed the inner transaction instead of the outer --- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 731b8ea8b..ee1c33feb 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1384,7 +1384,7 @@ namespace Duplicati.Library.Main.Database more = await rd.ReadAsync(); } } - await m_db.Transaction.Transaction.CommitAsync(); + await m_db.Transaction.CommitAsync(); } public async IAsyncEnumerable MetadataWithMissingBlocks() From 1861d904502132546388b58255f401d186865b90 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 09:23:34 +0200 Subject: [PATCH 229/362] Made the DirectBlockMarker hold a reference to the database instead of it's inner transaction to more closely resemble the other helper classes --- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index ee1c33feb..8530df194 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1794,7 +1794,7 @@ namespace Duplicati.Library.Main.Database private SqliteCommand m_updateAsRestoredCommand = null!; private SqliteCommand m_updateFileAsDataVerifiedCommand = null!; private SqliteCommand m_statUpdateCommand = null!; - private ReusableTransaction m_rtr = null!; + private LocalDatabase m_db = null!; private bool m_hasUpdates = false; private string m_blocktablename = null!; @@ -1812,7 +1812,7 @@ namespace Duplicati.Library.Main.Database { var dbm = new DirectBlockMarker() { - m_rtr = db.Transaction, + m_db = db, m_blocktablename = blocktablename, m_filetablename = filetablename, From 0d37608cb7083f10d4d4c86070ad031408736737 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 09:25:23 +0200 Subject: [PATCH 230/362] Each method in the DirectBlockMarker now sets the transaction upon calling to ensure that other outer methods don't commit (and thus invalidate the reference to) the transaction --- .../Main/Database/LocalRestoreDatabase.cs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 8530df194..14939023a 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1867,12 +1867,6 @@ namespace Duplicati.Library.Main.Database ") }; - dbm.m_insertblockCommand.SetTransaction(db.Transaction); - dbm.m_resetfileCommand.SetTransaction(db.Transaction); - dbm.m_updateAsRestoredCommand.SetTransaction(db.Transaction); - dbm.m_updateFileAsDataVerifiedCommand.SetTransaction(db.Transaction); - dbm.m_statUpdateCommand.SetTransaction(db.Transaction); - return dbm; } @@ -1882,7 +1876,9 @@ namespace Duplicati.Library.Main.Database return; m_hasUpdates = false; - using var rd = await m_statUpdateCommand.ExecuteReaderAsync(); + using var rd = await m_statUpdateCommand + .SetTransaction(m_db.Transaction) + .ExecuteReaderAsync(); var filesprocessed = 0L; var processedsize = 0L; @@ -1898,7 +1894,9 @@ namespace Duplicati.Library.Main.Database public async Task SetAllBlocksMissing(long targetfileid) { m_hasUpdates = true; - m_resetfileCommand.SetParameterValue("@TargetFileId", targetfileid); + m_resetfileCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid); var r = await m_resetfileCommand.ExecuteNonQueryAsync(); if (r <= 0) @@ -1908,7 +1906,9 @@ namespace Duplicati.Library.Main.Database public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata) { m_hasUpdates = true; - m_updateAsRestoredCommand.SetParameterValue("@TargetFileId", targetfileid) + m_updateAsRestoredCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid) .SetParameterValue("@Metadata", includeMetadata ? 1 : 0); var r = await m_updateAsRestoredCommand.ExecuteNonQueryAsync(); @@ -1919,7 +1919,9 @@ namespace Duplicati.Library.Main.Database public async Task SetFileDataVerified(long targetfileid) { m_hasUpdates = true; - m_updateFileAsDataVerifiedCommand.SetParameterValue("@TargetFileId", targetfileid); + m_updateFileAsDataVerifiedCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid); var r = await m_updateFileAsDataVerifiedCommand.ExecuteNonQueryAsync(); if (r != 1) @@ -1929,7 +1931,9 @@ namespace Duplicati.Library.Main.Database public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata) { m_hasUpdates = true; - m_insertblockCommand.SetParameterValue("@TargetFileId", targetfileid) + m_insertblockCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid) .SetParameterValue("@Index", index) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) @@ -1943,7 +1947,7 @@ namespace Duplicati.Library.Main.Database public async Task CommitAsync() { using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) - await m_rtr.CommitAsync(); + await m_db.Transaction.CommitAsync(); } public void Dispose() From 259f3dcf740e96b4a61e24ed6f5776ad7c1b6f28 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 09:39:35 +0200 Subject: [PATCH 231/362] Fixed typo in SQL query parameter names --- .../Library/Main/Database/LocalListBrokenFilesDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 88d64ce9a..6808ac0d1 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -345,7 +345,7 @@ namespace Duplicati.Library.Main.Database ""FileLookup"", ""FilesetEntry"" WHERE - ""FilesetEntry"".""FilesetId"" = @FilesetId + ""FilesetEntry"".""FilesetId"" = @FilesetID AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" ) AND ""BlocksetID"" NOT IN ( @@ -355,8 +355,8 @@ namespace Duplicati.Library.Main.Database ) ") .SetTransaction(m_rtr) - .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) - .SetParameterValue("@FilesetId", filesetId); + .SetParameterValue("@EmptyBlocksetID", emptyBlocksetId) + .SetParameterValue("@FilesetID", filesetId); return await cmd.ExecuteNonQueryAsync(); } From adcf3a7db41a7f207e73c0aa5dcd640712ab81d9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 09:40:18 +0200 Subject: [PATCH 232/362] The databases themselves handles disposing the transactions --- Duplicati/Library/Main/Operation/PurgeFilesHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 3064765f5..9daa4ae97 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -235,7 +235,7 @@ namespace Duplicati.Library.Main.Operation .DoCompactAsync(cdb, true, backendManager) .ConfigureAwait(false); - await cdb.Transaction.CommitAsync("PostCompact", restart: false); + await cdb.Transaction.CommitAsync("PostCompact"); } } From 44c0754495c6fa9a682c93b9f25c1aee38bd8366 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 09:40:30 +0200 Subject: [PATCH 233/362] Removed whitespace --- Duplicati/UnitTest/RepairHandlerTests.cs | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 98d636c85..bffb059e2 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; From f9428fee5e448e4c8af621580baff1da40f7a354 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 09:41:07 +0200 Subject: [PATCH 234/362] Updated RepairHandlerTests.RepairExtraIndexFiles to use the new async database methods --- Duplicati/UnitTest/RepairHandlerTests.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index bffb059e2..554cc4349 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -25,6 +25,7 @@ using System.IO; using System.IO.Compression; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; @@ -401,7 +402,7 @@ namespace Duplicati.UnitTest [Category("RepairHandler")] [TestCase("true")] [TestCase("false")] - public void RepairExtraIndexFiles(string noEncryption) + public async Task RepairExtraIndexFiles(string noEncryption) { // Extra index files will be added to the database and should have a correct link established Dictionary options = new Dictionary(this.TestOptions) { ["no-encryption"] = noEncryption }; @@ -425,18 +426,18 @@ namespace Duplicati.UnitTest } // Check database block link - using (LocalDatabase db = new LocalDatabase(DBFILE, "Test", true, 0)) + using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, 0)) { - var indexVolumeId = db.GetRemoteVolumeID(Path.GetFileName(origFile)); - var duplicateVolumeId = db.GetRemoteVolumeID(Path.GetFileName(dupFile)); + var indexVolumeId = await db.GetRemoteVolumeID(Path.GetFileName(origFile)); + var duplicateVolumeId = await db.GetRemoteVolumeID(Path.GetFileName(dupFile)); Assert.AreNotEqual(-1, indexVolumeId); Assert.AreNotEqual(-1, duplicateVolumeId); using (var cmd = db.Connection.CreateCommand()) { var sql = @"SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" = @VolumeId"; - var linkedIndexId = cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", indexVolumeId).ExecuteScalarInt64(-1); - var linkedDuplicateId = cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", duplicateVolumeId).ExecuteScalarInt64(-1); + var linkedIndexId = await cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", indexVolumeId).ExecuteScalarInt64Async(-1); + var linkedDuplicateId = await cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", duplicateVolumeId).ExecuteScalarInt64Async(-1); Assert.AreEqual(linkedIndexId, linkedDuplicateId); } } From fcf36d7714a13e69717ce250abfcca6e33e68699 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 11:28:41 +0200 Subject: [PATCH 235/362] Found async statements that weren't awaited --- Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 267957223..307e93a38 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -59,8 +59,8 @@ namespace Duplicati.Library.Main.Operation if (await db.PartiallyRecreated()) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); if (sets == null) From d2a34b600f4cfa2d08e6d15f07454a99d71039fd Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 27 May 2025 18:37:09 +0200 Subject: [PATCH 236/362] Fixed typo on which integer was the one being saved during recreatedatabase --- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index dad1a582e..f6c9a6340 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -822,7 +822,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .SetParameterValue("@VolumeId", currentVolumeId) + .SetParameterValue("@VolumeId", volumeID) .ExecuteNonQueryAsync(); return (anyChange, false); From c4d550572a31e793b7e07747d60730020dcd4c04 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 28 May 2025 05:53:14 +0200 Subject: [PATCH 237/362] Added missing quotes around table name containing a dash (-), which led to a syntax error --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index fcd3c5e79..424b51ef9 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -401,7 +401,7 @@ namespace Duplicati.Library.Main.Database DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId; - DELETE FROM {dbnew.m_tempDeletedBlockTable} + DELETE FROM ""{dbnew.m_tempDeletedBlockTable}"" WHERE ""ID"" = @DeletedBlockId; SELECT last_insert_rowid() From ff2e8ddd68e6c7b850c0dacc7665821661c37b93 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 28 May 2025 05:53:44 +0200 Subject: [PATCH 238/362] Found an SQL query missing the transaction --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 424b51ef9..851f847a6 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -665,6 +665,7 @@ namespace Duplicati.Library.Main.Database { // No transaction on the temporary table var id = await m_findindeletedCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) .ExecuteScalarInt64Async(m_logQueries, -1); From 04ffb4fb42d78479e5a36557051254d5713e924f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 2 Jun 2025 09:16:25 +0200 Subject: [PATCH 239/362] Fixed incorrectly merged changes in LocalDatabase --- .../Library/Main/Database/LocalDatabase.cs | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 3344f3da9..a44b897ee 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2048,19 +2048,26 @@ namespace Duplicati.Library.Main.Database throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}"); // Grab the type of entry - cmd.SetCommandAndParameters(@" - SELECT ""Type"" - FROM ""Remotevolume"" - WHERE ""Name"" = @Name - ") - .SetParameterValue("@Name", newname); var type = (RemoteVolumeType)Enum.Parse( - typeof(RemoteVolumeType), (await cmd.ExecuteScalarAsync())?.ToString() ?? "", - true); + typeof(RemoteVolumeType), + (await cmd.SetCommandAndParameters(@" + SELECT ""Type"" + FROM ""Remotevolume"" + WHERE ""Name"" = @Name + ") + .SetParameterValue("@Name", newname) + .ExecuteScalarAsync()) + ?.ToString() ?? "", + true + ); //Create a fake new entry with the old name and mark as deleting // as this ensures we will remove it, if it shows up in some later listing - await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting); + var newvolId = await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting); + + // IF needed, also create an empty fileset, so the validation works + if (type == RemoteVolumeType.Files) + await CreateFileset(newvolId, DateTime.UnixEpoch); await m_rtr.CommitAsync(); } From caa3c13dbe9b9024e60f8531b29cc530fbb73fa9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 2 Jun 2025 11:50:35 +0200 Subject: [PATCH 240/362] Post master merge fixes --- .../Library/Main/Database/LocalDatabase.cs | 10 +- .../Main/Database/LocalTestDatabase.cs | 295 +++++++++++++----- .../Library/Main/Operation/TestHandler.cs | 21 +- Duplicati/UnitTest/DisruptionTests.cs | 4 +- 4 files changed, 241 insertions(+), 89 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index a44b897ee..a293575fb 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1235,7 +1235,9 @@ namespace Duplicati.Library.Main.Database ""CalcLen"", ""Length"", ""A"".""BlocksetID"", ""File"".""Path"" - FROM ({combinedLengths}) A, ""File"" + FROM + ({combinedLengths}) A, + ""File"" WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" AND ""A"".""CalcLen"" != ""A"".""Length"" @@ -1729,7 +1731,7 @@ namespace Duplicati.Library.Main.Database ) L LEFT OUTER JOIN ""BlocklistHash"" M ON ""M"".""BlocksetID"" = ""L"".""MetablocksetID"" -"; + "; public const string LIST_FOLDERS_AND_SYMLINKS = @" SELECT @@ -2175,7 +2177,9 @@ namespace Duplicati.Library.Main.Database ""BlocklistHash"".""BlocksetID"", ""AllBlocksInVolume"".""Hash"", ""BlocklistHash"".""Index"" - FROM ""BlocklistHash"",""AllBlocksInVolume"" + FROM + ""BlocklistHash"", + ""AllBlocksInVolume"" WHERE ""BlocklistHash"".""Hash"" = ""AllBlocksInVolume"".""Hash"" GROUP BY ""AllBlocksInVolume"".""Hash"", diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 47ba39820..0add365c0 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -617,14 +617,14 @@ namespace Duplicati.Library.Main.Database public interface IBlocklist : IDisposable { - void AddBlock(string key, long value); - IEnumerable> Compare(); + Task AddBlock(string key, long value); + IAsyncEnumerable> Compare(); } public interface IBlocklistHashList : IDisposable { - void AddBlockHash(string hash, long size); - IEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize); + Task AddBlockHash(string hash, long size); + IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize); } private class Blocklist : Basiclist, IBlocklist @@ -633,50 +633,151 @@ namespace Duplicati.Library.Main.Database private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; - public Blocklist(IDbConnection connection, string volumename, ReusableTransaction rtr) - : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) - { } - - public void AddBlock(string hash, long size) + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] + public Blocklist(SqliteConnection connection, string volumename, ReusableTransaction rtr) { - m_insertCommand.SetTransaction(m_rtr.Transaction) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + throw new NotSupportedException("Use CreateAsync method instead."); } - public IEnumerable> Compare() + private Blocklist() { } + + public static async Task CreateAsync(LocalDatabase db, string volumename) + { + var bl = new Blocklist(); + return (Blocklist)await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + } + + public async Task AddBlock(string hash, long size) + { + await m_insertCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(); + } + + public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var curBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""Remotevolume"", ""Block"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"""; - var duplBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""DuplicateBlock"", ""Block"" WHERE ""DuplicateBlock"".""VolumeID"" = (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name) AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"""; - var delBlocks = @"SELECT ""DeletedBlock"".""Hash"" AS ""Hash"", ""DeletedBlock"".""Size"" AS ""Size"" FROM ""DeletedBlock"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = @Name AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"""; - var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT DISTINCT ""Hash"" AS ""Hash"", ""Size"" AS ""Size"" FROM ({curBlocks} UNION {delBlocks} UNION {duplBlocks})"); - var extra = FormatInvariant($@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{cmpName}"" )"); - var missing = FormatInvariant($@"SELECT @TypeMissing AS ""Type"", ""Hash"" AS ""Hash"" FROM ""{cmpName}"" WHERE ""Hash"" NOT IN (SELECT ""Hash"" FROM ""{m_tablename}"")"); - var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "); - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + var curBlocks = @" + SELECT + ""Block"".""Hash"" AS ""Hash"", + ""Block"".""Size"" AS ""Size"" + FROM + ""Remotevolume"", + ""Block"" + WHERE + ""Remotevolume"".""Name"" = @Name + AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"" + "; + + var duplBlocks = @" + SELECT + ""Block"".""Hash"" AS ""Hash"", + ""Block"".""Size"" AS ""Size"" + FROM + ""DuplicateBlock"", + ""Block"" + WHERE + ""DuplicateBlock"".""VolumeID"" = ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" = @Name + ) + AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"" + "; + + var delBlocks = @" + SELECT + ""DeletedBlock"".""Hash"" AS ""Hash"", + ""DeletedBlock"".""Size"" AS ""Size"" + FROM + ""DeletedBlock"", + ""RemoteVolume"" + WHERE + ""RemoteVolume"".""Name"" = @Name + AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"" + "; + + var create = $@" + CREATE TEMPORARY TABLE ""{cmpName}"" AS + SELECT DISTINCT + ""Hash"" AS ""Hash"", + ""Size"" AS ""Size"" + FROM ( + {curBlocks} + UNION {delBlocks} + UNION {duplBlocks} + ) + "; + + var extra = $@" + SELECT + @TypeExtra AS ""Type"", + ""{m_tablename}"".""Hash"" AS ""Hash"" + FROM ""{m_tablename}"" + WHERE ""{m_tablename}"".""Hash"" NOT IN ( + SELECT ""Hash"" + FROM ""{cmpName}"" + ) + "; + + var missing = $@" + SELECT + @TypeMissing AS ""Type"", + ""Hash"" AS ""Hash"" + FROM ""{cmpName}"" + WHERE ""Hash"" NOT IN ( + SELECT ""Hash"" + FROM ""{m_tablename}"" + ) + "; + + var modified = $@" + SELECT + @TypeModified AS ""Type"", + ""E"".""Hash"" AS ""Hash"" + FROM + ""{m_tablename}"" E, + ""{cmpName}"" D + WHERE + ""D"".""Hash"" = ""E"".""Hash"" + AND ""D"".""Size"" != ""E"".""Size"" + "; + + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; + + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { try { - cmd.SetCommandAndParameters(create) + await cmd + .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); - cmd.SetCommandAndParameters(FormatInvariant($"{extra} UNION {missing} UNION {modified}")) + cmd + .SetCommandAndParameters($@" + {extra} + UNION {missing} + UNION {modified} + ") .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } finally { - try { cmd.ExecuteNonQuery(drop); } + try + { + await cmd.ExecuteNonQueryAsync(drop); + } catch { } } } @@ -689,30 +790,46 @@ namespace Duplicati.Library.Main.Database private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; - public BlocklistHashList(IDbConnection connection, string volumename, ReusableTransaction rtr) - : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) - { } - - public void AddBlockHash(string hash, long size) + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] + public BlocklistHashList(SqliteConnection connection, string volumename, ReusableTransaction rtr) { - m_insertCommand.SetTransaction(m_rtr.Transaction) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + throw new NotSupportedException("Use CreateAsync method instead."); } - public IEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize) + private BlocklistHashList() { } + + public static async Task CreateAsync(LocalDatabase db, string volumename) + { + var bl = new BlocklistHashList(); + return (BlocklistHashList)await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + } + + public async Task AddBlockHash(string hash, long size) + { + await m_insertCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(); + } + + public async IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = FormatInvariant($@" + var create = $@" CREATE TEMPORARY TABLE ""{cmpName}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL ); - INSERT INTO ""{cmpName}"" (""Hash"", ""Size"") - SELECT b.""Hash"", b.""Size"" + INSERT INTO ""{cmpName}"" ( + ""Hash"", + ""Size"" + ) + SELECT + b.""Hash"", + b.""Size"" FROM Block b JOIN ( SELECT @@ -724,78 +841,108 @@ namespace Duplicati.Library.Main.Database ELSE {hashSize} * {hashesPerBlock} END AS ""Size"" FROM BlocklistHash blh - JOIN Blockset bs ON bs.""ID"" = blh.""BlocksetID"" - ) expected ON b.""Hash"" = expected.""Hash"" AND b.""Size"" = expected.""Size"" + JOIN Blockset bs + ON bs.""ID"" = blh.""BlocksetID"" + ) expected + ON + b.""Hash"" = expected.""Hash"" + AND b.""Size"" = expected.""Size"" WHERE b.""VolumeID"" IN ( SELECT ibl.""BlockVolumeID"" FROM Remotevolume idx - JOIN IndexBlockLink ibl ON ibl.""IndexVolumeID"" = idx.""ID"" + JOIN IndexBlockLink ibl + ON ibl.""IndexVolumeID"" = idx.""ID"" WHERE idx.""Name"" = @Name ); - "); + "; - var compare = FormatInvariant($@" + var compare = $@" WITH Expected AS ( - SELECT ""Hash"", ""Size"" FROM ""{cmpName}"" + SELECT + ""Hash"", + ""Size"" + FROM ""{cmpName}"" ), Actual AS ( - SELECT ""Hash"", ""Size"" FROM ""{m_tablename}"" + SELECT + ""Hash"", + ""Size"" + FROM ""{m_tablename}"" ), Extra AS ( SELECT @TypeExtra AS Type, a.""Hash"" FROM Actual a - LEFT JOIN Expected e ON a.""Hash"" = e.""Hash"" AND a.""Size"" = e.""Size"" + LEFT JOIN Expected e + ON + a.""Hash"" = e.""Hash"" + AND a.""Size"" = e.""Size"" WHERE e.""Hash"" IS NULL ), Missing AS ( SELECT @TypeMissing AS Type, e.""Hash"" FROM Expected e - LEFT JOIN Actual a ON a.""Hash"" = e.""Hash"" AND a.""Size"" = e.""Size"" + LEFT JOIN Actual a + ON + a.""Hash"" = e.""Hash"" + AND a.""Size"" = e.""Size"" WHERE a.""Hash"" IS NULL ), Modified AS ( SELECT @TypeModified AS Type, a.""Hash"" FROM Actual a - JOIN Expected e ON a.""Hash"" = e.""Hash"" - WHERE a.""Size"" != e.""Size"" - AND NOT EXISTS ( - SELECT 1 FROM Extra x WHERE x.""Hash"" = a.""Hash"" - ) + JOIN Expected e + ON a.""Hash"" = e.""Hash"" + WHERE + a.""Size"" != e.""Size"" + AND NOT EXISTS ( + SELECT 1 + FROM Extra x + WHERE x.""Hash"" = a.""Hash"" + ) ) - SELECT * FROM Extra + SELECT * + FROM Extra UNION - SELECT * FROM Missing + SELECT * + FROM Missing UNION - SELECT * FROM Modified; - "); + SELECT * + FROM Modified; + "; - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"""); + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"""; - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { try { // Create expected hash+size table filtered by volume - cmd.SetCommandAndParameters(create) + await cmd + .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(); // Compare against actual values inserted into temp table - cmd.SetCommandAndParameters(compare) + cmd + .SetCommandAndParameters(compare) .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) yield return new KeyValuePair( (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } finally { - try { cmd.ExecuteNonQuery(drop); } catch { } + try + { + await cmd.ExecuteNonQueryAsync(drop); + } + catch { } } } } @@ -806,19 +953,19 @@ namespace Duplicati.Library.Main.Database return await Filelist.CreateAsync(this, name); } - public IIndexlist CreateIndexlist(string name, ReusableTransaction rtr) + public async Task CreateIndexlist(string name) { - return new Indexlist(m_connection, name, rtr); + return await Indexlist.CreateAsync(this, name); } - public IBlocklist CreateBlocklist(string name, ReusableTransaction rtr) + public async Task CreateBlocklist(string name) { - return new Blocklist(m_connection, name, rtr); + return await Blocklist.CreateAsync(this, name); } - public IBlocklistHashList CreateBlocklistHashList(string name, ReusableTransaction rtr) + public async Task CreateBlocklistHashList(string name) { - return new BlocklistHashList(m_connection, name, rtr); + return await BlocklistHashList.CreateAsync(this, name); } } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 5443c4ba3..cdd539d25 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -154,7 +154,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.ReplaceFaultyIndexFiles) { Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, "Found {0} faulty index files, repairing now", faultyIndexFiles.Count); - await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, rtr, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_results.TaskControl.ProgressToken).ConfigureAwait(false); } else Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, "Found {0} faulty index files, use the option {1} to repair them", faultyIndexFiles.Count, "--replace-faulty-index-files"); @@ -280,19 +280,19 @@ namespace Duplicati.Library.Main.Operation foreach (var h in v.Blocks) await bl.AddBlock(h.Key, h.Value); - combined.AddRange(bl.Compare()); + combined.AddRange(await bl.Compare().ToListAsync()); } } if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) { var hashesPerBlock = options.Blocksize / options.BlockhashSize; - using (var bl = db.CreateBlocklistHashList(vol.Name, rtr)) + using (var bl = await db.CreateBlocklistHashList(vol.Name)) { foreach (var b in rd.BlockLists) - bl.AddBlockHash(b.Hash, b.Length); + await bl.AddBlockHash(b.Hash, b.Length); - combined.AddRange(bl.Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize)); + combined.AddRange(await bl.Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize).ToListAsync()); } } } @@ -347,9 +347,10 @@ namespace Duplicati.Library.Main.Operation return new KeyValuePair>>(vol.Name, null); } - private async Task ReplaceFaultyIndexFilesAsync(List faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, ReusableTransaction rtr, CancellationToken cancellationToken) + private async Task ReplaceFaultyIndexFilesAsync(List faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, CancellationToken cancellationToken) { - using var repairdb = new LocalRepairDatabase(db); + using var repairdb = await LocalRepairDatabase.CreateAsync(db); + foreach (var vol in faultyIndexFiles) { if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) @@ -362,7 +363,7 @@ namespace Duplicati.Library.Main.Operation try { var w = newEntry = new IndexVolumeWriter(m_options); - await RepairHandler.RunRepairDindex(backendManager, repairdb, rtr, w, vol, m_options, cancellationToken).ConfigureAwait(false); + await RepairHandler.RunRepairDindex(backendManager, repairdb, w, vol, m_options, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) { Logging.Log.WriteDryrunMessage(LOGTAG, "ReplaceFaultyIndexFile", "Would replace faulty index file {0} with {1}", vol.Name, w.RemoteFilename); @@ -370,8 +371,8 @@ namespace Duplicati.Library.Main.Operation else { await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(repairdb, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - rtr.Commit("ReplaceFaultyIndexFileCommit"); + await backendManager.WaitForEmptyAsync(repairdb, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await repairdb.Transaction.CommitAsync("ReplaceFaultyIndexFileCommit"); } } catch (Exception ex) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index c4a8f53e2..36410967e 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -731,7 +731,7 @@ namespace Duplicati.UnitTest [Test] [Category("Disruption")] - public void TestUploadExceptionWithNoRetries() + public async Task TestUploadExceptionWithNoRetries() { var testopts = TestOptions; testopts["number-of-retries"] = "0"; @@ -805,7 +805,7 @@ namespace Duplicati.UnitTest } catch (TestUtils.TestVerificationException e) { - using var db = new LocalDatabase(testopts["dbpath"], "test", true, 0); + using var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, 0).ConfigureAwait(false); using var cmd = db.Connection.CreateCommand(); var sb = new StringBuilder(); From 239d7370cd6b27b7067d27baa28dec3b3c9d1afb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 2 Jun 2025 11:50:44 +0200 Subject: [PATCH 241/362] Removed whitespace --- Duplicati/UnitTest/DisruptionTests.cs | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 36410967e..18982f1d9 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -1096,7 +1096,7 @@ namespace Duplicati.UnitTest // If we prevent the file from being uploaded, // it will be a temporary file, and a new synthetic file // will be attempted, but that fails as well - // If it is after, then the upload completes, + // If it is after, then the upload completes, // and it will be promoted to a regular uploaded backup expectedFilesets += before ? 1 : runs; From 85e6f45338e6427fd4257234d1c88b939ebea647 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 3 Jun 2025 21:44:55 +0200 Subject: [PATCH 242/362] Moved some of the commits to transactions outside of database methods, as this more closely mimics the commit flow of the old IDb backend. This was especially needed due to the Microsoft.Data.Sqlite backend not supporting nested transactions. --- .../BackendManager.DatabaseCollector.cs | 4 + .../Library/Main/Database/LocalDatabase.cs | 18 +- .../Main/Database/LocalPurgeDatabase.cs | 2 + .../Main/Operation/Backup/BackupDatabase.cs | 7 +- .../Main/Operation/PurgeFilesHandler.cs | 2 + .../Main/Operation/RecreateDatabaseHandler.cs | 27 +- .../Library/Main/Operation/RepairHandler.cs | 645 +++++++++--------- Duplicati/Library/Main/Utility.cs | 3 + 8 files changed, 381 insertions(+), 327 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs index ed66c61e6..86303cb1e 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs @@ -150,7 +150,11 @@ partial class BackendManager // Finally remove volumes from DB. if (volsRemoved.Count > 0) + { await db.RemoveRemoteVolumes(volsRemoved); + // TODO this commit is to mimic the old behavior, in which RemoveRemoteVolumes started a new transaction, if none was passed down. + await db.Transaction.CommitAsync(); + } return true; } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index a293575fb..c2c693606 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -635,8 +635,6 @@ namespace Duplicati.Library.Main.Database if (c != 1) throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); - - await m_rtr.CommitAsync(); } public async Task RemoveRemoteVolume(string name) @@ -920,8 +918,6 @@ namespace Duplicati.Library.Main.Database "); if (nonAttachedFiles > 0) throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); - - await m_rtr.CommitAsync(); } public async Task Vacuum() @@ -961,8 +957,6 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@ArchiveTime", 0) .ExecuteScalarInt64Async(); - await m_rtr.CommitAsync(); - return r; } @@ -1067,7 +1061,6 @@ namespace Duplicati.Library.Main.Database var res = await GetDbOptionList() .ToDictionaryAsync(x => x.Key, x => x.Value); - await m_rtr.CommitAsync(); return res; } @@ -1162,8 +1155,6 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Value", kp.Value) .ExecuteNonQueryAsync(); } - - await m_rtr.CommitAsync(); } public async Task GetBlocksLargerThan(long fhblocksize) @@ -2069,9 +2060,10 @@ namespace Duplicati.Library.Main.Database // IF needed, also create an empty fileset, so the validation works if (type == RemoteVolumeType.Files) + { await CreateFileset(newvolId, DateTime.UnixEpoch); - - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync(); + } } /// @@ -2104,8 +2096,6 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP) .ExecuteScalarInt64Async(-1); - await m_rtr.CommitAsync(); - return id; } @@ -2243,8 +2233,6 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", fileSetId) .SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP) .ExecuteNonQueryAsync(); - - await m_rtr.CommitAsync(); } /// diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index f7426f897..e2bec5722 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -287,6 +287,8 @@ namespace Duplicati.Library.Main.Database var remotevolid = await m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary); var filesetid = await m_db.CreateFileset(remotevolid, timestamp); await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); + // TODO this commit mimics the old way of not passing down a transaction to UpdateFullBackupStateInFileset, but maybe it is not needed. + await m_db.Transaction.CommitAsync(); using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) await cmd.SetCommandAndParameters($@" diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index e28dcec6f..5876702e3 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -229,7 +229,12 @@ namespace Duplicati.Library.Main.Operation.Backup public Task CreateFilesetAsync(long volumeID, DateTime fileTime) { - return RunOnMain(async () => await m_database.CreateFileset(volumeID, fileTime)); + return RunOnMain(async () => + { + var fs = await m_database.CreateFileset(volumeID, fileTime); + await m_database.Transaction.CommitAsync(); + return fs; + }); } public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid) diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 9daa4ae97..c6fe3312f 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -203,6 +203,8 @@ namespace Duplicati.Library.Main.Operation await backendManager.PutAsync(vol, null, null, true, async () => { await backendManager.FlushPendingMessagesAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + // TODO This commit mimics the old way of not passing down a transaction, but maybe is not needed anymore. + await db.Transaction.CommitAsync(); }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 3d8a737c8..7ce2c9b70 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -193,7 +193,7 @@ namespace Duplicati.Library.Main.Operation var hasUpdatedOptions = false; - //Record all blocksets and files needed + // Record all blocksets and files needed var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count); @@ -217,6 +217,8 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); return; } @@ -263,18 +265,25 @@ namespace Duplicati.Library.Main.Operation if (ex.IsAbortException()) { m_result.EndTime = DateTime.UtcNow; + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); throw; } if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException) { m_result.EndTime = DateTime.UtcNow; + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); throw; } if (m_options.UnittestMode) + { + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); throw; - + } } } @@ -313,6 +322,8 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); return; } @@ -392,11 +403,17 @@ namespace Duplicati.Library.Main.Operation if (ex.IsAbortException()) { m_result.EndTime = DateTime.UtcNow; + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); throw; } if (m_options.UnittestMode) + { + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); throw; + } } } @@ -456,6 +473,8 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); return; } @@ -514,7 +533,11 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message); if (m_options.UnittestMode) + { + // TODO implicit rollback + await restoredb.Transaction.RollBackAsync(); throw; + } } } } diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 32bfee31b..2c6a6bc60 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -130,392 +130,416 @@ namespace Duplicati.Library.Main.Operation using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - - if (await db.PartiallyRecreated()) - throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - - if (await db.RepairInProgress()) - throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); - - // Ensure the database is consistent before we start fixing the remote - await db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true); - - // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist - var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); - await db.Transaction.CommitAsync("CommitRemoteListAnalysisTransaction"); - - var buffer = new byte[m_options.Blocksize]; - var hashsize = m_options.BlockhashSize; - - var missingRemoteFilesets = await db.MissingRemoteFilesets().ToListAsync(cancellationToken: cancellationToken); - var missingLocalFilesets = await db.MissingLocalFilesets().ToListAsync(cancellationToken: cancellationToken); - var emptyIndexFiles = await db.EmptyIndexFiles().ToListAsync(cancellationToken: cancellationToken); - - var progress = 0; - var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; - - // Find the most recent timestamp from either a fileset or a remote volume - var mostRecentLocal = await db.GetRemoteVolumes() - .Where(x => x.Type == RemoteVolumeType.Files) - .Select(x => VolumeBase.ParseFilename(x.Name).Time.ToLocalTime()) - .Concat(db.FilesetTimes().Select(x => x.Value.ToLocalTime())) - .Append(DateTime.MinValue) - .MaxAsync(); - - var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); - if (mostRecentLocal < DateTime.UnixEpoch) - throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); - if (mostRecentRemote > mostRecentLocal) + try { - if (m_options.RepairIgnoreOutdatedDatabase) - Logging.Log.WriteWarningMessage(LOGTAG, "RemoteFilesNewerThanLocalDatabase", null, "The remote files are newer ({0}) than the local database ({1}), this is likely because the database is outdated. Continuing as the options force ignoring this.", mostRecentRemote, mostRecentLocal); - else - throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again. If this is expected, set the option \"--repair-ignore-outdated-database\" ", "RemoteFilesNewerThanLocalDatabase"); - } - if (m_options.Dryrun) - { - if (!tp.ParsedVolumes.Any() && tp.OtherVolumes.Any()) + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + + if (await db.PartiallyRecreated()) + throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); + + if (await db.RepairInProgress()) + throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); + + // Ensure the database is consistent before we start fixing the remote + await db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true); + + // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist + var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + await db.Transaction.CommitAsync("CommitRemoteListAnalysisTransaction"); + + var buffer = new byte[m_options.Blocksize]; + var hashsize = m_options.BlockhashSize; + + var missingRemoteFilesets = await db.MissingRemoteFilesets().ToListAsync(cancellationToken: cancellationToken); + var missingLocalFilesets = await db.MissingLocalFilesets().ToListAsync(cancellationToken: cancellationToken); + var emptyIndexFiles = await db.EmptyIndexFiles().ToListAsync(cancellationToken: cancellationToken); + + var progress = 0; + var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; + + // Find the most recent timestamp from either a fileset or a remote volume + var mostRecentLocal = await db.GetRemoteVolumes() + .Where(x => x.Type == RemoteVolumeType.Files) + .Select(x => VolumeBase.ParseFilename(x.Name).Time.ToLocalTime()) + .Concat(db.FilesetTimes().Select(x => x.Value.ToLocalTime())) + .Append(DateTime.MinValue) + .MaxAsync(); + + var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); + if (mostRecentLocal < DateTime.UnixEpoch) + throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); + if (mostRecentRemote > mostRecentLocal) { - if (tp.BackupPrefixes.Length == 1) - throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", m_options.Prefix, tp.BackupPrefixes[0]), "RemoteFolderEmptyWithPrefix"); + if (m_options.RepairIgnoreOutdatedDatabase) + Logging.Log.WriteWarningMessage(LOGTAG, "RemoteFilesNewerThanLocalDatabase", null, "The remote files are newer ({0}) than the local database ({1}), this is likely because the database is outdated. Continuing as the options force ignoring this.", mostRecentRemote, mostRecentLocal); else - throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)), "RemoteFolderEmptyWithPrefix"); + throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again. If this is expected, set the option \"--repair-ignore-outdated-database\" ", "RemoteFilesNewerThanLocalDatabase"); } - else if (!tp.ParsedVolumes.Any() && tp.ExtraVolumes.Any()) + + if (m_options.Dryrun) { - throw new UserInformationException(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()), "NoRemoteFilesMissing"); + if (!tp.ParsedVolumes.Any() && tp.OtherVolumes.Any()) + { + if (tp.BackupPrefixes.Length == 1) + throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", m_options.Prefix, tp.BackupPrefixes[0]), "RemoteFolderEmptyWithPrefix"); + else + throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)), "RemoteFolderEmptyWithPrefix"); + } + else if (!tp.ParsedVolumes.Any() && tp.ExtraVolumes.Any()) + { + throw new UserInformationException(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()), "NoRemoteFilesMissing"); + } } - } - if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any() || missingRemoteFilesets.Any() || missingLocalFilesets.Any() || emptyIndexFiles.Any()) - { - if (tp.VerificationRequiredVolumes.Any()) + if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any() || missingRemoteFilesets.Any() || missingLocalFilesets.Any() || emptyIndexFiles.Any()) { - using var testdb = await LocalTestDatabase.CreateAsync(db); + if (tp.VerificationRequiredVolumes.Any()) + { + using var testdb = await LocalTestDatabase.CreateAsync(db); - foreach (var n in tp.VerificationRequiredVolumes) + foreach (var n in tp.VerificationRequiredVolumes) + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(testdb, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + await testdb.Transaction.CommitAsync("CommitEarlyExit", false); + else + await testdb.Transaction.RollBackAsync(); + return; + } + + progress++; + m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + + KeyValuePair>> res; + var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); + using (tf) + res = await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1); + + if (res.Value.Any()) + throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); + + if (!m_options.Dryrun) + { + Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); + await db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash); + } + + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileVerificationError", ex, "Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message); + if (ex.IsAbortException()) + throw; + } + + await db.Transaction.CommitAsync("CommitVerificationTransaction"); + } + + // TODO: It is actually possible to use the extra files if we parse them + foreach (var n in tp.ExtraVolumes) try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(testdb, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await testdb.Transaction.CommitAsync("CommitEarlyExit", false); + await db.Transaction.CommitAsync("CommitEarlyExit", false); + else + await db.Transaction.RollBackAsync(); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - KeyValuePair>> res; - var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); - using (tf) - res = await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1); + // If this is a new index file, we can accept it if it matches our local data + // This makes it possible to augment the remote store with new index data + if (n.FileType == RemoteVolumeType.Index && m_options.IndexfilePolicy != Options.IndexFileStrategy.None) + { + try + { + (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.File.Name, null, n.File.Size, cancellationToken).ConfigureAwait(false); + using (tf) + using (var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize)) + { + foreach (var rv in ifr.Volumes) + { + var entry = await db.GetRemoteVolume(rv.Filename); + if (entry.ID < 0) + throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); - if (res.Value.Any()) - throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); + if (!new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) + throw new Exception(string.Format("Volume {0} has local state {1}", rv.Filename, entry.State)); + + if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) + throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); + + await db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks); + } + + var blocksize = m_options.Blocksize; + foreach (var ixb in ifr.BlockLists) + await db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize); + + // Register the new index file and link it to the block files + var selfid = await db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0)); + foreach (var rv in ifr.Volumes) + { + // Guard against unknown block files + long id = await db.GetRemoteVolumeID(rv.Filename); + if (id == -1) + Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); + else + await db.AddIndexBlockLink(selfid, id); + } + if (!m_options.Dryrun) + await db.Transaction.CommitAsync("CommitIndexFileTransaction"); + } + + // All checks fine, we accept the new index file + Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); + await db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash); + continue; + } + catch (Exception rex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FailedNewIndexFile", rex, "Failed to accept new index file: {0}, message: {1}", n.File.Name, rex.Message); + } + } if (!m_options.Dryrun) { - Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); - await db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash); + await db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); + await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); } - + else + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "would delete file {0}", n.File.Name); } catch (Exception ex) { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileVerificationError", ex, "Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message); + Logging.Log.WriteErrorMessage(LOGTAG, "FailedExtraFileCleanup", ex, "Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message); if (ex.IsAbortException()) throw; } - await db.Transaction.CommitAsync("CommitVerificationTransaction"); - } + var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks); + if (!m_options.RebuildMissingDblockFiles && missingDblocks.Count() > 0) + throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); - // TODO: It is actually possible to use the extra files if we parse them - foreach (var n in tp.ExtraVolumes) - try + var anyDlistUploads = false; + foreach (var (filesetId, timestamp, isfull) in missingRemoteFilesets) { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction.CommitAsync("CommitEarlyExit", false); + else + await db.Transaction.RollBackAsync(); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); - // If this is a new index file, we can accept it if it matches our local data - // This makes it possible to augment the remote store with new index data - if (n.FileType == RemoteVolumeType.Index && m_options.IndexfilePolicy != Options.IndexFileStrategy.None) + var fsw = new FilesetVolumeWriter(m_options, fileTime); + Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); + + if (!string.IsNullOrEmpty(m_options.ControlFiles)) + foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); + + fsw.CreateFilesetFile(isfull); + await db.WriteFileset(fsw, filesetId); + fsw.Close(); + + + fsw.VolumeID = await db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); + await db.LinkFilesetToVolume(filesetId, fsw.VolumeID); + if (m_options.Dryrun) { - try - { - (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.File.Name, null, n.File.Size, cancellationToken).ConfigureAwait(false); - using (tf) - using (var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize)) - { - foreach (var rv in ifr.Volumes) - { - var entry = await db.GetRemoteVolume(rv.Filename); - if (entry.ID < 0) - throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); - - if (!new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) - throw new Exception(string.Format("Volume {0} has local state {1}", rv.Filename, entry.State)); - - if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) - throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); - - await db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks); - } - - var blocksize = m_options.Blocksize; - foreach (var ixb in ifr.BlockLists) - await db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize); - - // Register the new index file and link it to the block files - var selfid = await db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0)); - foreach (var rv in ifr.Volumes) - { - // Guard against unknown block files - long id = await db.GetRemoteVolumeID(rv.Filename); - if (id == -1) - Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); - else - await db.AddIndexBlockLink(selfid, id); - } - if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitIndexFileTransaction"); - } - - // All checks fine, we accept the new index file - Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); - await db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash); - continue; - } - catch (Exception rex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FailedNewIndexFile", rex, "Failed to accept new index file: {0}, message: {1}", n.File.Name, rex.Message); - } + fsw.Dispose(); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}", fsw.RemoteFilename); + continue; } - - if (!m_options.Dryrun) - { - await db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); - await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); - } - else - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "would delete file {0}", n.File.Name); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FailedExtraFileCleanup", ex, "Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message); - if (ex.IsAbortException()) - throw; + await backendManager.PutAsync(fsw, null, null, false, null, cancellationToken).ConfigureAwait(false); } - var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks); - if (!m_options.RebuildMissingDblockFiles && missingDblocks.Count() > 0) - throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); - - var anyDlistUploads = false; - foreach (var (filesetId, timestamp, isfull) in missingRemoteFilesets) - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (anyDlistUploads) { await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitEarlyExit", false); - return; + await db.Transaction.CommitAsync("CommitFilesetTransaction"); } - progress++; - m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); - - var fsw = new FilesetVolumeWriter(m_options, fileTime); - Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); - - if (!string.IsNullOrEmpty(m_options.ControlFiles)) - foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) - fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); - - fsw.CreateFilesetFile(isfull); - await db.WriteFileset(fsw, filesetId); - fsw.Close(); - - - fsw.VolumeID = await db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); - await db.LinkFilesetToVolume(filesetId, fsw.VolumeID); - if (m_options.Dryrun) + foreach (var volumename in missingLocalFilesets) { - fsw.Dispose(); - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}", fsw.RemoteFilename); - continue; - } - await backendManager.PutAsync(fsw, null, null, false, null, cancellationToken).ConfigureAwait(false); - } - - if (anyDlistUploads) - { - await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); - await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); - if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitFilesetTransaction"); - } - - foreach (var volumename in missingLocalFilesets) - { - var remoteVolume = await db.GetRemoteVolume(volumename); - using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) - { - var parsed = VolumeBase.ParseFilename(remoteVolume.Name); - using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) - using (var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options)) + var remoteVolume = await db.GetRemoteVolume(volumename); + using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) { - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + var parsed = VolumeBase.ParseFilename(remoteVolume.Name); + using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) + using (var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options)) + { + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - var filesetid = await db.CreateFileset(remoteVolume.ID, parsed.Time); + var filesetid = await db.CreateFileset(remoteVolume.ID, parsed.Time); - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()); - if (!m_options.Dryrun) - await recreatedb.Transaction.CommitAsync("CommitRecreateFilesetTransaction"); + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()); + if (!m_options.Dryrun) + await recreatedb.Transaction.CommitAsync("CommitRecreateFilesetTransaction"); + } } } - } - if (!m_options.Dryrun && tp.MissingVolumes.Any()) - await db.TerminatedWithActiveUploads(true); + if (!m_options.Dryrun && tp.MissingVolumes.Any()) + await db.TerminatedWithActiveUploads(true); - if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) - throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); + if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) + throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); - // Process each of the missing volumes in the order of blocks, files and index - // It is important that we process the blocks first, as the index files are derived from the blocks + // Process each of the missing volumes in the order of blocks, files and index + // It is important that we process the blocks first, as the index files are derived from the blocks - void incrementProgress() - { - progress++; - m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - } - - // Blocks are recreated with the entire list of missing files to handle cases where partial recreation is needed - await RunRepairDblocks(backendManager, db, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); - - // Filesets are recreated one at a time - foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Files)) - { - FilesetVolumeWriter newEntry = null; - incrementProgress(); - - try + void incrementProgress() { - var timestamp = VolumeBase.ParseFilename(n.Name).Time; - var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); - var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); - - await RunRepairDlist(backendManager, db, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - if (newEntry != null) - try { newEntry?.Dispose(); } - catch { } - - Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); - - if (ex.IsAbortException()) - throw; - } - } - - // Index files are recreated one at a time, as they are derived from the blocks - foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Index)) - { - IndexVolumeWriter newEntry = null; - incrementProgress(); - - try - { - // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = (await db.GetRemoteVolume(n.Name)).State; - if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) - continue; - - var w = newEntry = new IndexVolumeWriter(m_options); - await RunRepairDindex(backendManager, db, w, n, m_options, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - if (newEntry != null) - try { newEntry?.Dispose(); } - catch { } - - Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); - - if (ex.IsAbortException()) - throw; - } - } - - foreach (var emptyIndexFile in emptyIndexFiles) - { - try - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); - return; - } - progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + } - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteEmptyIndexFile", "would delete empty index file {0}", emptyIndexFile.Name); - else + // Blocks are recreated with the entire list of missing files to handle cases where partial recreation is needed + await RunRepairDblocks(backendManager, db, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); + + // Filesets are recreated one at a time + foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Files)) + { + FilesetVolumeWriter newEntry = null; + incrementProgress(); + + try { - if (emptyIndexFile.Size > 2048) - { - Logging.Log.WriteWarningMessage(LOGTAG, "LargeEmptyIndexFile", null, "The empty index file {0} is larger than expected ({1} bytes), choosing not to delete it", emptyIndexFile.Name, emptyIndexFile.Size); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); - await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); - await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); - } + var timestamp = VolumeBase.ParseFilename(n.Name).Time; + var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); + var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); + + await RunRepairDlist(backendManager, db, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (newEntry != null) + try { newEntry?.Dispose(); } + catch { } + + Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); + + if (ex.IsAbortException()) + throw; } } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "CleanupEmptyIndexFileError", ex, "Failed to perform cleanup for empty index file: {0}, message: {1}", emptyIndexFile.Name, ex.Message); - if (ex.IsAbortException()) - throw; + // Index files are recreated one at a time, as they are derived from the blocks + foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Index)) + { + IndexVolumeWriter newEntry = null; + incrementProgress(); + + try + { + // Check if the index file has already been deleted, beause the dblock was recreated + var currentState = (await db.GetRemoteVolume(n.Name)).State; + if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) + continue; + + var w = newEntry = new IndexVolumeWriter(m_options); + await RunRepairDindex(backendManager, db, w, n, m_options, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (newEntry != null) + try { newEntry?.Dispose(); } + catch { } + + Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); + + if (ex.IsAbortException()) + throw; + } + } + + foreach (var emptyIndexFile in emptyIndexFiles) + { + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); + // TODO implicit rollback + await db.Transaction.RollBackAsync(); + return; + } + + progress++; + m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteEmptyIndexFile", "would delete empty index file {0}", emptyIndexFile.Name); + else + { + if (emptyIndexFile.Size > 2048) + { + Logging.Log.WriteWarningMessage(LOGTAG, "LargeEmptyIndexFile", null, "The empty index file {0} is larger than expected ({1} bytes), choosing not to delete it", emptyIndexFile.Name, emptyIndexFile.Size); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); + await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); + // TODO this commit mimics the old way of not passing down a transaction to FlushPendingMessagesAsync, but maybe it is not needed anymore. + await db.Transaction.CommitAsync(); + } + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "CleanupEmptyIndexFileError", ex, "Failed to perform cleanup for empty index file: {0}, message: {1}", emptyIndexFile.Name, ex.Message); + + if (ex.IsAbortException()) + throw; + } } } - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "DatabaseIsSynchronized", "Destination and database are synchronized, not making any changes"); - } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "DatabaseIsSynchronized", "Destination and database are synchronized, not making any changes"); + } - m_result.OperationProgressUpdater.UpdateProgress(1); - await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); - if (!m_options.Dryrun) + m_result.OperationProgressUpdater.UpdateProgress(1); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + { + await db.TerminatedWithActiveUploads(false); + await db.Transaction.CommitAsync("CommitRepairTransaction"); + } + else + { + await db.Transaction.RollBackAsync("RollbackRepairTransaction"); + } + } + catch (Exception) { - await db.TerminatedWithActiveUploads(false); - await db.Transaction.CommitAsync("CommitRepairTransaction"); + // TODO Implicit rollback + await db.Transaction.RollBackAsync(); + throw; } } @@ -542,6 +566,9 @@ namespace Duplicati.Library.Main.Operation volumeWriter.CreateFilesetFile(isPrevFull); var newFilesetID = await db.CreateFileset(volumeWriter.VolumeID, filesetTime); await db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull); + // TODO this commit mimics the old way of not passing down a transaction to UpdateFullBackupStateInFileset, but maybe it is not needed anymore. + //await db.Transaction.CommitAsync(); + // TODO The above only mimics it to some degree -the original implementation makes a new transaciton alongside the global reusable transaction, which means that this particular SQL statement is comitted to the database. As such, if this new implementation commits the global reusable transaction, all of the statements are comitted - which they shouldn't be! Instead, only whatever that function is trying to do should be committed. This happens at other points as well - all of which should be double checked wrt. whether they should be using the global one, or a new one. This is not supported in the new engine. await db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID); await db.MoveFilesFromFileset(newFilesetID, prevFilesetId); diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index 28efaa721..9c5ffd159 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -125,6 +125,9 @@ namespace Duplicati.Library.Main internal static async Task ContainsOptionsForVerification(LocalDatabase db) { var opts = await db.GetDbOptions(); + // TODO commit mimics old behavior of not passing down a transaction, + // but maybe it shouldn't be necessary? + await db.Transaction.CommitAsync(); return new[] { "blocksize", "blockhash", "filehash", "passphrase" }.Any(opts.ContainsKey); } From 3bfe6eb34973e86090e75915225c165e14c8c7cd Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 3 Jun 2025 21:46:19 +0200 Subject: [PATCH 243/362] Added additional context to commits without messages --- Duplicati/Library/Main/Database/ReusableTransaction.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 1d2461f55..e560b2ba3 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -89,7 +89,8 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran if (m_disposed) throw new InvalidOperationException("Transaction is already disposed"); - using (var timer = string.IsNullOrWhiteSpace(message) ? null : new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) + message ??= "Unnamed commit"; + using (var timer = new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) await m_transaction.CommitAsync(); await m_transaction.DisposeAsync(); From abcb3569ffe1277a3406c13dedb027d7c06fcd8f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 3 Jun 2025 21:46:50 +0200 Subject: [PATCH 244/362] Added additional logging of SQL executions --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 5b9dd5dde..35fd61b6d 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -249,11 +249,8 @@ public static partial class ExtensionMethods public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue = -1) { - using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) - return ConvertValueToInt64(rd, 0, defaultvalue); + return await ExecuteScalarInt64Async(self, true, defaultvalue).ConfigureAwait(false); - return defaultvalue; } public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string cmdtext, long defaultvalue = -1) @@ -263,7 +260,7 @@ public static partial class ExtensionMethods return await ExecuteScalarInt64Async(self, defaultvalue).ConfigureAwait(false); } - [Obsolete("Technically this method isn't async, but it is used in an async context and to differntiate this with the IDb version. ")] + //[Obsolete("Technically this method isn't async, but it is used in an async context and to differntiate this with the IDb version. ")] public static SqliteCommand ExpandInClauseParameterAsync(this SqliteCommand cmd, string originalParamName, IEnumerable values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) From d78d7c2739e8794d2e1b9de76ba252b6a5049f06 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 09:11:34 +0200 Subject: [PATCH 245/362] Added additional logging of SQL executions --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 35fd61b6d..c3760f872 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -178,9 +178,10 @@ public static partial class ExtensionMethods public static async IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self) { - using var rd = await self.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - yield return rd; + using (new Logging.Timer(LOGTAG, "ExecuteReaderEnumerableAsync", $"ExecuteReaderEnumerableAsync: {self.GetPrintableCommandText()}")) + using (var rd = await self.ExecuteReaderAsync()) + while (await rd.ReadAsync()) + yield return rd; } public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext) From beb0515a27ad4305425e336d1c7c8f781efdf568 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 09:19:38 +0200 Subject: [PATCH 246/362] The old IDb database backend disposed the transaction post running resulting in an implicit rollback. The new backend should do the same, especially since a list operation shouldn't modify the database. --- Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 33981faa2..732ba47ac 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -57,6 +57,7 @@ namespace Duplicati.Library.Main.Operation await Utility.UpdateOptionsFromDb(db, m_options); await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); + await db.Transaction.RollBackAsync().ConfigureAwait(false); } } From 80389f7bc1f4bdf055d8e6242f5e29c06c61dcdd Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 09:33:12 +0200 Subject: [PATCH 247/362] Incorrectly mimicked old way of executing SQL queries without a transaction --- Duplicati/Library/Main/Database/LocalPurgeDatabase.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index e2bec5722..f8223d6fd 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -288,7 +288,8 @@ namespace Duplicati.Library.Main.Database var filesetid = await m_db.CreateFileset(remotevolid, timestamp); await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); // TODO this commit mimics the old way of not passing down a transaction to UpdateFullBackupStateInFileset, but maybe it is not needed. - await m_db.Transaction.CommitAsync(); + //await m_db.Transaction.CommitAsync(); + // TODO not exactly, since the old way was using a nested transaction, which the new backend doesn't support. using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) await cmd.SetCommandAndParameters($@" From 0024e2315314251b1bc5d6b18be02643ed9d3556 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 09:34:19 +0200 Subject: [PATCH 248/362] When disposing a database, it should run PRAGMA optimize, which requires a transaction. Thus all operations should let the reusable transaction restart. --- .../Main/Operation/PurgeBrokenFilesHandler.cs | 408 +++++++++--------- 1 file changed, 204 insertions(+), 204 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 04e49236d..b92c9b1a0 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -1,204 +1,204 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Duplicati.Library.Interface; -using Duplicati.Library.Main.Database; -using Duplicati.Library.Utility; - -namespace Duplicati.Library.Main.Operation -{ - internal class PurgeBrokenFilesHandler - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(PurgeBrokenFilesHandler)); - protected readonly Options m_options; - protected readonly PurgeBrokenFilesResults m_result; - - public PurgeBrokenFilesHandler(Options options, PurgeBrokenFilesResults result) - { - m_options = options; - m_result = result; - } - - public async Task RunAsync(IBackendManager backendManager, Library.Utility.IFilter filter) - { - if (!System.IO.File.Exists(m_options.Dbpath)) - throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - - if (filter != null && !filter.Empty) - throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - - using (var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) - { - if (await db.PartiallyRecreated()) - throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - - var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); - if (sets == null) - return; - - if (sets.Length == 0) - { - if (missing == null) - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets"); - else if (missing.Count == 0) - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files"); - else - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count)); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.RemoveCount)); - - var pgoffset = 0.0f; - var pgspan = 0.95f / sets.Length; - - var filesets = await db.FilesetTimes().ToListAsync(); - - var compare_list = sets.Select(x => new - { - FilesetID = x.FilesetID, - Timestamp = x.FilesetTime, - RemoveCount = x.RemoveCount, - Version = filesets.FindIndex(y => y.Key == x.FilesetID), - // TODO if we await this properly, we'll have to await - // it again when using it. - SetCount = db.GetFilesetFileCount(x.FilesetID).Await() - }).ToArray(); - - var replacementMetadataBlocksetId = -1L; - if (!m_options.DisableReplaceMissingMetadata) - { - var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); - replacementMetadataBlocksetId = await db.GetEmptyMetadataBlocksetId((missing ?? []).Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length); - if (replacementMetadataBlocksetId < 0) - throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); - } - - var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); - var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); - - if (fully_emptied.Length == await db.FilesetTimes().CountAsync()) - throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); - - if (!m_options.Dryrun) - await db.Transaction.CommitAsync(); - - - if (fully_emptied.Length != 0) - { - if (fully_emptied.Length == 1) - Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount); - else - Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); - - m_result.DeleteResults = new DeleteResults(m_result); - using (var rmdb = await LocalDeleteDatabase.CreateAsync(db)) - { - var opts = new Options(new Dictionary(m_options.RawOptions)); - opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); - opts.RawOptions.Remove("time"); - opts.RawOptions["no-auto-compact"] = "true"; - - await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) - .DoRunAsync(rmdb, true, false, backendManager).ConfigureAwait(false); - - if (!m_options.Dryrun) - await rmdb.Transaction.CommitAsync("CommitDelete", restart: false); - } - - pgoffset += (pgspan * fully_emptied.Length); - m_result.OperationProgressUpdater.UpdateProgress(pgoffset); - } - - if (to_purge.Length > 0) - { - m_result.PurgeResults = new PurgeFilesResults(m_result); - - foreach (var bs in to_purge) - { - Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); - var opts = new Options(new Dictionary(m_options.RawOptions)); - - using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db)) - { - // Recompute the version number after we deleted the versions before - filesets = await pgdb.FilesetTimes().ToListAsync(); - var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); - if (thisversion < 0) - throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); - - opts.RawOptions["version"] = thisversion.ToString(); - opts.RawOptions.Remove("time"); - opts.RawOptions["no-auto-compact"] = "true"; - - await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, async (cmd, filesetid, tablename) => - { - if (filesetid != bs.FilesetID) - throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); - - // Update entries that would be removed because of missing metadata - var updatedEntries = 0; - if (!m_options.DisableReplaceMissingMetadata) - updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId); - - await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID"); - return updatedEntries; - }).ConfigureAwait(false); - } - - pgoffset += pgspan; - m_result.OperationProgressUpdater.UpdateProgress(pgoffset); - } - } - } - - m_result.OperationProgressUpdater.UpdateProgress(0.95f); - - if (!m_options.Dryrun && await db.RepairInProgress()) - { - Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); - Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); - await db.RepairInProgress(false); - } - else - { - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); - } - - m_result.OperationProgressUpdater.UpdateProgress(1.0f); - - } - } - } -} +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Main.Database; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.Main.Operation +{ + internal class PurgeBrokenFilesHandler + { + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(PurgeBrokenFilesHandler)); + protected readonly Options m_options; + protected readonly PurgeBrokenFilesResults m_result; + + public PurgeBrokenFilesHandler(Options options, PurgeBrokenFilesResults result) + { + m_options = options; + m_result = result; + } + + public async Task RunAsync(IBackendManager backendManager, Library.Utility.IFilter filter) + { + if (!System.IO.File.Exists(m_options.Dbpath)) + throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); + + if (filter != null && !filter.Empty) + throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); + + using (var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) + { + if (await db.PartiallyRecreated()) + throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); + + await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + + var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); + if (sets == null) + return; + + if (sets.Length == 0) + { + if (missing == null) + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets"); + else if (missing.Count == 0) + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files"); + else + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count)); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.RemoveCount)); + + var pgoffset = 0.0f; + var pgspan = 0.95f / sets.Length; + + var filesets = await db.FilesetTimes().ToListAsync(); + + var compare_list = sets.Select(x => new + { + FilesetID = x.FilesetID, + Timestamp = x.FilesetTime, + RemoveCount = x.RemoveCount, + Version = filesets.FindIndex(y => y.Key == x.FilesetID), + // TODO if we await this properly, we'll have to await + // it again when using it. + SetCount = db.GetFilesetFileCount(x.FilesetID).Await() + }).ToArray(); + + var replacementMetadataBlocksetId = -1L; + if (!m_options.DisableReplaceMissingMetadata) + { + var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); + replacementMetadataBlocksetId = await db.GetEmptyMetadataBlocksetId((missing ?? []).Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length); + if (replacementMetadataBlocksetId < 0) + throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); + } + + var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); + var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); + + if (fully_emptied.Length == await db.FilesetTimes().CountAsync()) + throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); + + if (!m_options.Dryrun) + await db.Transaction.CommitAsync(); + + + if (fully_emptied.Length != 0) + { + if (fully_emptied.Length == 1) + Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount); + else + Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); + + m_result.DeleteResults = new DeleteResults(m_result); + using (var rmdb = await LocalDeleteDatabase.CreateAsync(db)) + { + var opts = new Options(new Dictionary(m_options.RawOptions)); + opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); + opts.RawOptions.Remove("time"); + opts.RawOptions["no-auto-compact"] = "true"; + + await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) + .DoRunAsync(rmdb, true, false, backendManager).ConfigureAwait(false); + + if (!m_options.Dryrun) + await rmdb.Transaction.CommitAsync("CommitDelete"); + } + + pgoffset += (pgspan * fully_emptied.Length); + m_result.OperationProgressUpdater.UpdateProgress(pgoffset); + } + + if (to_purge.Length > 0) + { + m_result.PurgeResults = new PurgeFilesResults(m_result); + + foreach (var bs in to_purge) + { + Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); + var opts = new Options(new Dictionary(m_options.RawOptions)); + + using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db)) + { + // Recompute the version number after we deleted the versions before + filesets = await pgdb.FilesetTimes().ToListAsync(); + var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); + if (thisversion < 0) + throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); + + opts.RawOptions["version"] = thisversion.ToString(); + opts.RawOptions.Remove("time"); + opts.RawOptions["no-auto-compact"] = "true"; + + await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, async (cmd, filesetid, tablename) => + { + if (filesetid != bs.FilesetID) + throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); + + // Update entries that would be removed because of missing metadata + var updatedEntries = 0; + if (!m_options.DisableReplaceMissingMetadata) + updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId); + + await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID"); + return updatedEntries; + }).ConfigureAwait(false); + } + + pgoffset += pgspan; + m_result.OperationProgressUpdater.UpdateProgress(pgoffset); + } + } + } + + m_result.OperationProgressUpdater.UpdateProgress(0.95f); + + if (!m_options.Dryrun && await db.RepairInProgress()) + { + Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); + Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); + await db.RepairInProgress(false); + } + else + { + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); + } + + m_result.OperationProgressUpdater.UpdateProgress(1.0f); + + } + } + } +} From 2b61cf22bc8c0e6cf779045293e95d60189a4078 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 09:35:22 +0200 Subject: [PATCH 249/362] Dryrun shouldn't modify the database - at least not the core of it. --- Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index b92c9b1a0..e26dc8ca9 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -193,6 +193,7 @@ namespace Duplicati.Library.Main.Operation } else { + await db.Transaction.RollBackAsync(); await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); } From 086a0c7914c8c9823204b43734a6a9e06243b3fe Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 10:31:53 +0200 Subject: [PATCH 250/362] Made Issue5066 test legacy restore as well --- Duplicati/UnitTest/Issue5066.cs | 302 ++++++++++++++++---------------- 1 file changed, 152 insertions(+), 150 deletions(-) diff --git a/Duplicati/UnitTest/Issue5066.cs b/Duplicati/UnitTest/Issue5066.cs index 354573f0e..38e226461 100644 --- a/Duplicati/UnitTest/Issue5066.cs +++ b/Duplicati/UnitTest/Issue5066.cs @@ -1,150 +1,152 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Duplicati.Library.Utility; -using NUnit.Framework; - -namespace Duplicati.UnitTest -{ - public class Issue5066 : BasicSetupHelper - { - - private static string CalculateFileHash(string filename) - { - using (var fs = File.OpenRead(filename)) - using (var hasher = HashFactory.CreateHasher("SHA256")) - return Convert.ToBase64String(hasher.ComputeHash(fs)); - } - - [Test] - [Category("Targeted")] - [TestCase(true)] - [TestCase(false)] - public void TestDuplicatedBlocklists1(bool deleteAllIndexFiles) - { - var testopts = TestOptions.Expand(new { blocksize = "1kb" }); - var hashes = new List(); - - // Full blocklist with zeroes - var data = new byte[32769]; - data.AsSpan().Fill((byte)'a'); - File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); - hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - TestUtils.AssertResults(c.Backup([DATAFOLDER])); - - // Make the first blocklist different - data[0] = (byte)'b'; - File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); - hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); - - // Record existing dindex files - var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList(); - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - TestUtils.AssertResults(c.Backup([DATAFOLDER])); - - // Delete new index files - foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly)) - { - if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles) - File.Delete(file); - } - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - TestUtils.AssertResults(c.Repair()); - TestUtils.AssertResults(c.Test()); - } - - for (var version = 0; version < 2; version++) - { - File.Delete(Path.Combine(DATAFOLDER, "a")); - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null)) - { - TestUtils.AssertResults(c.Restore(null)); - var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a")); - Assert.AreEqual(hashes[version], hash, "Hash mismatch for version " + version); - } - } - } - - [Test] - [Category("Targeted")] - [TestCase(true)] - [TestCase(false)] - public void TestDuplicatedBlocklists2(bool deleteAllIndexFiles) - { - var testopts = TestOptions.Expand(new { blocksize = "1kb" }); - var hashes = new List(); - - // Full blocklist with zeroes - var data = new byte[32769]; - data.AsSpan().Fill((byte)'a'); - File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); - hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - TestUtils.AssertResults(c.Backup([DATAFOLDER])); - - // Expand to an new set - data = new byte[32769 * 2 + 1]; - data.AsSpan().Fill((byte)'b'); - data[data.Length - 1] = (byte)'a'; - File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); - hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); - - // Record existing dindex files - var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList(); - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - TestUtils.AssertResults(c.Backup([DATAFOLDER])); - - // Delete new index files - foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly)) - { - if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles) - File.Delete(file); - } - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - TestUtils.AssertResults(c.Repair()); - TestUtils.AssertResults(c.Test()); - } - - for (var version = 0; version < 2; version++) - { - File.Delete(Path.Combine(DATAFOLDER, "a")); - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null)) - { - TestUtils.AssertResults(c.Restore(null)); - var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a")); - Assert.AreEqual(hashes[version], hash, "Hash mismatch for version " + version); - } - } - } - } -} - +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Duplicati.Library.Utility; +using NUnit.Framework; + +namespace Duplicati.UnitTest +{ + public class Issue5066 : BasicSetupHelper + { + + private static string CalculateFileHash(string filename) + { + using (var fs = File.OpenRead(filename)) + using (var hasher = HashFactory.CreateHasher("SHA256")) + return Convert.ToBase64String(hasher.ComputeHash(fs)); + } + + [Test] + [Category("Targeted")] + public void TestDuplicatedBlocklists1([Values] bool deleteAllIndexFiles, [Values] bool restore_legacy) + { + var testopts = TestOptions.Expand(new + { + blocksize = "1kb", + restore_legacy = restore_legacy.ToString() + }); + var hashes = new List(); + + // Full blocklist with zeroes + var data = new byte[32769]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // Make the first blocklist different + data[0] = (byte)'b'; + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); + + // Record existing dindex files + var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList(); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // Delete new index files + foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly)) + { + if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles) + File.Delete(file); + } + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + + for (var version = 0; version < 2; version++) + { + File.Delete(Path.Combine(DATAFOLDER, "a")); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null)) + { + TestUtils.AssertResults(c.Restore(null)); + var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a")); + Assert.AreEqual(hashes[version], hash, "Hash mismatch for version " + version); + } + } + } + + [Test] + [Category("Targeted")] + [TestCase(true)] + [TestCase(false)] + public void TestDuplicatedBlocklists2(bool deleteAllIndexFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb" }); + var hashes = new List(); + + // Full blocklist with zeroes + var data = new byte[32769]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // Expand to an new set + data = new byte[32769 * 2 + 1]; + data.AsSpan().Fill((byte)'b'); + data[data.Length - 1] = (byte)'a'; + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); + + // Record existing dindex files + var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList(); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // Delete new index files + foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly)) + { + if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles) + File.Delete(file); + } + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + + for (var version = 0; version < 2; version++) + { + File.Delete(Path.Combine(DATAFOLDER, "a")); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null)) + { + TestUtils.AssertResults(c.Restore(null)); + var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a")); + Assert.AreEqual(hashes[version], hash, "Hash mismatch for version " + version); + } + } + } + } +} + From 8bf1b8da42b9de58bf5af4dc8ed5761c335ad113 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 10:31:59 +0200 Subject: [PATCH 251/362] Removed whitespace --- Duplicati/UnitTest/Issue5066.cs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/UnitTest/Issue5066.cs b/Duplicati/UnitTest/Issue5066.cs index 38e226461..4b647bcc8 100644 --- a/Duplicati/UnitTest/Issue5066.cs +++ b/Duplicati/UnitTest/Issue5066.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 6c6378c014d9f3f5ee47978d85a0a5db347a10cb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 10:32:19 +0200 Subject: [PATCH 252/362] Fixed missing dollar sign --- .../Main/Database/LocalRestoreDatabase.cs | 4424 ++++++++--------- 1 file changed, 2212 insertions(+), 2212 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index f750deebd..7b0901a62 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1,2212 +1,2212 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Duplicati.Library.Common.IO; -using Duplicati.Library.Main.Operation.Restore; -using Duplicati.Library.Main.Volumes; -using Duplicati.Library.SQLiteHelper; -using Duplicati.Library.Utility; -using Microsoft.Data.Sqlite; - -namespace Duplicati.Library.Main.Database -{ - internal class LocalRestoreDatabase : LocalDatabase - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRestoreDatabase)); - - protected readonly string m_temptabsetguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - /// - /// The name of the temporary table in the database, which is used to store the list of files to restore. - /// - protected string? m_tempfiletable; - protected string? m_tempblocktable; - protected ConcurrentBag<(SqliteConnection, ReusableTransaction)> m_connection_pool = []; - protected string? m_latestblocktable; - protected string? m_fileprogtable; - protected string? m_totalprogtable; - protected string? m_filesnewlydonetable; - - protected DateTime m_restoreTime; - - public DateTime RestoreTime { get { return m_restoreTime; } } - - public static async Task CreateAsync(string path, long pagecachesize, LocalRestoreDatabase? dbnew = null) - { - dbnew ??= new LocalRestoreDatabase(); - - dbnew = (LocalRestoreDatabase)await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew); - dbnew.ShouldCloseConnection = true; - - return dbnew; - } - - public static async Task CreateAsync(LocalDatabase dbparent, LocalRestoreDatabase? dbnew = null) - { - dbnew ??= new LocalRestoreDatabase(); - - return (LocalRestoreDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); - } - - /// - /// Create tables and triggers for automatic tracking of progress during a restore operation. - /// This replaces continuous requerying of block progress by iterating over blocks table. - /// SQLite is much faster keeping information up to date with internal triggers. - /// - /// This allows to create another table that keeps track - /// of all files that are done (all data blocks restored). - /// - /// The method is prepared to create a table that keeps track of all files being done completely. - /// That means, it fires for files where the number of restored blocks equals the number of all blocks. - /// It is intended to be used for fast identification of fully restored files to trigger their verification. - /// It should be read after a commit and truncated after putting the files to a verification queue. - /// Note: If a file is done once and then set back to a none restored state, the file is not automatically removed. - /// But if it reaches a restored state later, it will be re-added (trigger will fire) - /// - public async Task CreateProgressTracker(bool createFilesNewlyDoneTracker) - { - m_fileprogtable = "FileProgress-" + m_temptabsetguid; - m_totalprogtable = "TotalProgress-" + m_temptabsetguid; - m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; - - using var cmd = m_connection.CreateCommand() - .SetTransaction(m_rtr); - - // How to handle METADATA? - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" "); - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( - ""FileId"" INTEGER PRIMARY KEY, - ""TotalBlocks"" INTEGER NOT NULL, - ""TotalSize"" INTEGER NOT NULL, - ""BlocksRestored"" INTEGER NOT NULL, - ""SizeRestored"" INTEGER NOT NULL - ) - "); - - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" "); - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( - ""TotalFiles"" INTEGER NOT NULL, - ""TotalBlocks"" INTEGER NOT NULL, - ""TotalSize"" INTEGER NOT NULL, - ""FilesFullyRestored"" INTEGER NOT NULL, - ""FilesPartiallyRestored"" INTEGER NOT NULL, - ""BlocksRestored"" INTEGER NOT NULL, - ""SizeRestored"" INTEGER NOT NULL - ) - "); - - if (createFilesNewlyDoneTracker) - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" "); - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( - ""ID"" INTEGER PRIMARY KEY - ) - "); - } - - try - { - // Initialize statistics with File- and Block-Data (it is valid to already have restored blocks in files) - // A rebuild with this function should be valid anytime. - // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. - - // We use a LEFT JOIN to allow for empty files (no data Blocks) - - // Will be one row per file. - await cmd.ExecuteNonQueryAsync($@" - INSERT INTO ""{m_fileprogtable}"" ( - ""FileId"", - ""TotalBlocks"", - ""TotalSize"", - ""BlocksRestored"", - ""SizeRestored"" - ) - SELECT - ""F"".""ID"", - IFNULL(COUNT(""B"".""ID""), 0), - IFNULL(SUM(""B"".""Size""), 0), - IFNULL( - COUNT( - CASE ""B"".""Restored"" - WHEN 1 - THEN ""B"".""ID"" - ELSE NULL - END - ), - 0 - ), - IFNULL( - SUM( - CASE ""B"".""Restored"" - WHEN 1 - THEN ""B"".""Size"" - ELSE 0 - END - ), - 0 - ) - FROM ""{m_tempfiletable}"" ""F"" - LEFT JOIN ""{m_tempblocktable}"" ""B"" - ON ""B"".""FileID"" = ""F"".""ID"" - WHERE ""B"".""Metadata"" IS NOT 1 - GROUP BY ""F"".""ID"" - "); - - // Will result in a single line (no support to also track metadata) - await cmd.ExecuteNonQueryAsync($@" - INSERT INTO ""{m_totalprogtable}"" ( - ""TotalFiles"", - ""TotalBlocks"", - ""TotalSize"", - ""FilesFullyRestored"", - ""FilesPartiallyRestored"", - ""BlocksRestored"", - ""SizeRestored"" - ) - SELECT - IFNULL(COUNT(""P"".""FileId""), 0), - IFNULL(SUM(""P"".""TotalBlocks""), 0), - IFNULL(SUM(""P"".""TotalSize""), 0), - IFNULL( - COUNT( - CASE - WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" - THEN 1 - ELSE NULL - END - ), - 0 - ), - IFNULL( - COUNT( - CASE - WHEN - ""P"".""BlocksRestored"" BETWEEN 1 - AND ""P"".""TotalBlocks"" - 1 - THEN 1 - ELSE NULL - END - ), - 0 - ), - IFNULL(SUM(""P"".""BlocksRestored""), 0), - IFNULL(SUM(""P"".""SizeRestored""), 0) - FROM ""{m_fileprogtable}"" ""P"" - "); - - // Finally we create TRIGGERs to keep all our statistics up to date. - // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic. - // Note: We do assume that neither files nor blocks will be added or deleted during restore process - // and that the size of each block stays constant so there is no need to track that information - // with additional INSERT and DELETE triggers. - - // A trigger to update the file-stat entry each time a block changes restoration state. - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" - AFTER UPDATE OF ""Restored"" - ON ""{m_tempblocktable}"" - WHEN OLD.""Restored"" != NEW.""Restored"" - AND NEW.""Metadata"" = 0 - BEGIN UPDATE ""{m_fileprogtable}"" - SET - ""BlocksRestored"" = - ""{m_fileprogtable}"".""BlocksRestored"" - + (NEW.""Restored"" - OLD.""Restored""), - ""SizeRestored"" = - ""{m_fileprogtable}"".""SizeRestored"" - + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) - WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" - ; END - "); - - // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" - AFTER UPDATE ON ""{m_fileprogtable}"" - BEGIN UPDATE ""{m_totalprogtable}"" - SET - ""FilesFullyRestored"" = - ""{m_totalprogtable}"".""FilesFullyRestored"" - + (CASE - WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" - THEN 1 - ELSE 0 - END) - - (CASE - WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" - THEN 1 - ELSE 0 - END), - ""FilesPartiallyRestored"" = - ""{m_totalprogtable}"".""FilesPartiallyRestored"" - + (CASE - WHEN - NEW.""BlocksRestored"" BETWEEN 1 - AND NEW.""TotalBlocks"" - 1 - THEN 1 - ELSE 0 - END) - - (CASE - WHEN - OLD.""BlocksRestored"" BETWEEN 1 - AND OLD.""TotalBlocks"" - 1 - THEN 1 - ELSE 0 - END), - ""BlocksRestored"" = - ""{m_totalprogtable}"".""BlocksRestored"" - + NEW.""BlocksRestored"" - - OLD.""BlocksRestored"", - ""SizeRestored"" = - ""{m_totalprogtable}"".""SizeRestored"" - + NEW.""SizeRestored"" - - OLD.""SizeRestored"" - ; END - "); - - - if (createFilesNewlyDoneTracker) - { - // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" - AFTER UPDATE OF - ""BlocksRestored"", - ""TotalBlocks"" - ON ""{m_fileprogtable}"" - WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" - BEGIN - INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") - VALUES (NEW.""FileId""); - END - "); - } - - } - catch (Exception ex) - { - m_fileprogtable = null; - m_totalprogtable = null; - Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables"); - throw; - } - finally - { - await m_rtr.CommitAsync(); - } - } - - public async Task> PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter) - { - m_tempfiletable = "Fileset-" + m_temptabsetguid; - m_tempblocktable = "Blocks-" + m_temptabsetguid; - - using (var cmd = m_connection.CreateCommand()) - { - cmd.SetTransaction(m_rtr); - var filesetIds = await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToListAsync(); - while (filesetIds.Count > 0) - { - var filesetId = filesetIds[0]; - filesetIds.RemoveAt(0); - - cmd.SetCommandAndParameters(@" - SELECT ""Timestamp"" - FROM ""Fileset"" - WHERE ""ID"" = @FilesetId - ") - .SetParameterValue("@FilesetId", filesetId); - m_restoreTime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); - - var ix = await FilesetTimes() - .Select((value, index) => new { value.Key, index }) - .Where(n => n.Key == filesetId) - .Select(pair => pair.index + 1) - .FirstOrDefaultAsync() - 1; - - Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime); - - cmd.Parameters.Clear(); - - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" "); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" "); - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TABLE ""{m_tempfiletable}"" ( - ""ID"" INTEGER PRIMARY KEY, - ""Path"" TEXT NOT NULL, - ""BlocksetID"" INTEGER NOT NULL, - ""MetadataID"" INTEGER NOT NULL, - ""TargetPath"" TEXT NULL, - ""DataVerified"" BOOLEAN NOT NULL, - ""LatestBlocksetId"" INTEGER, - ""LocalSourceExists"" BOOLEAN - ) - "); - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TABLE ""{m_tempblocktable}"" ( - ""ID"" INTEGER PRIMARY KEY, - ""FileID"" INTEGER NOT NULL, - ""Index"" INTEGER NOT NULL, - ""Hash"" TEXT NOT NULL, - ""Size"" INTEGER NOT NULL, - ""Restored"" BOOLEAN NOT NULL, - ""Metadata"" BOOLEAN NOT NULL, - ""VolumeID"" INTEGER NOT NULL, - ""BlockID"" INTEGER NOT NULL - ) - "); - - // TODO: Optimize to use the path prefix - - if (filter == null || filter.Empty) - { - // Simple case, restore everything - await cmd.SetCommandAndParameters($@" - INSERT INTO ""{m_tempfiletable}"" ( - ""ID"", - ""Path"", - ""BlocksetID"", - ""MetadataID"", - ""DataVerified"" - ) - SELECT - ""File"".""ID"", - ""File"".""Path"", - ""File"".""BlocksetID"", - ""File"".""MetadataID"", - 0 - FROM - ""File"", - ""FilesetEntry"" - WHERE - ""File"".""ID"" = ""FilesetEntry"".""FileID"" - AND ""FilesetEntry"".""FilesetID"" = @FilesetId - ") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync(); - } - else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple) - { - using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset")) - await m_rtr.CommitAsync(); - cmd.SetTransaction(m_rtr); - // If we get a list of filenames, the lookup table is faster - // unfortunately we cannot do this if the filesystem is case sensitive as - // SQLite only supports ASCII compares - var p = expression.GetSimpleList(); - var m_filenamestable = "Filenames-" + m_temptabsetguid; - await cmd.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TABLE ""{m_filenamestable}"" ( - ""Path"" TEXT NOT NULL - ) - "); - cmd.SetCommandAndParameters($@" - INSERT INTO ""{m_filenamestable}"" (""Path"") - VALUES (@Path) - "); - - foreach (var s in p) - { - await cmd.SetParameterValue("@Path", s) - .ExecuteNonQueryAsync(); - } - - var c = await cmd.SetCommandAndParameters($@" - INSERT INTO ""{m_tempfiletable}"" ( - ""ID"", - ""Path"", - ""BlocksetID"", - ""MetadataID"", - ""DataVerified"" - ) - SELECT - ""File"".""ID"", - ""File"".""Path"", - ""File"".""BlocksetID"", - ""File"".""MetadataID"", - 0 - FROM - ""File"", - ""FilesetEntry"" - WHERE - ""File"".""ID"" = ""FilesetEntry"".""FileID"" - AND ""FilesetEntry"".""FilesetID"" = @FilesetId - AND ""Path"" IN ( - SELECT DISTINCT ""Path"" - FROM ""{m_filenamestable}"" - ) - ") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync(); - - if (c != p.Length && c != 0) - { - var sb = new StringBuilder(); - sb.AppendLine(); - - cmd.SetCommandAndParameters($@" - SELECT ""Path"" - FROM ""{m_filenamestable}"" - WHERE ""Path"" NOT IN ( - SELECT ""Path"" - FROM ""{m_tempfiletable}"" - ) - "); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) - sb.AppendLine(rd.ConvertValueToString(0)); - - cmd.SetCommandAndParameters(@" - SELECT ""Timestamp"" - FROM ""Fileset"" - WHERE ""ID"" = @FilesetId - ") - .SetParameterValue("@FilesetId", filesetId); - var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); - - Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); - cmd.Parameters.Clear(); - } - - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" "); - - using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset")) - await m_rtr.CommitAsync(); - cmd.SetTransaction(m_rtr); - } - else - { - // Restore but filter elements based on the filter expression - // If this is too slow, we could add a special handler for wildcard searches too - cmd.SetCommandAndParameters(@" - SELECT - ""File"".""ID"", - ""File"".""Path"", - ""File"".""BlocksetID"", - ""File"".""MetadataID"" - FROM - ""File"", - ""FilesetEntry"" - WHERE - ""File"".""ID"" = ""FilesetEntry"".""FileID"" - AND ""FilesetID"" = @FilesetId - ") - .SetParameterValue("@FilesetId", filesetId); - - object[] values = new object[4]; - using var cmd2 = m_connection.CreateCommand($@" - INSERT INTO ""{m_tempfiletable}"" ( - ""ID"", - ""Path"", - ""BlocksetID"", - ""MetadataID"", - ""DataVerified"" - ) - VALUES ( - @ID, - @Path, - @BlocksetID, - @MetadataID, - 0 - ) - "); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - { - rd.GetValues(values); - var path = values[1] as string; - if (path != null && FilterExpression.Matches(filter, path.ToString()!)) - { - await cmd2.SetParameterValue("@ID", values[0]) - .SetParameterValue("@Path", values[1]) - .SetParameterValue("@BlocksetID", values[2]) - .SetParameterValue("@MetadataID", values[3]) - .ExecuteNonQueryAsync(); - } - } - } - - //creating indexes after insertion is much faster - await cmd.ExecuteNonQueryAsync($@" - CREATE INDEX ""{m_tempfiletable}_ID"" - ON ""{m_tempfiletable}"" (""ID"") - "); - await cmd.ExecuteNonQueryAsync($@" - CREATE INDEX ""{m_tempfiletable}_TargetPath"" - ON ""{m_tempfiletable}"" (""TargetPath"") - "); - await cmd.ExecuteNonQueryAsync($@" - CREATE INDEX ""{m_tempfiletable}_Path"" - ON ""{m_tempfiletable}"" (""Path"") - "); - - cmd.SetCommandAndParameters($@" - SELECT - COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), - SUM(""Blockset"".""Length"") - FROM - ""{m_tempfiletable}"", - ""Blockset"" - WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" - "); - using (var rd = await cmd.ExecuteReaderAsync()) - { - var filecount = 0L; - var filesize = 0L; - - if (await rd.ReadAsync()) - { - filecount = rd.ConvertValueToInt64(0, 0); - filesize = rd.ConvertValueToInt64(1, 0); - } - - if (filecount > 0) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreTargetFileCount", "Needs to restore {0} files ({1})", filecount, Library.Utility.Utility.FormatSizeString(filesize)); - return new Tuple(filecount, filesize); - } - } - } - - await m_rtr.CommitAsync(); - } - - return new Tuple(0, 0); - } - - public async Task GetFirstPath() - { - using var cmd = m_connection.CreateCommand($@" - SELECT ""Path"" - FROM ""{m_tempfiletable}"" - ORDER BY LENGTH(""Path"") DESC - LIMIT 1 - ") - .SetTransaction(m_rtr); - - var v0 = await cmd.ExecuteScalarAsync(); - if (v0 == null || v0 == DBNull.Value) - return null; - - await m_rtr.CommitAsync(); - - return v0.ToString(); - } - - public async Task GetLargestPrefix() - { - using var cmd = m_connection.CreateCommand($@" - SELECT ""Path"" - FROM ""{m_tempfiletable}"" - ORDER BY LENGTH(""Path"") DESC - LIMIT 1 - ") - .SetTransaction(m_rtr); - - var v0 = await cmd.ExecuteScalarAsync(); - var maxpath = ""; - if (v0 != null && v0 != DBNull.Value) - maxpath = v0.ToString()!; - - var dirsep = Util.GuessDirSeparator(maxpath); - - var filecount = await cmd.ExecuteScalarInt64Async($@" - SELECT COUNT(*) - FROM ""{m_tempfiletable}"" - ", -1); - var foundfiles = -1L; - - //TODO: Handle FS case-sensitive? - cmd.SetCommandAndParameters($@" - SELECT COUNT(*) - FROM ""{m_tempfiletable}"" - WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix - "); - - while (filecount != foundfiles && maxpath.Length > 0) - { - var mp = Util.AppendDirSeparator(maxpath, dirsep); - foundfiles = await cmd.SetParameterValue("@PrefixLength", mp.Length) - .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64Async(-1); - - if (filecount != foundfiles) - { - var oldlen = maxpath.Length; - - var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); - maxpath = maxpath.Substring(0, lix + 1); - if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen) - maxpath = ""; - } - } - - await m_rtr.CommitAsync(); - - return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); - } - - public async Task SetTargetPaths(string largest_prefix, string destination) - { - var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? await GetFirstPath() : largest_prefix); - - using var cmd = m_connection.CreateCommand() - .SetTransaction(m_rtr); - if (string.IsNullOrEmpty(destination)) - { - //The string fixing here is meant to provide some non-random - // defaults when restoring cross OS, e.g. backup on Linux, restore on Windows - //This is mostly meaningless, and the user really should use --restore-path - - if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") - { - // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder - await cmd.ExecuteNonQueryAsync($@" - UPDATE ""{m_tempfiletable}"" - SET ""Targetpath"" = - CASE - WHEN SUBSTR(""Path"", 2, 1) == ':' - THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) - ELSE ""Path"" - END - "); - await cmd.ExecuteNonQueryAsync($@" - UPDATE ""{m_tempfiletable}"" - SET ""Targetpath"" = - CASE - WHEN SUBSTR(""Path"", 1, 2) == '\\' - THEN '\\' || SUBSTR(""Path"", 2) - ELSE ""Path"" - END - "); - - } - else if (OperatingSystem.IsWindows() && dirsep == "/") - { - // For Linux -> Win, we use the temporary folder's drive as the root path - await cmd.SetCommandAndParameters($@" - UPDATE ""{m_tempfiletable}"" - SET ""Targetpath"" = - CASE - WHEN SUBSTR(""Path"", 1, 1) == '/' - THEN @Path || SUBSTR(""Path"", 2) - ELSE ""Path"" - END - ") - .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) - .ExecuteNonQueryAsync(); - } - else - { - // Same OS, just use the path directly - await cmd.ExecuteNonQueryAsync($@" - UPDATE ""{m_tempfiletable}"" - SET ""Targetpath"" = ""Path"" - "); - } - } - else - { - if (string.IsNullOrEmpty(largest_prefix)) - { - //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) - - // We use the format / / - await cmd.ExecuteNonQueryAsync($@" - UPDATE ""{m_tempfiletable}"" - SET ""TargetPath"" = - CASE - WHEN SUBSTR(""Path"", 2, 1) == ':' - THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) - ELSE ""Path"" - END - "); - - // For UNC paths, we use \\server\folder -> / / - await cmd.ExecuteNonQueryAsync($@" - UPDATE ""{m_tempfiletable}"" - SET ""TargetPath"" = - CASE - WHEN SUBSTR(""Path"", 1, 2) == '\\' - THEN SUBSTR(""Path"", 2) - ELSE ""TargetPath"" - END - "); - } - else - { - largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); - await cmd.SetCommandAndParameters($@" - UPDATE ""{m_tempfiletable}"" - SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength) - ") - .SetParameterValue("@PrefixLength", largest_prefix.Length + 1) - .ExecuteNonQueryAsync(); - } - } - - // Cross-os path remapping support - if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") - // For Win paths on Linux - await cmd.ExecuteNonQueryAsync($@" - UPDATE ""{m_tempfiletable}"" - SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/') - "); - else if (OperatingSystem.IsWindows() && dirsep == "/") - // For Linux paths on Windows - await cmd.ExecuteNonQueryAsync($@" - UPDATE ""{m_tempfiletable}"" - SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\') - "); - - if (!string.IsNullOrEmpty(destination)) - { - // Paths are now relative with target-os naming system - // so we prefix them with the target path - await cmd.SetCommandAndParameters($@" - UPDATE ""{m_tempfiletable}"" - SET ""TargetPath"" = @Destination || ""TargetPath"" - ") - .SetParameterValue("@Destination", Util.AppendDirSeparator(destination)) - .ExecuteNonQueryAsync(); - } - - await m_rtr.CommitAsync(); - } - - public async Task FindMissingBlocks(bool skipMetadata) - { - using var cmd = m_connection.CreateCommand() - .SetTransaction(m_rtr); - var p1 = await cmd.ExecuteNonQueryAsync($@" - INSERT INTO ""{m_tempblocktable}"" ( - ""FileID"", - ""Index"", - ""Hash"", - ""Size"", - ""Restored"", - ""Metadata"", - ""VolumeId"", - ""BlockId"" - ) - SELECT DISTINCT - ""{m_tempfiletable}"".""ID"", - ""BlocksetEntry"".""Index"", - ""Block"".""Hash"", - ""Block"".""Size"", - 0, - 0, - ""Block"".""VolumeID"", - ""Block"".""ID"" - FROM - ""{m_tempfiletable}"", - ""BlocksetEntry"", - ""Block"" - WHERE - ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - "); - - var p2 = 0; - if (!skipMetadata) - p2 = await cmd.ExecuteNonQueryAsync($@" - INSERT INTO ""{m_tempblocktable}"" ( - ""FileID"", - ""Index"", - ""Hash"", - ""Size"", - ""Restored"", - ""Metadata"", - ""VolumeId"", - ""BlockId"" - ) - SELECT DISTINCT - ""{m_tempfiletable}"".""ID"", - ""BlocksetEntry"".""Index"", - ""Block"".""Hash"", - ""Block"".""Size"", - 0, - 1, - ""Block"".""VolumeID"", - ""Block"".""ID"" - FROM - ""{m_tempfiletable}"", - ""BlocksetEntry"", - ""Block"", - ""Metadataset"" - WHERE - ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" - AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - "); - - //creating indexes after insertion is much faster - await cmd.ExecuteNonQueryAsync($@" - CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" - ON ""{m_tempblocktable}"" (""Hash"", ""Size"") - "); - // better suited to speed up commit on UpdateBlocks - await cmd.ExecuteNonQueryAsync($@" - CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" - ON ""{m_tempblocktable}"" (""FileId"", ""Index"") - "); - - var size = await cmd.ExecuteScalarInt64Async($@" - SELECT SUM(""Size"") - FROM ""{m_tempblocktable}"" - ", 0); - Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); - - await m_rtr.CommitAsync(); - } - - public async Task UpdateTargetPath(long ID, string newname) - { - using var cmd = m_connection.CreateCommand($@" - UPDATE ""{m_tempfiletable}"" - SET ""TargetPath"" = @TargetPath - WHERE ""ID"" = @ID - "); - await cmd.SetTransaction(m_rtr) - .SetParameterValue("@TargetPath", newname) - .SetParameterValue("@ID", ID) - .ExecuteNonQueryAsync(); - - await m_rtr.CommitAsync(); - } - - public interface IExistingFileBlock - { - string Hash { get; } - long Index { get; } - long Size { get; } - } - - public interface IExistingFile - { - string TargetPath { get; } - string TargetHash { get; } - long TargetFileID { get; } - long Length { get; } - IAsyncEnumerable Blocks(); - } - - public interface IBlockSource - { - string Path { get; } - long Offset { get; } - bool IsMetadata { get; } - } - - public interface IBlockDescriptor - { - string Hash { get; } - long Size { get; } - long Offset { get; } - long Index { get; } - bool IsMetadata { get; } - IAsyncEnumerable BlockSources(); - } - - public interface ILocalBlockSource - { - string TargetPath { get; } - long TargetFileID { get; } - IAsyncEnumerable Blocks(); - } - - /// - /// Interface for an object describing a file to restore. - /// - public interface IFileToRestore - { - string Path { get; } - string Hash { get; } - long Length { get; } - } - - public interface IPatchBlock - { - long Offset { get; } - long Size { get; } - string Key { get; } - } - - public interface IVolumePatch - { - string Path { get; } - long FileID { get; } - IAsyncEnumerable Blocks(); - } - - private class ExistingFile : IExistingFile - { - private readonly SqliteDataReader m_reader; - - public ExistingFile(SqliteDataReader rd) { m_reader = rd; HasMore = true; } - - public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } - public string TargetHash { get { return m_reader.ConvertValueToString(1) ?? ""; } } - public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } } - public long Length { get { return m_reader.ConvertValueToInt64(3); } } - - public bool HasMore { get; private set; } - - private class ExistingFileBlock : IExistingFileBlock - { - private readonly SqliteDataReader m_reader; - - public ExistingFileBlock(SqliteDataReader rd) { m_reader = rd; } - - public string Hash { get { return m_reader.ConvertValueToString(4) ?? ""; } } - public long Index { get { return m_reader.ConvertValueToInt64(5); } } - public long Size { get { return m_reader.ConvertValueToInt64(6); } } - } - - public async IAsyncEnumerable Blocks() - { - string p = TargetPath; - while (HasMore && p == TargetPath) - { - yield return new ExistingFileBlock(m_reader); - HasMore = await m_reader.ReadAsync(); - } - } - - public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename) - { - using var cmd = db.Connection.CreateCommand($@" - SELECT - ""{tablename}"".""TargetPath"", - ""Blockset"".""FullHash"", - ""{tablename}"".""ID"", - ""Blockset"".""Length"", - ""Block"".""Hash"", - ""BlocksetEntry"".""Index"", - ""Block"".""Size"" - FROM - ""{tablename}"", - ""Blockset"", - ""BlocksetEntry"", - ""Block"" - WHERE - ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" - AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" - AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - ORDER BY - ""{tablename}"".""TargetPath"", - ""BlocksetEntry"".""Index"" - ") - .SetTransaction(db.Transaction); - using var rd = await cmd.ExecuteReaderAsync(); - if (await rd.ReadAsync()) - { - var more = true; - while (more) - { - var f = new ExistingFile(rd); - string current = f.TargetPath; - yield return f; - - more = f.HasMore; - while (more && current == f.TargetPath) - more = await rd.ReadAsync(); - } - } - await db.Transaction.CommitAsync(); - } - } - - public IAsyncEnumerable GetExistingFilesWithBlocks() - { - if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) - throw new InvalidOperationException("No temporary file table set up for this restore."); - return ExistingFile.GetExistingFilesWithBlocks(this, m_tempfiletable); - } - - private class LocalBlockSource : ILocalBlockSource - { - private class BlockDescriptor : IBlockDescriptor - { - private class BlockSource : IBlockSource - { - private readonly SqliteDataReader m_reader; - public BlockSource(SqliteDataReader rd) { m_reader = rd; } - - public string Path { get { return m_reader.ConvertValueToString(6) ?? ""; } } - public long Offset { get { return m_reader.ConvertValueToInt64(7); } } - public bool IsMetadata { get { return false; } } - } - - private readonly SqliteDataReader m_reader; - public BlockDescriptor(SqliteDataReader rd) { m_reader = rd; HasMore = true; } - - private string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } - - public string Hash { get { return m_reader.ConvertValueToString(2) ?? ""; } } - public long Offset { get { return m_reader.ConvertValueToInt64(3); } } - public long Index { get { return m_reader.ConvertValueToInt64(4); } } - public long Size { get { return m_reader.ConvertValueToInt64(5); } } - public bool IsMetadata { get { return !(m_reader.ConvertValueToInt64(9) == 0); } } - - public bool HasMore { get; private set; } - - public async IAsyncEnumerable BlockSources() - { - var p = TargetPath; - var h = Hash; - var s = Size; - - while (HasMore && p == TargetPath && h == Hash && s == Size) - { - yield return new BlockSource(m_reader); - HasMore = await m_reader.ReadAsync(); - } - } - - } - - private readonly SqliteDataReader m_reader; - public LocalBlockSource(SqliteDataReader rd) { m_reader = rd; HasMore = true; } - - public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } - public long TargetFileID { get { return m_reader.ConvertValueToInt64(1); } } - - public bool HasMore { get; private set; } - - public async IAsyncEnumerable Blocks() - { - var p = TargetPath; - while (HasMore && p == TargetPath) - { - var c = new BlockDescriptor(m_reader); - var h = c.Hash; - var s = c.Size; - - yield return c; - - HasMore = c.HasMore; - while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) - HasMore = await m_reader.ReadAsync(); - } - } - - public static async IAsyncEnumerable GetFilesAndSourceBlocks(LocalDatabase db, string filetablename, string blocktablename, long blocksize, bool skipMetadata) - { - // TODO: Skip metadata as required - // Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows - using var cmd = db.Connection.CreateCommand($@" - SELECT DISTINCT - ""A"".""TargetPath"", - ""A"".""ID"", - ""B"".""Hash"", - (""B"".""Index"" * {blocksize}), - ""B"".""Index"", - ""B"".""Size"", - ""C"".""Path"", - (""D"".""Index"" * {blocksize}), - ""E"".""Size"", - ""B"".""Metadata"" - FROM - ""{filetablename}"" ""A"", - ""{blocktablename}"" ""B"", - ""File"" ""C"", - ""BlocksetEntry"" ""D"", - ""Block"" E - WHERE - ""A"".""ID"" = ""B"".""FileID"" - AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" - AND ""D"".""BlockID"" = ""E"".""ID"" - AND ""B"".""Hash"" = ""E"".""Hash"" - AND ""B"".""Size"" = ""E"".""Size"" - AND ""B"".""Restored"" = 0 - ORDER BY - ""A"".""TargetPath"", - ""B"".""Index"" - ") - .SetTransaction(db.Transaction); - using var rd = await cmd.ExecuteReaderAsync(); - var more = await rd.ReadAsync(); - - while (more) - { - var f = new LocalBlockSource(rd); - string current = f.TargetPath; - yield return f; - - more = f.HasMore; - while (more && current == f.TargetPath) - more = await rd.ReadAsync(); - } - - await db.Transaction.CommitAsync(); - } - } - - public IAsyncEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize) - { - if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) - throw new InvalidOperationException("No temporary file table set up for this restore."); - return LocalBlockSource.GetFilesAndSourceBlocks(this, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata); - } - - public async IAsyncEnumerable GetMissingVolumes() - { - using var cmd = m_connection.CreateCommand($@" - SELECT - ""RV"".""Name"", - ""RV"".""Hash"", - ""RV"".""Size"", - ""BB"".""MaxIndex"" - FROM ""RemoteVolume"" ""RV"" - INNER JOIN ( - SELECT - ""TB"".""VolumeID"", - MAX(""TB"".""Index"") as ""MaxIndex"" - FROM ""{m_tempblocktable}"" ""TB"" - WHERE ""TB"".""Restored"" = 0 - GROUP BY ""TB"".""VolumeID"" - ) as ""BB"" - ON ""RV"".""ID"" = ""BB"".""VolumeID"" - ORDER BY ""BB"".""MaxIndex"" - ") - .SetTransaction(m_rtr); - - // Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore. - // If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file. - // If we manage to get the volumes in an order restoring front blocks first, this can save time. - // An optimal algorithm would build a dependency net with cycle resolution to find the best near topological - // order of volumes, but this is a bit too fancy here. - // We will just put a very simple heuristic to work, that will try to prefer volumes containing lower block indexes: - // We just order all volumes by the maximum block index they contain. This query is slow, but should be worth the effort. - // Now it is likely to restore all files from front to back. Large files will always be done last. - // One could also use like the average block number in a volume, that needs to be measured. - - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - { - yield return new RemoteVolume( - rd.ConvertValueToString(0), - rd.ConvertValueToString(1), - rd.ConvertValueToInt64(2, -1) - ); - } - - await m_rtr.CommitAsync(); - } - - public interface IFilesAndMetadata : IDisposable - { - IAsyncEnumerable FilesWithMissingBlocks(); - IAsyncEnumerable MetadataWithMissingBlocks(); - } - - private class FilesAndMetadata : IFilesAndMetadata - { - private string m_tmptable = null!; - private string m_filetablename = null!; - private string m_blocktablename = null!; - private long m_blocksize; - - private LocalDatabase m_db = null!; - - [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] - public FilesAndMetadata(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) - { - throw new NotImplementedException("Use CreateAsync instead of the constructor"); - } - - private FilesAndMetadata() { } - - public static async Task CreateAsync(LocalDatabase db, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) - { - var fam = new FilesAndMetadata() - { - m_db = db, - m_filetablename = filetablename, - m_blocktablename = blocktablename, - m_blocksize = blocksize, - }; - - using var c = db.Connection.CreateCommand() - .SetTransaction(db.Transaction); - fam.m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - await c.ExecuteNonQueryAsync($@" - CREATE TEMPORARY TABLE ""{fam.m_tmptable}"" ( - ""Hash"" TEXT NOT NULL, - ""Size"" INTEGER NOT NULL - ) - "); - - c.SetCommandAndParameters($@" - INSERT INTO ""{fam.m_tmptable}"" ( - ""Hash"", - ""Size"" - ) - VALUES ( - @Hash, - @Size) - "); - foreach (var s in curvolume.Blocks) - { - await c.SetParameterValue("@Hash", s.Key) - .SetParameterValue("@Size", s.Value) - .ExecuteNonQueryAsync(); - } - - // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks - - await db.Transaction.CommitAsync(); - - return fam; - } - - public void Dispose() - { - DisposeAsync().Await(); - } - - public async Task DisposeAsync() - { - if (m_tmptable != null) - { - using var c = m_db.Connection.CreateCommand(@"DROP TABLE IF EXISTS ""{m_tmptable}""") - .SetTransaction(m_db.Transaction); - await c.ExecuteNonQueryAsync(); - await m_db.Transaction.CommitAsync(); - } - } - - private class VolumePatch : IVolumePatch - { - private class PatchBlock : IPatchBlock - { - private readonly SqliteDataReader m_reader; - public PatchBlock(SqliteDataReader rd) { m_reader = rd; } - - public long Offset { get { return m_reader.ConvertValueToInt64(2); } } - public long Size { get { return m_reader.ConvertValueToInt64(3); } } - public string Key { get { return m_reader.ConvertValueToString(4) ?? ""; } } - } - - private readonly SqliteDataReader m_reader; - public VolumePatch(SqliteDataReader rd) { m_reader = rd; HasMore = true; } - - public string Path { get { return m_reader.ConvertValueToString(0) ?? ""; } } - public long FileID { get { return m_reader.ConvertValueToInt64(1); } } - public bool HasMore { get; private set; } - - public async IAsyncEnumerable Blocks() - { - string p = Path; - while (HasMore && p == Path) - { - yield return new PatchBlock(m_reader); - HasMore = await m_reader.ReadAsync(); - } - } - } - - public async IAsyncEnumerable FilesWithMissingBlocks() - { - // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - using var cmd = m_db.Connection.CreateCommand($@" - SELECT DISTINCT - ""A"".""TargetPath"", - ""BB"".""FileID"", - (""BB"".""Index"" * {m_blocksize}), - ""BB"".""Size"", - ""BB"".""Hash"" - FROM - ""{m_filetablename}"" ""A"", - ""{m_blocktablename}"" ""BB"" - WHERE - ""A"".""ID"" = ""BB"".""FileID"" - AND ""BB"".""Restored"" = 0 - AND ""BB"".""Metadata"" = {"0"} - AND ""BB"".""ID"" IN ( - SELECT ""B"".""ID"" - FROM - ""{m_blocktablename}"" ""B"", - ""{m_tmptable}"" ""C"" - WHERE - ""B"".""Hash"" = ""C"".""Hash"" - AND ""B"".""Size"" = ""C"".""Size"" - ) - ORDER BY - ""A"".""TargetPath"", - ""BB"".""Index"" - ") - .SetTransaction(m_db.Transaction); - - using var rd = await cmd.ExecuteReaderAsync(); - if (await rd.ReadAsync()) - { - var more = true; - while (more) - { - var f = new VolumePatch(rd); - var current = f.Path; - yield return f; - - more = f.HasMore; - while (more && current == f.Path) - more = await rd.ReadAsync(); - } - } - await m_db.Transaction.CommitAsync(); - } - - public async IAsyncEnumerable MetadataWithMissingBlocks() - { - // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - using var cmd = m_db.Connection.CreateCommand($@" - SELECT DISTINCT - ""A"".""TargetPath"", - ""BB"".""FileID"", - (""BB"".""Index"" * {m_blocksize}), - ""BB"".""Size"", - ""BB"".""Hash"" - FROM - ""{m_filetablename}"" ""A"", - ""{m_blocktablename}"" ""BB"" - WHERE - ""A"".""ID"" = ""BB"".""FileID"" - AND ""BB"".""Restored"" = 0 - AND ""BB"".""Metadata"" = {"1"} - AND ""BB"".""ID"" IN ( - SELECT ""B"".""ID"" - FROM - ""{m_blocktablename}"" ""B"", - ""{m_tmptable}"" ""C"" - WHERE - ""B"".""Hash"" = ""C"".""Hash"" - AND ""B"".""Size"" = ""C"".""Size"" - ) - ORDER BY - ""A"".""TargetPath"", - ""BB"".""Index"" - ") - .SetTransaction(m_db.Transaction); - - using var rd = await cmd.ExecuteReaderAsync(); - if (await rd.ReadAsync()) - { - var more = true; - while (more) - { - var f = new VolumePatch(rd); - string current = f.Path; - yield return f; - - more = f.HasMore; - while (more && current == f.Path) - more = await rd.ReadAsync(); - } - } - } - } - - public async Task GetMissingBlockData(BlockVolumeReader curvolume, long blocksize) - { - if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) - throw new InvalidOperationException("No temporary file table set up for this restore."); - return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume); - } - - /// - /// Returns a connection from the connection pool. - /// - /// A connection from the connection pool. - public async Task<(SqliteConnection, ReusableTransaction)> GetConnectionFromPool() - { - if (!m_connection_pool.TryTake(out var entry)) - { - var connection = await SQLiteLoader.LoadConnectionAsync(); - connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared"; - await connection.OpenAsync(); - await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize); - var transaction = new ReusableTransaction(connection); - - return (connection, transaction); - } - - return entry; - } - - private class FileToRestore : IFileToRestore - { - public string Path { get; private set; } - public string Hash { get; private set; } - public long Length { get; private set; } - - public FileToRestore(long id, string path, string hash, long length) - { - Path = path; - Hash = hash; - Length = length; - } - } - - public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified) - { - using var cmd = m_connection.CreateCommand($@" - SELECT - ""{m_tempfiletable}"".""ID"", - ""{m_tempfiletable}"".""TargetPath"", - ""Blockset"".""FullHash"", - ""Blockset"".""Length"" - FROM - ""{m_tempfiletable}"", - ""Blockset"" - WHERE - ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" - AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified - ") - .SetTransaction(m_rtr) - .SetParameterValue("@Verified", !onlyNonVerified); - - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - yield return new FileToRestore( - rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); - - await m_rtr.CommitAsync(); - } - - /// - /// Returns a list of files and symlinks to restore. - /// - /// A list of files and symlinks to restore. - public async IAsyncEnumerable GetFilesAndSymlinksToRestore() - { - // Order by length descending, so that larger files are restored first. - using var cmd = m_connection.CreateCommand($@" - SELECT - F.ID, - F.Path, - F.TargetPath, - IFNULL(B.FullHash, ''), - IFNULL(B.Length, 0) AS ""Length"", - F.BlocksetID - FROM ""{m_tempfiletable}"" F - LEFT JOIN Blockset B - ON F.BlocksetID = B.ID - WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID} - ORDER BY ""Length"" DESC - ") - .SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); - - await m_rtr.CommitAsync(); - } - - /// - /// Returns a list of folders to restore. Used to restore folder metadata. - /// - /// A list of folders to restore. - public async IAsyncEnumerable GetFolderMetadataToRestore() - { - using var cmd = m_connection.CreateCommand(); - cmd.SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync($@" - SELECT - F.ID, - '', - F.TargetPath, - '', - 0, - {FOLDER_BLOCKSET_ID} - FROM ""{m_tempfiletable}"" F - WHERE - F.BlocksetID = {FOLDER_BLOCKSET_ID} - AND F.MetadataID IS NOT NULL - AND F.MetadataID >= 0 - "); - - while (await rd.ReadAsync()) - yield return new FileRequest( - rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1), - rd.ConvertValueToString(2), - rd.ConvertValueToString(3), - rd.ConvertValueToInt64(4), - rd.ConvertValueToInt64(5) - ); - } - - /// - /// Returns a list of blocks and their volume IDs. Used by the to keep track of blocks and volumes to automatically evict them from the respective caches. - /// - /// Flag indicating whether the returned blocks should exclude the metadata blocks. - /// A list of tuples containing the block ID and the volume ID of the block. - public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) - { - var metadata_query = skipMetadata ? "" : $@" - UNION ALL - SELECT - ""Block"".""ID"", - ""Block"".""VolumeID"" - FROM ""{m_tempfiletable}"" - INNER JOIN ""Metadataset"" - ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" - INNER JOIN ""BlocksetEntry"" - ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - "; - using var cmd = Connection.CreateCommand($@" - SELECT - ""Block"".""ID"", - ""Block"".""VolumeID"" - FROM ""BlocksetEntry"" - INNER JOIN ""{m_tempfiletable}"" - ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" - INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - {metadata_query} - ") - .SetTransaction(m_rtr); - using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - yield return ( - reader.ConvertValueToInt64(0), - reader.ConvertValueToInt64(1) - ); - - await m_rtr.CommitAsync(); - } - - /// - /// Returns a list of for the given blockset ID. It is used by the to restore the blocks of a file. - /// - /// The BlocksetID of the file. - /// A list of needed to restore the given file. - public async IAsyncEnumerable GetBlocksFromFile(long blocksetID) - { - var (connection, transaction) = await GetConnectionFromPool(); - try - { - - using var cmd = connection.CreateCommand(@$" - SELECT - ""Block"".""ID"", - ""Block"".""Hash"", - ""Block"".""Size"", - ""Block"".""VolumeID"" - FROM ""BlocksetEntry"" - INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID - ") - .SetTransaction(transaction) - .SetParameterValue("@BlocksetID", blocksetID); - - using var reader = await cmd.ExecuteReaderAsync(); - for (long i = 0; await reader.ReadAsync(); i++) - yield return new BlockRequest( - reader.ConvertValueToInt64(0), - i, - reader.ConvertValueToString(1), - reader.ConvertValueToInt64(2), - reader.ConvertValueToInt64(3), - false - ); - } - finally - { - // Return the connection to the pool - m_connection_pool.Add((connection, transaction)); - } - } - - /// - /// Returns a list of for the metadata blocks of the given file. It is used by the to restore the metadata of a file. - /// - /// The ID of the file. - /// A list of needed to restore the metadata of the given file. - public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID) - { - var (connection, transaction) = await GetConnectionFromPool(); - try - { - using var cmd = connection.CreateCommand($@" - SELECT - ""Block"".""ID"", - ""Block"".""Hash"", - ""Block"".""Size"", - ""Block"".""VolumeID"" - FROM ""File"" - INNER JOIN ""Metadataset"" - ON ""File"".""MetadataID"" = ""Metadataset"".""ID"" - INNER JOIN ""BlocksetEntry"" - ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - WHERE ""File"".""ID"" = @FileID - ") - .SetTransaction(transaction) - .SetParameterValue("@FileID", fileID); - - using var reader = await cmd.ExecuteReaderAsync(); - for (long i = 0; await reader.ReadAsync(); i++) - { - yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); - } - } - finally - { - // Return the connection to the pool - m_connection_pool.Add((connection, transaction)); - } - } - - /// - /// Returns the volume information for the given volume ID. It is used by the to get the volume information for the given volume ID. - /// - /// The ID of the volume. - /// A tuple containing the name, size, and hash of the volume. - public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) - { - using var cmd = m_connection.CreateCommand(@" - SELECT - Name, - Size, - Hash - FROM RemoteVolume - WHERE ID = @VolumeID - ") - .SetTransaction(m_rtr) - .SetParameterValue("@VolumeID", VolumeID); - - using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - yield return (reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), reader.ConvertValueToString(2) ?? ""); - - await m_rtr.CommitAsync(); - } - - public async Task DropRestoreTable() - { - using var cmd = m_connection.CreateCommand() - .SetTransaction(m_rtr); - - if (m_tempfiletable != null) - try - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"""); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_tempfiletable = null; } - - if (m_tempblocktable != null) - try - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"""); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_tempblocktable = null; } - - if (m_latestblocktable != null) - try - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"""); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_latestblocktable = null; } - - if (m_fileprogtable != null) - try - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"""); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_fileprogtable = null; } - - if (m_totalprogtable != null) - try - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"""); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_totalprogtable = null; } - - if (m_filesnewlydonetable != null) - try - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"""); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_filesnewlydonetable = null; } - - await m_rtr.CommitAsync(); - } - - public interface IBlockMarker : IDisposable - { - Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata); - Task SetAllBlocksMissing(long targetfileid); - Task SetAllBlocksRestored(long targetfileid, bool includeMetadata); - Task SetFileDataVerified(long targetfileid); - Task CommitAsync(); - Task UpdateProcessed(IOperationProgressUpdater writer); - } - - /// - /// A new implementation of IBlockMarker, marking the blocks directly in the blocks table as restored - /// and reading statistics about progress from DB (kept up-to-date by triggers). - /// There is no negative influence on performance, esp. since the block table is temporary anyway. - /// - private class DirectBlockMarker : IBlockMarker - { - private SqliteCommand m_insertblockCommand = null!; - private SqliteCommand m_resetfileCommand = null!; - private SqliteCommand m_updateAsRestoredCommand = null!; - private SqliteCommand m_updateFileAsDataVerifiedCommand = null!; - private SqliteCommand m_statUpdateCommand = null!; - private LocalDatabase m_db = null!; - private bool m_hasUpdates = false; - - private string m_blocktablename = null!; - private string m_filetablename = null!; - - [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] - public DirectBlockMarker(SqliteConnection connection, string blocktablename, string filetablename, string statstablename) - { - throw new NotImplementedException("Use CreateAsync instead of the constructor"); - } - - private DirectBlockMarker() { } - - public static async Task CreateAsync(LocalDatabase db, string blocktablename, string filetablename, string statstablename) - { - var dbm = new DirectBlockMarker() - { - m_db = db, - m_blocktablename = blocktablename, - m_filetablename = filetablename, - - m_insertblockCommand = await db.Connection.CreateCommandAsync($@" - UPDATE ""{blocktablename}"" SET ""Restored"" = 1 - WHERE - ""FileID"" = @TargetFileId - AND ""Index"" = @Index - AND ""Hash"" = @Hash - AND ""Size"" = @Size - AND ""Metadata"" = @Metadata - AND ""Restored"" = 0 - "), - - m_resetfileCommand = await db.Connection.CreateCommandAsync($@" - UPDATE ""{blocktablename}"" - SET ""Restored"" = 0 - WHERE ""FileID"" = @TargetFileId - "), - - m_updateAsRestoredCommand = await db.Connection.CreateCommandAsync($@" - UPDATE ""{blocktablename}"" - SET ""Restored"" = 1 - WHERE - ""FileID"" = @TargetFileId - AND ""Metadata"" <= @Metadata - "), - - m_updateFileAsDataVerifiedCommand = await db.Connection.CreateCommandAsync($@" - UPDATE ""{filetablename}"" - SET ""DataVerified"" = 1 - WHERE ""ID"" = @TargetFileId - "), - - m_statUpdateCommand = statstablename == null ? - // very slow fallback if stats tables were not created - await db.Connection.CreateCommandAsync($@" - SELECT - COUNT(DISTINCT ""FileID""), - SUM(""Size"") - FROM ""{blocktablename}"" - WHERE ""Restored"" = 1 - ") - : - // Fields in Stats: TotalFiles, TotalBlocks, TotalSize - // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored - await db.Connection.CreateCommandAsync($@" - SELECT - SUM(""FilesFullyRestored""), - SUM(""SizeRestored"") - FROM ""{statstablename}"" - ") - }; - - return dbm; - } - - public async Task UpdateProcessed(IOperationProgressUpdater updater) - { - if (!m_hasUpdates) - return; - - m_hasUpdates = false; - using var rd = await m_statUpdateCommand - .SetTransaction(m_db.Transaction) - .ExecuteReaderAsync(); - var filesprocessed = 0L; - var processedsize = 0L; - - if (rd.Read()) - { - filesprocessed += rd.ConvertValueToInt64(0, 0); - processedsize += rd.ConvertValueToInt64(1, 0); - } - - updater.UpdatefilesProcessed(filesprocessed, processedsize); - } - - public async Task SetAllBlocksMissing(long targetfileid) - { - m_hasUpdates = true; - m_resetfileCommand - .SetTransaction(m_db.Transaction) - .SetParameterValue("@TargetFileId", targetfileid); - - var r = await m_resetfileCommand.ExecuteNonQueryAsync(); - if (r <= 0) - throw new Exception("Unexpected reset result"); - } - - public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata) - { - m_hasUpdates = true; - m_updateAsRestoredCommand - .SetTransaction(m_db.Transaction) - .SetParameterValue("@TargetFileId", targetfileid) - .SetParameterValue("@Metadata", includeMetadata ? 1 : 0); - - var r = await m_updateAsRestoredCommand.ExecuteNonQueryAsync(); - if (r <= 0) - throw new Exception("Unexpected reset result"); - } - - public async Task SetFileDataVerified(long targetfileid) - { - m_hasUpdates = true; - m_updateFileAsDataVerifiedCommand - .SetTransaction(m_db.Transaction) - .SetParameterValue("@TargetFileId", targetfileid); - - var r = await m_updateFileAsDataVerifiedCommand.ExecuteNonQueryAsync(); - if (r != 1) - throw new Exception("Unexpected result when marking file as verified."); - } - - public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata) - { - m_hasUpdates = true; - m_insertblockCommand - .SetTransaction(m_db.Transaction) - .SetParameterValue("@TargetFileId", targetfileid) - .SetParameterValue("@Index", index) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .SetParameterValue("@Metadata", metadata); - - var r = await m_insertblockCommand.ExecuteNonQueryAsync(); - if (r != 1) - throw new Exception("Unexpected result when marking block."); - } - - public async Task CommitAsync() - { - using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) - await m_db.Transaction.CommitAsync(); - } - - public void Dispose() - { - m_insertblockCommand?.Dispose(); - m_resetfileCommand?.Dispose(); - m_updateAsRestoredCommand?.Dispose(); - m_updateFileAsDataVerifiedCommand?.Dispose(); - m_statUpdateCommand?.Dispose(); - } - } - - [Obsolete("Calling this constructor will throw an exception. Use CreateBlockMarkerAsync instead.")] - public IBlockMarker CreateBlockMarker() - { - throw new NotImplementedException("Use CreateBlockMarkerAsync instead of the constructor"); - } - - public async Task CreateBlockMarkerAsync() - { - if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) - throw new InvalidOperationException("No temporary file table set up for this restore."); - if (string.IsNullOrWhiteSpace(m_totalprogtable)) - throw new InvalidOperationException("No progress table set up for this restore."); - return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable); - } - - public override void Dispose() - { - DisposeAsync().Await(); - } - - public override async Task DisposeAsync() - { - await DisposePoolAsync(); - await DropRestoreTable(); - await base.DisposeAsync(); - } - - public async Task DisposePoolAsync() - { - foreach (var (connection, transaction) in m_connection_pool) - { - await transaction.DisposeAsync(); - await connection.CloseAsync(); - await connection.DisposeAsync(); - } - m_connection_pool.Clear(); - } - - public async IAsyncEnumerable GetTargetFolders() - { - using var cmd = m_connection.CreateCommand($@" - SELECT ""TargetPath"" - FROM ""{m_tempfiletable}"" - WHERE ""BlocksetID"" == @BlocksetID - ") - .SetTransaction(m_rtr) - .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); - - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - yield return rd.ConvertValueToString(0) ?? ""; - - await m_rtr.CommitAsync(); - } - - public interface IFastSource - { - string TargetPath { get; } - long TargetFileID { get; } - string SourcePath { get; } - IAsyncEnumerable Blocks(); - } - - public interface IBlockEntry - { - long Offset { get; } - long Size { get; } - long Index { get; } - string Hash { get; } - } - - private class FastSource : IFastSource - { - private class BlockEntry : IBlockEntry - { - private readonly SqliteDataReader m_rd; - private readonly long m_blocksize; - public BlockEntry(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } - public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } } - public long Index { get { return m_rd.ConvertValueToInt64(3); } } - public long Size { get { return m_rd.ConvertValueToInt64(5); } } - public string Hash { get { return m_rd.ConvertValueToString(4) ?? ""; } } - } - - private readonly SqliteDataReader m_rd; - private readonly long m_blocksize; - public FastSource(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } - public bool MoreData { get; private set; } - public string TargetPath { get { return m_rd.ConvertValueToString(0) ?? ""; } } - public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } - public string SourcePath { get { return m_rd.ConvertValueToString(1) ?? ""; } } - - public async IAsyncEnumerable Blocks() - { - var tid = TargetFileID; - - do - { - yield return new BlockEntry(m_rd, m_blocksize); - } while ((MoreData = await m_rd.ReadAsync()) && tid == TargetFileID); - } - } - - public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize) - { - using (var cmdReader = m_connection.CreateCommand()) - using (var cmd = m_connection.CreateCommand()) - { - cmdReader.SetTransaction(m_rtr); - cmd.SetTransaction(m_rtr); - cmd.SetCommandAndParameters($@" - UPDATE ""{m_tempfiletable}"" - SET ""LocalSourceExists"" = 1 - WHERE Path = @Path - "); - cmdReader.SetCommandAndParameters($@" - SELECT DISTINCT ""{m_tempfiletable}"".""Path"" - FROM ""{m_tempfiletable}"" - "); - using (var rd = await cmdReader.ExecuteReaderAsync()) - { - while (await rd.ReadAsync()) - { - var sourcepath = rd.ConvertValueToString(0); - if (SystemIO.IO_OS.FileExists(sourcepath)) - { - await cmd.SetParameterValue("@Path", sourcepath) - .ExecuteNonQueryAsync(); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); - } - } - } - - //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" - await cmd.ExecuteNonQueryAsync($@" - CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" - ON ""{m_tempfiletable}"" (""LocalSourceExists"") - "); - - await m_rtr.CommitAsync(); - } - - m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; - - var whereclause = $@" - ""{m_tempfiletable}"".""LocalSourceExists"" = 1 - AND ""{m_tempblocktable}"".""Restored"" = 0 - AND ""{m_tempblocktable}"".""Metadata"" = 0 - AND ""{m_tempfiletable}"".""TargetPath"" != ""{m_tempfiletable}"".""Path"" - "; - - var latestBlocksetIds = $@" - SELECT - ""File"".""Path"" AS ""PATH"", - ""File"".""BlocksetID"" AS ""BlocksetID"", - MAX(""Fileset"".""Timestamp"") AS ""Timestamp"" - FROM - ""File"", - ""FilesetEntry"", - ""Fileset"" - WHERE - ""File"".""ID"" = ""FilesetEntry"".""FileID"" - AND ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" - AND ""File"".""Path"" IN ( - SELECT DISTINCT ""{m_tempfiletable}"".""Path"" - FROM - ""{m_tempfiletable}"", - ""{m_tempblocktable}"" - WHERE - ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" - AND {whereclause} - ) - GROUP BY ""File"".""Path"" - "; - - using (var cmd = m_connection.CreateCommand()) - { - cmd.SetTransaction(m_rtr); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" "); - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}"); - await cmd.ExecuteNonQueryAsync($@" - CREATE INDEX ""{m_latestblocktable}_path"" - ON ""{m_latestblocktable}"" (""Path"") - "); - - await cmd.ExecuteNonQueryAsync($@" - UPDATE ""{m_tempfiletable}"" - SET LatestBlocksetId = ( - SELECT BlocksetId - FROM ""{m_latestblocktable}"" - WHERE Path = ""{m_tempfiletable}"".Path - ) - "); - } - - var sources = $@" - SELECT DISTINCT - ""{m_tempfiletable}"".""TargetPath"", - ""{m_tempfiletable}"".""Path"", - ""{m_tempfiletable}"".""ID"", - ""{m_tempblocktable}"".""Index"", - ""{m_tempblocktable}"".""Hash"", - ""{m_tempblocktable}"".""Size"" - FROM - ""{m_tempfiletable}"", - ""{m_tempblocktable}"", - ""BlocksetEntry"" - WHERE - ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" - AND ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" - AND ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" - AND ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" - AND {whereclause} - ORDER BY - ""{m_tempfiletable}"".""ID"", - ""{m_tempblocktable}"".""Index"" - "; - - using (var cmd = m_connection.CreateCommand()) - { - cmd.SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync(sources); - if (await rd.ReadAsync()) - { - bool more; - do - { - var n = new FastSource(rd, blocksize); - var tid = n.TargetFileID; - yield return n; - - more = n.MoreData; - while (more && n.TargetFileID == tid) - more = await rd.ReadAsync(); - - } while (more); - } - } - - await m_rtr.CommitAsync(); - } - - } -} +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Main.Operation.Restore; +using Duplicati.Library.Main.Volumes; +using Duplicati.Library.SQLiteHelper; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; + +namespace Duplicati.Library.Main.Database +{ + internal class LocalRestoreDatabase : LocalDatabase + { + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRestoreDatabase)); + + protected readonly string m_temptabsetguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + /// + /// The name of the temporary table in the database, which is used to store the list of files to restore. + /// + protected string? m_tempfiletable; + protected string? m_tempblocktable; + protected ConcurrentBag<(SqliteConnection, ReusableTransaction)> m_connection_pool = []; + protected string? m_latestblocktable; + protected string? m_fileprogtable; + protected string? m_totalprogtable; + protected string? m_filesnewlydonetable; + + protected DateTime m_restoreTime; + + public DateTime RestoreTime { get { return m_restoreTime; } } + + public static async Task CreateAsync(string path, long pagecachesize, LocalRestoreDatabase? dbnew = null) + { + dbnew ??= new LocalRestoreDatabase(); + + dbnew = (LocalRestoreDatabase)await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew); + dbnew.ShouldCloseConnection = true; + + return dbnew; + } + + public static async Task CreateAsync(LocalDatabase dbparent, LocalRestoreDatabase? dbnew = null) + { + dbnew ??= new LocalRestoreDatabase(); + + return (LocalRestoreDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + } + + /// + /// Create tables and triggers for automatic tracking of progress during a restore operation. + /// This replaces continuous requerying of block progress by iterating over blocks table. + /// SQLite is much faster keeping information up to date with internal triggers. + /// + /// This allows to create another table that keeps track + /// of all files that are done (all data blocks restored). + /// + /// The method is prepared to create a table that keeps track of all files being done completely. + /// That means, it fires for files where the number of restored blocks equals the number of all blocks. + /// It is intended to be used for fast identification of fully restored files to trigger their verification. + /// It should be read after a commit and truncated after putting the files to a verification queue. + /// Note: If a file is done once and then set back to a none restored state, the file is not automatically removed. + /// But if it reaches a restored state later, it will be re-added (trigger will fire) + /// + public async Task CreateProgressTracker(bool createFilesNewlyDoneTracker) + { + m_fileprogtable = "FileProgress-" + m_temptabsetguid; + m_totalprogtable = "TotalProgress-" + m_temptabsetguid; + m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; + + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + + // How to handle METADATA? + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( + ""FileId"" INTEGER PRIMARY KEY, + ""TotalBlocks"" INTEGER NOT NULL, + ""TotalSize"" INTEGER NOT NULL, + ""BlocksRestored"" INTEGER NOT NULL, + ""SizeRestored"" INTEGER NOT NULL + ) + "); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( + ""TotalFiles"" INTEGER NOT NULL, + ""TotalBlocks"" INTEGER NOT NULL, + ""TotalSize"" INTEGER NOT NULL, + ""FilesFullyRestored"" INTEGER NOT NULL, + ""FilesPartiallyRestored"" INTEGER NOT NULL, + ""BlocksRestored"" INTEGER NOT NULL, + ""SizeRestored"" INTEGER NOT NULL + ) + "); + + if (createFilesNewlyDoneTracker) + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + "); + } + + try + { + // Initialize statistics with File- and Block-Data (it is valid to already have restored blocks in files) + // A rebuild with this function should be valid anytime. + // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. + + // We use a LEFT JOIN to allow for empty files (no data Blocks) + + // Will be one row per file. + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_fileprogtable}"" ( + ""FileId"", + ""TotalBlocks"", + ""TotalSize"", + ""BlocksRestored"", + ""SizeRestored"" + ) + SELECT + ""F"".""ID"", + IFNULL(COUNT(""B"".""ID""), 0), + IFNULL(SUM(""B"".""Size""), 0), + IFNULL( + COUNT( + CASE ""B"".""Restored"" + WHEN 1 + THEN ""B"".""ID"" + ELSE NULL + END + ), + 0 + ), + IFNULL( + SUM( + CASE ""B"".""Restored"" + WHEN 1 + THEN ""B"".""Size"" + ELSE 0 + END + ), + 0 + ) + FROM ""{m_tempfiletable}"" ""F"" + LEFT JOIN ""{m_tempblocktable}"" ""B"" + ON ""B"".""FileID"" = ""F"".""ID"" + WHERE ""B"".""Metadata"" IS NOT 1 + GROUP BY ""F"".""ID"" + "); + + // Will result in a single line (no support to also track metadata) + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_totalprogtable}"" ( + ""TotalFiles"", + ""TotalBlocks"", + ""TotalSize"", + ""FilesFullyRestored"", + ""FilesPartiallyRestored"", + ""BlocksRestored"", + ""SizeRestored"" + ) + SELECT + IFNULL(COUNT(""P"".""FileId""), 0), + IFNULL(SUM(""P"".""TotalBlocks""), 0), + IFNULL(SUM(""P"".""TotalSize""), 0), + IFNULL( + COUNT( + CASE + WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" + THEN 1 + ELSE NULL + END + ), + 0 + ), + IFNULL( + COUNT( + CASE + WHEN + ""P"".""BlocksRestored"" BETWEEN 1 + AND ""P"".""TotalBlocks"" - 1 + THEN 1 + ELSE NULL + END + ), + 0 + ), + IFNULL(SUM(""P"".""BlocksRestored""), 0), + IFNULL(SUM(""P"".""SizeRestored""), 0) + FROM ""{m_fileprogtable}"" ""P"" + "); + + // Finally we create TRIGGERs to keep all our statistics up to date. + // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic. + // Note: We do assume that neither files nor blocks will be added or deleted during restore process + // and that the size of each block stays constant so there is no need to track that information + // with additional INSERT and DELETE triggers. + + // A trigger to update the file-stat entry each time a block changes restoration state. + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" + AFTER UPDATE OF ""Restored"" + ON ""{m_tempblocktable}"" + WHEN OLD.""Restored"" != NEW.""Restored"" + AND NEW.""Metadata"" = 0 + BEGIN UPDATE ""{m_fileprogtable}"" + SET + ""BlocksRestored"" = + ""{m_fileprogtable}"".""BlocksRestored"" + + (NEW.""Restored"" - OLD.""Restored""), + ""SizeRestored"" = + ""{m_fileprogtable}"".""SizeRestored"" + + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) + WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" + ; END + "); + + // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" + AFTER UPDATE ON ""{m_fileprogtable}"" + BEGIN UPDATE ""{m_totalprogtable}"" + SET + ""FilesFullyRestored"" = + ""{m_totalprogtable}"".""FilesFullyRestored"" + + (CASE + WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" + THEN 1 + ELSE 0 + END) + - (CASE + WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" + THEN 1 + ELSE 0 + END), + ""FilesPartiallyRestored"" = + ""{m_totalprogtable}"".""FilesPartiallyRestored"" + + (CASE + WHEN + NEW.""BlocksRestored"" BETWEEN 1 + AND NEW.""TotalBlocks"" - 1 + THEN 1 + ELSE 0 + END) + - (CASE + WHEN + OLD.""BlocksRestored"" BETWEEN 1 + AND OLD.""TotalBlocks"" - 1 + THEN 1 + ELSE 0 + END), + ""BlocksRestored"" = + ""{m_totalprogtable}"".""BlocksRestored"" + + NEW.""BlocksRestored"" + - OLD.""BlocksRestored"", + ""SizeRestored"" = + ""{m_totalprogtable}"".""SizeRestored"" + + NEW.""SizeRestored"" + - OLD.""SizeRestored"" + ; END + "); + + + if (createFilesNewlyDoneTracker) + { + // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" + AFTER UPDATE OF + ""BlocksRestored"", + ""TotalBlocks"" + ON ""{m_fileprogtable}"" + WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" + BEGIN + INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") + VALUES (NEW.""FileId""); + END + "); + } + + } + catch (Exception ex) + { + m_fileprogtable = null; + m_totalprogtable = null; + Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables"); + throw; + } + finally + { + await m_rtr.CommitAsync(); + } + } + + public async Task> PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter) + { + m_tempfiletable = "Fileset-" + m_temptabsetguid; + m_tempblocktable = "Blocks-" + m_temptabsetguid; + + using (var cmd = m_connection.CreateCommand()) + { + cmd.SetTransaction(m_rtr); + var filesetIds = await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToListAsync(); + while (filesetIds.Count > 0) + { + var filesetId = filesetIds[0]; + filesetIds.RemoveAt(0); + + cmd.SetCommandAndParameters(@" + SELECT ""Timestamp"" + FROM ""Fileset"" + WHERE ""ID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); + m_restoreTime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); + + var ix = await FilesetTimes() + .Select((value, index) => new { value.Key, index }) + .Where(n => n.Key == filesetId) + .Select(pair => pair.index + 1) + .FirstOrDefaultAsync() - 1; + + Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime); + + cmd.Parameters.Clear(); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_tempfiletable}"" ( + ""ID"" INTEGER PRIMARY KEY, + ""Path"" TEXT NOT NULL, + ""BlocksetID"" INTEGER NOT NULL, + ""MetadataID"" INTEGER NOT NULL, + ""TargetPath"" TEXT NULL, + ""DataVerified"" BOOLEAN NOT NULL, + ""LatestBlocksetId"" INTEGER, + ""LocalSourceExists"" BOOLEAN + ) + "); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_tempblocktable}"" ( + ""ID"" INTEGER PRIMARY KEY, + ""FileID"" INTEGER NOT NULL, + ""Index"" INTEGER NOT NULL, + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Restored"" BOOLEAN NOT NULL, + ""Metadata"" BOOLEAN NOT NULL, + ""VolumeID"" INTEGER NOT NULL, + ""BlockID"" INTEGER NOT NULL + ) + "); + + // TODO: Optimize to use the path prefix + + if (filter == null || filter.Empty) + { + // Simple case, restore everything + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"", + 0 + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(); + } + else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple) + { + using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset")) + await m_rtr.CommitAsync(); + cmd.SetTransaction(m_rtr); + // If we get a list of filenames, the lookup table is faster + // unfortunately we cannot do this if the filesystem is case sensitive as + // SQLite only supports ASCII compares + var p = expression.GetSimpleList(); + var m_filenamestable = "Filenames-" + m_temptabsetguid; + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_filenamestable}"" ( + ""Path"" TEXT NOT NULL + ) + "); + cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_filenamestable}"" (""Path"") + VALUES (@Path) + "); + + foreach (var s in p) + { + await cmd.SetParameterValue("@Path", s) + .ExecuteNonQueryAsync(); + } + + var c = await cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"", + 0 + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId + AND ""Path"" IN ( + SELECT DISTINCT ""Path"" + FROM ""{m_filenamestable}"" + ) + ") + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(); + + if (c != p.Length && c != 0) + { + var sb = new StringBuilder(); + sb.AppendLine(); + + cmd.SetCommandAndParameters($@" + SELECT ""Path"" + FROM ""{m_filenamestable}"" + WHERE ""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ) + "); + using (var rd = await cmd.ExecuteReaderAsync()) + while (await rd.ReadAsync()) + sb.AppendLine(rd.ConvertValueToString(0)); + + cmd.SetCommandAndParameters(@" + SELECT ""Timestamp"" + FROM ""Fileset"" + WHERE ""ID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); + var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); + + Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); + cmd.Parameters.Clear(); + } + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" "); + + using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset")) + await m_rtr.CommitAsync(); + cmd.SetTransaction(m_rtr); + } + else + { + // Restore but filter elements based on the filter expression + // If this is too slow, we could add a special handler for wildcard searches too + cmd.SetCommandAndParameters(@" + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"" + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); + + object[] values = new object[4]; + using var cmd2 = m_connection.CreateCommand($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + VALUES ( + @ID, + @Path, + @BlocksetID, + @MetadataID, + 0 + ) + "); + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + { + rd.GetValues(values); + var path = values[1] as string; + if (path != null && FilterExpression.Matches(filter, path.ToString()!)) + { + await cmd2.SetParameterValue("@ID", values[0]) + .SetParameterValue("@Path", values[1]) + .SetParameterValue("@BlocksetID", values[2]) + .SetParameterValue("@MetadataID", values[3]) + .ExecuteNonQueryAsync(); + } + } + } + + //creating indexes after insertion is much faster + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_ID"" + ON ""{m_tempfiletable}"" (""ID"") + "); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_TargetPath"" + ON ""{m_tempfiletable}"" (""TargetPath"") + "); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_Path"" + ON ""{m_tempfiletable}"" (""Path"") + "); + + cmd.SetCommandAndParameters($@" + SELECT + COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), + SUM(""Blockset"".""Length"") + FROM + ""{m_tempfiletable}"", + ""Blockset"" + WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + "); + using (var rd = await cmd.ExecuteReaderAsync()) + { + var filecount = 0L; + var filesize = 0L; + + if (await rd.ReadAsync()) + { + filecount = rd.ConvertValueToInt64(0, 0); + filesize = rd.ConvertValueToInt64(1, 0); + } + + if (filecount > 0) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreTargetFileCount", "Needs to restore {0} files ({1})", filecount, Library.Utility.Utility.FormatSizeString(filesize)); + return new Tuple(filecount, filesize); + } + } + } + + await m_rtr.CommitAsync(); + } + + return new Tuple(0, 0); + } + + public async Task GetFirstPath() + { + using var cmd = m_connection.CreateCommand($@" + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + ") + .SetTransaction(m_rtr); + + var v0 = await cmd.ExecuteScalarAsync(); + if (v0 == null || v0 == DBNull.Value) + return null; + + await m_rtr.CommitAsync(); + + return v0.ToString(); + } + + public async Task GetLargestPrefix() + { + using var cmd = m_connection.CreateCommand($@" + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + ") + .SetTransaction(m_rtr); + + var v0 = await cmd.ExecuteScalarAsync(); + var maxpath = ""; + if (v0 != null && v0 != DBNull.Value) + maxpath = v0.ToString()!; + + var dirsep = Util.GuessDirSeparator(maxpath); + + var filecount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{m_tempfiletable}"" + ", -1); + var foundfiles = -1L; + + //TODO: Handle FS case-sensitive? + cmd.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ""{m_tempfiletable}"" + WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix + "); + + while (filecount != foundfiles && maxpath.Length > 0) + { + var mp = Util.AppendDirSeparator(maxpath, dirsep); + foundfiles = await cmd.SetParameterValue("@PrefixLength", mp.Length) + .SetParameterValue("@Prefix", mp) + .ExecuteScalarInt64Async(-1); + + if (filecount != foundfiles) + { + var oldlen = maxpath.Length; + + var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); + maxpath = maxpath.Substring(0, lix + 1); + if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen) + maxpath = ""; + } + } + + await m_rtr.CommitAsync(); + + return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); + } + + public async Task SetTargetPaths(string largest_prefix, string destination) + { + var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? await GetFirstPath() : largest_prefix); + + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + if (string.IsNullOrEmpty(destination)) + { + //The string fixing here is meant to provide some non-random + // defaults when restoring cross OS, e.g. backup on Linux, restore on Windows + //This is mostly meaningless, and the user really should use --restore-path + + if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") + { + // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 2, 1) == ':' + THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) + ELSE ""Path"" + END + "); + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 1, 2) == '\\' + THEN '\\' || SUBSTR(""Path"", 2) + ELSE ""Path"" + END + "); + + } + else if (OperatingSystem.IsWindows() && dirsep == "/") + { + // For Linux -> Win, we use the temporary folder's drive as the root path + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 1, 1) == '/' + THEN @Path || SUBSTR(""Path"", 2) + ELSE ""Path"" + END + ") + .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) + .ExecuteNonQueryAsync(); + } + else + { + // Same OS, just use the path directly + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = ""Path"" + "); + } + } + else + { + if (string.IsNullOrEmpty(largest_prefix)) + { + //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) + + // We use the format / / + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = + CASE + WHEN SUBSTR(""Path"", 2, 1) == ':' + THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) + ELSE ""Path"" + END + "); + + // For UNC paths, we use \\server\folder -> / / + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = + CASE + WHEN SUBSTR(""Path"", 1, 2) == '\\' + THEN SUBSTR(""Path"", 2) + ELSE ""TargetPath"" + END + "); + } + else + { + largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength) + ") + .SetParameterValue("@PrefixLength", largest_prefix.Length + 1) + .ExecuteNonQueryAsync(); + } + } + + // Cross-os path remapping support + if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") + // For Win paths on Linux + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/') + "); + else if (OperatingSystem.IsWindows() && dirsep == "/") + // For Linux paths on Windows + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\') + "); + + if (!string.IsNullOrEmpty(destination)) + { + // Paths are now relative with target-os naming system + // so we prefix them with the target path + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = @Destination || ""TargetPath"" + ") + .SetParameterValue("@Destination", Util.AppendDirSeparator(destination)) + .ExecuteNonQueryAsync(); + } + + await m_rtr.CommitAsync(); + } + + public async Task FindMissingBlocks(bool skipMetadata) + { + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + var p1 = await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_tempblocktable}"" ( + ""FileID"", + ""Index"", + ""Hash"", + ""Size"", + ""Restored"", + ""Metadata"", + ""VolumeId"", + ""BlockId"" + ) + SELECT DISTINCT + ""{m_tempfiletable}"".""ID"", + ""BlocksetEntry"".""Index"", + ""Block"".""Hash"", + ""Block"".""Size"", + 0, + 0, + ""Block"".""VolumeID"", + ""Block"".""ID"" + FROM + ""{m_tempfiletable}"", + ""BlocksetEntry"", + ""Block"" + WHERE + ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + "); + + var p2 = 0; + if (!skipMetadata) + p2 = await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_tempblocktable}"" ( + ""FileID"", + ""Index"", + ""Hash"", + ""Size"", + ""Restored"", + ""Metadata"", + ""VolumeId"", + ""BlockId"" + ) + SELECT DISTINCT + ""{m_tempfiletable}"".""ID"", + ""BlocksetEntry"".""Index"", + ""Block"".""Hash"", + ""Block"".""Size"", + 0, + 1, + ""Block"".""VolumeID"", + ""Block"".""ID"" + FROM + ""{m_tempfiletable}"", + ""BlocksetEntry"", + ""Block"", + ""Metadataset"" + WHERE + ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" + AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + "); + + //creating indexes after insertion is much faster + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" + ON ""{m_tempblocktable}"" (""Hash"", ""Size"") + "); + // better suited to speed up commit on UpdateBlocks + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" + ON ""{m_tempblocktable}"" (""FileId"", ""Index"") + "); + + var size = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_tempblocktable}"" + ", 0); + Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); + + await m_rtr.CommitAsync(); + } + + public async Task UpdateTargetPath(long ID, string newname) + { + using var cmd = m_connection.CreateCommand($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = @TargetPath + WHERE ""ID"" = @ID + "); + await cmd.SetTransaction(m_rtr) + .SetParameterValue("@TargetPath", newname) + .SetParameterValue("@ID", ID) + .ExecuteNonQueryAsync(); + + await m_rtr.CommitAsync(); + } + + public interface IExistingFileBlock + { + string Hash { get; } + long Index { get; } + long Size { get; } + } + + public interface IExistingFile + { + string TargetPath { get; } + string TargetHash { get; } + long TargetFileID { get; } + long Length { get; } + IAsyncEnumerable Blocks(); + } + + public interface IBlockSource + { + string Path { get; } + long Offset { get; } + bool IsMetadata { get; } + } + + public interface IBlockDescriptor + { + string Hash { get; } + long Size { get; } + long Offset { get; } + long Index { get; } + bool IsMetadata { get; } + IAsyncEnumerable BlockSources(); + } + + public interface ILocalBlockSource + { + string TargetPath { get; } + long TargetFileID { get; } + IAsyncEnumerable Blocks(); + } + + /// + /// Interface for an object describing a file to restore. + /// + public interface IFileToRestore + { + string Path { get; } + string Hash { get; } + long Length { get; } + } + + public interface IPatchBlock + { + long Offset { get; } + long Size { get; } + string Key { get; } + } + + public interface IVolumePatch + { + string Path { get; } + long FileID { get; } + IAsyncEnumerable Blocks(); + } + + private class ExistingFile : IExistingFile + { + private readonly SqliteDataReader m_reader; + + public ExistingFile(SqliteDataReader rd) { m_reader = rd; HasMore = true; } + + public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } + public string TargetHash { get { return m_reader.ConvertValueToString(1) ?? ""; } } + public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } } + public long Length { get { return m_reader.ConvertValueToInt64(3); } } + + public bool HasMore { get; private set; } + + private class ExistingFileBlock : IExistingFileBlock + { + private readonly SqliteDataReader m_reader; + + public ExistingFileBlock(SqliteDataReader rd) { m_reader = rd; } + + public string Hash { get { return m_reader.ConvertValueToString(4) ?? ""; } } + public long Index { get { return m_reader.ConvertValueToInt64(5); } } + public long Size { get { return m_reader.ConvertValueToInt64(6); } } + } + + public async IAsyncEnumerable Blocks() + { + string p = TargetPath; + while (HasMore && p == TargetPath) + { + yield return new ExistingFileBlock(m_reader); + HasMore = await m_reader.ReadAsync(); + } + } + + public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename) + { + using var cmd = db.Connection.CreateCommand($@" + SELECT + ""{tablename}"".""TargetPath"", + ""Blockset"".""FullHash"", + ""{tablename}"".""ID"", + ""Blockset"".""Length"", + ""Block"".""Hash"", + ""BlocksetEntry"".""Index"", + ""Block"".""Size"" + FROM + ""{tablename}"", + ""Blockset"", + ""BlocksetEntry"", + ""Block"" + WHERE + ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + ORDER BY + ""{tablename}"".""TargetPath"", + ""BlocksetEntry"".""Index"" + ") + .SetTransaction(db.Transaction); + using var rd = await cmd.ExecuteReaderAsync(); + if (await rd.ReadAsync()) + { + var more = true; + while (more) + { + var f = new ExistingFile(rd); + string current = f.TargetPath; + yield return f; + + more = f.HasMore; + while (more && current == f.TargetPath) + more = await rd.ReadAsync(); + } + } + await db.Transaction.CommitAsync(); + } + } + + public IAsyncEnumerable GetExistingFilesWithBlocks() + { + if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) + throw new InvalidOperationException("No temporary file table set up for this restore."); + return ExistingFile.GetExistingFilesWithBlocks(this, m_tempfiletable); + } + + private class LocalBlockSource : ILocalBlockSource + { + private class BlockDescriptor : IBlockDescriptor + { + private class BlockSource : IBlockSource + { + private readonly SqliteDataReader m_reader; + public BlockSource(SqliteDataReader rd) { m_reader = rd; } + + public string Path { get { return m_reader.ConvertValueToString(6) ?? ""; } } + public long Offset { get { return m_reader.ConvertValueToInt64(7); } } + public bool IsMetadata { get { return false; } } + } + + private readonly SqliteDataReader m_reader; + public BlockDescriptor(SqliteDataReader rd) { m_reader = rd; HasMore = true; } + + private string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } + + public string Hash { get { return m_reader.ConvertValueToString(2) ?? ""; } } + public long Offset { get { return m_reader.ConvertValueToInt64(3); } } + public long Index { get { return m_reader.ConvertValueToInt64(4); } } + public long Size { get { return m_reader.ConvertValueToInt64(5); } } + public bool IsMetadata { get { return !(m_reader.ConvertValueToInt64(9) == 0); } } + + public bool HasMore { get; private set; } + + public async IAsyncEnumerable BlockSources() + { + var p = TargetPath; + var h = Hash; + var s = Size; + + while (HasMore && p == TargetPath && h == Hash && s == Size) + { + yield return new BlockSource(m_reader); + HasMore = await m_reader.ReadAsync(); + } + } + + } + + private readonly SqliteDataReader m_reader; + public LocalBlockSource(SqliteDataReader rd) { m_reader = rd; HasMore = true; } + + public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } + public long TargetFileID { get { return m_reader.ConvertValueToInt64(1); } } + + public bool HasMore { get; private set; } + + public async IAsyncEnumerable Blocks() + { + var p = TargetPath; + while (HasMore && p == TargetPath) + { + var c = new BlockDescriptor(m_reader); + var h = c.Hash; + var s = c.Size; + + yield return c; + + HasMore = c.HasMore; + while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) + HasMore = await m_reader.ReadAsync(); + } + } + + public static async IAsyncEnumerable GetFilesAndSourceBlocks(LocalDatabase db, string filetablename, string blocktablename, long blocksize, bool skipMetadata) + { + // TODO: Skip metadata as required + // Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows + using var cmd = db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""A"".""ID"", + ""B"".""Hash"", + (""B"".""Index"" * {blocksize}), + ""B"".""Index"", + ""B"".""Size"", + ""C"".""Path"", + (""D"".""Index"" * {blocksize}), + ""E"".""Size"", + ""B"".""Metadata"" + FROM + ""{filetablename}"" ""A"", + ""{blocktablename}"" ""B"", + ""File"" ""C"", + ""BlocksetEntry"" ""D"", + ""Block"" E + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" + AND ""D"".""BlockID"" = ""E"".""ID"" + AND ""B"".""Hash"" = ""E"".""Hash"" + AND ""B"".""Size"" = ""E"".""Size"" + AND ""B"".""Restored"" = 0 + ORDER BY + ""A"".""TargetPath"", + ""B"".""Index"" + ") + .SetTransaction(db.Transaction); + using var rd = await cmd.ExecuteReaderAsync(); + var more = await rd.ReadAsync(); + + while (more) + { + var f = new LocalBlockSource(rd); + string current = f.TargetPath; + yield return f; + + more = f.HasMore; + while (more && current == f.TargetPath) + more = await rd.ReadAsync(); + } + + await db.Transaction.CommitAsync(); + } + } + + public IAsyncEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize) + { + if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) + throw new InvalidOperationException("No temporary file table set up for this restore."); + return LocalBlockSource.GetFilesAndSourceBlocks(this, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata); + } + + public async IAsyncEnumerable GetMissingVolumes() + { + using var cmd = m_connection.CreateCommand($@" + SELECT + ""RV"".""Name"", + ""RV"".""Hash"", + ""RV"".""Size"", + ""BB"".""MaxIndex"" + FROM ""RemoteVolume"" ""RV"" + INNER JOIN ( + SELECT + ""TB"".""VolumeID"", + MAX(""TB"".""Index"") as ""MaxIndex"" + FROM ""{m_tempblocktable}"" ""TB"" + WHERE ""TB"".""Restored"" = 0 + GROUP BY ""TB"".""VolumeID"" + ) as ""BB"" + ON ""RV"".""ID"" = ""BB"".""VolumeID"" + ORDER BY ""BB"".""MaxIndex"" + ") + .SetTransaction(m_rtr); + + // Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore. + // If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file. + // If we manage to get the volumes in an order restoring front blocks first, this can save time. + // An optimal algorithm would build a dependency net with cycle resolution to find the best near topological + // order of volumes, but this is a bit too fancy here. + // We will just put a very simple heuristic to work, that will try to prefer volumes containing lower block indexes: + // We just order all volumes by the maximum block index they contain. This query is slow, but should be worth the effort. + // Now it is likely to restore all files from front to back. Large files will always be done last. + // One could also use like the average block number in a volume, that needs to be measured. + + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + { + yield return new RemoteVolume( + rd.ConvertValueToString(0), + rd.ConvertValueToString(1), + rd.ConvertValueToInt64(2, -1) + ); + } + + await m_rtr.CommitAsync(); + } + + public interface IFilesAndMetadata : IDisposable + { + IAsyncEnumerable FilesWithMissingBlocks(); + IAsyncEnumerable MetadataWithMissingBlocks(); + } + + private class FilesAndMetadata : IFilesAndMetadata + { + private string m_tmptable = null!; + private string m_filetablename = null!; + private string m_blocktablename = null!; + private long m_blocksize; + + private LocalDatabase m_db = null!; + + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] + public FilesAndMetadata(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + { + throw new NotImplementedException("Use CreateAsync instead of the constructor"); + } + + private FilesAndMetadata() { } + + public static async Task CreateAsync(LocalDatabase db, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + { + var fam = new FilesAndMetadata() + { + m_db = db, + m_filetablename = filetablename, + m_blocktablename = blocktablename, + m_blocksize = blocksize, + }; + + using var c = db.Connection.CreateCommand() + .SetTransaction(db.Transaction); + fam.m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await c.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{fam.m_tmptable}"" ( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + ) + "); + + c.SetCommandAndParameters($@" + INSERT INTO ""{fam.m_tmptable}"" ( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size) + "); + foreach (var s in curvolume.Blocks) + { + await c.SetParameterValue("@Hash", s.Key) + .SetParameterValue("@Size", s.Value) + .ExecuteNonQueryAsync(); + } + + // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks + + await db.Transaction.CommitAsync(); + + return fam; + } + + public void Dispose() + { + DisposeAsync().Await(); + } + + public async Task DisposeAsync() + { + if (m_tmptable != null) + { + using var c = m_db.Connection.CreateCommand(@$"DROP TABLE IF EXISTS ""{m_tmptable}""") + .SetTransaction(m_db.Transaction); + await c.ExecuteNonQueryAsync(); + await m_db.Transaction.CommitAsync(); + } + } + + private class VolumePatch : IVolumePatch + { + private class PatchBlock : IPatchBlock + { + private readonly SqliteDataReader m_reader; + public PatchBlock(SqliteDataReader rd) { m_reader = rd; } + + public long Offset { get { return m_reader.ConvertValueToInt64(2); } } + public long Size { get { return m_reader.ConvertValueToInt64(3); } } + public string Key { get { return m_reader.ConvertValueToString(4) ?? ""; } } + } + + private readonly SqliteDataReader m_reader; + public VolumePatch(SqliteDataReader rd) { m_reader = rd; HasMore = true; } + + public string Path { get { return m_reader.ConvertValueToString(0) ?? ""; } } + public long FileID { get { return m_reader.ConvertValueToInt64(1); } } + public bool HasMore { get; private set; } + + public async IAsyncEnumerable Blocks() + { + string p = Path; + while (HasMore && p == Path) + { + yield return new PatchBlock(m_reader); + HasMore = await m_reader.ReadAsync(); + } + } + } + + public async IAsyncEnumerable FilesWithMissingBlocks() + { + // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! + using var cmd = m_db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""BB"".""FileID"", + (""BB"".""Index"" * {m_blocksize}), + ""BB"".""Size"", + ""BB"".""Hash"" + FROM + ""{m_filetablename}"" ""A"", + ""{m_blocktablename}"" ""BB"" + WHERE + ""A"".""ID"" = ""BB"".""FileID"" + AND ""BB"".""Restored"" = 0 + AND ""BB"".""Metadata"" = {"0"} + AND ""BB"".""ID"" IN ( + SELECT ""B"".""ID"" + FROM + ""{m_blocktablename}"" ""B"", + ""{m_tmptable}"" ""C"" + WHERE + ""B"".""Hash"" = ""C"".""Hash"" + AND ""B"".""Size"" = ""C"".""Size"" + ) + ORDER BY + ""A"".""TargetPath"", + ""BB"".""Index"" + ") + .SetTransaction(m_db.Transaction); + + using var rd = await cmd.ExecuteReaderAsync(); + if (await rd.ReadAsync()) + { + var more = true; + while (more) + { + var f = new VolumePatch(rd); + var current = f.Path; + yield return f; + + more = f.HasMore; + while (more && current == f.Path) + more = await rd.ReadAsync(); + } + } + await m_db.Transaction.CommitAsync(); + } + + public async IAsyncEnumerable MetadataWithMissingBlocks() + { + // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! + using var cmd = m_db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""BB"".""FileID"", + (""BB"".""Index"" * {m_blocksize}), + ""BB"".""Size"", + ""BB"".""Hash"" + FROM + ""{m_filetablename}"" ""A"", + ""{m_blocktablename}"" ""BB"" + WHERE + ""A"".""ID"" = ""BB"".""FileID"" + AND ""BB"".""Restored"" = 0 + AND ""BB"".""Metadata"" = {"1"} + AND ""BB"".""ID"" IN ( + SELECT ""B"".""ID"" + FROM + ""{m_blocktablename}"" ""B"", + ""{m_tmptable}"" ""C"" + WHERE + ""B"".""Hash"" = ""C"".""Hash"" + AND ""B"".""Size"" = ""C"".""Size"" + ) + ORDER BY + ""A"".""TargetPath"", + ""BB"".""Index"" + ") + .SetTransaction(m_db.Transaction); + + using var rd = await cmd.ExecuteReaderAsync(); + if (await rd.ReadAsync()) + { + var more = true; + while (more) + { + var f = new VolumePatch(rd); + string current = f.Path; + yield return f; + + more = f.HasMore; + while (more && current == f.Path) + more = await rd.ReadAsync(); + } + } + } + } + + public async Task GetMissingBlockData(BlockVolumeReader curvolume, long blocksize) + { + if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) + throw new InvalidOperationException("No temporary file table set up for this restore."); + return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume); + } + + /// + /// Returns a connection from the connection pool. + /// + /// A connection from the connection pool. + public async Task<(SqliteConnection, ReusableTransaction)> GetConnectionFromPool() + { + if (!m_connection_pool.TryTake(out var entry)) + { + var connection = await SQLiteLoader.LoadConnectionAsync(); + connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared"; + await connection.OpenAsync(); + await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize); + var transaction = new ReusableTransaction(connection); + + return (connection, transaction); + } + + return entry; + } + + private class FileToRestore : IFileToRestore + { + public string Path { get; private set; } + public string Hash { get; private set; } + public long Length { get; private set; } + + public FileToRestore(long id, string path, string hash, long length) + { + Path = path; + Hash = hash; + Length = length; + } + } + + public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified) + { + using var cmd = m_connection.CreateCommand($@" + SELECT + ""{m_tempfiletable}"".""ID"", + ""{m_tempfiletable}"".""TargetPath"", + ""Blockset"".""FullHash"", + ""Blockset"".""Length"" + FROM + ""{m_tempfiletable}"", + ""Blockset"" + WHERE + ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Verified", !onlyNonVerified); + + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + yield return new FileToRestore( + rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); + + await m_rtr.CommitAsync(); + } + + /// + /// Returns a list of files and symlinks to restore. + /// + /// A list of files and symlinks to restore. + public async IAsyncEnumerable GetFilesAndSymlinksToRestore() + { + // Order by length descending, so that larger files are restored first. + using var cmd = m_connection.CreateCommand($@" + SELECT + F.ID, + F.Path, + F.TargetPath, + IFNULL(B.FullHash, ''), + IFNULL(B.Length, 0) AS ""Length"", + F.BlocksetID + FROM ""{m_tempfiletable}"" F + LEFT JOIN Blockset B + ON F.BlocksetID = B.ID + WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID} + ORDER BY ""Length"" DESC + ") + .SetTransaction(m_rtr); + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); + + await m_rtr.CommitAsync(); + } + + /// + /// Returns a list of folders to restore. Used to restore folder metadata. + /// + /// A list of folders to restore. + public async IAsyncEnumerable GetFolderMetadataToRestore() + { + using var cmd = m_connection.CreateCommand(); + cmd.SetTransaction(m_rtr); + using var rd = await cmd.ExecuteReaderAsync($@" + SELECT + F.ID, + '', + F.TargetPath, + '', + 0, + {FOLDER_BLOCKSET_ID} + FROM ""{m_tempfiletable}"" F + WHERE + F.BlocksetID = {FOLDER_BLOCKSET_ID} + AND F.MetadataID IS NOT NULL + AND F.MetadataID >= 0 + "); + + while (await rd.ReadAsync()) + yield return new FileRequest( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToString(3), + rd.ConvertValueToInt64(4), + rd.ConvertValueToInt64(5) + ); + } + + /// + /// Returns a list of blocks and their volume IDs. Used by the to keep track of blocks and volumes to automatically evict them from the respective caches. + /// + /// Flag indicating whether the returned blocks should exclude the metadata blocks. + /// A list of tuples containing the block ID and the volume ID of the block. + public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) + { + var metadata_query = skipMetadata ? "" : $@" + UNION ALL + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""{m_tempfiletable}"" + INNER JOIN ""Metadataset"" + ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""BlocksetEntry"" + ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + "; + using var cmd = Connection.CreateCommand($@" + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""BlocksetEntry"" + INNER JOIN ""{m_tempfiletable}"" + ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + {metadata_query} + ") + .SetTransaction(m_rtr); + using var reader = await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + yield return ( + reader.ConvertValueToInt64(0), + reader.ConvertValueToInt64(1) + ); + + await m_rtr.CommitAsync(); + } + + /// + /// Returns a list of for the given blockset ID. It is used by the to restore the blocks of a file. + /// + /// The BlocksetID of the file. + /// A list of needed to restore the given file. + public async IAsyncEnumerable GetBlocksFromFile(long blocksetID) + { + var (connection, transaction) = await GetConnectionFromPool(); + try + { + + using var cmd = connection.CreateCommand(@$" + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""BlocksetEntry"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID + ") + .SetTransaction(transaction) + .SetParameterValue("@BlocksetID", blocksetID); + + using var reader = await cmd.ExecuteReaderAsync(); + for (long i = 0; await reader.ReadAsync(); i++) + yield return new BlockRequest( + reader.ConvertValueToInt64(0), + i, + reader.ConvertValueToString(1), + reader.ConvertValueToInt64(2), + reader.ConvertValueToInt64(3), + false + ); + } + finally + { + // Return the connection to the pool + m_connection_pool.Add((connection, transaction)); + } + } + + /// + /// Returns a list of for the metadata blocks of the given file. It is used by the to restore the metadata of a file. + /// + /// The ID of the file. + /// A list of needed to restore the metadata of the given file. + public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID) + { + var (connection, transaction) = await GetConnectionFromPool(); + try + { + using var cmd = connection.CreateCommand($@" + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""File"" + INNER JOIN ""Metadataset"" + ON ""File"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""BlocksetEntry"" + ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE ""File"".""ID"" = @FileID + ") + .SetTransaction(transaction) + .SetParameterValue("@FileID", fileID); + + using var reader = await cmd.ExecuteReaderAsync(); + for (long i = 0; await reader.ReadAsync(); i++) + { + yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); + } + } + finally + { + // Return the connection to the pool + m_connection_pool.Add((connection, transaction)); + } + } + + /// + /// Returns the volume information for the given volume ID. It is used by the to get the volume information for the given volume ID. + /// + /// The ID of the volume. + /// A tuple containing the name, size, and hash of the volume. + public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) + { + using var cmd = m_connection.CreateCommand(@" + SELECT + Name, + Size, + Hash + FROM RemoteVolume + WHERE ID = @VolumeID + ") + .SetTransaction(m_rtr) + .SetParameterValue("@VolumeID", VolumeID); + + using var reader = await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + yield return (reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), reader.ConvertValueToString(2) ?? ""); + + await m_rtr.CommitAsync(); + } + + public async Task DropRestoreTable() + { + using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + + if (m_tempfiletable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"""); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_tempfiletable = null; } + + if (m_tempblocktable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"""); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_tempblocktable = null; } + + if (m_latestblocktable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"""); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_latestblocktable = null; } + + if (m_fileprogtable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"""); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_fileprogtable = null; } + + if (m_totalprogtable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"""); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_totalprogtable = null; } + + if (m_filesnewlydonetable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"""); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_filesnewlydonetable = null; } + + await m_rtr.CommitAsync(); + } + + public interface IBlockMarker : IDisposable + { + Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata); + Task SetAllBlocksMissing(long targetfileid); + Task SetAllBlocksRestored(long targetfileid, bool includeMetadata); + Task SetFileDataVerified(long targetfileid); + Task CommitAsync(); + Task UpdateProcessed(IOperationProgressUpdater writer); + } + + /// + /// A new implementation of IBlockMarker, marking the blocks directly in the blocks table as restored + /// and reading statistics about progress from DB (kept up-to-date by triggers). + /// There is no negative influence on performance, esp. since the block table is temporary anyway. + /// + private class DirectBlockMarker : IBlockMarker + { + private SqliteCommand m_insertblockCommand = null!; + private SqliteCommand m_resetfileCommand = null!; + private SqliteCommand m_updateAsRestoredCommand = null!; + private SqliteCommand m_updateFileAsDataVerifiedCommand = null!; + private SqliteCommand m_statUpdateCommand = null!; + private LocalDatabase m_db = null!; + private bool m_hasUpdates = false; + + private string m_blocktablename = null!; + private string m_filetablename = null!; + + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] + public DirectBlockMarker(SqliteConnection connection, string blocktablename, string filetablename, string statstablename) + { + throw new NotImplementedException("Use CreateAsync instead of the constructor"); + } + + private DirectBlockMarker() { } + + public static async Task CreateAsync(LocalDatabase db, string blocktablename, string filetablename, string statstablename) + { + var dbm = new DirectBlockMarker() + { + m_db = db, + m_blocktablename = blocktablename, + m_filetablename = filetablename, + + m_insertblockCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" SET ""Restored"" = 1 + WHERE + ""FileID"" = @TargetFileId + AND ""Index"" = @Index + AND ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""Metadata"" = @Metadata + AND ""Restored"" = 0 + "), + + m_resetfileCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" + SET ""Restored"" = 0 + WHERE ""FileID"" = @TargetFileId + "), + + m_updateAsRestoredCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" + SET ""Restored"" = 1 + WHERE + ""FileID"" = @TargetFileId + AND ""Metadata"" <= @Metadata + "), + + m_updateFileAsDataVerifiedCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{filetablename}"" + SET ""DataVerified"" = 1 + WHERE ""ID"" = @TargetFileId + "), + + m_statUpdateCommand = statstablename == null ? + // very slow fallback if stats tables were not created + await db.Connection.CreateCommandAsync($@" + SELECT + COUNT(DISTINCT ""FileID""), + SUM(""Size"") + FROM ""{blocktablename}"" + WHERE ""Restored"" = 1 + ") + : + // Fields in Stats: TotalFiles, TotalBlocks, TotalSize + // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored + await db.Connection.CreateCommandAsync($@" + SELECT + SUM(""FilesFullyRestored""), + SUM(""SizeRestored"") + FROM ""{statstablename}"" + ") + }; + + return dbm; + } + + public async Task UpdateProcessed(IOperationProgressUpdater updater) + { + if (!m_hasUpdates) + return; + + m_hasUpdates = false; + using var rd = await m_statUpdateCommand + .SetTransaction(m_db.Transaction) + .ExecuteReaderAsync(); + var filesprocessed = 0L; + var processedsize = 0L; + + if (rd.Read()) + { + filesprocessed += rd.ConvertValueToInt64(0, 0); + processedsize += rd.ConvertValueToInt64(1, 0); + } + + updater.UpdatefilesProcessed(filesprocessed, processedsize); + } + + public async Task SetAllBlocksMissing(long targetfileid) + { + m_hasUpdates = true; + m_resetfileCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid); + + var r = await m_resetfileCommand.ExecuteNonQueryAsync(); + if (r <= 0) + throw new Exception("Unexpected reset result"); + } + + public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata) + { + m_hasUpdates = true; + m_updateAsRestoredCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid) + .SetParameterValue("@Metadata", includeMetadata ? 1 : 0); + + var r = await m_updateAsRestoredCommand.ExecuteNonQueryAsync(); + if (r <= 0) + throw new Exception("Unexpected reset result"); + } + + public async Task SetFileDataVerified(long targetfileid) + { + m_hasUpdates = true; + m_updateFileAsDataVerifiedCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid); + + var r = await m_updateFileAsDataVerifiedCommand.ExecuteNonQueryAsync(); + if (r != 1) + throw new Exception("Unexpected result when marking file as verified."); + } + + public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata) + { + m_hasUpdates = true; + m_insertblockCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid) + .SetParameterValue("@Index", index) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .SetParameterValue("@Metadata", metadata); + + var r = await m_insertblockCommand.ExecuteNonQueryAsync(); + if (r != 1) + throw new Exception("Unexpected result when marking block."); + } + + public async Task CommitAsync() + { + using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) + await m_db.Transaction.CommitAsync(); + } + + public void Dispose() + { + m_insertblockCommand?.Dispose(); + m_resetfileCommand?.Dispose(); + m_updateAsRestoredCommand?.Dispose(); + m_updateFileAsDataVerifiedCommand?.Dispose(); + m_statUpdateCommand?.Dispose(); + } + } + + [Obsolete("Calling this constructor will throw an exception. Use CreateBlockMarkerAsync instead.")] + public IBlockMarker CreateBlockMarker() + { + throw new NotImplementedException("Use CreateBlockMarkerAsync instead of the constructor"); + } + + public async Task CreateBlockMarkerAsync() + { + if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) + throw new InvalidOperationException("No temporary file table set up for this restore."); + if (string.IsNullOrWhiteSpace(m_totalprogtable)) + throw new InvalidOperationException("No progress table set up for this restore."); + return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable); + } + + public override void Dispose() + { + DisposeAsync().Await(); + } + + public override async Task DisposeAsync() + { + await DisposePoolAsync(); + await DropRestoreTable(); + await base.DisposeAsync(); + } + + public async Task DisposePoolAsync() + { + foreach (var (connection, transaction) in m_connection_pool) + { + await transaction.DisposeAsync(); + await connection.CloseAsync(); + await connection.DisposeAsync(); + } + m_connection_pool.Clear(); + } + + public async IAsyncEnumerable GetTargetFolders() + { + using var cmd = m_connection.CreateCommand($@" + SELECT ""TargetPath"" + FROM ""{m_tempfiletable}"" + WHERE ""BlocksetID"" == @BlocksetID + ") + .SetTransaction(m_rtr) + .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); + + using var rd = await cmd.ExecuteReaderAsync(); + while (await rd.ReadAsync()) + yield return rd.ConvertValueToString(0) ?? ""; + + await m_rtr.CommitAsync(); + } + + public interface IFastSource + { + string TargetPath { get; } + long TargetFileID { get; } + string SourcePath { get; } + IAsyncEnumerable Blocks(); + } + + public interface IBlockEntry + { + long Offset { get; } + long Size { get; } + long Index { get; } + string Hash { get; } + } + + private class FastSource : IFastSource + { + private class BlockEntry : IBlockEntry + { + private readonly SqliteDataReader m_rd; + private readonly long m_blocksize; + public BlockEntry(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } + public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } } + public long Index { get { return m_rd.ConvertValueToInt64(3); } } + public long Size { get { return m_rd.ConvertValueToInt64(5); } } + public string Hash { get { return m_rd.ConvertValueToString(4) ?? ""; } } + } + + private readonly SqliteDataReader m_rd; + private readonly long m_blocksize; + public FastSource(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } + public bool MoreData { get; private set; } + public string TargetPath { get { return m_rd.ConvertValueToString(0) ?? ""; } } + public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } + public string SourcePath { get { return m_rd.ConvertValueToString(1) ?? ""; } } + + public async IAsyncEnumerable Blocks() + { + var tid = TargetFileID; + + do + { + yield return new BlockEntry(m_rd, m_blocksize); + } while ((MoreData = await m_rd.ReadAsync()) && tid == TargetFileID); + } + } + + public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize) + { + using (var cmdReader = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) + { + cmdReader.SetTransaction(m_rtr); + cmd.SetTransaction(m_rtr); + cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""LocalSourceExists"" = 1 + WHERE Path = @Path + "); + cmdReader.SetCommandAndParameters($@" + SELECT DISTINCT ""{m_tempfiletable}"".""Path"" + FROM ""{m_tempfiletable}"" + "); + using (var rd = await cmdReader.ExecuteReaderAsync()) + { + while (await rd.ReadAsync()) + { + var sourcepath = rd.ConvertValueToString(0); + if (SystemIO.IO_OS.FileExists(sourcepath)) + { + await cmd.SetParameterValue("@Path", sourcepath) + .ExecuteNonQueryAsync(); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); + } + } + } + + //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" + ON ""{m_tempfiletable}"" (""LocalSourceExists"") + "); + + await m_rtr.CommitAsync(); + } + + m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; + + var whereclause = $@" + ""{m_tempfiletable}"".""LocalSourceExists"" = 1 + AND ""{m_tempblocktable}"".""Restored"" = 0 + AND ""{m_tempblocktable}"".""Metadata"" = 0 + AND ""{m_tempfiletable}"".""TargetPath"" != ""{m_tempfiletable}"".""Path"" + "; + + var latestBlocksetIds = $@" + SELECT + ""File"".""Path"" AS ""PATH"", + ""File"".""BlocksetID"" AS ""BlocksetID"", + MAX(""Fileset"".""Timestamp"") AS ""Timestamp"" + FROM + ""File"", + ""FilesetEntry"", + ""Fileset"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" + AND ""File"".""Path"" IN ( + SELECT DISTINCT ""{m_tempfiletable}"".""Path"" + FROM + ""{m_tempfiletable}"", + ""{m_tempblocktable}"" + WHERE + ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" + AND {whereclause} + ) + GROUP BY ""File"".""Path"" + "; + + using (var cmd = m_connection.CreateCommand()) + { + cmd.SetTransaction(m_rtr); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" "); + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}"); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_latestblocktable}_path"" + ON ""{m_latestblocktable}"" (""Path"") + "); + + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET LatestBlocksetId = ( + SELECT BlocksetId + FROM ""{m_latestblocktable}"" + WHERE Path = ""{m_tempfiletable}"".Path + ) + "); + } + + var sources = $@" + SELECT DISTINCT + ""{m_tempfiletable}"".""TargetPath"", + ""{m_tempfiletable}"".""Path"", + ""{m_tempfiletable}"".""ID"", + ""{m_tempblocktable}"".""Index"", + ""{m_tempblocktable}"".""Hash"", + ""{m_tempblocktable}"".""Size"" + FROM + ""{m_tempfiletable}"", + ""{m_tempblocktable}"", + ""BlocksetEntry"" + WHERE + ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" + AND ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" + AND {whereclause} + ORDER BY + ""{m_tempfiletable}"".""ID"", + ""{m_tempblocktable}"".""Index"" + "; + + using (var cmd = m_connection.CreateCommand()) + { + cmd.SetTransaction(m_rtr); + using var rd = await cmd.ExecuteReaderAsync(sources); + if (await rd.ReadAsync()) + { + bool more; + do + { + var n = new FastSource(rd, blocksize); + var tid = n.TargetFileID; + yield return n; + + more = n.MoreData; + while (more && n.TargetFileID == tid) + more = await rd.ReadAsync(); + + } while (more); + } + } + + await m_rtr.CommitAsync(); + } + + } +} From 3db2010b1a5facfb2735c74dc548d4a65a07db11 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 4 Jun 2025 10:32:49 +0200 Subject: [PATCH 253/362] Ordering of the list of blocks to restore was not ensured to be in the correct order. --- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 7b0901a62..1e658cee4 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1631,6 +1631,7 @@ namespace Duplicati.Library.Main.Database INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID + ORDER BY ""BlocksetEntry"".""Index"" ") .SetTransaction(transaction) .SetParameterValue("@BlocksetID", blocksetID); From 507087df1dae25015466754bb90431e787ff969c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 5 Jun 2025 09:13:40 +0200 Subject: [PATCH 254/362] Calling close on the connection, as that calls the backend directly, and not through the IDb interface --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index c2c693606..9168e434b 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2414,7 +2414,7 @@ namespace Duplicati.Library.Main.Database } } - await m_connection.CloseAsync(); + m_connection.Close(); } await m_connection.DisposeAsync(); diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 1e658cee4..0beb374b8 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1996,7 +1996,7 @@ namespace Duplicati.Library.Main.Database foreach (var (connection, transaction) in m_connection_pool) { await transaction.DisposeAsync(); - await connection.CloseAsync(); + connection.Close(); await connection.DisposeAsync(); } m_connection_pool.Clear(); From db7b81d3a1d7326665fa8860fb5c12614c205a93 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 5 Jun 2025 09:14:17 +0200 Subject: [PATCH 255/362] Disposing a reusable transaction implicitly rolls it back --- Duplicati/Library/Main/Database/ReusableTransaction.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index e560b2ba3..d5f1d3995 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -119,8 +119,8 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran { try { - using (var timer = new Logging.Timer(LOGTAG, "Dispose", "Commit during transaction dispose")) - await m_transaction.CommitAsync(); + using (var timer = new Logging.Timer(LOGTAG, "Dispose", "Rollback during transaction dispose")) + await m_transaction.RollbackAsync(); } catch (Exception ex) { From 23509a27eb494e70fe9df7cac2693ab6c7b87837 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 5 Jun 2025 09:15:17 +0200 Subject: [PATCH 256/362] Disposing the database disposes the reusable transaction and creates it's own transaction for PRAGMA optimize --- Duplicati/Library/Main/Database/LocalDatabase.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 9168e434b..2c4e237ee 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2395,18 +2395,20 @@ namespace Duplicati.Library.Main.Database if (ShouldCloseConnection && m_connection != null) { + await m_rtr.DisposeAsync(); if (m_connection.State == ConnectionState.Open && !m_hasExecutedVacuum) { using (var command = m_connection.CreateCommand()) + using (var tr = m_connection.BeginTransaction()) { // SQLite recommends that PRAGMA optimize is run just before closing each database connection. await command - .SetTransaction(m_rtr) + .SetTransaction(tr) .ExecuteNonQueryAsync("PRAGMA optimize"); try { - await m_rtr.CommitAsync(restart: false); + await tr.CommitAsync(); } catch (Exception ex) { From bc52522f3bffb9272e33c024ee393761c61907fc Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 5 Jun 2025 14:44:25 +0200 Subject: [PATCH 257/362] A lot of the SELECT SQL queries don't have to commit. Especially not the ones returning an IAsyncEnumerable as that can leave a query execution as open, while the transaction is committed elsewhere. --- .../Main/Database/LocalRestoreDatabase.cs | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 0beb374b8..69f6c1592 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -619,8 +619,6 @@ namespace Duplicati.Library.Main.Database if (v0 == null || v0 == DBNull.Value) return null; - await m_rtr.CommitAsync(); - return v0.ToString(); } @@ -672,8 +670,6 @@ namespace Duplicati.Library.Main.Database } } - await m_rtr.CommitAsync(); - return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); } @@ -1044,7 +1040,6 @@ namespace Duplicati.Library.Main.Database more = await rd.ReadAsync(); } } - await db.Transaction.CommitAsync(); } } @@ -1169,8 +1164,6 @@ namespace Duplicati.Library.Main.Database while (more && current == f.TargetPath) more = await rd.ReadAsync(); } - - await db.Transaction.CommitAsync(); } } @@ -1222,8 +1215,6 @@ namespace Duplicati.Library.Main.Database rd.ConvertValueToInt64(2, -1) ); } - - await m_rtr.CommitAsync(); } public interface IFilesAndMetadata : IDisposable @@ -1385,7 +1376,6 @@ namespace Duplicati.Library.Main.Database more = await rd.ReadAsync(); } } - await m_db.Transaction.CommitAsync(); } public async IAsyncEnumerable MetadataWithMissingBlocks() @@ -1501,8 +1491,6 @@ namespace Duplicati.Library.Main.Database while (await rd.ReadAsync()) yield return new FileToRestore( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); - - await m_rtr.CommitAsync(); } /// @@ -1530,8 +1518,6 @@ namespace Duplicati.Library.Main.Database using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); - - await m_rtr.CommitAsync(); } /// @@ -1606,8 +1592,6 @@ namespace Duplicati.Library.Main.Database reader.ConvertValueToInt64(0), reader.ConvertValueToInt64(1) ); - - await m_rtr.CommitAsync(); } /// @@ -1716,8 +1700,6 @@ namespace Duplicati.Library.Main.Database using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) yield return (reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), reader.ConvertValueToString(2) ?? ""); - - await m_rtr.CommitAsync(); } public async Task DropRestoreTable() @@ -2015,8 +1997,6 @@ namespace Duplicati.Library.Main.Database using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) yield return rd.ConvertValueToString(0) ?? ""; - - await m_rtr.CommitAsync(); } public interface IFastSource From 73c8471f8e688a8ca09d3465ab539c1daf718548 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 07:37:22 +0200 Subject: [PATCH 258/362] Made BlocklistHashEnumerable and the code using it async --- .../Library/Main/Database/LocalDatabase.cs | 25 +++++++++---------- .../Main/Volumes/FilesetVolumeWriter.cs | 16 ++++++------ 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 2c4e237ee..4e0d4555e 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -33,6 +33,7 @@ using Duplicati.Library.Modules.Builtin.ResultSerialization; using Duplicati.Library.Utility; using Duplicati.Library.Interface; using Microsoft.Data.Sqlite; +using System.Threading; // Expose internal classes to UnitTests, so that Database classes can be tested [assembly: InternalsVisibleTo("Duplicati.UnitTest")] @@ -1554,10 +1555,9 @@ namespace Duplicati.Library.Main.Database yield return new Block(rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), rd.ConvertValueToInt64(1)); } - // TODO: Replace this with an enumerable method - private class BlocklistHashEnumerable : IEnumerable + private class BlocklistHashEnumerable : IAsyncEnumerable { - private class BlocklistHashEnumerator : IEnumerator + private class BlocklistHashEnumerator : IAsyncEnumerator { private readonly SqliteDataReader m_reader; private readonly BlocklistHashEnumerable m_parent; @@ -1573,13 +1573,12 @@ namespace Duplicati.Library.Main.Database public string Current { get { return m_current!; } } - public void Dispose() - { - } + public void Dispose() { } +#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + public async ValueTask DisposeAsync() { } +#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - object System.Collections.IEnumerator.Current { get { return Current; } } - - public bool MoveNext() + public async ValueTask MoveNextAsync() { m_first = false; @@ -1594,7 +1593,7 @@ namespace Duplicati.Library.Main.Database if (m_current == null) return false; - if (!m_reader.ReadAsync().Await()) + if (!await m_reader.ReadAsync()) { m_current = null; m_parent.MoreData = false; @@ -1632,12 +1631,12 @@ namespace Duplicati.Library.Main.Database public bool MoreData { get; protected set; } - public IEnumerator GetEnumerator() + public IAsyncEnumerator GetEnumerator() { return new BlocklistHashEnumerator(this, m_reader); } - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken token) { return GetEnumerator(); } @@ -1833,7 +1832,7 @@ namespace Duplicati.Library.Main.Database if (metablockhash == metahash) metablockhash = null; - filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); + await filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); if (blrd == null) more = await rd.ReadAsync(); else diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 7347a0ad8..7480b2f6b 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -82,17 +82,17 @@ namespace Duplicati.Library.Main.Volumes } } - public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) + public async Task AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) { - AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); + await AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); } - public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) + public async Task AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) { - AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); + await AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); } - private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) + private async Task AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) { m_filecount++; m_writer.WriteStartObject(); @@ -112,14 +112,14 @@ namespace Duplicati.Library.Main.Volumes if (blocklisthashes != null) { //Slightly awkward, but we avoid writing if there are no entries - using (var en = blocklisthashes.GetEnumerator()) + await using (var en = blocklisthashes.GetAsyncEnumerator()) { - if (en.MoveNext() && !string.IsNullOrEmpty(en.Current)) + if (await en.MoveNextAsync() && !string.IsNullOrEmpty(en.Current)) { m_writer.WritePropertyName("blocklists"); m_writer.WriteStartArray(); m_writer.WriteValue(en.Current); - while (en.MoveNext()) + while (await en.MoveNextAsync()) m_writer.WriteValue(en.Current); m_writer.WriteEndArray(); } From 66ed9600b86ae5ee354a57b58ae786702cd389c6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 07:38:24 +0200 Subject: [PATCH 259/362] Removed unused synchronous helper method --- Duplicati/Library/Main/Database/LocalDatabase.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 4e0d4555e..f23bdfac4 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -99,11 +99,6 @@ namespace Duplicati.Library.Main.Database throw new NotImplementedException("Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead"); } - protected static SqliteConnection CreateConnection(string path, long pagecachesize) - { - return CreateConnectionAsync(path, pagecachesize).Await(); - } - protected static async Task CreateConnectionAsync(string path, long pagecachesize) { path = Path.GetFullPath(path); From 087ff656463a83e1ff6c59644e438e214b88b8a9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 07:41:03 +0200 Subject: [PATCH 260/362] Removed synchronous helper method --- .../Library/Main/Database/LocalBackupDatabase.cs | 2 +- .../Library/Main/Database/ReusableTransaction.cs | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 851f847a6..1bdedb90e 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -848,7 +848,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@BlocksetId", blocksetid) .ExecuteScalarInt64Async(m_logQueries); - m_rtr.Commit(); + await m_rtr.CommitAsync(); return (true, metadataid); } diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index d5f1d3995..c8bb256d6 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -63,20 +63,6 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran /// public SqliteTransaction Transaction => m_disposed ? throw new InvalidOperationException("Transaction is disposed") : m_transaction; - /// - /// Commits the current transaction and optionally restarts it. - /// - /// - /// Calls the Async version of this method and awaits it. - /// - /// The log message to use. - /// True if the transaction should be restarted. - /// If the transaction is already Disposed. - public void Commit(string? message = null, bool restart = true) - { - CommitAsync(message, restart).Await(); - } - /// /// Async version of Commit: /// From 14abda1c8b420903c123d126b25ee47cf1a7ba8c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:03:00 +0200 Subject: [PATCH 261/362] Made file count retrieval async in PurgeBrokenFilesHandler --- .../Library/Main/Operation/PurgeBrokenFilesHandler.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index e26dc8ca9..69ed321ac 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -84,16 +84,16 @@ namespace Duplicati.Library.Main.Operation var filesets = await db.FilesetTimes().ToListAsync(); - var compare_list = sets.Select(x => new + var compare_list = sets.Select(async x => new { FilesetID = x.FilesetID, Timestamp = x.FilesetTime, RemoveCount = x.RemoveCount, Version = filesets.FindIndex(y => y.Key == x.FilesetID), - // TODO if we await this properly, we'll have to await - // it again when using it. - SetCount = db.GetFilesetFileCount(x.FilesetID).Await() - }).ToArray(); + SetCount = await db.GetFilesetFileCount(x.FilesetID) + }) + .Select(x => x.Result) + .ToArray(); var replacementMetadataBlocksetId = -1L; if (!m_options.DisableReplaceMissingMetadata) From fa872d703e4722b060b643f21f647d6799709b28 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:07:44 +0200 Subject: [PATCH 262/362] Made InProgressDblockVolumes constructor calling async methods to use async factory-like methods --- Duplicati/Library/Main/Operation/RepairHandler.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 2c6a6bc60..62011ad65 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -693,14 +693,19 @@ namespace Duplicati.Library.Main.Operation /// The options for the current operation /// The database to use for registering a volume in progress /// The transaction to use for the operation - public InProgressDblockVolumes(Options options, LocalDatabase db) + private InProgressDblockVolumes(Options options, LocalDatabase db) { m_database = db; m_options = options; m_maxVolumeSize = options.VolumeSize - m_options.Blocksize; m_activeWriter = new BlockVolumeWriter(options); - // TODO this constructor should be converted to a factory method - m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero).Await(); + } + + public static async Task CreateAsync(Options options, LocalDatabase db) + { + var ipdv = new InProgressDblockVolumes(options, db); + ipdv.m_activeWriter.VolumeID = await db.RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero); + return ipdv; } /// @@ -773,7 +778,7 @@ namespace Duplicati.Library.Main.Operation /// >A task representing the asynchronous operation private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) { - var currentVolume = new InProgressDblockVolumes(m_options, db); + var currentVolume = await InProgressDblockVolumes.CreateAsync(m_options, db); foreach (var n in missingDblockFiles) { From 471309aa159893fcc27b03790374374675eba79f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:12:23 +0200 Subject: [PATCH 263/362] Made RepairHandler.InProgressDblockVolumes.AddBlock async --- Duplicati/Library/Main/Operation/RepairHandler.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 62011ad65..49e9bc197 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -742,12 +742,12 @@ namespace Duplicati.Library.Main.Operation /// The offset in the buffer where the block data starts /// The size of the block data /// The compression hint for the block - public void AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) + public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) { m_activeWriter.AddBlock(hash, buffer, offset, size, hint); m_anyData = true; if (IsFull) - StartNewVolume().Await(); + await StartNewVolume(); } /// @@ -951,7 +951,7 @@ namespace Duplicati.Library.Main.Operation if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) { recoveredSourceBlocks++; - pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); + await pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); } } catch (Exception ex) @@ -1000,7 +1000,7 @@ namespace Duplicati.Library.Main.Operation if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) { recoveredSourceBlocks++; - pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); + await pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); } } catch (Exception ex) @@ -1035,7 +1035,7 @@ namespace Duplicati.Library.Main.Operation if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) { if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) - pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); + await pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); } else { @@ -1072,7 +1072,7 @@ namespace Duplicati.Library.Main.Operation foreach (var b in f.Blocks) if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) if (f.ReadBlock(b.Key, buffer) == b.Value) - pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); + await pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); } catch (Exception ex) { From f32a5fa7002685ecbc0d99e4300469ce58ee0c59 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:15:30 +0200 Subject: [PATCH 264/362] Removed whitespace --- .../Operation/Backup/DataBlockProcessor.cs | 30 ++++++------- .../Operation/Backup/SpillCollectorProcess.cs | 30 ++++++------- .../Library/Main/Volumes/BlockVolumeWriter.cs | 44 +++++++++---------- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 01e4f123a..0f41c881f 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; diff --git a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs index 46d7fb02c..2b27031cc 100644 --- a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; diff --git a/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs b/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs index 865c91807..4d351b3d8 100644 --- a/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; @@ -33,7 +33,7 @@ namespace Duplicati.Library.Main.Volumes public long Blocks { get { return m_blocks; } } public long SourceSize { get { return m_sourcesize; } } - + public override RemoteVolumeType FileType { get { return RemoteVolumeType.Blocks; } } public BlockVolumeWriter(Options options) @@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Volumes m_blocks++; m_sourcesize += size; - //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, + //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, using (var s = m_compression.CreateFile(Library.Utility.Utility.Base64PlainToBase64Url(hash), hint, DateTime.UtcNow)) s.Write(data, offset, size); } From 8cff30f99b5d333c25a5c8b24a9fa09a50570e61 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:17:04 +0200 Subject: [PATCH 265/362] Made BlockVolumeWriter.AddBlock async --- .../Library/Main/Operation/Backup/DataBlockProcessor.cs | 2 +- .../Library/Main/Operation/Backup/SpillCollectorProcess.cs | 2 +- Duplicati/Library/Main/Operation/CompactHandler.cs | 2 +- Duplicati/Library/Main/Operation/RepairHandler.cs | 2 +- Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs | 5 +++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 0f41c881f..2412fbfce 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -114,7 +114,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (newBlock) { - blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint); + await blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint); if (indexvolume != null) indexvolume.AddBlock(b.HashKey, b.Size); diff --git a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs index 2b27031cc..922c664c3 100644 --- a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Operation.Backup target = await GetNextTarget(source).ConfigureAwait(false); var len = rd.ReadBlock(file.Key, buffer); - target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default); + await target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default); await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false); if (target.IndexVolume != null) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 39a313a54..2c1812160 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -173,7 +173,7 @@ namespace Duplicati.Library.Main.Operation if (s != e.Value) throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value)); - newvol.AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible); + await newvol.AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible); if (newvolindex != null) newvolindex.AddBlock(e.Key, e.Value); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 49e9bc197..6a9fc28a0 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -744,7 +744,7 @@ namespace Duplicati.Library.Main.Operation /// The compression hint for the block public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) { - m_activeWriter.AddBlock(hash, buffer, offset, size, hint); + await m_activeWriter.AddBlock(hash, buffer, offset, size, hint); m_anyData = true; if (IsFull) await StartNewVolume(); diff --git a/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs b/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs index 4d351b3d8..e399f23f2 100644 --- a/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs @@ -22,6 +22,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Volumes @@ -41,14 +42,14 @@ namespace Duplicati.Library.Main.Volumes { } - public void AddBlock(string hash, byte[] data, int offset, int size, CompressionHint hint) + public async Task AddBlock(string hash, byte[] data, int offset, int size, CompressionHint hint) { m_blocks++; m_sourcesize += size; //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, using (var s = m_compression.CreateFile(Library.Utility.Utility.Base64PlainToBase64Url(hash), hint, DateTime.UtcNow)) - s.Write(data, offset, size); + await s.WriteAsync(data.AsMemory(offset, size)); } } } From eda3d5321ba167819e2e31c1f3cb6d505d3edf11 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:56:58 +0200 Subject: [PATCH 266/362] Removed whitespace --- .../DatabaseTool/Commands/Downgrade.cs | 30 +++++++++---------- .../DatabaseTool/Commands/Upgrade.cs | 30 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs index 6d1b0c37c..dc3adbf04 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.NamingConventionBinder; diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs index 5454ffcf0..90c0a817e 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.NamingConventionBinder; From 17cabeb3293b05f744488d3995471c679b0ed64a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:57:14 +0200 Subject: [PATCH 267/362] Updated the DatabaseTool to use the new async database backend --- .../DatabaseTool/Commands/Downgrade.cs | 26 +++++++++------ .../DatabaseTool/Commands/Upgrade.cs | 23 +++++++------ Duplicati/CommandLine/DatabaseTool/Helper.cs | 32 ++++++++++++++----- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs index dc3adbf04..7627a037a 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs @@ -24,6 +24,7 @@ using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; +using Duplicati.Library.Utility; namespace Duplicati.CommandLine.DatabaseTool.Commands; @@ -50,7 +51,7 @@ public static class Downgrade } .WithHandler(CommandHandler.Create((databases, serverdatafolder, serverversion, localversion, nobackups, includeuntrackeddatabases) => { - databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases); + databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases).Await(); if (databases.Length == 0) { Console.WriteLine("No databases found to downgrade"); @@ -100,7 +101,7 @@ public static class Downgrade bool isserverdb; try { - (version, isserverdb) = Helper.ExamineDatabase(db); + (version, isserverdb) = Helper.ExamineDatabase(db).Await(); } catch (Exception ex) { @@ -112,7 +113,8 @@ public static class Downgrade ApplyDowngrade(db, version, isserverdb ? serverversion : localversion, isserverdb ? serverVersions : localVersions, - nobackups); + nobackups) + .Await(); } })); @@ -132,7 +134,7 @@ public static class Downgrade /// The target database version /// The downgrade scripts /// Flag to disable backups - private static void ApplyDowngrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) + private static async Task ApplyDowngrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) { if (dbversion > targetversion) { @@ -158,7 +160,7 @@ public static class Downgrade if (!nobackups) Helper.CreateFileBackup(db); - using var con = SQLiteLoader.LoadConnection(db, 0); + using var con = await SQLiteLoader.LoadConnectionAsync(db, 0); using var tr = con.BeginTransaction(); using var cmd = con.CreateCommand(tr); foreach (var script in downgradeScripts) @@ -167,12 +169,16 @@ public static class Downgrade try { cmd.SetCommandAndParameters(script.Content); - var r = cmd.ExecuteScalar(); + var r = await cmd.ExecuteScalarAsync(); if (r != null) throw new UserInformationException($"{r}", "DowngradeScriptFailure"); - cmd.SetCommandAndParameters("UPDATE Version SET Version = @Version") - .SetParameterValue("@Version", script.Version - 1); - cmd.ExecuteNonQuery(); + + await cmd.SetCommandAndParameters(@" + UPDATE Version + SET Version = @Version + ") + .SetParameterValue("@Version", script.Version - 1) + .ExecuteNonQueryAsync(); } catch (Exception ex) { @@ -180,7 +186,7 @@ public static class Downgrade } } - tr.Commit(); + await tr.CommitAsync(); Console.WriteLine($"Downgraded {db} to version {targetversion}"); } else if (dbversion == targetversion) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs index 90c0a817e..28be63e26 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs @@ -24,6 +24,7 @@ using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; +using Duplicati.Library.Utility; namespace Duplicati.CommandLine.DatabaseTool.Commands; @@ -50,7 +51,7 @@ public static class Upgrade } .WithHandler(CommandHandler.Create((databases, serverdatafolder, serverversion, localversion, nobackups, includeuntrackeddatabases) => { - databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases); + databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases).Await(); if (databases.Length == 0) { Console.WriteLine("No databases found to upgrade"); @@ -83,7 +84,7 @@ public static class Upgrade bool isserverdb; try { - (version, isserverdb) = Helper.ExamineDatabase(db); + (version, isserverdb) = Helper.ExamineDatabase(db).Await(); } catch (Exception ex) { @@ -95,7 +96,8 @@ public static class Upgrade ApplyUpgrade(db, version, isserverdb ? serverversion : localversion, isserverdb ? serverVersions : localVersions, - nobackups); + nobackups) + .Await(); } })); @@ -145,7 +147,7 @@ public static class Upgrade /// The target database version /// The upgrade scripts /// Flag to disable backups - private static void ApplyUpgrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) + private static async Task ApplyUpgrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) { if (targetversion > dbversion) { @@ -170,7 +172,7 @@ public static class Upgrade if (!nobackups) Helper.CreateFileBackup(db); - using var con = SQLiteLoader.LoadConnection(db, 0); + using var con = await SQLiteLoader.LoadConnectionAsync(db, 0); using var tr = con.BeginTransaction(); using var cmd = con.CreateCommand(tr); foreach (var script in upgradeScripts) @@ -179,12 +181,15 @@ public static class Upgrade try { cmd.SetCommandAndParameters(script.Content); - var r = cmd.ExecuteScalar(); + var r = await cmd.ExecuteScalarAsync(); if (r != null) throw new UserInformationException($"{r}", "UpgradeScriptFailure"); - cmd.SetCommandAndParameters("UPDATE Version SET Version = @Version") - .SetParameterValue("@Version", script.Version); - cmd.ExecuteNonQuery(); + await cmd.SetCommandAndParameters(@" + UPDATE Version + SET Version = @Version + ") + .SetParameterValue("@Version", script.Version) + .ExecuteNonQueryAsync(); } catch (Exception ex) { diff --git a/Duplicati/CommandLine/DatabaseTool/Helper.cs b/Duplicati/CommandLine/DatabaseTool/Helper.cs index bba1578c8..7024fe94f 100644 --- a/Duplicati/CommandLine/DatabaseTool/Helper.cs +++ b/Duplicati/CommandLine/DatabaseTool/Helper.cs @@ -42,7 +42,7 @@ public static class Helper /// The folder to scan /// Whether to scan for extra databases /// The list of databases - public static string[] FindAllDatabases(string[]? databases, string datafolder, bool scanExtra) + public static async Task FindAllDatabases(string[]? databases, string datafolder, bool scanExtra) { databases ??= []; if (databases.Length != 0) @@ -60,9 +60,12 @@ public static class Helper { try { - using var con = SQLiteLoader.LoadConnection(serverdb, 0); - using var cmd = con.CreateCommand(); - foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""DBPath"" FROM ""Backup""")) + using var con = await SQLiteLoader.LoadConnectionAsync(serverdb, 0); + using var cmd = con.CreateCommand(@" + SELECT ""DBPath"" + FROM ""Backup"" + "); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) dbpaths.Add(rd.ConvertValueToString(0) ?? ""); } catch @@ -87,13 +90,26 @@ public static class Helper /// /// The path to the database /// A tuple containing the version and whether it is a server database - public static (int Version, bool isserver) ExamineDatabase(string db) + public static async Task<(int Version, bool isserver)> ExamineDatabase(string db) { - using (var con = SQLiteLoader.LoadConnection(db, 0)) + using (var con = await SQLiteLoader.LoadConnectionAsync(db, 0)) { using var cmd = con.CreateCommand(); - var isserverdb = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='Backup' OR name='Schedule'") == 2; - var version = (int)cmd.ExecuteScalarInt64("SELECT MAX(Version) FROM Version"); + + var isserverdb = await cmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM sqlite_master + WHERE + type='table' + AND name='Backup' + OR name='Schedule' + ") == 2; + + var version = (int)await cmd.ExecuteScalarInt64Async(@" + SELECT MAX(Version) + FROM Version + "); + return (version, isserverdb); } } From 94420be76d593d926dade6749c6d7e1f0c26118d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:57:29 +0200 Subject: [PATCH 268/362] Removed whitespace --- Duplicati/Server/Program.cs | 30 ++++++++++++------------- Duplicati/UnitTest/ImportExportTests.cs | 30 ++++++++++++------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index d8530f950..7b1c514c9 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index 1d61312f3..2fd4443e6 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 90721af98dd2e49aaff71eaa2320f2766407753a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 08:58:32 +0200 Subject: [PATCH 269/362] Updated Server and ImportExportTests to use the async database backend --- Duplicati/Server/Program.cs | 2 +- Duplicati/UnitTest/ImportExportTests.cs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 7b1c514c9..4f9f66432 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -761,7 +761,7 @@ namespace Duplicati.Server System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(databasePath)); // Attempt to open the database, removing any encryption present - Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)); + Library.SQLiteHelper.SQLiteLoader.OpenDatabaseAsync(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)).Await(); Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, databasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); } diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index 2fd4443e6..f0b0a61c9 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -22,6 +22,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Threading.Tasks; using Duplicati.Library.RestAPI; using Duplicati.Library.SQLiteHelper; using Duplicati.Server; @@ -123,7 +124,7 @@ namespace Duplicati.UnitTest [Test] [Category("ImportExport")] - public void RoundTrip() + public async Task RoundTrip() { var metadata = new Dictionary { { "SourceFilesCount", "1" } }; var advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; @@ -133,7 +134,7 @@ namespace Duplicati.UnitTest File.Delete(dbpath); var con = SQLiteLoader.LoadConnection(); - SQLiteLoader.OpenDatabase(con, dbpath, null); + await SQLiteLoader.OpenDatabaseAsync(con, dbpath, null); DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); using (var connection = new Connection(con, true, null, this.serverDatafolder, null)) From 51b86b06fff1dee73ce6e16940c54efeeffc1636 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:00:26 +0200 Subject: [PATCH 270/362] Made the SQLiteLoader open async again --- Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index acd9e7026..5c2bc7ecf 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -276,7 +276,7 @@ namespace Duplicati.Library.SQLiteHelper private static async Task OpenSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con, string path) { con.ConnectionString = $"Data Source={path};Pooling=false"; - con.Open(); + await con.OpenAsync(); // TODO legacy configuration? //if (con is Microsoft.Data.Sqlite.SqliteConnection sqlitecon && !OperatingSystem.IsMacOS()) //{ From ebdddebeb1b072abd4674ed64f5f132ca620e770 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:01:12 +0200 Subject: [PATCH 271/362] Pragmas that change the safety level of a connection cannot be performed within a transaction --- Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 5c2bc7ecf..b91db4d51 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -147,9 +147,7 @@ namespace Duplicati.Library.SQLiteHelper return con; using (var cmd = con.CreateCommand()) - using (var transaction = con.BeginTransaction(deferred: true)) { - cmd.Transaction = transaction; foreach (var opt in opts.Split([';'], StringSplitOptions.RemoveEmptyEntries)) { Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt); @@ -163,7 +161,6 @@ namespace Duplicati.Library.SQLiteHelper Logging.Log.WriteWarningMessage(LOGTAG, "CustomSQLiteOption", ex, @"Error setting custom SQLite option '{0}'.", opt); } } - await transaction.CommitAsync(); } return con; } From 51f2d271664cd99fe673d9391400b68902a0b2c9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:01:46 +0200 Subject: [PATCH 272/362] Removed unused helper methods --- .../Library/SQLiteHelper/SQLiteLoader.cs | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index b91db4d51..c9b1fe6be 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -42,11 +42,6 @@ namespace Duplicati.Library.SQLiteHelper /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(SQLiteLoader)); - public static void OpenDatabase(Microsoft.Data.Sqlite.SqliteConnection con, string databasePath, string? decryptionPassword) - { - OpenDatabaseAsync(con, databasePath, decryptionPassword).Await(); - } - /// /// Helper method with logic to handle opening a database in possibly encrypted format /// @@ -117,11 +112,6 @@ namespace Duplicati.Library.SQLiteHelper return con ?? throw new InvalidOperationException("Failed to load connection"); } - public static Microsoft.Data.Sqlite.SqliteConnection ApplyCustomPragmas(Microsoft.Data.Sqlite.SqliteConnection con, long pagecachesize) - { - return ApplyCustomPragmasAsync(con, pagecachesize).Await(); - } - /// /// Applies user-supplied custom pragmas to the SQLite connection /// @@ -244,11 +234,6 @@ namespace Duplicati.Library.SQLiteHelper Environment.SetEnvironmentVariable("TEMP", Utility.TempFolder.SystemTempPath); } - private static void DisposeConnection(Microsoft.Data.Sqlite.SqliteConnection? con) - { - DisposeConnectionAsync(con).Await(); - } - /// /// Wrapper to dispose the SQLite connection /// @@ -260,11 +245,6 @@ namespace Duplicati.Library.SQLiteHelper catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "ConnectionDisposeError", ex, "Failed to dispose connection"); } } - private static void OpenSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con, string path) - { - OpenSQLiteFileAsync(con, path).Await(); - } - /// /// Opens the SQLite file in the given connection, creating the file if required /// @@ -274,13 +254,6 @@ namespace Duplicati.Library.SQLiteHelper { con.ConnectionString = $"Data Source={path};Pooling=false"; await con.OpenAsync(); - // TODO legacy configuration? - //if (con is Microsoft.Data.Sqlite.SqliteConnection sqlitecon && !OperatingSystem.IsMacOS()) - //{ - // // These configuration options crash on MacOS (arm64), but the other platforms should be enough to detect incorrect SQL - // sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DDL, false); - // sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DML, false); - //} // Make the file only accessible by the current user, unless opting out if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile))) @@ -288,11 +261,6 @@ namespace Duplicati.Library.SQLiteHelper catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "SQLiteFilePermissionError", ex, "Failed to set permissions on SQLite file '{0}'", path); } } - private static void TestSQLiteFile(Microsoft.Data.Sqlite.SqliteConnection con) - { - TestSQLiteFileAsync(con).Await(); - } - /// /// Tests the SQLite connection, throwing an exception if the connection does not work /// From f411260c03b706c5ad5eb96ae88d0607cb4d7f72 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:11:35 +0200 Subject: [PATCH 273/362] Removed TODO in comments on newly introduced commits due to how the pre Microsoft.Data.Sqlite backend would make nested transactions --- .../Main/Backend/BackendManager.DatabaseCollector.cs | 4 +++- Duplicati/Library/Main/Database/LocalPurgeDatabase.cs | 3 --- Duplicati/Library/Main/Operation/PurgeFilesHandler.cs | 5 ++++- Duplicati/Library/Main/Operation/RepairHandler.cs | 9 +++++---- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs index 86303cb1e..4903e9d63 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs @@ -152,7 +152,9 @@ partial class BackendManager if (volsRemoved.Count > 0) { await db.RemoveRemoteVolumes(volsRemoved); - // TODO this commit is to mimic the old behavior, in which RemoveRemoteVolumes started a new transaction, if none was passed down. + // This commit is to mimic the pre Microsoft.Data.Sqlite backend + // behavior, in which RemoveRemoteVolumes started a new + // transaction, if none was passed down. await db.Transaction.CommitAsync(); } diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index f8223d6fd..f7426f897 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -287,9 +287,6 @@ namespace Duplicati.Library.Main.Database var remotevolid = await m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary); var filesetid = await m_db.CreateFileset(remotevolid, timestamp); await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); - // TODO this commit mimics the old way of not passing down a transaction to UpdateFullBackupStateInFileset, but maybe it is not needed. - //await m_db.Transaction.CommitAsync(); - // TODO not exactly, since the old way was using a nested transaction, which the new backend doesn't support. using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) await cmd.SetCommandAndParameters($@" diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index c6fe3312f..b743c7d31 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -203,7 +203,10 @@ namespace Duplicati.Library.Main.Operation await backendManager.PutAsync(vol, null, null, true, async () => { await backendManager.FlushPendingMessagesAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - // TODO This commit mimics the old way of not passing down a transaction, but maybe is not needed anymore. + // This commit mimics the pre- + // Microsoft.Data.Sqlite backend way of not + // passing down a transaction, but maybe is + // not needed anymore. await db.Transaction.CommitAsync(); }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 6a9fc28a0..8b1f6222d 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -504,7 +504,11 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); - // TODO this commit mimics the old way of not passing down a transaction to FlushPendingMessagesAsync, but maybe it is not needed anymore. + // This commit mimics the pre + // Microsoft.Data.Sqlite backend way of not + // passing down a transaction to + // FlushPendingMessagesAsync, but maybe it + // is not needed anymore. await db.Transaction.CommitAsync(); } } @@ -566,9 +570,6 @@ namespace Duplicati.Library.Main.Operation volumeWriter.CreateFilesetFile(isPrevFull); var newFilesetID = await db.CreateFileset(volumeWriter.VolumeID, filesetTime); await db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull); - // TODO this commit mimics the old way of not passing down a transaction to UpdateFullBackupStateInFileset, but maybe it is not needed anymore. - //await db.Transaction.CommitAsync(); - // TODO The above only mimics it to some degree -the original implementation makes a new transaciton alongside the global reusable transaction, which means that this particular SQL statement is comitted to the database. As such, if this new implementation commits the global reusable transaction, all of the statements are comitted - which they shouldn't be! Instead, only whatever that function is trying to do should be committed. This happens at other points as well - all of which should be double checked wrt. whether they should be using the global one, or a new one. This is not supported in the new engine. await db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID); await db.MoveFilesFromFileset(newFilesetID, prevFilesetId); From e946b3593a62cddb3cb3b3e0056944d7afedd49b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:32:28 +0200 Subject: [PATCH 274/362] Handled TODOs in the database extension methods --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index c3760f872..b9383b7a7 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -36,8 +36,6 @@ namespace Duplicati.Library.Main.Database; /// public static partial class ExtensionMethods { - // TODO parameter types and sizes - public static long ConvertValueToInt64(this SqliteDataReader reader, int index, long defaultvalue = -1) { try @@ -106,7 +104,6 @@ public static partial class ExtensionMethods { self.SetCommandAndParameters(cmdtext); - // TODO "late format string-ing" using (new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", $"ExecuteNonQueryAsync: {self.CommandText}")) return await self.ExecuteNonQueryAsync().ConfigureAwait(false); } @@ -400,7 +397,9 @@ public static partial class ExtensionMethods return self; } - // TODO commennt from here on + // TODO At some point, when the entire codebase is migrated to + // Microsoft.Data.Sqlite, the extension methods using IDb* should be + // removed. Currently, they're kept for compatibility. public static IDbCommand AddNamedParameter(this IDbCommand self, string name, object? value = null) { @@ -933,5 +932,5 @@ public static partial class ExtensionMethods cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause().Await(), StringComparison.OrdinalIgnoreCase); return cmd; } - // TODO comment to here + } \ No newline at end of file From c1710cd6a47a8262933a0dbbf3059cebbdd23c98 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:33:30 +0200 Subject: [PATCH 275/362] Updated the database helper method for formatting SQL queries for logging --- .../Library/Main/Database/ExtensionMethods.cs | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index b9383b7a7..1f09f64ba 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -467,23 +467,28 @@ public static partial class ExtensionMethods public static string GetPrintableCommandText(this IDbCommand self) { var txt = self.CommandText; - // TODO not adjusted to named parameters - it's using positional parameters. - foreach (var p in self.Parameters.Cast()) + // Replace each named parameter with its value + foreach (IDataParameter p in self.Parameters) { - var ix = txt.IndexOf('?'); - if (ix >= 0) - { - string v; - if (p.Value is string) - v = string.Format("\"{0}\"", p.Value); - else if (p.Value == null) - v = "NULL"; - else - v = string.Format("{0}", p.Value); + var paramName = p.ParameterName; + if (string.IsNullOrEmpty(paramName)) + continue; - txt = string.Concat(txt.AsSpan(0, ix), v, txt.AsSpan(ix + 1)); - } + string v; + if (p.Value == null || p.Value == DBNull.Value) + v = "NULL"; + else if (p.Value is string) + v = $"\"{p.Value}\""; + else if (p.Value is DateTime dt) + v = $"\"{dt:O}\""; + else if (p.Value is byte[] bytes) + v = $"X'{BitConverter.ToString(bytes).Replace("-", "")}'"; + else + v = p.Value.ToString() ?? "NULL"; + + // Replace all occurrences of the parameter (with word boundary) + txt = Regex.Replace(txt, $@"\B{Regex.Escape(paramName)}\b", v, RegexOptions.IgnoreCase); } return txt; From 1ab303caea0c63544c5ff1de05a7d4fdafc94026 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:34:50 +0200 Subject: [PATCH 276/362] Reordered the call trace of ExecuteReaderAsync --- .../Library/Main/Database/ExtensionMethods.cs | 40 +++++-------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 1f09f64ba..781afa9a7 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -125,46 +125,26 @@ public static partial class ExtensionMethods return await self.ExecuteNonQueryAsync(); } - //public static SqliteDataReader ExecuteReader(this SqliteCommand self, string cmd) - //{ - // return ExecuteReader(self, true, cmd); - //} - - //public static SqliteDataReader ExecuteReader(this SqliteCommand self, bool writeLog, string? cmd) - //{ - // if (cmd != null) - // self.SetCommandAndParameters(cmd); - // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) - // return self.ExecuteReader(); - //} - - public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmd) + public static async Task ExecuteReaderAsync(this SqliteCommand self, string? cmdtext) { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReaderAsync", string.Format("ExecuteReaderAsync: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteReaderAsync(); + return await ExecuteReaderAsync(self, true, cmdtext, null).ConfigureAwait(false); } - public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext) + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext) { - self.SetCommandAndParameters(cmdtext); - - // TODO "late format string-ing" - using (new Logging.Timer(LOGTAG, "ExecuteReaderAsync", $"ExecuteReaderAsync: {self.CommandText}")) - return await self.ExecuteReaderAsync().ConfigureAwait(false); + return await ExecuteReaderAsync(self, writeLog, cmdtext, null).ConfigureAwait(false); } - public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmd, Dictionary? values) + + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary? values) { - return await ExecuteReaderAsync(self, true, cmd, values); + return await ExecuteReaderAsync(self, true, cmdtext, values).ConfigureAwait(false); } - public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values) + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext, Dictionary? values) { - if (cmd != null) - self.SetCommandAndParameters(cmd); + if (cmdtext != null) + self.SetCommandAndParameters(cmdtext); if (values != null && values.Count > 0) self.SetParameterValues(values); From 7c9516eea04fadfd9ae1bcc7441195f150e05964 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:42:51 +0200 Subject: [PATCH 277/362] Reordered the call trace of ExecuteNonQueryAsync --- .../Library/Main/Database/ExtensionMethods.cs | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 781afa9a7..8e8c527f7 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -88,24 +88,17 @@ public static partial class ExtensionMethods public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog) { - return await ExecuteNonQueryAsync(self, writeLog, null); + return await ExecuteNonQueryAsync(self, writeLog, null, null); + } + + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string? cmdtext) + { + return await ExecuteNonQueryAsync(self, true, cmdtext, null); } public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd) { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", string.Format("ExecuteNonQueryAsync: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteNonQueryAsync(); - } - - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmdtext) - { - self.SetCommandAndParameters(cmdtext); - - using (new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", $"ExecuteNonQueryAsync: {self.CommandText}")) - return await self.ExecuteNonQueryAsync().ConfigureAwait(false); + return await ExecuteNonQueryAsync(self, writeLog, cmd, null); } public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary values) @@ -113,7 +106,7 @@ public static partial class ExtensionMethods return await ExecuteNonQueryAsync(self, true, cmd, values); } - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary values) + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values) { if (cmd != null) self.SetCommandAndParameters(cmd); From 544ec85d877c19f62990cc170edee7be15c5f200 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:48:09 +0200 Subject: [PATCH 278/362] Added missing combination of parameters to ExecuterReaderAsync --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 8e8c527f7..7c5559371 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -123,12 +123,16 @@ public static partial class ExtensionMethods return await ExecuteReaderAsync(self, true, cmdtext, null).ConfigureAwait(false); } + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog) + { + return await ExecuteReaderAsync(self, writeLog, null, null).ConfigureAwait(false); + } + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext) { return await ExecuteReaderAsync(self, writeLog, cmdtext, null).ConfigureAwait(false); } - public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary? values) { return await ExecuteReaderAsync(self, true, cmdtext, values).ConfigureAwait(false); From 16fdb633be851130be8237e5fdc48f47074d9b6a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 09:50:28 +0200 Subject: [PATCH 279/362] Reordered the call trace of ExecuteReaderEnumerableAsync --- .../Library/Main/Database/ExtensionMethods.cs | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 7c5559371..04a9ddf8c 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -150,19 +150,33 @@ public static partial class ExtensionMethods return await self.ExecuteReaderAsync(); } - public static async IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self) + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self) { - using (new Logging.Timer(LOGTAG, "ExecuteReaderEnumerableAsync", $"ExecuteReaderEnumerableAsync: {self.GetPrintableCommandText()}")) - using (var rd = await self.ExecuteReaderAsync()) - while (await rd.ReadAsync()) - yield return rd; + return ExecuteReaderEnumerableAsync(self, true, null, null); } public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext) { - self.SetCommandAndParameters(cmdtext); + return ExecuteReaderEnumerableAsync(self, true, cmdtext, null); + } - return ExecuteReaderEnumerableAsync(self); + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, Dictionary? values) + { + return ExecuteReaderEnumerableAsync(self, true, cmdtext, values); + } + + public static async IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, bool writeLog, string? cmdtext = null, Dictionary? values = null) + { + if (cmdtext != null) + self.SetCommandAndParameters(cmdtext); + + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReaderEnumerableAsync", $"ExecuteReaderEnumerableAsync: {self.GetPrintableCommandText()}") : null) + using (var rd = await self.ExecuteReaderAsync()) + while (await rd.ReadAsync()) + yield return rd; } public static async Task ExecuteScalarAsync(this SqliteCommand self) From bbe409d5643add540841fbc3d8e8e3e161831248 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 10:01:40 +0200 Subject: [PATCH 280/362] Reordered the call trace of ExecuteScalarAsync --- .../Library/Main/Database/ExtensionMethods.cs | 46 +++++++------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 04a9ddf8c..b58ed952b 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -181,42 +181,30 @@ public static partial class ExtensionMethods public static async Task ExecuteScalarAsync(this SqliteCommand self) { - return await self.ExecuteScalarAsync().ConfigureAwait(false); - } - - public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd) - { - if (cmd != null) - self.SetCommandAndParameters(cmd); - - using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarAsync", string.Format("ExecuteScalarAsync: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteScalarAsync(); + return await self.ExecuteScalarAsync(true, null, null).ConfigureAwait(false); } public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext) { - self.SetCommandAndParameters(cmdtext); - - return await ExecuteScalarAsync(self).ConfigureAwait(false); + return await self.ExecuteScalarAsync(true, cmdtext, null).ConfigureAwait(false); } - //public static long ExecuteScalarInt64(this SqliteCommand self, long defaultvalue = -1) - //{ - // return ExecuteScalarInt64(self, true, null, defaultvalue); - //} + public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext, Dictionary? values) + { + return await self.ExecuteScalarAsync(true, cmdtext, values).ConfigureAwait(false); + } - //public static long ExecuteScalarInt64(this SqliteCommand self, bool writeLog, string? cmd, long defaultvalue) - //{ - // if (cmd != null) - // self.SetCommandAndParameters(cmd); - // - // using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())) : null) - // using (var rd = self.ExecuteReader()) - // if (rd.Read()) - // return ConvertValueToInt64(rd, 0, defaultvalue); - // - // return defaultvalue; - //} + public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarAsync", string.Format("ExecuteScalarAsync: {0}", self.GetPrintableCommandText())) : null) + return await self.ExecuteScalarAsync(); + } public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue = -1) { From 1db0fc77892c248a34353d98c9ae18894a78d116 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 10:06:55 +0200 Subject: [PATCH 281/362] Reordered the call trace of ExecuteScalarInt64Async --- .../Library/Main/Database/ExtensionMethods.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index b58ed952b..4b73fc31d 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -208,14 +208,27 @@ public static partial class ExtensionMethods public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue = -1) { - return await ExecuteScalarInt64Async(self, writeLog, null, defaultvalue); + return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue); } - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd, long defaultvalue) + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, long defaultvalue = -1) + { + return await ExecuteScalarInt64Async(self, true, cmd, null, defaultvalue); + } + + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary? values, long defaultvalue = -1) + { + return await ExecuteScalarInt64Async(self, true, cmd, values, defaultvalue); + } + + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null, long defaultvalue = -1) { if (cmd != null) self.SetCommandAndParameters(cmd); + if (values != null && values.Count > 0) + self.SetParameterValues(values); + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64Async", string.Format("ExecuteScalarInt64Async: {0}", self.GetPrintableCommandText())) : null) using (var rd = await self.ExecuteReaderAsync()) if (await rd.ReadAsync()) From 9c2252f36c2467e42c305a495c056e962a830b17 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 10:46:04 +0200 Subject: [PATCH 282/362] Ensured common order and interface for the async extension methods. --- .../Library/Main/Database/ExtensionMethods.cs | 33 +++++-------------- .../Main/Database/LocalBackupDatabase.cs | 6 ++-- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 4b73fc31d..2153518c4 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -86,27 +86,17 @@ public static partial class ExtensionMethods return self.CreateCommand(rtr.Transaction); } - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog) - { - return await ExecuteNonQueryAsync(self, writeLog, null, null); - } - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string? cmdtext) { return await ExecuteNonQueryAsync(self, true, cmdtext, null); } - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd) - { - return await ExecuteNonQueryAsync(self, writeLog, cmd, null); - } - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary values) { return await ExecuteNonQueryAsync(self, true, cmd, values); } - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values) + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null) { if (cmd != null) self.SetCommandAndParameters(cmd); @@ -118,27 +108,17 @@ public static partial class ExtensionMethods return await self.ExecuteNonQueryAsync(); } - public static async Task ExecuteReaderAsync(this SqliteCommand self, string? cmdtext) + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext) { return await ExecuteReaderAsync(self, true, cmdtext, null).ConfigureAwait(false); } - public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog) - { - return await ExecuteReaderAsync(self, writeLog, null, null).ConfigureAwait(false); - } - - public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext) - { - return await ExecuteReaderAsync(self, writeLog, cmdtext, null).ConfigureAwait(false); - } - public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary? values) { return await ExecuteReaderAsync(self, true, cmdtext, values).ConfigureAwait(false); } - public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext, Dictionary? values) + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext = null, Dictionary? values = null) { if (cmdtext != null) self.SetCommandAndParameters(cmdtext); @@ -206,7 +186,12 @@ public static partial class ExtensionMethods return await self.ExecuteScalarAsync(); } - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue = -1) + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue = -1) + { + return await ExecuteScalarInt64Async(self, true, null, null, defaultvalue).ConfigureAwait(false); + } + + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue) { return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue); } diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 1bdedb90e..38575f4b6 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -719,7 +719,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Fullhash", filehash) .SetParameterValue("@Length", size) - .ExecuteScalarInt64Async(m_logQueries, null, -1); + .ExecuteScalarInt64Async(m_logQueries, -1); if (blocksetid != -1) return (false, blocksetid); //Found it @@ -820,7 +820,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Hash", filehash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(m_logQueries, null, -1); + .ExecuteScalarInt64Async(m_logQueries, -1); return (metadataid != -1, metadataid); } @@ -961,7 +961,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries, null)) + using (var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries)) if (await rd.ReadAsync()) { length = -1; From f373880c5a46da71d458a23fb692e1b6dc5b2073 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 10:47:40 +0200 Subject: [PATCH 283/362] Renamed ExpandInClauseParameterAsync, as one of them isn't actually async. Postfix mssqlite has been added as well to not clash with the IDb implementation --- .../Library/Main/Database/ExtensionMethods.cs | 20 +++---------------- .../Main/Database/LocalBackupDatabase.cs | 6 +++--- .../Library/Main/Database/LocalDatabase.cs | 6 +++--- .../Main/Database/LocalDeleteDatabase.cs | 8 ++++---- .../Database/LocalListAffectedDatabase.cs | 8 ++++---- .../Database/LocalListBrokenFilesDatabase.cs | 8 ++++---- .../Main/Database/LocalListDatabase.cs | 16 +++++++-------- .../Main/Database/LocalRepairDatabase.cs | 6 +++--- .../Main/Database/LocalTestDatabase.cs | 6 +++--- 9 files changed, 35 insertions(+), 49 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 2153518c4..6943bbbe2 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -222,21 +222,7 @@ public static partial class ExtensionMethods return defaultvalue; } - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue = -1) - { - return await ExecuteScalarInt64Async(self, true, defaultvalue).ConfigureAwait(false); - - } - - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string cmdtext, long defaultvalue = -1) - { - self.SetCommandAndParameters(cmdtext); - - return await ExecuteScalarInt64Async(self, defaultvalue).ConfigureAwait(false); - } - - //[Obsolete("Technically this method isn't async, but it is used in an async context and to differntiate this with the IDb version. ")] - public static SqliteCommand ExpandInClauseParameterAsync(this SqliteCommand cmd, string originalParamName, IEnumerable values) + public static SqliteCommand ExpandInClauseParameterMssqlite(this SqliteCommand cmd, string originalParamName, IEnumerable values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); @@ -264,13 +250,13 @@ public static partial class ExtensionMethods return cmd; } - internal static async Task ExpandInClauseParameterAsync(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) + internal static async Task ExpandInClauseParameterMssqliteAsync(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); if (!values.IsTableCreated) - return ExpandInClauseParameterAsync(cmd, originalParamName, values.Values); + return ExpandInClauseParameterMssqlite(cmd, originalParamName, values.Values); // We have a temporary table, so we need to replace the parameter with the table name cmd.CommandText = cmd.CommandText.Replace(originalParamName, await values.GetInClause(), StringComparison.OrdinalIgnoreCase); diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 38575f4b6..3587652f4 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -321,7 +321,7 @@ namespace Duplicati.Library.Main.Database ""Hash"", ""Size"" ") - .ExpandInClauseParameterAsync("@States", [ + .ExpandInClauseParameterMssqlite("@States", [ RemoteVolumeState.Deleted, RemoteVolumeState.Deleting ]) @@ -1244,7 +1244,7 @@ namespace Duplicati.Library.Main.Database ) ") .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameterAsync("@Paths", tmplist)) + .ExpandInClauseParameterMssqliteAsync("@Paths", tmplist)) .ExecuteNonQueryAsync(m_logQueries); } @@ -1415,7 +1415,7 @@ namespace Duplicati.Library.Main.Database AND ""State"" IN (@States) ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameterAsync("@States", [ + .ExpandInClauseParameterMssqlite("@States", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() ]); diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index f23bdfac4..3160e7197 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -519,7 +519,7 @@ namespace Duplicati.Library.Main.Database "); cmd.SetTransaction(m_rtr); using var tmptable = await TemporaryDbValueList.CreateAsync(this, files); - await cmd.ExpandInClauseParameterAsync("@Name", tmptable); + await cmd.ExpandInClauseParameterMssqliteAsync("@Name", tmptable); using var rd = await cmd.ExecuteReaderAsync(); while (await rd.ReadAsync()) @@ -660,7 +660,7 @@ namespace Duplicati.Library.Main.Database FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames) ") - .ExpandInClauseParameterAsync("@VolumeNames", [.. names]) + .ExpandInClauseParameterMssqlite("@VolumeNames", [.. names]) .ExecuteNonQueryAsync(); var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" "; @@ -859,7 +859,7 @@ namespace Duplicati.Library.Main.Database AND ""State"" IN (@AllowedStates) ") .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) - .ExpandInClauseParameterAsync("@AllowedStates", [ + .ExpandInClauseParameterMssqlite("@AllowedStates", [ RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index d01aa4765..885a211b5 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -86,7 +86,7 @@ namespace Duplicati.Library.Main.Database DELETE FROM ""Fileset"" WHERE ""Timestamp"" IN (@Timestamps) ") - .ExpandInClauseParameterAsync("@Timestamps", tempTable)) + .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable)) .ExecuteNonQueryAsync(); if (deleted != toDelete.Length) @@ -204,7 +204,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) .SetParameterValue("@CurrentType", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameterAsync("@AllowedStates", [ + .ExpandInClauseParameterMssqlite("@AllowedStates", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString(), @@ -427,7 +427,7 @@ namespace Duplicati.Library.Main.Database { await cmd.SetCommandAndParameters(createtable) .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameterAsync("@AllowedStates", [ + .ExpandInClauseParameterMssqlite("@AllowedStates", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() ]) @@ -714,7 +714,7 @@ namespace Duplicati.Library.Main.Database ) GROUP BY ""BlockID"" ") - .ExpandInClauseParameterAsync("@VolumeIds", tempTable)) + .ExpandInClauseParameterMssqliteAsync("@VolumeIds", tempTable)) .ExecuteNonQueryAsync(); var targetCount = await cmd.ExecuteScalarInt64Async($@" diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 2b317e945..a88e84cbd 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -107,7 +107,7 @@ namespace Duplicati.Library.Main.Database "; using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items)) - using (var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterAsync("@Names", tmptable)) + using (var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterMssqliteAsync("@Names", tmptable)) using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) { @@ -181,7 +181,7 @@ namespace Duplicati.Library.Main.Database "; using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items)) - using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterAsync("@Names", tmptable)) + using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterMssqliteAsync("@Names", tmptable)) using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) yield return new ListResultFile() @@ -220,7 +220,7 @@ namespace Duplicati.Library.Main.Database "; cmd.SetCommandAndParameters(sql) - .ExpandInClauseParameter("@Paths", items.ToArray()); + .ExpandInClauseParameterMssqlite("@Paths", items.ToArray()); foreach ((var x, var i) in items.Select((x, i) => (x, i))) cmd.SetParameterValue($"@Message{i}", "%" + x + "%"); @@ -301,7 +301,7 @@ namespace Duplicati.Library.Main.Database ) "; - using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterAsync("@Names", items.ToArray())) + using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterMssqlite("@Names", items.ToArray())) using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) yield return new ListResultRemoteVolume() diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 5220e4614..3811af305 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -275,7 +275,7 @@ namespace Duplicati.Library.Main.Database ) ") .SetTransaction(m_rtr) - .ExpandInClauseParameterAsync("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds.Cast()) .SetParameterValue("@EmptyHash", emptyHash) .SetParameterValue("@EmptyHashSize", emptyHashSize); @@ -298,7 +298,7 @@ namespace Duplicati.Library.Main.Database AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ) ") - .ExpandInClauseParameterAsync("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds.Cast()) .SetParameterValue("@EmptyHashSize", 0) .ExecuteScalarInt64Async(-1); @@ -319,7 +319,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Blockset"".""Length"" ASC LIMIT 1 ") - .ExpandInClauseParameterAsync("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds.Cast()) .ExecuteScalarInt64Async(-1); return res; @@ -383,7 +383,7 @@ namespace Duplicati.Library.Main.Database FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names) ") - .ExpandInClauseParameterAsync("@Names", tmptable)) + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable)) .ExecuteNonQueryAsync(); var volIdsSubQuery = $@" diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index ef081349c..962e07ede 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -789,7 +789,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds.Cast()) .SetParameterValue("@filesetid", filesetid) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); @@ -835,7 +835,7 @@ namespace Duplicati.Library.Main.Database fe.FilesetID = @filesetid AND fl.PrefixID IN (@PrefixIds) ") - .ExpandInClauseParameterAsync("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds.Cast()) .SetParameterValue("@filesetid", filesetid) .ExecuteScalarInt64Async(0); } @@ -899,7 +899,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameter("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds.Cast()) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); @@ -938,7 +938,7 @@ namespace Duplicati.Library.Main.Database AND fl.""PrefixId"" IN (@PrefixIds) ") .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameterAsync("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds.Cast()) .ExecuteScalarInt64Async(0); } @@ -1143,7 +1143,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@offset", offset); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds.Cast()); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds.Cast()); var results = new List(); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -1192,7 +1192,7 @@ namespace Duplicati.Library.Main.Database .ExpandInClauseParameter("@Paths", paths); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds.Cast()); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds.Cast()); totalCount = await cmd.ExecuteScalarInt64Async(0); } @@ -1403,7 +1403,7 @@ namespace Duplicati.Library.Main.Database cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds.Cast()); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds.Cast()); var results = new List(); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -1453,7 +1453,7 @@ namespace Duplicati.Library.Main.Database cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds.Cast()); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds.Cast()); totalCount = await cmd.ExecuteScalarInt64Async(0); } diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 0b2fbadaf..580b2a650 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1628,7 +1628,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameterAsync("@States", [ + .ExpandInClauseParameterMssqlite("@States", [ RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString() ]); @@ -1655,7 +1655,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameterAsync("@States", [ + .ExpandInClauseParameterMssqlite("@States", [ RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString() ]); @@ -1686,7 +1686,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) - .ExpandInClauseParameterAsync("@States", [ + .ExpandInClauseParameterMssqlite("@States", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() ]); diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 0add365c0..d1dfbb5ec 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -154,7 +154,7 @@ namespace Duplicati.Library.Main.Database { // Select any broken items cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) AND (""ArchiveTime"" = 0)") - .ExpandInClauseParameter("@States", [RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Uploaded.ToString()]); + .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Uploaded.ToString()]); using (var rd = cmd.ExecuteReader()) while (rd.Read()) @@ -197,7 +197,7 @@ namespace Duplicati.Library.Main.Database AND ""ArchiveTime"" = 0 ") .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) @@ -226,7 +226,7 @@ namespace Duplicati.Library.Main.Database AND ""ArchiveTime"" = 0 ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); using (var rd = await cmd.ExecuteReaderAsync()) while (await rd.ReadAsync()) From fd5823a2f46e26c4a1f84a75a2357ec463f78d0a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 10:54:02 +0200 Subject: [PATCH 284/362] Found unformatted SQL queries in LocalTestDatabase --- .../Main/Database/LocalTestDatabase.cs | 58 +++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index d1dfbb5ec..4a63e5ead 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -148,13 +148,36 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand(m_rtr)) { var files = new List(); - var max = cmd.ExecuteScalarInt64(@"SELECT MAX(""VerificationCount"") FROM ""RemoteVolume""", 0); + var max = cmd.ExecuteScalarInt64(@" + SELECT MAX(""VerificationCount"") + FROM ""RemoteVolume"" + ", 0); if (options.FullRemoteVerification != Options.RemoteTestStrategy.IndexesOnly) { // Select any broken items - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) AND (""ArchiveTime"" = 0)") - .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Uploaded.ToString()]); + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM + ""Remotevolume"" + WHERE + (""State"" IN (@States)) + AND ( + ""Hash"" = '' + OR ""Hash"" IS NULL + OR ""Size"" <= 0 + ) + AND (""ArchiveTime"" = 0) + ") + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Verified.ToString(), + RemoteVolumeState.Uploaded.ToString() + ]); using (var rd = cmd.ExecuteReader()) while (rd.Read()) @@ -162,7 +185,34 @@ namespace Duplicati.Library.Main.Database //First we select some filesets var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); - using (var rd = cmd.SetCommandAndParameters(@"SELECT ""A"".""VolumeID"", ""A"".""Name"", ""A"".""Size"", ""A"".""Hash"", ""A"".""VerificationCount"" FROM (SELECT ""ID"" AS ""VolumeID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""ArchiveTime"" = 0 AND ""State"" IN (@State1, @State2)) A, ""Fileset"" " + whereClause + @" ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" ") + using (var rd = cmd.SetCommandAndParameters(@$" + SELECT + ""A"".""VolumeID"", + ""A"".""Name"", + ""A"".""Size"", + ""A"".""Hash"", + ""A"".""VerificationCount"" + FROM + ( + SELECT + ""ID"" AS ""VolumeID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + ""ArchiveTime"" = 0 + AND ""State"" IN ( + @State1, + @State2 + ) + ) A, + ""Fileset"" + {whereClause} + ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" + ORDER BY ""Fileset"".""Timestamp"" + ") .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) .SetParameterValues(tp.Item2) From be550de0816a40a4c98c39b0481f551195266847 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 10:56:29 +0200 Subject: [PATCH 285/362] Wrapped a database VACUUM in a transaction --- Duplicati/Library/Main/Database/LocalBugReportDatabase.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 338967313..41330b294 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -145,8 +145,9 @@ namespace Duplicati.Library.Main.Database using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) await m_rtr.CommitAsync(); - // TODO maybe this needs a transaction. + cmd.SetTransaction(m_rtr); await cmd.ExecuteNonQueryAsync("VACUUM"); + await m_rtr.CommitAsync(); } } } From da9334d1742c11da1208d73a5e3ecef52e961fcd Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:07:41 +0200 Subject: [PATCH 286/362] Made ExpandInClaseParameterMssqlite generic to avoid casting --- Duplicati/Library/Main/Database/ExtensionMethods.cs | 2 +- .../Library/Main/Database/LocalListBrokenFilesDatabase.cs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 6943bbbe2..d154c766b 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -222,7 +222,7 @@ public static partial class ExtensionMethods return defaultvalue; } - public static SqliteCommand ExpandInClauseParameterMssqlite(this SqliteCommand cmd, string originalParamName, IEnumerable values) + public static SqliteCommand ExpandInClauseParameterMssqlite(this SqliteCommand cmd, string originalParamName, IEnumerable values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 3811af305..a3ac29e16 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -257,7 +257,6 @@ namespace Duplicati.Library.Main.Database /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize) { - // TODO the casting to object is a typing fix for the extension method for now. Should be fixed properly. using var cmd = Connection.CreateCommand(@" SELECT ""ID"" FROM ""Blockset"" @@ -275,7 +274,7 @@ namespace Duplicati.Library.Main.Database ) ") .SetTransaction(m_rtr) - .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) .SetParameterValue("@EmptyHash", emptyHash) .SetParameterValue("@EmptyHashSize", emptyHashSize); From 09251284ab851f6529128065c818449a07f61160 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:07:54 +0200 Subject: [PATCH 287/362] Removed TODOs --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 -- Duplicati/Library/Main/Database/LocalDeleteDatabase.cs | 1 - 2 files changed, 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 3160e7197..25403e5a8 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -58,7 +58,6 @@ namespace Duplicati.Library.Main.Database protected long m_operationid = -1; protected long m_pagecachesize; private bool m_hasExecutedVacuum; - // TODO dispose should check this, and report if it was not disposed prior to calling dispose. Maybe the rtr itself should check this during dispose? And just throw a warning during runtime. protected ReusableTransaction m_rtr = null!; public ReusableTransaction Transaction { get { return m_rtr; } } @@ -360,7 +359,6 @@ namespace Duplicati.Library.Main.Database } /// - /// TODO Remove from here and call the utility function directly - or maybe rename this so that the utility function makes more sense? /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value /// public static DateTime ParseFromEpochSeconds(long seconds) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 885a211b5..76892a47e 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -57,7 +57,6 @@ namespace Duplicati.Library.Main.Database return dbnew; } - // TODO Second argumnet is optional, as the original constructor call chain will have initialized the object if the constructor is called from another constructor. If not, the constructor will instantiate a new object. public static async Task CreateAsync(LocalDatabase dbparent, LocalDeleteDatabase? dbnew = null) { dbnew ??= new LocalDeleteDatabase(); From 95dc884ab38cf17eeb823f5257797db82f52fe4e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:27:15 +0200 Subject: [PATCH 288/362] Removed unnecessary casts --- .../Database/LocalListBrokenFilesDatabase.cs | 4 ++-- .../Library/Main/Database/LocalListDatabase.cs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index a3ac29e16..bd235bcb6 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -297,7 +297,7 @@ namespace Duplicati.Library.Main.Database AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ) ") - .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) .SetParameterValue("@EmptyHashSize", 0) .ExecuteScalarInt64Async(-1); @@ -318,7 +318,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Blockset"".""Length"" ASC LIMIT 1 ") - .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds.Cast()) + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) .ExecuteScalarInt64Async(-1); return res; diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 962e07ede..a94a23084 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -789,7 +789,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) .SetParameterValue("@filesetid", filesetid) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); @@ -835,7 +835,7 @@ namespace Duplicati.Library.Main.Database fe.FilesetID = @filesetid AND fl.PrefixID IN (@PrefixIds) ") - .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) .SetParameterValue("@filesetid", filesetid) .ExecuteScalarInt64Async(0); } @@ -899,7 +899,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); @@ -938,7 +938,7 @@ namespace Duplicati.Library.Main.Database AND fl.""PrefixId"" IN (@PrefixIds) ") .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds.Cast()) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) .ExecuteScalarInt64Async(0); } @@ -1143,7 +1143,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@offset", offset); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds.Cast()); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); var results = new List(); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -1192,7 +1192,7 @@ namespace Duplicati.Library.Main.Database .ExpandInClauseParameter("@Paths", paths); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds.Cast()); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); totalCount = await cmd.ExecuteScalarInt64Async(0); } @@ -1403,7 +1403,7 @@ namespace Duplicati.Library.Main.Database cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds.Cast()); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); var results = new List(); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) @@ -1453,7 +1453,7 @@ namespace Duplicati.Library.Main.Database cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds.Cast()); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); totalCount = await cmd.ExecuteScalarInt64Async(0); } From cb06275a8193a56d2ace13c5eb8972c49712b9a3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:30:13 +0200 Subject: [PATCH 289/362] Removed TODOs --- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 5 ----- Duplicati/Library/Main/Database/LocalRepairDatabase.cs | 5 ----- 2 files changed, 10 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index f6c9a6340..25aa81fc1 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -70,11 +70,6 @@ namespace Duplicati.Library.Main.Database private string m_tempblocklist = null!; private string m_tempsmalllist = null!; - // TODO why was this "new" ? - //private new readonly SqliteConnection m_connection; - - //public new bool RepairInProgress { get; set; } = false; - /// /// A lookup table that prevents multiple downloads of the same volume /// diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 580b2a650..42d41e9c6 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -357,11 +357,6 @@ namespace Duplicati.Library.Main.Database /// The name of the volume where blocks are missing /// private readonly string m_volumename; - // TODO remove - /// - /// The query to use for getting missing blocks - /// - //private readonly string m_missingBlocksQuery; /// /// Whether the object has been disposed From daac3ee24197670f2e92ecf44a2adb094b0ee622 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:35:03 +0200 Subject: [PATCH 290/362] Removed whitespace --- .../Backup/RecreateMissingIndexFiles.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs index 608ff7b7e..5dd344284 100644 --- a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs +++ b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Threading.Tasks; From 3042f311e27fc332802fcfd5c0075c8c6e3d6cae Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:35:32 +0200 Subject: [PATCH 291/362] Handled TODO stating that the call should use IAsyncEnumerable --- Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs | 5 ++--- .../Main/Operation/Backup/RecreateMissingIndexFiles.cs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index 5876702e3..c0dc09841 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -257,10 +257,9 @@ namespace Duplicati.Library.Main.Operation.Backup return RunOnMain(async () => await m_database.UpdateFullBackupStateInFileset(filesetid, true)); } - // TODO: Make IAsyncEnumerable or get rid of the SingleRunner setup - public Task GetMissingIndexFilesAsync() + public Task> GetMissingIndexFilesAsync() { - return RunOnMain(async () => await m_database.GetMissingIndexFiles().ToArrayAsync()); + return RunOnMain(() => m_database.GetMissingIndexFiles()); } public Task UpdateChangeStatisticsAsync(BackupResults result) diff --git a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs index 5dd344284..387b8431b 100644 --- a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs +++ b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs @@ -35,7 +35,7 @@ internal static class RecreateMissingIndexFiles { if (options.IndexfilePolicy != Options.IndexFileStrategy.None) { - foreach (var blockfile in await database.GetMissingIndexFilesAsync().ConfigureAwait(false)) + await foreach (var blockfile in await database.GetMissingIndexFilesAsync().ConfigureAwait(false)) { if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; From d78348de8186e9c16eaaa86dc882b436f62102ff Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:48:16 +0200 Subject: [PATCH 292/362] Removed TODOs --- .../Main/Operation/RecreateDatabaseHandler.cs | 18 +++++++++--------- .../Library/Main/Operation/RepairHandler.cs | 4 ++-- Duplicati/Library/Main/Utility.cs | 10 +++++++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 7ce2c9b70..77cd1a001 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -217,7 +217,7 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); return; } @@ -265,7 +265,7 @@ namespace Duplicati.Library.Main.Operation if (ex.IsAbortException()) { m_result.EndTime = DateTime.UtcNow; - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); throw; } @@ -273,14 +273,14 @@ namespace Duplicati.Library.Main.Operation if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException) { m_result.EndTime = DateTime.UtcNow; - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); throw; } if (m_options.UnittestMode) { - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); throw; } @@ -322,7 +322,7 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); return; } @@ -403,14 +403,14 @@ namespace Duplicati.Library.Main.Operation if (ex.IsAbortException()) { m_result.EndTime = DateTime.UtcNow; - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); throw; } if (m_options.UnittestMode) { - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); throw; } @@ -473,7 +473,7 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); return; } @@ -534,7 +534,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message); if (m_options.UnittestMode) { - // TODO implicit rollback + // Implicit rollback await restoredb.Transaction.RollBackAsync(); throw; } diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 8b1f6222d..23e7f0b2b 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -483,7 +483,7 @@ namespace Duplicati.Library.Main.Operation if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); - // TODO implicit rollback + // Implicit rollback await db.Transaction.RollBackAsync(); return; } @@ -541,7 +541,7 @@ namespace Duplicati.Library.Main.Operation } catch (Exception) { - // TODO Implicit rollback + // Implicit rollback await db.Transaction.RollBackAsync(); throw; } diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index 8a1ab24ce..c069c65f9 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -125,10 +125,14 @@ namespace Duplicati.Library.Main internal static async Task ContainsOptionsForVerification(LocalDatabase db) { var opts = await db.GetDbOptions(); - // TODO commit mimics old behavior of not passing down a transaction, - // but maybe it shouldn't be necessary? await db.Transaction.CommitAsync(); - return new[] { "blocksize", "blockhash", "filehash", "passphrase" }.Any(opts.ContainsKey); + return new[] { + "blocksize", + "blockhash", + "filehash", + "passphrase" + } + .Any(opts.ContainsKey); } /// From d56e8b2f4af5596bb592e84916ccae101d151538 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:58:07 +0200 Subject: [PATCH 293/362] Removed whitespace --- .../Library/SQLiteHelper/DatabaseUpgrader.cs | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs b/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs index 259eccdcd..aad2d6f59 100644 --- a/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs +++ b/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -36,24 +36,24 @@ namespace Duplicati.Library.SQLiteHelper /// Updates should have the form "1.Sample upgrade.sql". /// When the database schema changes, simply put a new file into the folder /// and set it to be embedded in the binary. - /// - /// Additionally, it's possible to execute custom code before and after + /// + /// Additionally, it's possible to execute custom code before and after /// the SQL is executed. To set up a custom upgrade stage, add your /// code to DbUpgradesRegistry along with the DB version to apply it with. - /// + /// /// Even if all the DB upgrade code is handled in C#, you still have to add /// a dummy SQL file to indicate the version ID is already taken. - /// + /// /// Each upgrade file should ONLY upgrade from the previous version. /// If done correctly, a user may be upgrade from the very first version /// to the very latest. - /// - /// The Schema.sql file should ALWAYS have the latest schema, as that will + /// + /// The Schema.sql file should ALWAYS have the latest schema, as that will /// ensure that new installs do not run upgrades after installation. - /// Also remember to update the last line in Schema.sql to insert the + /// Also remember to update the last line in Schema.sql to insert the /// current version number in the version table. - /// - /// Currently no upgrades may contain semicolons, except as the SQL statement + /// + /// Currently no upgrades may contain semicolons, except as the SQL statement /// delimiter. /// public static class DatabaseUpgrader From e5337e4d269ea331320293a52d4a66d79b5aaec2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 11:58:14 +0200 Subject: [PATCH 294/362] Formatted SQL queries --- .../Main/Database/LocalBackupDatabase.cs | 4 ++- .../Library/Main/Database/LocalDatabase.cs | 30 ++++++++++++++----- .../Database/LocalListBrokenFilesDatabase.cs | 4 ++- .../Main/Database/LocalRecreateDatabase.cs | 4 ++- .../Library/SQLiteHelper/DatabaseUpgrader.cs | 11 +++++-- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 3587652f4..e350aca60 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -592,7 +592,9 @@ namespace Duplicati.Library.Main.Database dbnew.m_selectfileHashCommand = dbnew.m_connection.CreateCommand(@" SELECT ""Blockset"".""Fullhash"" - FROM ""Blockset"", ""FileLookup"" + FROM + ""Blockset"", + ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = @FileId diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 25403e5a8..e314fac1a 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -270,7 +270,9 @@ namespace Duplicati.Library.Main.Database "); dbnew.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" - SELECT DISTINCT ""Name"", ""State"" + SELECT DISTINCT + ""Name"", + ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN ( @@ -432,7 +434,9 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable> FilesetTimes() { using var cmd = await m_connection.CreateCommandAsync(@" - SELECT ""ID"", ""Timestamp"" + SELECT + ""ID"", + ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC "); @@ -661,12 +665,17 @@ namespace Duplicati.Library.Main.Database .ExpandInClauseParameterMssqlite("@VolumeNames", [.. names]) .ExecuteNonQueryAsync(); - var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" "; + var volIdsSubQuery = $@" + SELECT ""ID"" + FROM ""{volidstable}"" + "; deletecmd.Parameters.Clear(); var bsIdsSubQuery = @$" SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" - FROM ""BlocksetEntry"", ""Block"" + FROM + ""BlocksetEntry"", + ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) @@ -698,7 +707,9 @@ namespace Duplicati.Library.Main.Database // Create a temp table to associate metadata that is being deleted to a fileset var metadataFilesetQuery = $@" - SELECT Metadataset.ID, FilesetEntry.FilesetID + SELECT + Metadataset.ID, + FilesetEntry.FilesetID FROM Metadataset INNER JOIN FileLookup ON FileLookup.MetadataID = Metadataset.ID @@ -732,7 +743,8 @@ namespace Duplicati.Library.Main.Database DELETE FROM FilesetEntry WHERE FilesetEntry.FilesetID IN ( - SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"" + SELECT DISTINCT FilesetID + FROM ""{metadataFilesetTable}"" ) AND FilesetEntry.FileID IN ( SELECT FilesetEntry.FileID @@ -740,7 +752,8 @@ namespace Duplicati.Library.Main.Database INNER JOIN FileLookup ON FileLookup.ID = FilesetEntry.FileID WHERE FileLookup.MetadataID IN ( - SELECT MetadataID FROM ""{metadataFilesetTable}"" + SELECT MetadataID + FROM ""{metadataFilesetTable}"" ) ) "); @@ -907,7 +920,8 @@ namespace Duplicati.Library.Main.Database SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN ( - SELECT ""ID"" FROM ""FileLookup"" + SELECT ""ID"" + FROM ""FileLookup"" ) "); if (nonAttachedFiles > 0) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index bd235bcb6..1e5709a70 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -349,7 +349,9 @@ namespace Duplicati.Library.Main.Database ) AND ""BlocksetID"" NOT IN ( SELECT ""BlocksetID"" - FROM ""BlocksetEntry"", ""Block"" + FROM + ""BlocksetEntry"", + ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ) ") diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 25aa81fc1..1d1a58459 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -274,7 +274,9 @@ namespace Duplicati.Library.Main.Database dbnew.m_findMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" SELECT ""Metadataset"".""ID"" - FROM ""Metadataset"",""Blockset"" + FROM + ""Metadataset"", + ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash diff --git a/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs b/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs index aad2d6f59..52cba922a 100644 --- a/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs +++ b/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs @@ -203,14 +203,21 @@ namespace Duplicati.Library.SQLiteHelper try { //See if the version table is present, - cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER WHERE Name LIKE 'Version'"; + cmd.CommandText = @" + SELECT COUNT(*) + FROM SQLITE_MASTER + WHERE Name LIKE 'Version' + "; int count = Convert.ToInt32(cmd.ExecuteScalar()); if (count == 0) dbversion = -1; //Empty else if (count == 1) { - cmd.CommandText = "SELECT max(Version) FROM Version"; + cmd.CommandText = @" + SELECT max(Version) + FROM Version + "; dbversion = Convert.ToInt32(cmd.ExecuteScalar()); } else From ef5df3bd7adf3929440087c8280a932b99f74644 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 11 Jun 2025 14:32:18 +0200 Subject: [PATCH 295/362] Changed property accessing database to be an async method --- .../Library/Main/Database/LocalListDatabase.cs | 15 ++++++--------- .../Library/Main/Operation/ListFilesHandler.cs | 10 ++++++---- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index a94a23084..2091dc5bb 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -47,7 +47,7 @@ namespace Duplicati.Library.Main.Database public interface IFileversion { string? Path { get; } - IEnumerable Sizes { get; } + IAsyncEnumerable Sizes(); } public interface IFileset @@ -148,15 +148,12 @@ namespace Duplicati.Library.Main.Database More = true; } - public IEnumerable Sizes + public async IAsyncEnumerable Sizes() { - get + while (More && Path == m_reader.ConvertValueToString(0)) { - while (More && Path == m_reader.ConvertValueToString(0)) - { - yield return m_reader.ConvertValueToInt64(1, -1); - More = m_reader.Read(); - } + yield return m_reader.ConvertValueToInt64(1, -1); + More = await m_reader.ReadAsync(); } } } @@ -164,7 +161,7 @@ namespace Duplicati.Library.Main.Database private class FileversionFixed : IFileversion { public string? Path { get; internal set; } - public IEnumerable Sizes { get { return new long[0]; } } + public IAsyncEnumerable Sizes() => AsyncEnumerable.Empty(); } public IAsyncEnumerable GetLargestPrefix(IFilter filter) diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 46409229b..8419eb04e 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -104,10 +104,12 @@ namespace Duplicati.Library.Main.Operation .ToArrayAsync(), files == null ? null - : (from n in files - select (Duplicati.Library.Interface.IListResultFile)(new ListResultFile(n.Path, - n.Sizes.ToArray()))) - .ToArray() + : + files.Select(async n => + new ListResultFile(n.Path, await n.Sizes().ToArrayAsync()) + ) + .Select(x => x.Result) + .Cast() ); } From 217ac39e0f9974fb06670f9ef9557c1b5310a8c4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 12 Jun 2025 07:13:59 +0200 Subject: [PATCH 296/362] Changed the constructor of restore's BlockManager.SleepableDictionary to use factory-like async construction to make the database accesses async --- .../Main/Operation/Restore/BlockManager.cs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index c334c63eb..2880f34f0 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -134,7 +134,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The database holding information about how many of each block this restore requires. /// Channel for submitting block requests from a volume. /// Number of readers accessing this dictionary. Used during shutdown / cleanup. - public SleepableDictionary(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers) + private SleepableDictionary(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers) { m_options = options; m_volume_request = volume_request; @@ -171,14 +171,21 @@ namespace Duplicati.Library.Main.Operation.Restore sw_set_set = options.InternalProfiling ? new() : null; sw_set_wake_get = options.InternalProfiling ? new() : null; sw_set_wake_set = options.InternalProfiling ? new() : null; + } - foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata).ToEnumerable()) + public static async Task CreateAsync(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers) + { + var sd = new SleepableDictionary(db, volume_request, options, readers); + + await foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata).ConfigureAwait(false)) { - var bc = m_blockcount.TryGetValue(block_id, out var c); - m_blockcount[block_id] = bc ? c + 1 : 1; - var vc = m_volumecount.TryGetValue(volume_id, out var v); - m_volumecount[volume_id] = vc ? v + 1 : 1; + var bc = sd.m_blockcount.TryGetValue(block_id, out var c); + sd.m_blockcount[block_id] = bc ? c + 1 : 1; + var vc = sd.m_volumecount.TryGetValue(volume_id, out var v); + sd.m_volumecount[volume_id] = vc ? v + 1 : 1; } + + return sd; } /// @@ -435,7 +442,9 @@ namespace Duplicati.Library.Main.Operation.Restore async self => { // Create a cache for the blocks, - using SleepableDictionary cache = new(db, self.Output, options, fp_requests.Length); + using SleepableDictionary cache = + await SleepableDictionary.CreateAsync(db, self.Output, options, fp_requests.Length) + .ConfigureAwait(false); // The volume consumer will read blocks from the input channel (data blocks from the volumes) and store them in the cache. var volume_consumer = Task.Run(async () => From 006a9b17ed4dc4766992078663df42e7cf522770 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 12 Jun 2025 08:17:37 +0200 Subject: [PATCH 297/362] Properly get the list results asynchronously and evaluate them to an array. Otherwise, the results will have a lazy loaded enumerable containing a hook into a disposed database, leading to errors when trying to do anything with the results. --- .../Main/Operation/ListFilesHandler.cs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 8419eb04e..30a3341b6 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -67,14 +67,14 @@ namespace Duplicati.Library.Main.Operation } } - IEnumerable files; + IAsyncEnumerable files; if (m_options.ListFolderContents) { - files = filesets.SelectFolderContents(filter).ToEnumerable(); + files = filesets.SelectFolderContents(filter); } else if (m_options.ListPrefixOnly) { - files = filesets.GetLargestPrefix(filter).ToEnumerable(); + files = filesets.GetLargestPrefix(filter); } else if (filter.Empty) { @@ -82,7 +82,7 @@ namespace Duplicati.Library.Main.Operation } else { - files = filesets.SelectFiles(filter).ToEnumerable(); + files = filesets.SelectFiles(filter); } if (m_options.ListSetsOnly) @@ -105,11 +105,19 @@ namespace Duplicati.Library.Main.Operation files == null ? null : - files.Select(async n => - new ListResultFile(n.Path, await n.Sizes().ToArrayAsync()) + await files.Select(async n => + new ListResultFile( + n.Path, + await n + .Sizes() + .ToArrayAsync() + .ConfigureAwait(false) + ) ) .Select(x => x.Result) .Cast() + .ToArrayAsync() + .ConfigureAwait(false) ); } From 3d11a054008b2013bc291a378d55911d5fd5bf42 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 12 Jun 2025 08:34:29 +0200 Subject: [PATCH 298/362] Added .ConfigureAwait(false) to all of the await statements --- .../DatabaseTool/Commands/Downgrade.cs | 10 +- .../DatabaseTool/Commands/Upgrade.cs | 9 +- Duplicati/CommandLine/DatabaseTool/Helper.cs | 13 +- .../BackendManager.DatabaseCollector.cs | 16 +- .../Library/Main/Backend/BackendManager.cs | 7 +- Duplicati/Library/Main/Controller.cs | 15 +- .../Library/Main/Database/ChangeStatistics.cs | 54 +- .../Library/Main/Database/ExtensionMethods.cs | 60 +- .../Main/Database/LocalBackupDatabase.cs | 340 +++++---- .../Main/Database/LocalBugReportDatabase.cs | 43 +- .../Library/Main/Database/LocalDatabase.cs | 670 ++++++++++++------ .../Main/Database/LocalDeleteDatabase.cs | 174 +++-- .../Database/LocalListAffectedDatabase.cs | 33 +- .../Database/LocalListBrokenFilesDatabase.cs | 83 ++- .../Main/Database/LocalListChangesDatabase.cs | 238 ++++--- .../Main/Database/LocalListDatabase.cs | 174 +++-- .../Main/Database/LocalPurgeDatabase.cs | 92 ++- .../Main/Database/LocalRecreateDatabase.cs | 200 ++++-- .../Main/Database/LocalRepairDatabase.cs | 258 ++++--- .../Main/Database/LocalRestoreDatabase.cs | 458 +++++++----- .../Main/Database/LocalTestDatabase.cs | 119 ++-- .../Main/Database/ReusableTransaction.cs | 12 +- .../Main/Database/TemporaryDbValueList.cs | 27 +- .../Main/Operation/Backup/BackupDatabase.cs | 233 ++++-- .../Operation/Backup/DataBlockProcessor.cs | 7 +- .../Backup/RecreateMissingIndexFiles.cs | 2 +- .../Operation/Backup/SpillCollectorProcess.cs | 2 +- .../Library/Main/Operation/BackupHandler.cs | 195 +++-- .../Main/Operation/Common/DatabaseCommon.cs | 70 +- .../Library/Main/Operation/CompactHandler.cs | 129 +++- .../Main/Operation/CreateBugReportHandler.cs | 6 +- .../Library/Main/Operation/DeleteHandler.cs | 37 +- .../Main/Operation/FilelistProcessor.cs | 65 +- .../Library/Main/Operation/ListAffected.cs | 36 +- .../Main/Operation/ListBrokenFilesHandler.cs | 51 +- .../Main/Operation/ListChangesHandler.cs | 37 +- .../Main/Operation/ListControlFilesHandler.cs | 6 +- .../Main/Operation/ListFileVersionsHandler.cs | 14 +- .../Main/Operation/ListFilesHandler.cs | 19 +- .../Main/Operation/ListFilesetsHandler.cs | 10 +- .../Main/Operation/ListFolderHandler.cs | 28 +- .../Main/Operation/PurgeBrokenFilesHandler.cs | 58 +- .../Main/Operation/PurgeFilesHandler.cs | 91 ++- .../Main/Operation/RecreateDatabaseHandler.cs | 255 +++++-- .../Library/Main/Operation/RepairHandler.cs | 461 ++++++++---- .../Main/Operation/Restore/BlockManager.cs | 2 +- .../Main/Operation/Restore/FileLister.cs | 11 +- .../Main/Operation/Restore/FileProcessor.cs | 78 +- .../Operation/Restore/VolumeDownloader.cs | 5 +- .../Operation/RestoreControlFilesHandler.cs | 6 +- .../Library/Main/Operation/RestoreHandler.cs | 188 +++-- .../Main/Operation/SearchEntriesHandler.cs | 14 +- .../Library/Main/Operation/TestHandler.cs | 120 +++- .../Library/Main/Operation/VacuumHandler.cs | 6 +- Duplicati/Library/Main/ResultClasses.cs | 8 +- Duplicati/Library/Main/Utility.cs | 12 +- .../Library/Main/Volumes/BlockVolumeWriter.cs | 2 +- .../Main/Volumes/FilesetVolumeWriter.cs | 13 +- .../Library/SQLiteHelper/SQLiteLoader.cs | 26 +- Duplicati/UnitTest/DirectListHandlerTests.cs | 32 +- Duplicati/UnitTest/ImportExportTests.cs | 4 +- Duplicati/UnitTest/RepairHandlerTests.cs | 22 +- Duplicati/UnitTest/SQLiteTests.cs | 31 +- 63 files changed, 3771 insertions(+), 1726 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs index 7627a037a..cc08250fb 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs @@ -160,7 +160,8 @@ public static class Downgrade if (!nobackups) Helper.CreateFileBackup(db); - using var con = await SQLiteLoader.LoadConnectionAsync(db, 0); + using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) + .ConfigureAwait(false); using var tr = con.BeginTransaction(); using var cmd = con.CreateCommand(tr); foreach (var script in downgradeScripts) @@ -169,7 +170,7 @@ public static class Downgrade try { cmd.SetCommandAndParameters(script.Content); - var r = await cmd.ExecuteScalarAsync(); + var r = await cmd.ExecuteScalarAsync().ConfigureAwait(false); if (r != null) throw new UserInformationException($"{r}", "DowngradeScriptFailure"); @@ -178,7 +179,8 @@ public static class Downgrade SET Version = @Version ") .SetParameterValue("@Version", script.Version - 1) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } catch (Exception ex) { @@ -186,7 +188,7 @@ public static class Downgrade } } - await tr.CommitAsync(); + await tr.CommitAsync().ConfigureAwait(false); Console.WriteLine($"Downgraded {db} to version {targetversion}"); } else if (dbversion == targetversion) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs index 28be63e26..12ada136f 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs @@ -172,7 +172,8 @@ public static class Upgrade if (!nobackups) Helper.CreateFileBackup(db); - using var con = await SQLiteLoader.LoadConnectionAsync(db, 0); + using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) + .ConfigureAwait(false); using var tr = con.BeginTransaction(); using var cmd = con.CreateCommand(tr); foreach (var script in upgradeScripts) @@ -181,7 +182,8 @@ public static class Upgrade try { cmd.SetCommandAndParameters(script.Content); - var r = await cmd.ExecuteScalarAsync(); + var r = await cmd.ExecuteScalarAsync() + .ConfigureAwait(false); if (r != null) throw new UserInformationException($"{r}", "UpgradeScriptFailure"); await cmd.SetCommandAndParameters(@" @@ -189,7 +191,8 @@ public static class Upgrade SET Version = @Version ") .SetParameterValue("@Version", script.Version) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } catch (Exception ex) { diff --git a/Duplicati/CommandLine/DatabaseTool/Helper.cs b/Duplicati/CommandLine/DatabaseTool/Helper.cs index 7024fe94f..2b71aa549 100644 --- a/Duplicati/CommandLine/DatabaseTool/Helper.cs +++ b/Duplicati/CommandLine/DatabaseTool/Helper.cs @@ -60,12 +60,13 @@ public static class Helper { try { - using var con = await SQLiteLoader.LoadConnectionAsync(serverdb, 0); + using var con = await SQLiteLoader.LoadConnectionAsync(serverdb, 0) + .ConfigureAwait(false); using var cmd = con.CreateCommand(@" SELECT ""DBPath"" FROM ""Backup"" "); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) dbpaths.Add(rd.ConvertValueToString(0) ?? ""); } catch @@ -92,7 +93,7 @@ public static class Helper /// A tuple containing the version and whether it is a server database public static async Task<(int Version, bool isserver)> ExamineDatabase(string db) { - using (var con = await SQLiteLoader.LoadConnectionAsync(db, 0)) + using (var con = await SQLiteLoader.LoadConnectionAsync(db, 0).ConfigureAwait(false)) { using var cmd = con.CreateCommand(); @@ -103,12 +104,14 @@ public static class Helper type='table' AND name='Backup' OR name='Schedule' - ") == 2; + ") + .ConfigureAwait(false) == 2; var version = (int)await cmd.ExecuteScalarInt64Async(@" SELECT MAX(Version) FROM Version - "); + ") + .ConfigureAwait(false); return (version, isserverdb); } diff --git a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs index 4903e9d63..a53e68159 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs @@ -135,27 +135,31 @@ partial class BackendManager // As we replace the list, we can now freely access the elements without locking foreach (var e in entries) if (e is RemoteOperationLogEntry operation) - await db.LogRemoteOperation(operation.Action, operation.File, operation.Result); + await db.LogRemoteOperation(operation.Action, operation.File, operation.Result) + .ConfigureAwait(false); else if (e is RemoteVolumeUpdate update && update.State == RemoteVolumeState.Deleted) { - await db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null); + await db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null) + .ConfigureAwait(false); volsRemoved.Add(update.Remotename); } else if (e is RemoteVolumeUpdate dbUpdate) - await db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash); + await db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash) + .ConfigureAwait(false); else if (e is RenameRemoteVolume rename) - await db.RenameRemoteFile(rename.Oldname, rename.Newname); + await db.RenameRemoteFile(rename.Oldname, rename.Newname) + .ConfigureAwait(false); else if (e != null) Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e); // Finally remove volumes from DB. if (volsRemoved.Count > 0) { - await db.RemoveRemoteVolumes(volsRemoved); + await db.RemoveRemoteVolumes(volsRemoved).ConfigureAwait(false); // This commit is to mimic the pre Microsoft.Data.Sqlite backend // behavior, in which RemoveRemoteVolumes started a new // transaction, if none was passed down. - await db.Transaction.CommitAsync(); + await db.Transaction.CommitAsync().ConfigureAwait(false); } return true; diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index 1f7995261..560d14a37 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -300,7 +300,7 @@ internal partial class BackendManager : IBackendManager /// public async Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken) { - await context.Database.FlushPendingMessages(database); + await context.Database.FlushPendingMessages(database).ConfigureAwait(false); } /// @@ -367,7 +367,8 @@ internal partial class BackendManager : IBackendManager yield break; // Get the first volume, so we do not have pending parallel transfers - var prevResult = await GetWithInfoAsync(prevVolume.Name, prevVolume.Hash, prevVolume.Size, cancelToken); + var prevResult = await GetWithInfoAsync(prevVolume.Name, prevVolume.Hash, prevVolume.Size, cancelToken) + .ConfigureAwait(false); foreach (var volume in volumes.Skip(1)) { @@ -380,7 +381,7 @@ internal partial class BackendManager : IBackendManager // Set up for next iteration prevVolume = volume; - prevResult = await nextTask; + prevResult = await nextTask.ConfigureAwait(false); } // Return the last result diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 588cbbc41..3d0a25912 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -253,7 +253,7 @@ namespace Duplicati.Library.Main return RunAction(new ListRemoteResults(), async (result, backendManager) => { using (var tf = File.Exists(m_options.Dbpath) ? null : new TempFile()) - using (var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache)) + using (var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false)) result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); }); } @@ -277,11 +277,16 @@ namespace Duplicati.Library.Main // and uses the same prefix (see issues #2678, #3845, and #4244). if (File.Exists(m_options.Dbpath)) { - using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache)) + using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false)) { var dbRemoteVolumes = db.GetRemoteVolumes(); - var dbRemoteFiles = await dbRemoteVolumes.Select(x => x.Name).ToHashSetAsync(); - list = list.Where(x => dbRemoteFiles.Contains(x.File.Name)).ToList(); + var dbRemoteFiles = await dbRemoteVolumes + .Select(x => x.Name) + .ToHashSetAsync() + .ConfigureAwait(false); + list = list.Where(x => + dbRemoteFiles.Contains(x.File.Name) + ).ToList(); } } @@ -763,7 +768,7 @@ namespace Duplicati.Library.Main private async Task ApplySecretProvider(CancellationToken cancellationToken) { var args = new[] { new Library.Utility.Uri(m_backendUrl) }; - await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken); + await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken).ConfigureAwait(false); // Write back the backend argument, if it was modified by the secret provider m_backendUrl = args[0].ToString(); } diff --git a/Duplicati/Library/Main/Database/ChangeStatistics.cs b/Duplicati/Library/Main/Database/ChangeStatistics.cs index 67a8e3ff3..0332882e7 100644 --- a/Duplicati/Library/Main/Database/ChangeStatistics.cs +++ b/Duplicati/Library/Main/Database/ChangeStatistics.cs @@ -77,7 +77,8 @@ public static class ChangeStatistics ") .SetParameterValue("@LastFilesetId", previousFilesetId) .SetParameterValue("@CurrentFilesetId", currentFilesetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); // Index for fast comparison await cmd.ExecuteNonQueryAsync($@" @@ -86,37 +87,48 @@ public static class ChangeStatistics ""PrefixID"", ""Path"" ) - "); + ") + .ConfigureAwait(false); // Added - results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); + results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID) + .ConfigureAwait(false); + results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID) + .ConfigureAwait(false); results.AddedFiles = await CountAdded(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID - ]); + ]) + .ConfigureAwait(false); // Deleted - results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); + results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID) + .ConfigureAwait(false); + results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID) + .ConfigureAwait(false); results.DeletedFiles = await CountDeleted(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID - ]); + ]) + .ConfigureAwait(false); // Modified - results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); + results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID) + .ConfigureAwait(false); + results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID) + .ConfigureAwait(false); results.ModifiedFiles = await CountModified(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID - ]); + ]) + .ConfigureAwait(false); } finally { try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";"); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";") + .ConfigureAwait(false); } catch (Exception ex) { @@ -147,7 +159,8 @@ public static class ChangeStatistics ) "; - return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets); + return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets) + .ConfigureAwait(false); } /// @@ -172,7 +185,8 @@ public static class ChangeStatistics ) "; - return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets); + return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets) + .ConfigureAwait(false); } /// @@ -225,8 +239,10 @@ public static class ChangeStatistics WHERE {conditions} "; - return await cmd.SetCommandAndParameters(sql) - .ExecuteScalarInt64Async(0); + return await cmd + .SetCommandAndParameters(sql) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); } /// @@ -252,8 +268,10 @@ public static class ChangeStatistics WHERE {fullCondition} "; - return await cmd.SetCommandAndParameters(sql) - .ExecuteScalarInt64Async(0); + return await cmd + .SetCommandAndParameters(sql) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); } /// diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index d154c766b..dca4b64fb 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -66,9 +66,9 @@ public static partial class ExtensionMethods public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext) { - var cmd = self.CreateCommand(); - cmd.SetCommandAndParameters(cmdtext); - await cmd.PrepareAsync(); + var cmd = self.CreateCommand() + .SetCommandAndParameters(cmdtext); + await cmd.PrepareAsync().ConfigureAwait(false); return cmd; } @@ -88,12 +88,14 @@ public static partial class ExtensionMethods public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string? cmdtext) { - return await ExecuteNonQueryAsync(self, true, cmdtext, null); + return await ExecuteNonQueryAsync(self, true, cmdtext, null) + .ConfigureAwait(false); } public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary values) { - return await ExecuteNonQueryAsync(self, true, cmd, values); + return await ExecuteNonQueryAsync(self, true, cmd, values) + .ConfigureAwait(false); } public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null) @@ -105,17 +107,19 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", string.Format("ExecuteNonQueryAsync: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteNonQueryAsync(); + return await self.ExecuteNonQueryAsync().ConfigureAwait(false); } public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext) { - return await ExecuteReaderAsync(self, true, cmdtext, null).ConfigureAwait(false); + return await ExecuteReaderAsync(self, true, cmdtext, null) + .ConfigureAwait(false); } public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary? values) { - return await ExecuteReaderAsync(self, true, cmdtext, values).ConfigureAwait(false); + return await ExecuteReaderAsync(self, true, cmdtext, values) + .ConfigureAwait(false); } public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext = null, Dictionary? values = null) @@ -127,7 +131,7 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteReaderAsync(); + return await self.ExecuteReaderAsync().ConfigureAwait(false); } public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self) @@ -154,24 +158,27 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReaderEnumerableAsync", $"ExecuteReaderEnumerableAsync: {self.GetPrintableCommandText()}") : null) - using (var rd = await self.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return rd; } public static async Task ExecuteScalarAsync(this SqliteCommand self) { - return await self.ExecuteScalarAsync(true, null, null).ConfigureAwait(false); + return await self.ExecuteScalarAsync(true, null, null) + .ConfigureAwait(false); } public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext) { - return await self.ExecuteScalarAsync(true, cmdtext, null).ConfigureAwait(false); + return await self.ExecuteScalarAsync(true, cmdtext, null) + .ConfigureAwait(false); } public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext, Dictionary? values) { - return await self.ExecuteScalarAsync(true, cmdtext, values).ConfigureAwait(false); + return await self.ExecuteScalarAsync(true, cmdtext, values) + .ConfigureAwait(false); } public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null) @@ -183,27 +190,31 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarAsync", string.Format("ExecuteScalarAsync: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteScalarAsync(); + return await self.ExecuteScalarAsync().ConfigureAwait(false); } public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue = -1) { - return await ExecuteScalarInt64Async(self, true, null, null, defaultvalue).ConfigureAwait(false); + return await ExecuteScalarInt64Async(self, true, null, null, defaultvalue) + .ConfigureAwait(false); } public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue) { - return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue); + return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue) + .ConfigureAwait(false); } public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, long defaultvalue = -1) { - return await ExecuteScalarInt64Async(self, true, cmd, null, defaultvalue); + return await ExecuteScalarInt64Async(self, true, cmd, null, defaultvalue) + .ConfigureAwait(false); } public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary? values, long defaultvalue = -1) { - return await ExecuteScalarInt64Async(self, true, cmd, values, defaultvalue); + return await ExecuteScalarInt64Async(self, true, cmd, values, defaultvalue) + .ConfigureAwait(false); } public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null, long defaultvalue = -1) @@ -215,8 +226,8 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64Async", string.Format("ExecuteScalarInt64Async: {0}", self.GetPrintableCommandText())) : null) - using (var rd = await self.ExecuteReaderAsync()) - if (await rd.ReadAsync()) + using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) return ConvertValueToInt64(rd, 0, defaultvalue); return defaultvalue; @@ -259,7 +270,12 @@ public static partial class ExtensionMethods return ExpandInClauseParameterMssqlite(cmd, originalParamName, values.Values); // We have a temporary table, so we need to replace the parameter with the table name - cmd.CommandText = cmd.CommandText.Replace(originalParamName, await values.GetInClause(), StringComparison.OrdinalIgnoreCase); + cmd.CommandText = cmd.CommandText.Replace( + originalParamName, + await values.GetInClause().ConfigureAwait(false), + StringComparison.OrdinalIgnoreCase + ); + return cmd; } diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index e350aca60..d708260ac 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -89,8 +89,10 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalBackupDatabase(); - dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(path, "Backup", false, options.SqlitePageCache, dbnew); - dbnew = await CreateAsync(dbnew, options); + dbnew = (LocalBackupDatabase) + await CreateLocalDatabaseAsync(path, "Backup", false, options.SqlitePageCache, dbnew) + .ConfigureAwait(false); + dbnew = await CreateAsync(dbnew, options).ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; @@ -100,7 +102,8 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalBackupDatabase(); - dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew) + .ConfigureAwait(false); dbnew.m_logQueries = options.ProfileAllDatabaseQueries; @@ -110,7 +113,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - "); + ") + .ConfigureAwait(false); dbnew.m_findblocksetCommand = await dbnew.Connection.CreateCommandAsync(@" SELECT ""ID"" @@ -118,7 +122,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Fullhash"" = @Fullhash AND ""Length"" = @Length - "); + ") + .ConfigureAwait(false); dbnew.m_findmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" SELECT ""A"".""ID"" @@ -131,7 +136,8 @@ namespace Duplicati.Library.Main.Database AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = @Hash AND ""C"".""Size"" = @Size - "); + ") + .ConfigureAwait(false); dbnew.m_findfilesetCommand = await dbnew.Connection.CreateCommandAsync(@" SELECT ""ID"" @@ -141,7 +147,8 @@ namespace Duplicati.Library.Main.Database AND ""MetadataID"" = @MetadataId AND ""Path"" = @Path AND ""PrefixID"" = @PrefixId - "); + ") + .ConfigureAwait(false); dbnew.m_insertblockCommand = await dbnew.Connection.CreateCommandAsync(@" INSERT INTO ""Block"" ( @@ -155,7 +162,8 @@ namespace Duplicati.Library.Main.Database @Size ); SELECT last_insert_rowid(); - "); + ") + .ConfigureAwait(false); dbnew.m_insertfileOperationCommand = await dbnew.Connection.CreateCommandAsync(@" INSERT INTO ""FilesetEntry"" ( @@ -168,7 +176,8 @@ namespace Duplicati.Library.Main.Database @FileId, @LastModified ) - "); + ") + .ConfigureAwait(false); dbnew.m_insertfileCommand = await dbnew.Connection.CreateCommandAsync(@" INSERT INTO ""FileLookup"" ( @@ -184,7 +193,8 @@ namespace Duplicati.Library.Main.Database @MetadataId ); SELECT last_insert_rowid(); - "); + ") + .ConfigureAwait(false); dbnew.m_insertblocksetCommand = await dbnew.Connection.CreateCommandAsync(@" INSERT INTO ""Blockset"" ( @@ -195,7 +205,8 @@ namespace Duplicati.Library.Main.Database @Length, @Fullhash ); - SELECT last_insert_rowid();"); + SELECT last_insert_rowid();") + .ConfigureAwait(false); dbnew.m_insertblocksetentryCommand = await dbnew.Connection.CreateCommandAsync(@" INSERT INTO ""BlocksetEntry"" ( @@ -210,7 +221,8 @@ namespace Duplicati.Library.Main.Database FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size - "); + ") + .ConfigureAwait(false); dbnew.m_insertblocklistHashesCommand = await dbnew.Connection.CreateCommandAsync(@" INSERT INTO ""BlocklistHash"" ( @@ -223,13 +235,15 @@ namespace Duplicati.Library.Main.Database @Index, @Hash ) - "); + ") + .ConfigureAwait(false); dbnew.m_insertmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid(); - "); + ") + .ConfigureAwait(false); dbnew.m_selectfilelastmodifiedCommand = await dbnew.Connection.CreateCommandAsync(@" SELECT @@ -245,7 +259,8 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId - "); + ") + .ConfigureAwait(false); dbnew.m_selectfilelastmodifiedWithSizeCommand = await dbnew.Connection.CreateCommandAsync(@" SELECT @@ -274,7 +289,8 @@ namespace Duplicati.Library.Main.Database ) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" - "); + ") + .ConfigureAwait(false); dbnew.m_selectfilemetadatahashandsizeCommand = await dbnew.Connection.CreateCommandAsync(@" SELECT @@ -288,7 +304,8 @@ namespace Duplicati.Library.Main.Database ""File"".""ID"" = @FileId AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" - "); + ") + .ConfigureAwait(false); // Experimental toggling of the deleted block cache // If the value is less than zero, the lookup is disabled @@ -325,17 +342,20 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Deleted, RemoteVolumeState.Deleting ]) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$" SELECT COUNT(*) FROM ""{dbnew.m_tempDeletedBlockTable}"" - ", 0); + ", 0) + .ConfigureAwait(false); // There are no deleted blocks, so we can drop the table if (deletedBlocks == 0) { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""") + .ConfigureAwait(false); dbnew.m_tempDeletedBlockTable = null; } @@ -349,10 +369,11 @@ namespace Duplicati.Library.Main.Database ""Hash"", ""Size"" FROM ""{dbnew.m_tempDeletedBlockTable}"" - "); + ") + .ConfigureAwait(false); - using (var reader = await cmd.ExecuteReaderAsync()) - while (await reader.ReadAsync()) + using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await reader.ReadAsync().ConfigureAwait(false)) { var id = reader.ConvertValueToInt64(0); var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null"); @@ -363,7 +384,8 @@ namespace Duplicati.Library.Main.Database sizes[size] = id; } - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""") + .ConfigureAwait(false); dbnew.m_tempDeletedBlockTable = null; } // The deleted blocks are too large to fit in memory, so we use a temporary table @@ -375,7 +397,8 @@ namespace Duplicati.Library.Main.Database ""Hash"", ""Size"" ) - "); + ") + .ConfigureAwait(false); dbnew.m_findindeletedCommand = await dbnew.Connection.CreateCommandAsync($@" SELECT ""ID"" @@ -383,7 +406,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - "); + ") + .ConfigureAwait(false); dbnew.m_moveblockfromdeletedCommand = await dbnew.Connection.CreateCommandAsync(@$" INSERT INTO ""Block"" ( @@ -405,7 +429,8 @@ namespace Duplicati.Library.Main.Database WHERE ""ID"" = @DeletedBlockId; SELECT last_insert_rowid() - "); + ") + .ConfigureAwait(false); } @@ -428,7 +453,8 @@ namespace Duplicati.Library.Main.Database WHERE ""ID"" = @DeletedBlockId; SELECT last_insert_rowid() - "); + ") + .ConfigureAwait(false); } } @@ -631,7 +657,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(m_logQueries, -1); + .ExecuteScalarInt64Async(m_logQueries, -1) + .ConfigureAwait(false); } /// @@ -642,7 +669,7 @@ namespace Duplicati.Library.Main.Database /// True if the block should be added to the current output public async Task AddBlock(string key, long size, long volumeid) { - var r = await FindBlockID(key, size); + var r = await FindBlockID(key, size).ConfigureAwait(false); if (r == -1L) { if (m_moveblockfromdeletedCommand != null) @@ -655,7 +682,8 @@ namespace Duplicati.Library.Main.Database await m_moveblockfromdeletedCommand .SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) - .ExecuteNonQueryAsync(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); sizes.Remove(size); if (sizes.Count == 0) @@ -670,14 +698,16 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(m_logQueries, -1); + .ExecuteScalarInt64Async(m_logQueries, -1) + .ConfigureAwait(false); if (id != -1) { var c = await m_moveblockfromdeletedCommand .SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) - .ExecuteNonQueryAsync(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); if (c != 2) throw new Exception($"Failed to move block {key} with size {size}, result count: {c}"); @@ -693,7 +723,9 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", key) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); + if (ins != 1) throw new Exception($"Failed to insert block {key} with size {size}, result count: {ins}"); @@ -721,7 +753,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Fullhash", filehash) .SetParameterValue("@Length", size) - .ExecuteScalarInt64Async(m_logQueries, -1); + .ExecuteScalarInt64Async(m_logQueries, -1) + .ConfigureAwait(false); if (blocksetid != -1) return (false, blocksetid); //Found it @@ -730,7 +763,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@Fullhash", filehash) - .ExecuteScalarInt64Async(m_logQueries); + .ExecuteScalarInt64Async(m_logQueries) + .ConfigureAwait(false); long ix = 0; if (blocklistHashes != null) @@ -744,9 +778,12 @@ namespace Duplicati.Library.Main.Database var c = await m_insertblocklistHashesCommand .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", bh) - .ExecuteNonQueryAsync(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); + if (c != 1) throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}"); + ix++; } } @@ -764,7 +801,9 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", h) .SetParameterValue("@Size", exsize) - .ExecuteNonQueryAsync(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); + if (c != 1) { Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size); @@ -776,7 +815,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash ") .SetParameterValue("@Hash", h) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (bid == -1) throw new Exception($"Could not find any blocks with the given hash: {h}"); @@ -788,7 +828,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Hash", h); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); } @@ -799,7 +839,7 @@ namespace Duplicati.Library.Main.Database remainsize -= blocksize; } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); return (true, blocksetid); } @@ -822,7 +862,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Hash", filehash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(m_logQueries, -1); + .ExecuteScalarInt64Async(m_logQueries, -1) + .ConfigureAwait(false); + return (metadataid != -1, metadataid); } @@ -841,16 +883,18 @@ namespace Duplicati.Library.Main.Database /// True if the set was added to the database, false otherwise public async Task<(bool, long)> AddMetadataset(string filehash, long size, long blocksetid) { - var (metadatafound, metadataid) = await GetMetadatasetID(filehash, size); + var (metadatafound, metadataid) = await GetMetadatasetID(filehash, size) + .ConfigureAwait(false); if (metadatafound) return (false, metadataid); metadataid = await m_insertmetadatasetCommand .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64Async(m_logQueries); + .ExecuteScalarInt64Async(m_logQueries) + .ConfigureAwait(false); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); return (true, metadataid); } @@ -872,7 +916,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@MetadataId", metadataID) .SetParameterValue("@Path", filename) .SetParameterValue("@PrefixId", pathprefixid) - .ExecuteScalarInt64Async(m_logQueries); + .ExecuteScalarInt64Async(m_logQueries) + .ConfigureAwait(false); if (fileidobj == -1) { @@ -881,12 +926,13 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", filename) .SetParameterValue("@BlocksetId", blocksetID) .SetParameterValue("@MetadataId", metadataID) - .ExecuteScalarInt64Async(m_logQueries); + .ExecuteScalarInt64Async(m_logQueries) + .ConfigureAwait(false); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } - await AddKnownFile(fileidobj, lastmodified); + await AddKnownFile(fileidobj, lastmodified).ConfigureAwait(false); } /// @@ -902,12 +948,13 @@ namespace Duplicati.Library.Main.Database var split = SplitIntoPrefixAndName(filename); await AddFile( - await GetOrCreatePathPrefix(split.Key), + await GetOrCreatePathPrefix(split.Key).ConfigureAwait(false), split.Value, lastmodified, blocksetID, metadataID - ); + ) + .ConfigureAwait(false); } /// @@ -923,17 +970,20 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", m_filesetId) .SetParameterValue("@FileId", fileid) .SetParameterValue("@LastModified", lastmodified.ToUniversalTime().Ticks) - .ExecuteNonQueryAsync(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); } public async Task AddDirectoryEntry(string path, long metadataID, DateTime lastmodified) { - await AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID); + await AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID) + .ConfigureAwait(false); } public async Task AddSymlinkEntry(string path, long metadataID, DateTime lastmodified) { - await AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID); + await AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID) + .ConfigureAwait(false); } public async Task<(long, DateTime, long)> GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength) @@ -948,8 +998,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null)) - if (await rd.ReadAsync()) + using (var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null).ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) { oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); length = rd.ConvertValueToInt64(2); @@ -963,8 +1013,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries)) - if (await rd.ReadAsync()) + using (var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries).ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) { length = -1; oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); @@ -991,13 +1041,14 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = await m_findfileCommand.ExecuteReaderAsync()) - if (await rd.ReadAsync()) + using (var rd = await m_findfileCommand.ExecuteReaderAsync().ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) { oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); lastFileSize = rd.ConvertValueToInt64(2); oldMetahash = rd.ConvertValueToString(3); oldMetasize = rd.ConvertValueToInt64(4); + return ( rd.ConvertValueToInt64(0), oldModified, @@ -1012,6 +1063,7 @@ namespace Duplicati.Library.Main.Database lastFileSize = -1; oldMetahash = null; oldMetasize = -1; + return ( -1, oldModified, @@ -1028,8 +1080,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid); - using (var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync()) - if (await rd.ReadAsync()) + using (var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync().ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) return ( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null") @@ -1044,7 +1096,8 @@ namespace Duplicati.Library.Main.Database var r = await m_selectfileHashCommand .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid) - .ExecuteScalarAsync(m_logQueries, null); + .ExecuteScalarAsync(m_logQueries, null) + .ConfigureAwait(false); if (r == null || r == DBNull.Value) return null; @@ -1063,15 +1116,16 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_connection.CreateCommand(m_rtr)) - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}"""); - await m_rtr.CommitAsync(); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}""") + .ConfigureAwait(false); + await m_rtr.CommitAsync().ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DropTempTableFailed", ex, "Failed to drop temporary table {0}: {1}", m_tempDeletedBlockTable, ex.Message); } - await base.DisposeAsync(); + await base.DisposeAsync().ConfigureAwait(false); } public async Task GetLastWrittenDBlockVolumeSize() @@ -1088,12 +1142,14 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); } private async Task GetPreviousFilesetID(SqliteCommand cmd) { - return await GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId); + return await GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId) + .ConfigureAwait(false); } private async Task GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid) @@ -1110,7 +1166,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) .SetParameterValue("@FilesetId", filesetid) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); } internal async Task> GetLastBackupFileCountAndSize() @@ -1122,7 +1179,8 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1 - "); + ") + .ConfigureAwait(false); var count = await cmd.SetCommandAndParameters(@" SELECT COUNT(*) @@ -1139,7 +1197,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", lastFilesetId) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); var size = await cmd.SetCommandAndParameters(@" SELECT SUM(""Blockset"".""Length"") @@ -1159,7 +1218,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", lastFilesetId) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); return new Tuple(count, size); } @@ -1169,8 +1229,10 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand(m_rtr)) { - var prevFileSetId = await GetPreviousFilesetID(cmd); - await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId); + var prevFileSetId = await GetPreviousFilesetID(cmd) + .ConfigureAwait(false); + await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId) + .ConfigureAwait(false); } } @@ -1182,7 +1244,8 @@ namespace Duplicati.Library.Main.Database /// List of deleted paths, or null public async Task AppendFilesFromPreviousSet(IEnumerable? deleted = null) { - await AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp); + await AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp) + .ConfigureAwait(false); } /// @@ -1199,7 +1262,11 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) using (var cmdDelete = m_connection.CreateCommand()) { - long lastFilesetId = prevId < 0 ? await GetPreviousFilesetID(cmd, timestamp, filesetid) : prevId; + long lastFilesetId = prevId < 0 ? + await GetPreviousFilesetID(cmd, timestamp, filesetid) + .ConfigureAwait(false) + : + prevId; await cmd.SetTransaction(m_rtr) .SetCommandAndParameters(@" @@ -1229,28 +1296,36 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@CurrentFilesetId", filesetid) .SetParameterValue("@PreviousFilesetId", lastFilesetId) - .ExecuteNonQueryAsync(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); if (deleted != null) { - using var tmplist = await TemporaryDbValueList.CreateAsync(this, deleted); - await (await cmdDelete.SetTransaction(m_rtr) - .SetCommandAndParameters(@" - DELETE FROM ""FilesetEntry"" - WHERE - ""FilesetID"" = @FilesetId - AND ""FileID"" IN ( - SELECT ""ID"" - FROM ""File"" - WHERE ""Path"" IN (@Paths) - ) - ") - .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameterMssqliteAsync("@Paths", tmplist)) - .ExecuteNonQueryAsync(m_logQueries); + using var tmplist = await TemporaryDbValueList + .CreateAsync(this, deleted) + .ConfigureAwait(false); + + await ( + await cmdDelete.SetTransaction(m_rtr) + .SetCommandAndParameters(@" + DELETE FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @FilesetId + AND ""FileID"" IN ( + SELECT ""ID"" + FROM ""File"" + WHERE ""Path"" IN (@Paths) + ) + ") + .SetParameterValue("@FilesetId", filesetid) + .ExpandInClauseParameterMssqliteAsync("@Paths", tmplist) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } } @@ -1263,7 +1338,8 @@ namespace Duplicati.Library.Main.Database /// Optional exclusion predicate (true = exclude file) public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate) { - await AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp); + await AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp) + .ConfigureAwait(false); } /// @@ -1280,14 +1356,19 @@ namespace Duplicati.Library.Main.Database { if (exclusionPredicate == null) { - await AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp); + await AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp) + .ConfigureAwait(false); return; } using (var cmd = m_connection.CreateCommand()) using (var cmdDelete = m_connection.CreateCommand()) { - long lastFilesetId = prevFileSetId < 0 ? await GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId; + long lastFilesetId = prevFileSetId < 0 ? + await GetPreviousFilesetID(cmd, timestamp, fileSetId) + .ConfigureAwait(false) + : + prevFileSetId; // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -1315,7 +1396,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@PreviousFilesetId", lastFilesetId) .SetParameterValue("@CurrentFilesetId", fileSetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); // now we need to remove, from the above, any entries that were enumerated by the // UNC-driven backup @@ -1344,14 +1426,15 @@ namespace Duplicati.Library.Main.Database ON f.""BlocksetID"" = bs.""ID""; "); - await foreach (var row in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var row in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path"); var size = row.ConvertValueToInt64(3); if (exclusionPredicate(path, size)) await cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } // now copy the temporary table into the FileSetEntry table @@ -1368,9 +1451,10 @@ namespace Duplicati.Library.Main.Database FROM ""{tempFileSetTable}"" ") .SetParameterValue("@FilesetId", fileSetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } } @@ -1382,22 +1466,29 @@ namespace Duplicati.Library.Main.Database /// An optional external transaction public override async Task CreateFileset(long volumeid, DateTime timestamp) { - return m_filesetId = await base.CreateFileset(volumeid, timestamp); + return m_filesetId = await base.CreateFileset(volumeid, timestamp) + .ConfigureAwait(false); } public async Task> GetTemporaryFilelistVolumeNames(bool latestOnly) { var incompleteFilesetIDs = GetIncompleteFilesets().OrderBy(x => x.Value).Select(x => x.Key); - if (!await incompleteFilesetIDs.AnyAsync()) + if (!await incompleteFilesetIDs.AnyAsync().ConfigureAwait(false)) return []; if (latestOnly) - incompleteFilesetIDs = new long[] { await incompleteFilesetIDs.LastAsync() }.ToAsyncEnumerable(); + incompleteFilesetIDs = new long[] { + await incompleteFilesetIDs.LastAsync().ConfigureAwait(false) + } + .ToAsyncEnumerable(); var volumeNames = new List(); - await foreach (var filesetID in incompleteFilesetIDs) - volumeNames.Add((await GetRemoteVolumeFromFilesetID(filesetID)).Name); + await foreach (var filesetID in incompleteFilesetIDs.ConfigureAwait(false)) + volumeNames.Add(( + await GetRemoteVolumeFromFilesetID(filesetID) + .ConfigureAwait(false) + ).Name); return volumeNames; } @@ -1422,8 +1513,8 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Verified.ToString() ]); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } @@ -1443,7 +1534,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", blockkey) .SetParameterValue("@Size", size) .SetParameterValue("@PreviousVolumeId", sourcevolumeid) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}"); @@ -1452,7 +1544,7 @@ namespace Duplicati.Library.Main.Database public async Task SafeDeleteRemoteVolume(string name) { - var volumeid = await GetRemoteVolumeID(name); + var volumeid = await GetRemoteVolumeID(name).ConfigureAwait(false); using (var cmd = m_connection.CreateCommand(m_rtr)) { @@ -1462,12 +1554,13 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", volumeid) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (c != 0) throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); - await RemoveRemoteVolume(name); + await RemoveRemoteVolume(name).ConfigureAwait(false); } } @@ -1491,7 +1584,8 @@ namespace Duplicati.Library.Main.Database return await cmd.ExecuteReaderEnumerableAsync() .Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash")) - .ToArrayAsync(); + .ToArrayAsync() + .ConfigureAwait(false); } } @@ -1504,7 +1598,8 @@ namespace Duplicati.Library.Main.Database FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1 - "); + ") + .ConfigureAwait(false); if (v0 == null || v0 == DBNull.Value) return null; @@ -1533,9 +1628,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId); - using (var rd = await cmd.ExecuteReaderAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) { - while (await rd.ReadAsync()) + while (await rd.ReadAsync().ConfigureAwait(false)) { yield return new Interface.USNJournalDataEntry { @@ -1580,14 +1675,15 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@JournalId", entry.JournalId) .SetParameterValue("@NextUsn", entry.NextUsn) .SetParameterValue("@ConfigHash", entry.ConfigHash) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) throw new Exception("Unable to add change journal entry"); } } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } /// @@ -1615,11 +1711,12 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", fileSetId) .SetParameterValue("@VolumeName", entry.Volume) .SetParameterValue("@JournalId", entry.JournalId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } /// @@ -1633,7 +1730,8 @@ namespace Duplicati.Library.Main.Database var res = await m_getfirstfilesetwithblockinblockset .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) - .ExecuteScalarInt64Async(); + .ExecuteScalarInt64Async() + .ConfigureAwait(false); if (res != -1 && res != m_filesetId) return true; diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 41330b294..6470afa09 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -38,7 +38,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalBugReportDatabase(); - dbnew = (LocalBugReportDatabase)await CreateLocalDatabaseAsync(path, "BugReportCreate", false, pagecachesize, dbnew); + dbnew = (LocalBugReportDatabase) + await CreateLocalDatabaseAsync(path, "BugReportCreate", false, pagecachesize, dbnew) + .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; @@ -60,14 +62,16 @@ namespace Duplicati.Library.Main.Database ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL ) - "); + ") + .ConfigureAwait(false); await upcmd.ExecuteNonQueryAsync($@" INSERT INTO ""{tablename}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" - "); + ") + .ConfigureAwait(false); await upcmd.SetCommandAndParameters($@" UPDATE ""{tablename}"" @@ -87,7 +91,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\") .SetParameterValue("@DirSep", Util.DirectorySeparatorString) .SetParameterValue("@FileExt", ".bin") - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); /*long id = 1; using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename))) @@ -108,7 +113,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" UPDATE ""LogData"" @@ -116,13 +122,15 @@ namespace Duplicati.Library.Main.Database WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE TABLE ""FixedFile"" AS @@ -135,19 +143,24 @@ namespace Duplicati.Library.Main.Database ""{tablename}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" - "); + ") + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" "); - await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" "); - await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" "); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" "); + await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ") + .ConfigureAwait(false); using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); cmd.SetTransaction(m_rtr); - await cmd.ExecuteNonQueryAsync("VACUUM"); - await m_rtr.CommitAsync(); + await cmd.ExecuteNonQueryAsync("VACUUM").ConfigureAwait(false); + await m_rtr.CommitAsync().ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index e314fac1a..9f7828edb 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -104,7 +104,8 @@ namespace Duplicati.Library.Main.Database if (!Directory.Exists(Path.GetDirectoryName(path))) Directory.CreateDirectory(Path.GetDirectoryName(path) ?? throw new DirectoryNotFoundException("Path was a root folder.")); - var c = await SQLiteHelper.SQLiteLoader.LoadConnectionAsync(path, pagecachesize); + var c = await SQLiteHelper.SQLiteLoader.LoadConnectionAsync(path, pagecachesize) + .ConfigureAwait(false); try { @@ -113,7 +114,7 @@ namespace Duplicati.Library.Main.Database catch { //Don't leak database connections when something goes wrong - await c.DisposeAsync(); + await c.DisposeAsync().ConfigureAwait(false); throw; } @@ -124,8 +125,10 @@ namespace Duplicati.Library.Main.Database { db ??= new LocalDatabase(); - var connection = await CreateConnectionAsync(path, pagecachesize); - db = await CreateLocalDatabaseAsync(connection, operation, db); + var connection = await CreateConnectionAsync(path, pagecachesize) + .ConfigureAwait(false); + db = await CreateLocalDatabaseAsync(connection, operation, db) + .ConfigureAwait(false); db.ShouldCloseConnection = shouldclose; db.m_pagecachesize = pagecachesize; @@ -139,7 +142,8 @@ namespace Duplicati.Library.Main.Database dbnew.m_connection = dbparent.m_connection; dbnew.m_rtr = dbparent.m_rtr; - dbnew = await CreateLocalDatabaseAsync(dbparent.m_connection, dbnew); + dbnew = await CreateLocalDatabaseAsync(dbparent.m_connection, dbnew) + .ConfigureAwait(false); dbnew.OperationTimestamp = dbparent.OperationTimestamp; dbnew.m_operationid = dbparent.m_operationid; @@ -151,12 +155,13 @@ namespace Duplicati.Library.Main.Database public static async Task CreateLocalDatabaseAsync(SqliteConnection connection, string operation, LocalDatabase? dbnew = null) { dbnew ??= new LocalDatabase(); - dbnew = await CreateLocalDatabaseAsync(connection, dbnew); + dbnew = await CreateLocalDatabaseAsync(connection, dbnew) + .ConfigureAwait(false); dbnew.OperationTimestamp = DateTime.UtcNow; if (dbnew.m_connection.State != ConnectionState.Open) - await dbnew.m_connection.OpenAsync(); + await dbnew.m_connection.OpenAsync().ConfigureAwait(false); using var cmd = dbnew.m_connection.CreateCommand() .SetTransaction(dbnew.m_rtr); @@ -175,7 +180,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Description", operation) .SetParameterValue("@Timestamp", dbnew.OperationTimestamp) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); } else { @@ -187,9 +193,10 @@ namespace Duplicati.Library.Main.Database FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1 - "); + ") + .ConfigureAwait(false); - if (!await rd.ReadAsync()) + if (!await rd.ReadAsync().ConfigureAwait(false)) throw new Exception("LocalDatabase does not contain a previous operation."); dbnew.m_operationid = rd.ConvertValueToInt64(0); @@ -233,7 +240,8 @@ namespace Duplicati.Library.Main.Database @Message, @Exception ) - "); + ") + .ConfigureAwait(false); dbnew.m_insertremotelogCommand = await connection.CreateCommandAsync(@" INSERT INTO ""RemoteOperation"" ( @@ -250,7 +258,8 @@ namespace Duplicati.Library.Main.Database @Path, @Data ) - "); + ") + .ConfigureAwait(false); dbnew.m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" UPDATE ""Remotevolume"" @@ -260,14 +269,18 @@ namespace Duplicati.Library.Main.Database ""Hash"" = @Hash, ""Size"" = @Size WHERE ""Name"" = @Name - "); + ") + .ConfigureAwait(false); - dbnew.m_selectremotevolumesCommand = await connection.CreateCommandAsync(selectremotevolumes_sql); + dbnew.m_selectremotevolumesCommand = + await connection.CreateCommandAsync(selectremotevolumes_sql) + .ConfigureAwait(false); dbnew.m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$" {selectremotevolumes_sql} WHERE ""Name"" = @Name - "); + ") + .ConfigureAwait(false); dbnew.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" SELECT DISTINCT @@ -281,7 +294,8 @@ namespace Duplicati.Library.Main.Database WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') ) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') - "); + ") + .ConfigureAwait(false); dbnew.m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" DELETE FROM ""Remotevolume"" @@ -291,7 +305,8 @@ namespace Duplicati.Library.Main.Database ""DeleteGraceTime"" < @Now OR ""State"" != @State ) - "); + ") + .ConfigureAwait(false); // >12 is to handle removal of old records that were in ticks dbnew.m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@" @@ -302,13 +317,15 @@ namespace Duplicati.Library.Main.Database ""DeleteGraceTime"" < @Now OR LENGTH(""DeleteGraceTime"") > 12 ) - "); + ") + .ConfigureAwait(false); dbnew.m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = @Name - "); + ") + .ConfigureAwait(false); dbnew.m_createremotevolumeCommand = await connection.CreateCommandAsync(@" INSERT INTO ""Remotevolume"" ( @@ -332,7 +349,8 @@ namespace Duplicati.Library.Main.Database @ArchiveTime ); SELECT last_insert_rowid(); - "); + ") + .ConfigureAwait(false); dbnew.m_insertIndexBlockLink = await connection.CreateCommandAsync(@" INSERT INTO ""IndexBlockLink"" ( @@ -343,19 +361,22 @@ namespace Duplicati.Library.Main.Database @IndexVolumeId, @BlockVolumeId ) - "); + ") + .ConfigureAwait(false); dbnew.m_findpathprefixCommand = await connection.CreateCommandAsync(@" SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = @Prefix - "); + ") + .ConfigureAwait(false); dbnew.m_insertpathprefixCommand = await connection.CreateCommandAsync(@" INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (@Prefix); SELECT last_insert_rowid(); - "); + ") + .ConfigureAwait(false); return dbnew; } @@ -370,12 +391,14 @@ namespace Duplicati.Library.Main.Database public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash) { - await UpdateRemoteVolume(name, state, size, hash, false); + await UpdateRemoteVolume(name, state, size, hash, false) + .ConfigureAwait(false); } public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup) { - await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null); + await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null) + .ConfigureAwait(false); } public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived) @@ -386,7 +409,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) { @@ -399,11 +423,14 @@ namespace Duplicati.Library.Main.Database UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = @DeleteGraceTime WHERE ""Name"" = @Name - "); + ") + .ConfigureAwait(false); + c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)) .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) throw new Exception($"Unexpected number of updates when recording remote volume updates: {c}!"); @@ -415,11 +442,14 @@ namespace Duplicati.Library.Main.Database UPDATE ""RemoteVolume"" SET ""ArchiveTime"" = @ArchiveTime WHERE ""Name"" = @Name - "); + ") + .ConfigureAwait(false); + c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0) .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) throw new Exception($"Unexpected number of updates when recording remote volume archive-time updates: {c}!"); @@ -427,7 +457,7 @@ namespace Duplicati.Library.Main.Database if (!suppressCleanup && state == RemoteVolumeState.Deleted) { - await RemoveRemoteVolume(name); + await RemoveRemoteVolume(name).ConfigureAwait(false); } } @@ -439,10 +469,15 @@ namespace Duplicati.Library.Main.Database ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC - "); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - yield return new KeyValuePair(rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime()); + ") + .ConfigureAwait(false); + + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() + ); } public async Task<(string Query, Dictionary Values)> GetFilelistWhereClause(DateTime time, long[]? versions, IEnumerable>? filesetslist = null, bool singleTimeMatch = false) @@ -451,7 +486,10 @@ namespace Duplicati.Library.Main.Database if (filesetslist != null) filesets = [.. filesetslist]; else - filesets = await FilesetTimes().ToArrayAsync(); + filesets = await FilesetTimes() + .ToArrayAsync() + .ConfigureAwait(false); + var query = new StringBuilder(); var args = new Dictionary(); if (time.Ticks > 0 || (versions != null && versions.Length > 0)) @@ -507,7 +545,8 @@ namespace Duplicati.Library.Main.Database return await m_selectremotevolumeIdCommand .SetTransaction(m_rtr) .SetParameterValue("@Name", file) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); } public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files) @@ -518,13 +557,17 @@ namespace Duplicati.Library.Main.Database ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name) - "); - cmd.SetTransaction(m_rtr); - using var tmptable = await TemporaryDbValueList.CreateAsync(this, files); - await cmd.ExpandInClauseParameterMssqliteAsync("@Name", tmptable); + ") + .ConfigureAwait(false); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + cmd.SetTransaction(m_rtr); + using var tmptable = await TemporaryDbValueList.CreateAsync(this, files) + .ConfigureAwait(false); + await cmd.ExpandInClauseParameterMssqliteAsync("@Name", tmptable) + .ConfigureAwait(false); + + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } @@ -534,8 +577,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Name", file); - using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync()) - if (await rd.ReadAsync()) + using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync().ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) return new RemoteVolumeEntry( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), @@ -554,11 +597,13 @@ namespace Duplicati.Library.Main.Database { m_selectduplicateRemoteVolumesCommand.SetTransaction(m_rtr); - await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync()) + await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { yield return new KeyValuePair( rd.ConvertValueToString(0) ?? throw new Exception("Name was null"), - (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(1) ?? "") + (RemoteVolumeState)Enum.Parse( + typeof(RemoteVolumeState), rd.ConvertValueToString(1) ?? "" + ) ); } } @@ -566,8 +611,11 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable GetRemoteVolumes() { m_selectremotevolumesCommand.SetTransaction(m_rtr); - using var rd = await m_selectremotevolumesCommand.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + using var rd = await m_selectremotevolumesCommand + .ExecuteReaderAsync() + .ConfigureAwait(false); + + while (await rd.ReadAsync().ConfigureAwait(false)) { yield return new RemoteVolumeEntry( rd.ConvertValueToInt64(0), @@ -597,7 +645,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Operation", operation) .SetParameterValue("@Path", path) .SetParameterValue("@Data", data) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } /// @@ -615,7 +664,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", type) .SetParameterValue("@Message", message) .SetParameterValue("@Exception", exception?.ToString()) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state) @@ -625,11 +675,14 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name AND ""State"" = @State - "); + ") + .ConfigureAwait(false); + var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@Name", name) .SetParameterValue("@State", state.ToString()) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); @@ -637,7 +690,7 @@ namespace Duplicati.Library.Main.Database public async Task RemoveRemoteVolume(string name) { - await RemoveRemoteVolumes([name]); + await RemoveRemoteVolumes([name]).ConfigureAwait(false); } public async Task RemoveRemoteVolumes(IEnumerable names) @@ -655,7 +708,9 @@ namespace Duplicati.Library.Main.Database CREATE TEMP TABLE ""{volidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) - "); + ") + .ConfigureAwait(false); + await deletecmd.SetCommandAndParameters($@" INSERT OR IGNORE INTO ""{volidstable}"" SELECT ""ID"" @@ -663,7 +718,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" IN (@VolumeNames) ") .ExpandInClauseParameterMssqlite("@VolumeNames", [.. names]) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); var volIdsSubQuery = $@" SELECT ""ID"" @@ -694,11 +750,15 @@ namespace Duplicati.Library.Main.Database CREATE TEMP TABLE ""{blocksetidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery} - "); + ") + .ConfigureAwait(false); + bsIdsSubQuery = $@" SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" @@ -729,14 +789,17 @@ namespace Duplicati.Library.Main.Database MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER ) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" INSERT OR IGNORE INTO ""{metadataFilesetTable}"" ( MetadataID, FilesetID ) {metadataFilesetQuery} - "); + ") + .ConfigureAwait(false); // Delete FilesetEntry rows that had their metadata deleted await deletecmd.ExecuteNonQueryAsync($@" @@ -756,7 +819,8 @@ namespace Duplicati.Library.Main.Database FROM ""{metadataFilesetTable}"" ) ) - "); + ") + .ConfigureAwait(false); // Delete FilesetEntry rows that had their blocks deleted await deletecmd.ExecuteNonQueryAsync($@" @@ -766,50 +830,68 @@ namespace Duplicati.Library.Main.Database FROM FileLookup WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}) ) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM FileLookup WHERE FileLookup.MetadataID IN ( SELECT MetadataID FROM ""{metadataFilesetTable}"" ) - "); + ") + .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery}) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery}) - "); + ") + .ConfigureAwait(false); // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery}) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN ( @@ -817,7 +899,9 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) ) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM FilesetEntry WHERE FilesetID IN ( @@ -825,19 +909,24 @@ namespace Duplicati.Library.Main.Database FROM Fileset WHERE VolumeID IN ({volIdsSubQuery}) ) - "); + ") + .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" CREATE TABLE ""{filesetidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) - "); + ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@" INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - "); + ") + .ConfigureAwait(false); + // Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore await deletecmd.ExecuteNonQueryAsync($@" INSERT OR IGNORE INTO ""{filesetidstable}"" @@ -852,7 +941,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""FilesetID"" FROM FilesetEntry ) - "); + ") + .ConfigureAwait(false); // Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later await deletecmd.SetCommandAndParameters($@" @@ -876,7 +966,8 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString() ]) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Fileset"" @@ -884,22 +975,31 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""{filesetidstable}"" ) - "); + ") + .ConfigureAwait(false); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // SQLite.SQLiteException (0x80004005): database table is locked - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{blocksetidstable}"" "); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" "); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{metadataFilesetTable}"" "); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{filesetidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{blocksetidstable}"" ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{metadataFilesetTable}"" ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{filesetidstable}"" ") + .ConfigureAwait(false); try { deletecmd.CommandTimeout = 2; - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" "); - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" "); - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" "); - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filesetidstable}"" "); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ") + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ") + .ConfigureAwait(false); } catch { /* Ignore, will be deleted on close anyway. */ } @@ -912,7 +1012,8 @@ namespace Duplicati.Library.Main.Database await m_removeremotevolumeCommand .SetTransaction(m_rtr) .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } // Validate before commiting changes @@ -923,7 +1024,9 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""FileLookup"" ) - "); + ") + .ConfigureAwait(false); + if (nonAttachedFiles > 0) throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); } @@ -933,22 +1036,25 @@ namespace Duplicati.Library.Main.Database m_hasExecutedVacuum = true; using var cmd = m_connection.CreateCommand(); - await cmd.ExecuteNonQueryAsync("VACUUM"); + await cmd.ExecuteNonQueryAsync("VACUUM").ConfigureAwait(false); } public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state) { - return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0)); + return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0)) + .ConfigureAwait(false); } public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state) { - return await RegisterRemoteVolume(name, type, state, new TimeSpan(0)); + return await RegisterRemoteVolume(name, type, state, new TimeSpan(0)) + .ConfigureAwait(false); } public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime) { - return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime); + return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime) + .ConfigureAwait(false); } public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime) @@ -963,7 +1069,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@VerificationCount", 0) .SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks) .SetParameterValue("@ArchiveTime", 0) - .ExecuteScalarInt64Async(); + .ExecuteScalarInt64Async() + .ConfigureAwait(false); return r; } @@ -973,7 +1080,9 @@ namespace Duplicati.Library.Main.Database if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); - (var wherequery, var values) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch); + (var wherequery, var values) = + await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch) + .ConfigureAwait(false); var res = new List(); using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters($@" @@ -983,8 +1092,9 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC ") .SetParameterValues(values); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); if (res.Count == 0) @@ -994,8 +1104,8 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" ORDER BY ""Timestamp"" DESC "); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); if (res.Count == 0) @@ -1013,7 +1123,9 @@ namespace Duplicati.Library.Main.Database if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); - var (wherequery, args) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true); + var (wherequery, args) = + await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true) + .ConfigureAwait(false); var res = new List(); using var cmd = m_connection.CreateCommand(); @@ -1024,8 +1136,8 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC ") .SetParameterValues(args); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); return res; @@ -1042,8 +1154,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)); - using var rd = await cmd.ExecuteReaderAsync(); - if (!await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + if (!await rd.ReadAsync().ConfigureAwait(false)) return false; var isFullBackup = rd.GetInt32(0); return isFullBackup == BackupType.FULL_BACKUP; @@ -1059,15 +1171,19 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? ""); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "" + ); } public async Task> GetDbOptions() { var res = await GetDbOptionList() - .ToDictionaryAsync(x => x.Key, x => x.Value); + .ToDictionaryAsync(x => x.Key, x => x.Value) + .ConfigureAwait(false); return res; } @@ -1079,15 +1195,15 @@ namespace Duplicati.Library.Main.Database /// The value to set private async Task UpdateDbOption(string key, bool value) { - var opts = await GetDbOptions(); + var opts = await GetDbOptions().ConfigureAwait(false); if (value) opts[key] = "true"; else opts.Remove(key); - await SetDbOptions(opts); - await m_rtr.CommitAsync(); + await SetDbOptions(opts).ConfigureAwait(false); + await m_rtr.CommitAsync().ConfigureAwait(false); } /// @@ -1097,11 +1213,13 @@ namespace Duplicati.Library.Main.Database { if (value is bool v) { - await UpdateDbOption("repair-in-progress", v); + await UpdateDbOption("repair-in-progress", v) + .ConfigureAwait(false); + return v; } - var opts = await GetDbOptions(); + var opts = await GetDbOptions().ConfigureAwait(false); return opts.ContainsKey("repair-in-progress"); } @@ -1112,11 +1230,13 @@ namespace Duplicati.Library.Main.Database { if (value is bool v) { - await UpdateDbOption("partially-recreated", v); + await UpdateDbOption("partially-recreated", v) + .ConfigureAwait(false); + return v; } - var opts = await GetDbOptions(); + var opts = await GetDbOptions().ConfigureAwait(false); return opts.ContainsKey("partially-recreated"); } @@ -1127,11 +1247,13 @@ namespace Duplicati.Library.Main.Database { if (value is bool v) { - await UpdateDbOption("terminated-with-active-uploads", v); + await UpdateDbOption("terminated-with-active-uploads", v) + .ConfigureAwait(false); + return v; } - var opts = await GetDbOptions(); + var opts = await GetDbOptions().ConfigureAwait(false); return opts.ContainsKey("terminated-with-active-uploads"); } @@ -1145,7 +1267,8 @@ namespace Duplicati.Library.Main.Database using var cmd = m_connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Configuration"" - "); + ") + .ConfigureAwait(false); foreach (var kp in options) { @@ -1161,7 +1284,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Key", kp.Key) .SetParameterValue("@Value", kp.Value) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } @@ -1171,10 +1295,13 @@ namespace Duplicati.Library.Main.Database SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size - "); + ") + .ConfigureAwait(false); - return await cmd.SetParameterValue("@Size", fhblocksize) - .ExecuteScalarInt64Async(-1); + return await cmd + .SetParameterValue("@Size", fhblocksize) + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); } /// @@ -1185,7 +1312,8 @@ namespace Duplicati.Library.Main.Database /// Also verify filelists (can be slow) /// The transaction to run in public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists) - => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false); + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false) + .ConfigureAwait(false); /// /// Verifies the consistency of the database prior to repair @@ -1195,7 +1323,8 @@ namespace Duplicati.Library.Main.Database /// Also verify filelists (can be slow) /// The transaction to run in public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists) - => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true); + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true) + .ConfigureAwait(false); /// /// Verifies the consistency of the database @@ -1242,8 +1371,8 @@ namespace Duplicati.Library.Main.Database AND ""A"".""CalcLen"" != ""A"".""Length"" "; - using (var rd = await cmd.ExecuteReaderAsync(reportDetails)) - if (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync(reportDetails).ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) { var sb = new StringBuilder(); sb.AppendLine("Found inconsistency in the following files while validating database: "); @@ -1253,7 +1382,7 @@ namespace Duplicati.Library.Main.Database if (c < 5) sb.AppendFormat("{0}, actual size {1}, dbsize {2}, blocksetid: {3}{4}", rd.GetValue(3), rd.GetValue(1), rd.GetValue(0), rd.GetValue(2), Environment.NewLine); c++; - } while (await rd.ReadAsync()); + } while (await rd.ReadAsync().ConfigureAwait(false)); c -= 5; if (c > 0) @@ -1266,7 +1395,9 @@ namespace Duplicati.Library.Main.Database var real_count = await cmd.ExecuteScalarInt64Async(@" SELECT Count(*) FROM ""BlocklistHash"" - ", 0); + ", 0) + .ConfigureAwait(false); + var unique_count = await cmd.ExecuteScalarInt64Async(@" SELECT Count(*) FROM ( @@ -1274,7 +1405,9 @@ namespace Duplicati.Library.Main.Database ""BlocksetID"", ""Index"" FROM ""BlocklistHash"" - )", 0); + ) + ", 0) + .ConfigureAwait(false); if (real_count != unique_count) throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); @@ -1311,7 +1444,9 @@ namespace Duplicati.Library.Main.Database ) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"" ) - ", 0); + ", 0) + .ConfigureAwait(false); + if (itemswithnoblocklisthash != 0) throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); @@ -1325,7 +1460,8 @@ namespace Duplicati.Library.Main.Database FROM ""BlocksetEntry"" ) "); - if (await cmd.ExecuteScalarInt64Async() != 0) + + if (await cmd.ExecuteScalarInt64Async().ConfigureAwait(false) != 0) throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); cmd.SetCommandAndParameters(@" @@ -1341,7 +1477,7 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); - if (await cmd.ExecuteScalarInt64Async(0) != 0) + if (await cmd.ExecuteScalarInt64Async(0).ConfigureAwait(false) != 0) throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!"); if (!laxVerifyForRepair) @@ -1359,7 +1495,9 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - var filesetsMissingVolumes = await cmd.ExecuteScalarInt64Async(0); + var filesetsMissingVolumes = + await cmd.ExecuteScalarInt64Async(0) + .ConfigureAwait(false); if (filesetsMissingVolumes != 0) { @@ -1381,8 +1519,9 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - using var reader = await cmd.ExecuteReaderAsync(); - if (await reader.ReadAsync()) + using var reader = await cmd.ExecuteReaderAsync() + .ConfigureAwait(false); + if (await reader.ReadAsync().ConfigureAwait(false)) throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); } @@ -1402,7 +1541,9 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - var volumesMissingFilests = await cmd.ExecuteScalarInt64Async(0); + var volumesMissingFilests = await cmd.ExecuteScalarInt64Async(0) + .ConfigureAwait(false); + if (volumesMissingFilests != 0) { if (volumesMissingFilests == 1) @@ -1423,8 +1564,9 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - using var reader = await cmd.ExecuteReaderAsync(); - if (await reader.ReadAsync()) + using var reader = await cmd.ExecuteReaderAsync() + .ConfigureAwait(false); + if (await reader.ReadAsync().ConfigureAwait(false)) throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); } @@ -1439,7 +1581,9 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""FileLookup"" ) - "); + ") + .ConfigureAwait(false); + if (nonAttachedFiles != 0) { // Attempt to create a better error message by finding the first 10 fileset ids with the issue @@ -1451,10 +1595,12 @@ namespace Duplicati.Library.Main.Database FROM ""FileLookup"" ) LIMIT 11 - "); + ") + .ConfigureAwait(false); + var filesetIds = new HashSet(); var overflow = false; - while (await filesetIdReader.ReadAsync()) + while (await filesetIdReader.ReadAsync().ConfigureAwait(false)) { if (filesetIds.Count >= 10) { @@ -1470,7 +1616,7 @@ namespace Duplicati.Library.Main.Database .Select(x => $"Fileset {x.Version}: {x.Time} (id = {x.FilesetId})"); // Fall back to a generic error message if we can't find the fileset ids - if (!await pairs.AnyAsync()) + if (!await pairs.AnyAsync().ConfigureAwait(false)) throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); if (overflow) @@ -1489,7 +1635,7 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""Fileset"" "); - await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync().Select(x => x.ConvertValueToInt64(0, -1))) + await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync().Select(x => x.ConvertValueToInt64(0, -1)).ConfigureAwait(false)) { var expandedlist = await cmd2.SetCommandAndParameters($@" SELECT COUNT(*) @@ -1504,7 +1650,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", filesetid) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != @FolderBlocksetId AND ""FileLookup"".""BlocksetID"" != @SymlinkBlocksetId"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); @@ -1514,14 +1661,25 @@ namespace Duplicati.Library.Main.Database WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetid) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); if (expandedlist != storedlist) { var filesetname = filesetid.ToString(); - var fileset = await FilesetTimes().Zip(AsyncEnumerable.Range(0, await FilesetTimes().CountAsync()), (a, b) => new Tuple(b, a.Key, a.Value)).FirstOrDefaultAsync(x => x.Item2 == filesetid); + var fileset = await FilesetTimes() + .Zip( + AsyncEnumerable.Range(0, await FilesetTimes() + .CountAsync() + .ConfigureAwait(false) + ), (a, b) => new Tuple(b, a.Key, a.Value) + ) + .FirstOrDefaultAsync(x => x.Item2 == filesetid) + .ConfigureAwait(false); + if (fileset != null) filesetname = $"version {fileset.Item1}: {fileset.Item3} (database id: {fileset.Item2})"; + anyError.Add($"Unexpected difference in fileset {filesetname}, found {expandedlist} entries, but expected {storedlist}"); } } @@ -1554,12 +1712,18 @@ namespace Duplicati.Library.Main.Database ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId - "); + ") + .ConfigureAwait(false); + cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - yield return new Block(rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), rd.ConvertValueToInt64(1)); + + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new Block( + rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), + rd.ConvertValueToInt64(1) + ); } private class BlocklistHashEnumerable : IAsyncEnumerable @@ -1600,7 +1764,7 @@ namespace Duplicati.Library.Main.Database if (m_current == null) return false; - if (!await m_reader.ReadAsync()) + if (!await m_reader.ReadAsync().ConfigureAwait(false)) { m_current = null; m_parent.MoreData = false; @@ -1787,8 +1951,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); string? lastpath = null; - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) { var blocksetID = rd.ConvertValueToInt64(0, -1); var path = rd.ConvertValueToString(2); @@ -1813,8 +1977,8 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(LIST_FILESETS); cmd.SetParameterValue("@FilesetId", filesetId); - using (var rd = await cmd.ExecuteReaderAsync()) - if (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) { var more = false; do @@ -1839,9 +2003,23 @@ namespace Duplicati.Library.Main.Database if (metablockhash == metahash) metablockhash = null; - await filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); + await filesetvolume.AddFile( + path, + filehash, + size, + lastmodified, + metahash, + metasize, + metablockhash, + blockhash, + blocksize, + blrd, + string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash } + ) + .ConfigureAwait(false); + if (blrd == null) - more = await rd.ReadAsync(); + more = await rd.ReadAsync().ConfigureAwait(false); else more = blrd.MoreData; @@ -1855,11 +2033,14 @@ namespace Duplicati.Library.Main.Database UPDATE ""Fileset"" SET ""VolumeID"" = @VolumeId WHERE ""ID"" = @FilesetId - "); + ") + .ConfigureAwait(false); + var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@FilesetId", filesetid) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); @@ -1883,10 +2064,12 @@ namespace Duplicati.Library.Main.Database ) "; - using var cmd = await m_connection.CreateCommandAsync(query); + using var cmd = await m_connection.CreateCommandAsync(query) + .ConfigureAwait(false); await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } /// @@ -1927,9 +2110,13 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{ftt.Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" - "); + ") + .ConfigureAwait(false); + await cmd.SetTransaction(db.Transaction) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + return ftt; } @@ -1940,31 +2127,39 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( ""Path"" TEXT NOT NULL ) - "); + ") + .ConfigureAwait(false); + await cmd.SetTransaction(db.Transaction) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); cmd.SetCommandAndParameters($@" INSERT INTO ""{ftt.Tablename}"" (""Path"") VALUES (@Path) "); + using var c2 = await db.Connection.CreateCommandAsync(@" SELECT DISTINCT ""Path"" FROM ""File"" - "); + ") + .ConfigureAwait(false); + c2.SetTransaction(db.Transaction); - using (var rd = await c2.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await c2.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) { var p = rd.ConvertValueToString(0) ?? ""; if (FilterExpression.Matches(filter, p)) { - await cmd.SetParameterValue("@Path", p) - .ExecuteNonQueryAsync(); + await cmd + .SetParameterValue("@Path", p) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } - await db.Transaction.CommitAsync(); + await db.Transaction.CommitAsync().ConfigureAwait(false); } else { @@ -1995,16 +2190,18 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( ""Path"" TEXT NOT NULL ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" INSERT INTO ""{ftt.Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb} - ", args); + ", args) + .ConfigureAwait(false); - await db.Transaction.CommitAsync(); + await db.Transaction.CommitAsync().ConfigureAwait(false); } return ftt; @@ -2023,7 +2220,8 @@ namespace Duplicati.Library.Main.Database using var cmd = m_db.Connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@$" DROP TABLE IF EXISTS ""{Tablename}"" - "); + ") + .ConfigureAwait(false); } catch { } finally { Tablename = null!; } @@ -2037,11 +2235,14 @@ namespace Duplicati.Library.Main.Database UPDATE ""Remotevolume"" SET ""Name"" = @Newname WHERE ""Name"" = @Oldname - "); + ") + .ConfigureAwait(false); + var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@Newname", newname) .SetParameterValue("@Oldname", oldname) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}"); @@ -2055,20 +2256,24 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name ") .SetParameterValue("@Name", newname) - .ExecuteScalarAsync()) - ?.ToString() ?? "", + .ExecuteScalarAsync() + .ConfigureAwait(false) + )?.ToString() ?? "", true ); //Create a fake new entry with the old name and mark as deleting // as this ensures we will remove it, if it shows up in some later listing - var newvolId = await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting); + var newvolId = + await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting) + .ConfigureAwait(false); // IF needed, also create an empty fileset, so the validation works if (type == RemoteVolumeType.Files) { - await CreateFileset(newvolId, DateTime.UnixEpoch); - await m_rtr.CommitAsync(); + await CreateFileset(newvolId, DateTime.UnixEpoch) + .ConfigureAwait(false); + await m_rtr.CommitAsync().ConfigureAwait(false); } } @@ -2094,13 +2299,17 @@ namespace Duplicati.Library.Main.Database @IsFullBackup ); SELECT last_insert_rowid(); - "); - var id = await cmd.SetTransaction(m_rtr) + ") + .ConfigureAwait(false); + + var id = await cmd + .SetTransaction(m_rtr) .SetParameterValue("@OperationId", m_operationid) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); return id; } @@ -2118,10 +2327,12 @@ namespace Duplicati.Library.Main.Database if (blockVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0."); - await m_insertIndexBlockLink.SetTransaction(m_rtr) + await m_insertIndexBlockLink + .SetTransaction(m_rtr) .SetParameterValue("@IndexVolumeId", indexVolumeID) .SetParameterValue("@BlockVolumeId", blockVolumeID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } /// @@ -2198,10 +2409,11 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(sql) .SetParameterValue("@VolumeId", volumeid) - .SetParameterValue("@HashesPerBlock", blocksize / hashsize); + .SetParameterValue("@HashesPerBlock", blocksize / hashsize) + .ConfigureAwait(false); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) { var blockhash = rd.ConvertValueToString(0); if (curHash != null && (blockhash != curHash || count + hashsize > buffer.Length)) @@ -2234,11 +2446,14 @@ namespace Duplicati.Library.Main.Database UPDATE ""Fileset"" SET ""IsFullBackup"" = @IsFullBackup WHERE ""ID"" = @FilesetId - "); + ") + .ConfigureAwait(false); + await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId) .SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } /// @@ -2251,10 +2466,13 @@ namespace Duplicati.Library.Main.Database using var cmd = await m_connection.CreateCommandAsync(@" DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId - "); + ") + .ConfigureAwait(false); + await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } /// @@ -2267,8 +2485,13 @@ namespace Duplicati.Library.Main.Database var candidates = GetIncompleteFilesets() .OrderBy(x => x.Value); - if (await candidates.AnyAsync()) - return await GetRemoteVolumeFromFilesetID((await candidates.LastAsync()).Key); + if (await candidates.AnyAsync().ConfigureAwait(false)) + return await GetRemoteVolumeFromFilesetID( + ( + await candidates.LastAsync().ConfigureAwait(false) + ).Key + ) + .ConfigureAwait(false); return default; } @@ -2297,11 +2520,14 @@ namespace Duplicati.Library.Main.Database ""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}' ) - "); + ") + .ConfigureAwait(false); using var rd = await cmd.SetTransaction(m_rtr) - .ExecuteReaderAsync(); - while (await rd.ReadAsync()) + .ExecuteReaderAsync() + .ConfigureAwait(false); + + while (await rd.ReadAsync().ConfigureAwait(false)) { yield return new KeyValuePair( rd.ConvertValueToInt64(0), @@ -2335,12 +2561,15 @@ namespace Duplicati.Library.Main.Database WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Fileset"".""ID"" = @FilesetId - "); + ") + .ConfigureAwait(false); using var rd = await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetID) - .ExecuteReaderAsync(); - if (await rd.ReadAsync()) + .ExecuteReaderAsync() + .ConfigureAwait(false); + + if (await rd.ReadAsync().ConfigureAwait(false)) return new RemoteVolumeEntry( rd.ConvertValueToInt64(0, -1), rd.ConvertValueToString(1), @@ -2365,16 +2594,18 @@ namespace Duplicati.Library.Main.Database WHERE ""Timestamp"" < @Timestamp ") .SetParameterValue("@Timestamp", t) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); await cmd.SetCommandAndParameters(@" DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < @Timestamp ") .SetParameterValue("@Timestamp", t) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } public async Task PurgeDeletedVolumes(DateTime threshold) @@ -2382,9 +2613,10 @@ namespace Duplicati.Library.Main.Database await m_removedeletedremotevolumeCommand .SetTransaction(m_rtr) .SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } public virtual void Dispose() @@ -2401,7 +2633,7 @@ namespace Duplicati.Library.Main.Database if (ShouldCloseConnection && m_connection != null) { - await m_rtr.DisposeAsync(); + await m_rtr.DisposeAsync().ConfigureAwait(false); if (m_connection.State == ConnectionState.Open && !m_hasExecutedVacuum) { using (var command = m_connection.CreateCommand()) @@ -2410,11 +2642,12 @@ namespace Duplicati.Library.Main.Database // SQLite recommends that PRAGMA optimize is run just before closing each database connection. await command .SetTransaction(tr) - .ExecuteNonQueryAsync("PRAGMA optimize"); + .ExecuteNonQueryAsync("PRAGMA optimize") + .ConfigureAwait(false); try { - await tr.CommitAsync(); + await tr.CommitAsync().ConfigureAwait(false); } catch (Exception ex) { @@ -2425,7 +2658,7 @@ namespace Duplicati.Library.Main.Database m_connection.Close(); } - await m_connection.DisposeAsync(); + await m_connection.DisposeAsync().ConfigureAwait(false); } IsDisposed = true; @@ -2487,7 +2720,7 @@ namespace Duplicati.Library.Main.Database { if (result is BasicResults basicResults) { - await basicResults.FlushLog(this); + await basicResults.FlushLog(this).ConfigureAwait(false); if (basicResults.EndTime.Ticks == 0) basicResults.EndTime = DateTime.UtcNow; } @@ -2496,7 +2729,8 @@ namespace Duplicati.Library.Main.Database await LogMessage("Result", serializer.SerializeResults(result), null - ); + ) + .ConfigureAwait(false); } } @@ -2525,15 +2759,19 @@ namespace Duplicati.Library.Main.Database return m_pathPrefixLookup[ix].Value; } - var id = await m_findpathprefixCommand.SetTransaction(m_rtr) + var id = await m_findpathprefixCommand + .SetTransaction(m_rtr) .SetParameterValue("@Prefix", prefix) - .ExecuteScalarInt64Async(); + .ExecuteScalarInt64Async() + .ConfigureAwait(false); if (id < 0) { - id = await m_insertpathprefixCommand.SetTransaction(m_rtr) + id = await m_insertpathprefixCommand + .SetTransaction(m_rtr) .SetParameterValue("@Prefix", prefix) - .ExecuteScalarInt64Async(); + .ExecuteScalarInt64Async() + .ConfigureAwait(false); } m_pathPrefixIndex = (m_pathPrefixIndex + 1) % m_pathPrefixLookup.Length; diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 76892a47e..c4d20dae2 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -50,9 +50,13 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalDeleteDatabase(); - dbnew = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(path, operation, true, pagecachesize, dbnew); + dbnew = (LocalDeleteDatabase) + await CreateLocalDatabaseAsync(path, operation, true, pagecachesize, dbnew) + .ConfigureAwait(false); - dbnew.m_registerDuplicateBlockCommand = await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND); + dbnew.m_registerDuplicateBlockCommand = + await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND) + .ConfigureAwait(false); return dbnew; } @@ -61,9 +65,13 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalDeleteDatabase(); - dbnew = (LocalDeleteDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + dbnew = (LocalDeleteDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew) + .ConfigureAwait(false); - dbnew.m_registerDuplicateBlockCommand = await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND); + dbnew.m_registerDuplicateBlockCommand = + await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND) + .ConfigureAwait(false); return dbnew; } @@ -80,13 +88,17 @@ namespace Duplicati.Library.Main.Database { var deleted = 0; - using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds))) - deleted += await (await cmd.SetCommandAndParameters(@" - DELETE FROM ""Fileset"" - WHERE ""Timestamp"" IN (@Timestamps) - ") - .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable)) - .ExecuteNonQueryAsync(); + using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds)).ConfigureAwait(false)) + deleted += await ( + await cmd.SetCommandAndParameters(@" + DELETE FROM ""Fileset"" + WHERE ""Timestamp"" IN (@Timestamps) + ") + .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (deleted != toDelete.Length) throw new Exception($"Unexpected number of deleted filesets {deleted} vs {toDelete.Length}"); @@ -98,7 +110,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""Fileset"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""ChangeJournalData"" @@ -106,7 +119,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""Fileset"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""FileLookup"" @@ -114,7 +128,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Metadataset"" @@ -122,7 +137,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""MetadataID"" FROM ""FileLookup"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Blockset"" @@ -133,7 +149,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""BlocksetEntry"" @@ -141,7 +158,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""Blockset"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""BlocklistHash"" @@ -149,7 +167,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""Blockset"" ) - "); + ") + .ConfigureAwait(false); //We save the block info for the remote files, before we delete it await cmd.ExecuteNonQueryAsync(@" @@ -173,7 +192,8 @@ namespace Duplicati.Library.Main.Database ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Block"" @@ -187,7 +207,8 @@ namespace Duplicati.Library.Main.Database ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ) - "); + ") + .ConfigureAwait(false); //Find all remote filesets that are no longer required, and mark them as deleting var updated = await cmd.SetCommandAndParameters(@" @@ -209,7 +230,8 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Deleting.ToString() ]) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (deleted != updated) throw new Exception($"Unexpected number of remote volumes marked as deleted. Found {deleted} filesets, but {updated} volumes"); @@ -226,8 +248,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) .SetParameterValue("@State", RemoteVolumeState.Deleting.ToString()); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1) @@ -256,10 +278,10 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC "); - using (var reader = await cmd.ExecuteReaderAsync()) + using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) { int version = 0; - while (await reader.ReadAsync()) + while (await reader.ReadAsync().ConfigureAwait(false)) { yield return new ListResultFileset( version++, @@ -424,13 +446,15 @@ namespace Duplicati.Library.Main.Database { try { - await cmd.SetCommandAndParameters(createtable) + await cmd + .SetCommandAndParameters(createtable) .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExpandInClauseParameterMssqlite("@AllowedStates", [ RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() ]) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); cmd.SetCommandAndParameters($@" SELECT @@ -445,8 +469,8 @@ namespace Duplicati.Library.Main.Database ORDER BY ""B"".""Sorttime"" ASC "); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new VolumeUsage( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), @@ -458,7 +482,9 @@ namespace Duplicati.Library.Main.Database { try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"" "); + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"" ") + .ConfigureAwait(false); } catch { } } @@ -565,7 +591,15 @@ namespace Duplicati.Library.Main.Database public async Task GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount) { - return new CompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, await GetWastedSpaceReport().ToListAsync()); + return new CompactReport( + volsize, + wastethreshold, + smallfilesize, + maxsmallfilecount, + await GetWastedSpaceReport() + .ToListAsync() + .ConfigureAwait(false) + ); } @@ -611,6 +645,7 @@ namespace Duplicati.Library.Main.Database ""Hash"" = @Hash AND ""Size"" = @Size ") + .ConfigureAwait(false) }; } @@ -621,7 +656,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (r == -1) { @@ -651,7 +687,7 @@ namespace Duplicati.Library.Main.Database /// public async Task CreateBlockQueryHelper() { - return await BlockQuery.CreateAsync(this); + return await BlockQuery.CreateAsync(this).ConfigureAwait(false); } /// @@ -668,7 +704,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@VolumeId", volumeID) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } /// @@ -679,7 +716,8 @@ namespace Duplicati.Library.Main.Database /// The transaction to execute the command in public async Task PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved) { - var deletedVolume = await GetRemoteVolume(filename); + var deletedVolume = await GetRemoteVolume(filename) + .ConfigureAwait(false); if (deletedVolume.Type != RemoteVolumeType.Blocks) return; @@ -696,30 +734,36 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved)) - await (await cmd.SetCommandAndParameters($@" - CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS - SELECT - ""BlockID"", - MAX(""VolumeID"") AS ""VolumeID"" - FROM ""DuplicateBlock"" - WHERE - ""VolumeID"" NOT IN (@VolumeIds) - AND ""BlockID"" IN ( - SELECT ""ID"" - FROM ""{updatedBlocks}"" - ) - GROUP BY ""BlockID"" - ") - .ExpandInClauseParameterMssqliteAsync("@VolumeIds", tempTable)) - .ExecuteNonQueryAsync(); + using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved).ConfigureAwait(false)) + await ( + await cmd.SetCommandAndParameters($@" + CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS + SELECT + ""BlockID"", + MAX(""VolumeID"") AS ""VolumeID"" + FROM ""DuplicateBlock"" + WHERE + ""VolumeID"" NOT IN (@VolumeIds) + AND ""BlockID"" IN ( + SELECT ""ID"" + FROM ""{updatedBlocks}"" + ) + GROUP BY ""BlockID"" + ") + .ExpandInClauseParameterMssqliteAsync("@VolumeIds", tempTable) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); var targetCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{updatedBlocks}"" - "); + ") + .ConfigureAwait(false); if (targetCount == 0) return; @@ -727,7 +771,8 @@ namespace Duplicati.Library.Main.Database var replacementCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{replacementBlocks}"" - "); + ") + .ConfigureAwait(false); var updateCount = await cmd.SetCommandAndParameters(@$" UPDATE ""Block"" @@ -741,7 +786,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Block"".""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); var deleteCount = await cmd.ExecuteNonQueryAsync(@$" DELETE FROM ""DuplicateBlock"" @@ -757,7 +803,8 @@ namespace Duplicati.Library.Main.Database || ""RB"".""VolumeID"" FROM ""{replacementBlocks}"" RB ) - "); + ") + .ConfigureAwait(false); if (targetCount != updateCount || replacementCount != deleteCount @@ -772,18 +819,21 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } finally { try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ") + .ConfigureAwait(false); } catch { } try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ") + .ConfigureAwait(false); } catch { } } @@ -823,8 +873,8 @@ namespace Duplicati.Library.Main.Database AND ""B"".""Size"" IS NOT NULL "); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) { var name = rd.ConvertValueToString(0) ?? ""; if (!lookupBlock.TryGetValue(name, out var indexfileList)) diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index a88e84cbd..423cd1436 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -34,7 +34,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalListAffectedDatabase(); - dbnew = (LocalListAffectedDatabase)await CreateLocalDatabaseAsync(path, "ListAffected", false, pagecachesize, dbnew); + dbnew = (LocalListAffectedDatabase) + await CreateLocalDatabaseAsync(path, "ListAffected", false, pagecachesize, dbnew) + .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; @@ -68,7 +70,10 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable GetFilesets(IEnumerable items) { - var filesets = await FilesetTimes().ToArrayAsync(); + var filesets = await FilesetTimes() + .ToArrayAsync() + .ConfigureAwait(false); + var dict = new Dictionary(); for (var i = 0; i < filesets.Length; i++) dict[filesets[i].Key] = i; @@ -106,10 +111,10 @@ namespace Duplicati.Library.Main.Database ) "; - using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items)) - using (var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterMssqliteAsync("@Names", tmptable)) - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false)) + using (var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false)) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) { var v = dict[rd.ConvertValueToInt64(0)]; yield return new ListResultFileset() @@ -180,10 +185,10 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Path"" "; - using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items)) - using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterMssqliteAsync("@Names", tmptable)) - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false)) + using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false)) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new ListResultFile() { Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"), @@ -224,8 +229,8 @@ namespace Duplicati.Library.Main.Database foreach ((var x, var i) in items.Select((x, i) => (x, i))) cmd.SetParameterValue($"@Message{i}", "%" + x + "%"); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new ListResultRemoteLog() { Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)), @@ -302,8 +307,8 @@ namespace Duplicati.Library.Main.Database "; using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterMssqlite("@Names", items.ToArray())) - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new ListResultRemoteVolume() { Name = rd.ConvertValueToString(0) ?? "" diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 1e5709a70..bae48a759 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -142,7 +142,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalListBrokenFilesDatabase(); - dbnew = (LocalListBrokenFilesDatabase)await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, pagecachesize, dbnew); + dbnew = (LocalListBrokenFilesDatabase) + await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, pagecachesize, dbnew) + .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; @@ -152,7 +154,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalListBrokenFilesDatabase(); - dbnew = (LocalListBrokenFilesDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + dbnew = (LocalListBrokenFilesDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew) + .ConfigureAwait(false); dbnew.ShouldCloseConnection = false; return dbnew; @@ -161,7 +165,9 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[]? versions) { var query = BROKEN_FILE_SETS; - var clause = await GetFilelistWhereClause(time, versions); + var clause = await GetFilelistWhereClause(time, versions) + .ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(clause.Query)) query += $@" AND ""A"".""FilesetID"" IN ( @@ -177,7 +183,7 @@ namespace Duplicati.Library.Main.Database .SetCommandAndParameters(query) .SetParameterValues(clause.Values); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) if (!rd.IsDBNull(0)) yield return ( ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), @@ -192,7 +198,7 @@ namespace Duplicati.Library.Main.Database .SetCommandAndParameters(BROKEN_FILE_NAMES) .SetParameterValue("@FilesetId", filesetid); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) if (!rd.IsDBNull(0)) yield return new Tuple( rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), @@ -222,7 +228,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), @@ -243,7 +249,8 @@ namespace Duplicati.Library.Main.Database .SetCommandAndParameters(INSERT_BROKEN_IDS(tablename, IDfieldname)) .SetParameterValue("@FilesetId", filesetid); - await cmd.ExecuteNonQueryAsync(); + await cmd.ExecuteNonQueryAsync() + .ConfigureAwait(false); } /// @@ -278,7 +285,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@EmptyHash", emptyHash) .SetParameterValue("@EmptyHashSize", emptyHashSize); - var res = await cmd.ExecuteScalarInt64Async(-1); + var res = await cmd.ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); // No empty block found, try to find a zero-length block instead if (res < 0 && emptyHashSize != 0) @@ -299,7 +307,8 @@ namespace Duplicati.Library.Main.Database ") .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) .SetParameterValue("@EmptyHashSize", 0) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); // No empty block found, pick the smallest one if (res < 0) @@ -319,7 +328,8 @@ namespace Duplicati.Library.Main.Database LIMIT 1 ") .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); return res; } @@ -358,7 +368,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@EmptyBlocksetID", emptyBlocksetId) .SetParameterValue("@FilesetID", filesetId); - return await cmd.ExecuteNonQueryAsync(); + + return await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } public async Task RemoveMissingBlocks(IEnumerable names) @@ -375,17 +386,22 @@ namespace Duplicati.Library.Main.Database CREATE TEMP TABLE ""{volidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) - "); + ") + .ConfigureAwait(false); - using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names)) - await (await deletecmd.SetCommandAndParameters($@" - INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") - SELECT ""ID"" - FROM ""RemoteVolume"" - WHERE ""Name"" IN (@Names) - ") - .ExpandInClauseParameterMssqliteAsync("@Names", tmptable)) - .ExecuteNonQueryAsync(); + using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names).ConfigureAwait(false)) + await ( + await deletecmd.SetCommandAndParameters($@" + INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ") + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); var volIdsSubQuery = $@" SELECT ""ID"" @@ -398,31 +414,40 @@ namespace Duplicati.Library.Main.Database WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery}) - "); + ") + .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - "); + ") + .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - "); + ") + .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - "); + ") + .ConfigureAwait(false); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // SQLite.SQLiteException (0x80004005): database table is locked - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" "); + await deletecmd + .ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ") + .ConfigureAwait(false); + try { deletecmd.CommandTimeout = 2; - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" "); + await deletecmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ") + .ConfigureAwait(false); } catch { /* Ignore, will be deleted on close anyway. */ } } @@ -438,7 +463,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetid); - return await cmd.ExecuteScalarInt64Async(0); + return await cmd + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index d40c9ca66..b1c0e240b 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -36,7 +36,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalListChangesDatabase(); - dbnew = (LocalListChangesDatabase)await CreateLocalDatabaseAsync(path, "ListChanges", false, pagecachesize, dbnew); + dbnew = (LocalListChangesDatabase) + await CreateLocalDatabaseAsync(path, "ListChanges", false, pagecachesize, dbnew) + .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; @@ -137,7 +139,8 @@ namespace Duplicati.Library.Main.Database ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" ( @@ -147,42 +150,47 @@ namespace Duplicati.Library.Main.Database ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); } - sh.m_insertPreviousElementCommand = await sh.m_db.Connection.CreateCommandAsync($@" - INSERT INTO ""{sh.m_previousTable}"" ( - ""Path"", - ""FileHash"", - ""MetaHash"", - ""Size"", - ""Type"" - ) - VALUES ( - @Path, - @FileHash, - @MetaHash, - @Size, - @Type - ) - "); + sh.m_insertPreviousElementCommand = await sh.m_db.Connection + .CreateCommandAsync($@" + INSERT INTO ""{sh.m_previousTable}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + VALUES ( + @Path, + @FileHash, + @MetaHash, + @Size, + @Type + ) + ") + .ConfigureAwait(false); - sh.m_insertCurrentElementCommand = await sh.m_db.Connection.CreateCommandAsync($@" - INSERT INTO ""{sh.m_currentTable}"" ( - ""Path"", - ""FileHash"", - ""MetaHash"", - ""Size"", - ""Type"" - ) - VALUES ( - @Path, - @FileHash, - @MetaHash, - @Size, - @Type - ) - "); + sh.m_insertCurrentElementCommand = await sh.m_db.Connection + .CreateCommandAsync($@" + INSERT INTO ""{sh.m_currentTable}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + VALUES ( + @Path, + @FileHash, + @MetaHash, + @Size, + @Type + ) + ") + .ConfigureAwait(false); return sh; } @@ -277,7 +285,8 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { @@ -290,17 +299,21 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{filenamestable}"" ( ""Path"" TEXT NOT NULL ) - "); + ") + .ConfigureAwait(false); await cmd.SetCommandAndParameters($@" INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path) ") - .PrepareAsync(); + .PrepareAsync() + .ConfigureAwait(false); foreach (var s in p) - await cmd.SetParameterValue("@Path", s) - .ExecuteNonQueryAsync(); + await cmd + .SetParameterValue("@Path", s) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); string whereClause; if (expression.Result) @@ -343,9 +356,12 @@ namespace Duplicati.Library.Main.Database WHERE {whereClause} ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ") + .ConfigureAwait(false); } else { @@ -362,7 +378,8 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) - .PrepareAsync(); + .PrepareAsync() + .ConfigureAwait(false); using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) .SetCommandAndParameters($@" @@ -381,10 +398,13 @@ namespace Duplicati.Library.Main.Database @Type ) "); - await cmd2.PrepareAsync(); + await cmd2.PrepareAsync().ConfigureAwait(false); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + using var rd = await cmd + .ExecuteReaderAsync() + .ConfigureAwait(false); + + while (await rd.ReadAsync().ConfigureAwait(false)) { rd.GetValues(values); var path = values[0] as string; @@ -396,7 +416,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@MetaHash", values[2]) .SetParameterValue("@Size", values[3]) .SetParameterValue("@Type", values[4]) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } } @@ -412,13 +433,14 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@MetaHash", metahash) .SetParameterValue("@Size", size) .SetParameterValue("@Type", (int)type) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } private static async IAsyncEnumerable ReaderToStringList(SqliteDataReader rd) { using (rd) - while (await rd.ReadAsync()) + while (await rd.ReadAsync().ConfigureAwait(false)) { var v = rd.GetValue(0); if (v == null || v == DBNull.Value) @@ -479,24 +501,28 @@ namespace Duplicati.Library.Main.Database PreviousSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_previousTable}"" - ", 0), + ", 0) + .ConfigureAwait(false), CurrentSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_currentTable}"" - ", 0), + ", 0) + .ConfigureAwait(false), AddedSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({Added}) - ", 0), + ", 0) + .ConfigureAwait(false), DeletedSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({Deleted}) ", 0) + .ConfigureAwait(false) }; return result; @@ -526,35 +552,51 @@ namespace Duplicati.Library.Main.Database var result = new ChangeCountReport { - AddedFolders = await cmd.SetCommandAndParameters(added) + AddedFolders = await cmd + .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0), - AddedSymlinks = await cmd.SetCommandAndParameters(added) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false), + AddedSymlinks = await cmd + .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0), - AddedFiles = await cmd.SetCommandAndParameters(added) - .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64Async(0), - - DeletedFolders = await cmd.SetCommandAndParameters(deleted) - .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0), - DeletedSymlinks = await cmd.SetCommandAndParameters(deleted) - .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0), - DeletedFiles = await cmd.SetCommandAndParameters(deleted) - .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64Async(0), - - ModifiedFolders = await cmd.SetCommandAndParameters(modified) - .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0), - ModifiedSymlinks = await cmd.SetCommandAndParameters(modified) - .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0), - ModifiedFiles = await cmd.SetCommandAndParameters(modified) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false), + AddedFiles = await cmd + .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) .ExecuteScalarInt64Async(0) + .ConfigureAwait(false), + DeletedFolders = await cmd + .SetCommandAndParameters(deleted) + .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false), + DeletedSymlinks = await cmd + .SetCommandAndParameters(deleted) + .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false), + DeletedFiles = await cmd + .SetCommandAndParameters(deleted) + .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false), + ModifiedFolders = await cmd + .SetCommandAndParameters(modified) + .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false), + ModifiedSymlinks = await cmd + .SetCommandAndParameters(modified) + .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false), + ModifiedFiles = await cmd + .SetCommandAndParameters(modified) + .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false) }; return result; @@ -576,15 +618,26 @@ namespace Duplicati.Library.Main.Database { cmd.SetCommandAndParameters(sql); foreach (var type in elTypes) - await foreach (var s in ReaderToStringList(await cmd.SetParameterValue("@Type", (int)type).ExecuteReaderAsync())) + await foreach (var s in ReaderToStringList(await cmd.SetParameterValue("@Type", (int)type).ExecuteReaderAsync().ConfigureAwait(false)).ConfigureAwait(false)) yield return new Tuple(changeType, type, s ?? ""); } - await foreach (var r in BuildResult(cmd, Added, Interface.ListChangesChangeType.Added)) + await foreach (var r in + BuildResult(cmd, Added, Interface.ListChangesChangeType.Added) + .ConfigureAwait(false) + ) yield return r; - await foreach (var r in BuildResult(cmd, Deleted, Interface.ListChangesChangeType.Deleted)) + + await foreach (var r in + BuildResult(cmd, Deleted, Interface.ListChangesChangeType.Deleted) + .ConfigureAwait(false) + ) yield return r; - await foreach (var r in BuildResult(cmd, Modified, Interface.ListChangesChangeType.Modified)) + + await foreach (var r in + BuildResult(cmd, Modified, Interface.ListChangesChangeType.Modified) + .ConfigureAwait(false) + ) yield return r; } } @@ -598,19 +651,34 @@ namespace Duplicati.Library.Main.Database { if (m_insertPreviousElementCommand != null) { - try { await m_insertPreviousElementCommand.DisposeAsync(); } + try + { + await m_insertPreviousElementCommand + .DisposeAsync() + .ConfigureAwait(false); + } catch { } finally { m_insertPreviousElementCommand = null!; } } if (m_insertCurrentElementCommand != null) { - try { await m_insertCurrentElementCommand.DisposeAsync(); } + try + { + await m_insertCurrentElementCommand + .DisposeAsync() + .ConfigureAwait(false); + } catch { } finally { m_insertCurrentElementCommand = null!; } } - try { await m_db.Transaction.RollBackAsync(); } + try + { + await m_db.Transaction + .RollBackAsync() + .ConfigureAwait(false); + } catch { } finally { @@ -622,7 +690,7 @@ namespace Duplicati.Library.Main.Database public async Task CreateStorageHelper() { - return await StorageHelper.CreateAsync(this); + return await StorageHelper.CreateAsync(this).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 2091dc5bb..3d8859b01 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -38,7 +38,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalListDatabase(); - dbnew = (LocalListDatabase)await CreateLocalDatabaseAsync(path, "List", false, pagecachesize, dbnew); + dbnew = (LocalListDatabase) + await CreateLocalDatabaseAsync(path, "List", false, pagecachesize, dbnew) + .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; @@ -88,10 +90,14 @@ namespace Duplicati.Library.Main.Database var fs = new FileSets { m_db = owner, - m_filesets = await owner.FilesetTimes().ToArrayAsync(), + m_filesets = await owner + .FilesetTimes() + .ToArrayAsync() + .ConfigureAwait(false), m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) }; - var tmp = await owner.GetFilelistWhereClause(time, versions, fs.m_filesets); + var tmp = await owner.GetFilelistWhereClause(time, versions, fs.m_filesets) + .ConfigureAwait(false); var query = tmp.Item1; var args = tmp.Item2; @@ -103,7 +109,9 @@ namespace Duplicati.Library.Main.Database ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"", ""Timestamp"" AS ""Timestamp"" - FROM ""Fileset"" {query}", args); + FROM ""Fileset"" {query} + ", args) + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{fs.m_tablename}_FilesetIDTimestampIndex"" @@ -111,7 +119,8 @@ namespace Duplicati.Library.Main.Database ""FilesetID"", ""Timestamp"" DESC ) - "); + ") + .ConfigureAwait(false); } return fs; @@ -153,7 +162,7 @@ namespace Duplicati.Library.Main.Database while (More && Path == m_reader.ConvertValueToString(0)) { yield return m_reader.ConvertValueToInt64(1, -1); - More = await m_reader.ReadAsync(); + More = await m_reader.ReadAsync().ConfigureAwait(false); } } } @@ -171,7 +180,7 @@ namespace Duplicati.Library.Main.Database private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) { - using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter)) + using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false)) using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets @@ -189,7 +198,8 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tablename}"" ) ) - "); + ") + .ConfigureAwait(false); //If we have a prefix rule, apply it if (!string.IsNullOrWhiteSpace(prefixrule)) @@ -198,7 +208,8 @@ namespace Duplicati.Library.Main.Database WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule ") .SetParameterValue("@Rule", prefixrule) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); // Then we recursively find the largest prefix var v0 = await cmd.ExecuteScalarAsync($@" @@ -206,7 +217,8 @@ namespace Duplicati.Library.Main.Database FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1 - "); + ") + .ConfigureAwait(false); var maxpath = ""; if (v0 != null) @@ -217,7 +229,8 @@ namespace Duplicati.Library.Main.Database var filecount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" - ", 0); + ", 0) + .ConfigureAwait(false); var foundfiles = -1L; @@ -227,7 +240,8 @@ namespace Duplicati.Library.Main.Database FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix ") - .PrepareAsync(); + .PrepareAsync() + .ConfigureAwait(false); while (filecount != foundfiles && maxpath.Length > 0) { @@ -235,7 +249,8 @@ namespace Duplicati.Library.Main.Database foundfiles = await cmd .SetParameterValue("@PrefixLength", mp.Length) .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); if (filecount != foundfiles) { @@ -276,7 +291,7 @@ namespace Duplicati.Library.Main.Database .FirstAsync()) .Distinct(); - await foreach (var el in result) + await foreach (var el in result.ConfigureAwait(false)) yield return await el; } @@ -297,8 +312,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""Path"" FROM ""{table}"" "); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) { var s = rd.ConvertValueToString(0) ?? ""; if (!s.StartsWith(prefix, StringComparison.Ordinal)) @@ -335,7 +350,7 @@ namespace Duplicati.Library.Main.Database if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); - using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true))) + using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true)).ConfigureAwait(false)) using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets @@ -353,7 +368,8 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tablename}"" ) ) - "); + ") + .ConfigureAwait(false); // If we had instr support this would work: /*var distinctPaths = @"SELECT DISTINCT :1 || " + @@ -368,7 +384,8 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{tbname}"" ( ""Path"" TEXT NOT NULL ) - "); + ") + .ConfigureAwait(false); using (var c2 = m_db.Connection.CreateCommand()) { @@ -377,15 +394,17 @@ namespace Duplicati.Library.Main.Database VALUES (@Path) "); - await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) + await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct().ConfigureAwait(false)) await c2 .SetParameterValue("@Path", n) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); await c2.ExecuteNonQueryAsync($@" CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"") - "); + ") + .ConfigureAwait(false); } //Then we select the matching results @@ -438,8 +457,11 @@ namespace Duplicati.Library.Main.Database AND ""C"".""Path"" = ""D"".""Path"" "; - using var rd = await cmd.ExecuteReaderAsync(query); - if (await rd.ReadAsync()) + using var rd = await cmd + .ExecuteReaderAsync(query) + .ConfigureAwait(false); + + if (await rd.ReadAsync().ConfigureAwait(false)) { bool more; do @@ -452,7 +474,8 @@ namespace Duplicati.Library.Main.Database } else { - more = rd.Read(); + more = await rd.ReadAsync() + .ConfigureAwait(false); } } while (more); @@ -464,7 +487,8 @@ namespace Duplicati.Library.Main.Database try { using (var c = m_db.Connection.CreateCommand()) - await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}"""); + await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}""") + .ConfigureAwait(false); } catch { @@ -474,7 +498,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable SelectFiles(IFilter filter) { - using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter)) + using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false)) using (var cmd = m_db.Connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets @@ -492,7 +516,8 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tablename}"" ) ) - "); + ") + .ConfigureAwait(false); //Then we select the matching results var filesets = $@" @@ -544,8 +569,9 @@ namespace Duplicati.Library.Main.Database AND ""C"".""Path"" = ""D"".""Path"" "; - using var rd = await cmd.ExecuteReaderAsync(query); - if (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync(query) + .ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) { bool more; do @@ -569,7 +595,8 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC LIMIT 1 ) - "); + ") + .ConfigureAwait(false); } public async IAsyncEnumerable QuickSets() @@ -585,14 +612,20 @@ namespace Duplicati.Library.Main.Database ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC - ")) - while (await rd.ReadAsync()) + ").ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) { var id = rd.ConvertValueToInt64(0); var backupType = rd.GetInt32(1); var e = dict[id]; - yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); + yield return new Fileset( + e, + backupType, + m_filesets[e].Value, + -1L, + -1L + ); } } @@ -631,9 +664,10 @@ namespace Duplicati.Library.Main.Database LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC - "); + ") + .ConfigureAwait(false); - while (await rd.ReadAsync()) + while (await rd.ReadAsync().ConfigureAwait(false)) { var id = rd.ConvertValueToInt64(0); var isFullBackup = rd.GetInt32(1); @@ -641,7 +675,13 @@ namespace Duplicati.Library.Main.Database var filecount = rd.ConvertValueToInt64(2, -1L); var filesizes = rd.ConvertValueToInt64(3, -1L); - yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); + yield return new Fileset( + e, + isFullBackup, + m_filesets[e].Value, + filecount, + filesizes + ); } } @@ -657,7 +697,8 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_db.Connection.CreateCommand()) - await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" "); + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ") + .ConfigureAwait(false); } catch { } finally { m_tablename = null!; } @@ -667,7 +708,8 @@ namespace Duplicati.Library.Main.Database public async Task SelectFileSets(DateTime time, long[] versions) { - return await FileSets.CreateAsync(this, time, versions); + return await FileSets.CreateAsync(this, time, versions) + .ConfigureAwait(false); } /// @@ -705,13 +747,14 @@ namespace Duplicati.Library.Main.Database "; using (var cmd = m_connection.CreateCommand(query)) - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var id = rd.ConvertValueToInt64(0); var time = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); var isFullBackup = rd.GetInt32(2) == BackupType.FULL_BACKUP; var filecount = rd.ConvertValueToInt64(3, -1L); var filesizes = rd.ConvertValueToInt64(4, -1L); + yield return new FilesetEntry( id, isFullBackup, @@ -792,7 +835,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@offset", offset); var results = new List(); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); @@ -832,9 +875,10 @@ namespace Duplicati.Library.Main.Database fe.FilesetID = @filesetid AND fl.PrefixID IN (@PrefixIds) ") - .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) - .SetParameterValue("@filesetid", filesetid) - .ExecuteScalarInt64Async(0); + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) + .SetParameterValue("@filesetid", filesetid) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); } // Return paginated results @@ -901,7 +945,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@offset", offset); var results = new List(); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); @@ -936,7 +980,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", filesetid) .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; @@ -1013,12 +1058,15 @@ namespace Duplicati.Library.Main.Database { using var cmd = m_connection.CreateCommand(); - var dirSeparator = Util.GuessDirSeparator((await cmd.SetCommandAndParameters(@" - SELECT ""Prefix"" - FROM ""PathPrefix"" - LIMIT 1 - ") - .ExecuteScalarAsync())?.ToString()); + var dirSeparator = Util.GuessDirSeparator(( + await cmd.SetCommandAndParameters(@" + SELECT ""Prefix"" + FROM ""PathPrefix"" + LIMIT 1 + ") + .ExecuteScalarAsync() + .ConfigureAwait(false) + )?.ToString()); cmd.SetCommandAndParameters(@" WITH Prefixes AS ( @@ -1095,9 +1143,11 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC "); - var versionMap = await cmd.ExecuteReaderEnumerableAsync() + var versionMap = await cmd + .ExecuteReaderEnumerableAsync() .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) - .ToDictionaryAsync(x => x.FilesetId, x => x.Version); + .ToDictionaryAsync(x => x.FilesetId, x => x.Version) + .ConfigureAwait(false); // Then fetch the actual data var fetchSql = @$" @@ -1143,7 +1193,7 @@ namespace Duplicati.Library.Main.Database cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); var results = new List(); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); @@ -1191,7 +1241,8 @@ namespace Duplicati.Library.Main.Database if (filesetIds != null && filesetIds.Length > 0) cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); - totalCount = await cmd.ExecuteScalarInt64Async(0); + totalCount = await cmd.ExecuteScalarInt64Async(0) + .ConfigureAwait(false); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; @@ -1348,9 +1399,11 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC "); - var versionMap = await cmd.ExecuteReaderEnumerableAsync() + var versionMap = await cmd + .ExecuteReaderEnumerableAsync() .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) - .ToDictionaryAsync(x => x.FilesetId, x => x.Version); + .ToDictionaryAsync(x => x.FilesetId, x => x.Version) + .ConfigureAwait(false); // Fetch results var fetchSql = $@" @@ -1403,7 +1456,7 @@ namespace Duplicati.Library.Main.Database cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); var results = new List(); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); @@ -1452,7 +1505,8 @@ namespace Duplicati.Library.Main.Database if (filesetIds != null && filesetIds.Length > 0) cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); - totalCount = await cmd.ExecuteScalarInt64Async(0); + totalCount = await cmd.ExecuteScalarInt64Async(0) + .ConfigureAwait(false); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index f7426f897..fd9049484 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -35,7 +35,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalPurgeDatabase(); - dbnew = (LocalPurgeDatabase)await LocalDeleteDatabase.CreateAsync(path, "Purge", pagecachesize, dbnew); + dbnew = (LocalPurgeDatabase) + await LocalDeleteDatabase.CreateAsync(path, "Purge", pagecachesize, dbnew) + .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -46,14 +48,17 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalPurgeDatabase(); - dbnew = (LocalPurgeDatabase)await LocalDeleteDatabase.CreateAsync(dbparent, dbnew); + dbnew = (LocalPurgeDatabase) + await LocalDeleteDatabase.CreateAsync(dbparent, dbnew) + .ConfigureAwait(false); return dbnew; } public async Task CreateTemporaryFileset(long parentid) { - return await TemporaryFileset.CreateAsync(parentid, this); + return await TemporaryFileset.CreateAsync(parentid, this) + .ConfigureAwait(false); } public async Task GetRemoteVolumeNameForFileset(long id) @@ -70,8 +75,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", id); - using (var rd = await cmd.ExecuteReaderAsync()) - if (!await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + if (!await rd.ReadAsync().ConfigureAwait(false)) throw new Exception($"No remote volume found for fileset with id {id}"); else return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); @@ -88,8 +93,8 @@ namespace Duplicati.Library.Main.Database FROM ""FilesetEntry"" ) "); - using var rd = await cmd.ExecuteReaderAsync(); - if (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) return rd.ConvertValueToInt64(0, 0); else return 0; @@ -140,7 +145,8 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{tempf.m_tablename}"" ( ""FileID"" INTEGER PRIMARY KEY ) - "); + ") + .ConfigureAwait(false); return tempf; } @@ -149,9 +155,10 @@ namespace Duplicati.Library.Main.Database { int updated; using (var cmd = m_db.Connection.CreateCommand()) - updated = await filtercommand(cmd, ParentID, m_tablename); + updated = await filtercommand(cmd, ParentID, m_tablename) + .ConfigureAwait(false); - await PostFilterChecks(updated); + await PostFilterChecks(updated).ConfigureAwait(false); } public async Task ApplyFilter(Library.Utility.IFilter filter) @@ -169,17 +176,20 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{filenamestable}"" ( ""Path"" TEXT NOT NULL ) - "); + ") + .ConfigureAwait(false); await cmd.SetCommandAndParameters($@" INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path) ") - .PrepareAsync(); + .PrepareAsync() + .ConfigureAwait(false); foreach (var s in p) await cmd.SetParameterValue("@Path", s) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); await cmd.SetCommandAndParameters($@" INSERT INTO ""{m_tablename}"" (""FileID"") @@ -193,9 +203,12 @@ namespace Duplicati.Library.Main.Database AND ""B"".""Path"" IN ""{filenamestable}"" ") .SetParameterValue("@FilesetId", ParentID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" "); + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ") + .ConfigureAwait(false); } } else @@ -209,7 +222,8 @@ namespace Duplicati.Library.Main.Database INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId) ") - .PrepareAsync(); + .PrepareAsync() + .ConfigureAwait(false); cmd.SetCommandAndParameters(@" SELECT @@ -224,8 +238,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", ParentID); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) { rd.GetValues(values); var path = values[0] as string; @@ -233,13 +247,14 @@ namespace Duplicati.Library.Main.Database { await cmd2 .SetParameterValue("@FileId", values[1]) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } } } - await PostFilterChecks(0); + await PostFilterChecks(0).ConfigureAwait(false); } private async Task PostFilterChecks(int updated) @@ -251,7 +266,8 @@ namespace Duplicati.Library.Main.Database RemovedFileCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{m_tablename}"" - ", 0); + ", 0) + .ConfigureAwait(false); RemovedFileSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""C"".""Length"") @@ -269,13 +285,15 @@ namespace Duplicati.Library.Main.Database AND ""D"".""BlocksetID"" = ""C"".""ID"" ) ) - ", 0); + ", 0) + .ConfigureAwait(false); var filesetcount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID} - ", 0); + ", 0) + .ConfigureAwait(false); if (filesetcount == RemovedFileCount) throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); @@ -284,9 +302,15 @@ namespace Duplicati.Library.Main.Database public async Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup) { - var remotevolid = await m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary); - var filesetid = await m_db.CreateFileset(remotevolid, timestamp); - await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup); + var remotevolid = + await m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary) + .ConfigureAwait(false); + + var filesetid = await m_db.CreateFileset(remotevolid, timestamp) + .ConfigureAwait(false); + + await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup) + .ConfigureAwait(false); using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) await cmd.SetCommandAndParameters($@" @@ -306,7 +330,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@TargetFilesetId", filesetid) .SetParameterValue("@SourceFilesetId", ParentID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); return new Tuple(remotevolid, filesetid); } @@ -325,9 +350,12 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" - ")) - while (await rd.ReadAsync()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + ").ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1) + ); } public void Dispose() @@ -340,7 +368,9 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_db.Connection.CreateCommand()) - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}"""); + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + .ConfigureAwait(false); } catch { } } diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 1d1a58459..9814ae6af 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -124,7 +124,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalRecreateDatabase(); - dbnew = (LocalRecreateDatabase)await LocalRestoreDatabase.CreateAsync(parentdb, dbnew); + dbnew = (LocalRecreateDatabase) + await LocalRestoreDatabase.CreateAsync(parentdb, dbnew) + .ConfigureAwait(false); dbnew.m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); dbnew.m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -139,12 +141,14 @@ namespace Duplicati.Library.Main.Database ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" ON ""{dbnew.m_tempblocklist}"" (""BlockListHash""); - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" ( @@ -152,7 +156,8 @@ namespace Duplicati.Library.Main.Database ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" @@ -160,7 +165,8 @@ namespace Duplicati.Library.Main.Database ""FileHash"", ""BlockSize"" ); - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" @@ -168,7 +174,8 @@ namespace Duplicati.Library.Main.Database ""BlockHash"", ""BlockSize"" ); - "); + ") + .ConfigureAwait(false); dbnew.m_insertFileCommand = await dbnew.m_connection.CreateCommandAsync(@" INSERT INTO ""FileLookup"" ( @@ -184,7 +191,8 @@ namespace Duplicati.Library.Main.Database @MetadataId ); SELECT last_insert_rowid(); - "); + ") + .ConfigureAwait(false); dbnew.m_insertFilesetEntryCommand = await dbnew.m_connection.CreateCommandAsync(@" INSERT INTO ""FilesetEntry"" ( @@ -197,13 +205,15 @@ namespace Duplicati.Library.Main.Database @FileId, @LastModified ) - "); + ") + .ConfigureAwait(false); dbnew.m_insertMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid(); - "); + ") + .ConfigureAwait(false); dbnew.m_insertBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@" INSERT INTO ""Blockset"" ( @@ -215,7 +225,8 @@ namespace Duplicati.Library.Main.Database @FullHash ); SELECT last_insert_rowid(); - "); + ") + .ConfigureAwait(false); dbnew.m_insertBlocklistHashCommand = await dbnew.m_connection.CreateCommandAsync(@" INSERT INTO ""BlocklistHash"" ( @@ -228,7 +239,8 @@ namespace Duplicati.Library.Main.Database @Index, @Hash ) - "); + ") + .ConfigureAwait(false); dbnew.m_updateBlockVolumeCommand = await dbnew.m_connection.CreateCommandAsync(@" UPDATE ""Block"" @@ -236,7 +248,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - "); + ") + .ConfigureAwait(false); dbnew.m_insertTempBlockListHash = await dbnew.m_connection.CreateCommandAsync($@" INSERT INTO ""{dbnew.m_tempblocklist}"" ( @@ -249,7 +262,8 @@ namespace Duplicati.Library.Main.Database @BlockHash, @Index ) - "); + ") + .ConfigureAwait(false); dbnew.m_insertSmallBlockset = await dbnew.m_connection.CreateCommandAsync($@" INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" ( @@ -262,7 +276,8 @@ namespace Duplicati.Library.Main.Database @BlockHash, @BlockSize ) - "); + ") + .ConfigureAwait(false); dbnew.m_findBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@" SELECT ""ID"" @@ -270,7 +285,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash - "); + ") + .ConfigureAwait(false); dbnew.m_findMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" SELECT ""Metadataset"".""ID"" @@ -281,7 +297,8 @@ namespace Duplicati.Library.Main.Database ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length - "); + ") + .ConfigureAwait(false); dbnew.m_findFilesetCommand = await dbnew.m_connection.CreateCommandAsync(@" SELECT ""ID"" @@ -291,13 +308,15 @@ namespace Duplicati.Library.Main.Database AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId - "); + ") + .ConfigureAwait(false); dbnew.m_findTempBlockListHashCommand = await dbnew.m_connection.CreateCommandAsync($@" SELECT DISTINCT ""BlockListHash"" FROM ""{dbnew.m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash - "); + ") + .ConfigureAwait(false); dbnew.m_findHashBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" SELECT ""VolumeID"" @@ -305,7 +324,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - "); + ") + .ConfigureAwait(false); dbnew.m_insertBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" INSERT INTO ""Block"" ( @@ -318,7 +338,8 @@ namespace Duplicati.Library.Main.Database @Size, @VolumeId ) - "); + ") + .ConfigureAwait(false); dbnew.m_insertDuplicateBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" INSERT OR IGNORE INTO ""DuplicateBlock"" ( @@ -335,7 +356,8 @@ namespace Duplicati.Library.Main.Database ), @VolumeId ) - "); + ") + .ConfigureAwait(false); return dbnew; } @@ -410,7 +432,8 @@ namespace Duplicati.Library.Main.Database "; // Insert all known blocks into block table with volumeid = -1 - await cmd.ExecuteNonQueryAsync(insertBlocksCommand); + await cmd.ExecuteNonQueryAsync(insertBlocksCommand) + .ConfigureAwait(false); // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati // this could be used instead @@ -488,7 +511,8 @@ namespace Duplicati.Library.Main.Database try { - await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand); + await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand) + .ConfigureAwait(false); } catch (Exception ex) { @@ -500,19 +524,21 @@ namespace Duplicati.Library.Main.Database CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" - "); + ") + .ConfigureAwait(false); await fixcmd.ExecuteNonQueryAsync($@" CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" - "); + ") + .ConfigureAwait(false); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } /// @@ -630,9 +656,12 @@ namespace Duplicati.Library.Main.Database try { // Insert discovered new blocks into block table with volumeid = -1 - await cmd.ExecuteNonQueryAsync(insertBlocksCommand); + await cmd.ExecuteNonQueryAsync(insertBlocksCommand) + .ConfigureAwait(false); + // Insert corresponding entries into blockset - await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand); + await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand) + .ConfigureAwait(false); } catch (Exception ex) { @@ -644,13 +673,15 @@ namespace Duplicati.Library.Main.Database CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" - "); + ") + .ConfigureAwait(false); await fixcmd.ExecuteNonQueryAsync($@" CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" - "); + ") + .ConfigureAwait(false); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -659,17 +690,20 @@ namespace Duplicati.Library.Main.Database public async Task AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid) { - await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid); + await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid) + .ConfigureAwait(false); } public async Task AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid) { - await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid); + await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid) + .ConfigureAwait(false); } public async Task AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid) { - await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid); + await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid) + .ConfigureAwait(false); } private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid) @@ -680,7 +714,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@MetadataId", metadataid) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (fileid < 0) { @@ -690,7 +725,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@MetadataId", metadataid) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); } await m_insertFilesetEntryCommand @@ -698,7 +734,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", filesetid) .SetParameterValue("@FileId", fileid) .SetParameterValue("@LastModified", time.ToUniversalTime().Ticks) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes) @@ -711,17 +748,21 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FullHash", metahash) .SetParameterValue("@Length", metahashsize) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (metadataid != -1) return metadataid; - var blocksetid = await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes); + var blocksetid = + await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes) + .ConfigureAwait(false); metadataid = await m_insertMetadatasetCommand .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); return metadataid; } @@ -732,7 +773,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@FullHash", fullhash) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (blocksetid != -1) return blocksetid; @@ -741,7 +783,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@FullHash", fullhash) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); long c = 0; if (blocklisthashes != null) @@ -761,7 +804,8 @@ namespace Duplicati.Library.Main.Database await m_insertBlocklistHashCommand .SetParameterValue("@Index", index++) .SetParameterValue("@Hash", hash) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } } @@ -780,7 +824,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(-2); + .ExecuteScalarInt64Async(-2) + .ConfigureAwait(false); if (currentVolumeId == volumeID) return (anyChange, false); @@ -795,7 +840,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@VolumeId", volumeID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); return (anyChange, true); } @@ -807,7 +853,9 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@VolumeId", volumeID) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + if (c != 1) throw new Exception($"Failed to update table, found {c} entries for key {hash} with size {size}"); @@ -820,7 +868,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@VolumeId", volumeID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); return (anyChange, false); } @@ -833,7 +882,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FileHash", filehash) .SetParameterValue("@BlockHash", blockhash) .SetParameterValue("@BlockSize", blocksize) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes) @@ -841,7 +891,8 @@ namespace Duplicati.Library.Main.Database var r = await m_findTempBlockListHashCommand .SetTransaction(m_rtr) .SetParameterValue("@BlocklistHash", hash) - .ExecuteScalarAsync(); + .ExecuteScalarAsync() + .ConfigureAwait(false); if (r != null && r != DBNull.Value) return false; @@ -857,7 +908,8 @@ namespace Duplicati.Library.Main.Database await m_insertTempBlockListHash .SetParameterValue("@BlockHash", s) .SetParameterValue("@Index", index++) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } return true; @@ -878,8 +930,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); - using var rd = await cmd.ExecuteReaderAsync(); - while (rd.Read()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? ""; } @@ -949,7 +1001,8 @@ namespace Duplicati.Library.Main.Database //On anything but the first pass, we check if we are done var r = await cmd .SetCommandAndParameters(countMissingInformation) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); if (r == 0 && !forceBlockUse) yield break; @@ -987,8 +1040,11 @@ namespace Duplicati.Library.Main.Database } } - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + using var rd = await cmd + .ExecuteReaderAsync() + .ConfigureAwait(false); + + while (await rd.ReadAsync().ConfigureAwait(false)) { var volumeID = rd.ConvertValueToInt64(3); @@ -1131,13 +1187,19 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand(m_rtr)) { - var cnt = await cmd.ExecuteScalarInt64Async(countsql); + var cnt = await cmd.ExecuteScalarInt64Async(countsql) + .ConfigureAwait(false); + if (cnt > 0) { try { - await cmd.ExecuteNonQueryAsync(sql); - var cnt2 = await cmd.ExecuteScalarInt64Async(countsql); + await cmd.ExecuteNonQueryAsync(sql) + .ConfigureAwait(false); + + var cnt2 = await cmd.ExecuteScalarInt64Async(countsql) + .ConfigureAwait(false); + Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); } catch (Exception ex) @@ -1179,7 +1241,8 @@ namespace Duplicati.Library.Main.Database SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" ) - "); + ") + .ConfigureAwait(false); // 2. Insert blocks into DeletedBlock table await cmd.ExecuteNonQueryAsync($@" @@ -1193,7 +1256,8 @@ namespace Duplicati.Library.Main.Database ""Size"", ""VolumeID"" FROM ""{tmptablename}"" - "); + ") + .ConfigureAwait(false); // 3. Remove blocks from Block table await cmd.ExecuteNonQueryAsync($@" @@ -1202,10 +1266,14 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""{tmptablename}"" ) - "); + ") + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"""); - await m_rtr.CommitAsync(); + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}""") + .ConfigureAwait(false); + + await m_rtr.CommitAsync().ConfigureAwait(false); } public override void Dispose() @@ -1221,7 +1289,8 @@ namespace Duplicati.Library.Main.Database if (m_tempblocklist != null) try { - await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}"""); + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""") + .ConfigureAwait(false); } catch (Exception ex) { @@ -1235,7 +1304,8 @@ namespace Duplicati.Library.Main.Database if (m_tempsmalllist != null) try { - await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}"""); + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""") + .ConfigureAwait(false); } catch (Exception ex) { @@ -1248,7 +1318,7 @@ namespace Duplicati.Library.Main.Database } - await base.DisposeAsync(); + await base.DisposeAsync().ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 42d41e9c6..331420983 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -45,7 +45,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalRepairDatabase(); - dbnew = (LocalRepairDatabase)await CreateLocalDatabaseAsync(path, "Repair", true, pagecachesize, dbnew); + dbnew = (LocalRepairDatabase) + await CreateLocalDatabaseAsync(path, "Repair", true, pagecachesize, dbnew) + .ConfigureAwait(false); return dbnew; } @@ -58,7 +60,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalRepairDatabase(); - return (LocalRepairDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + return (LocalRepairDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew) + .ConfigureAwait(false); } @@ -84,9 +88,10 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr) .SetParameterValue("@Name", filelist); - var rd = await cmd.ExecuteReaderAsync(); - if (!await rd.ReadAsync()) + var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + + if (!await rd.ReadAsync().ConfigureAwait(false)) throw new Exception($"No such remote file: {filelist}"); return ( @@ -119,7 +124,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@CurrentFilesetId", filesetid) .SetParameterValue("@PreviousFilesetId", prevFilesetId); - await cmd.ExecuteNonQueryAsync(); + await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } /// @@ -143,7 +148,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@BlockFileId", blockfileid) .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); } @@ -170,8 +175,8 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) { yield return new KeyValuePair( rd.ConvertValueToInt64(0), @@ -195,7 +200,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetid); - return await cmd.ExecuteNonQueryAsync(); + return await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } private class RemoteVolume : IRemoteVolume @@ -233,7 +238,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Name", indexName); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", @@ -389,7 +394,8 @@ namespace Duplicati.Library.Main.Database ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); cmd.SetCommandAndParameters($@" INSERT INTO ""{blocklist.m_tablename}"" ( @@ -407,7 +413,9 @@ namespace Duplicati.Library.Main.Database AND ""Remotevolume"".""Name"" = @Name ") .SetParameterValue("@Name", volumename); - var blockCount = await cmd.ExecuteNonQueryAsync(); + + var blockCount = await cmd.ExecuteNonQueryAsync() + .ConfigureAwait(false); if (blockCount == 0) throw new Exception($"Unexpected empty block volume: {0}"); @@ -419,7 +427,8 @@ namespace Duplicati.Library.Main.Database ""Size"", ""Restored"" ) - "); + ") + .ConfigureAwait(false); } blocklist.m_insertCommand = await connection.CreateCommandAsync($@" @@ -429,7 +438,8 @@ namespace Duplicati.Library.Main.Database ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue - "); + ") + .ConfigureAwait(false); blocklist.m_copyIntoDuplicatedBlocks = await connection.CreateCommandAsync(@" INSERT OR IGNORE INTO ""DuplicateBlock"" ( @@ -443,7 +453,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Block"".""Hash"" = @Hash AND ""Block"".""Size"" = @Size - "); + ") + .ConfigureAwait(false); blocklist.m_assignBlocksToNewVolume = await connection.CreateCommandAsync(@" UPDATE ""Block"" @@ -451,7 +462,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - "); + ") + .ConfigureAwait(false); var m_missingBlocksQuery = $@" SELECT @@ -459,12 +471,17 @@ namespace Duplicati.Library.Main.Database ""{blocklist.m_tablename}"".""Size"" FROM ""{blocklist.m_tablename}"" WHERE ""{blocklist.m_tablename}"".""Restored"" = @Restored "; - blocklist.m_missingBlocksCommand = await connection.CreateCommandAsync(m_missingBlocksQuery); - blocklist.m_missingBlocksCountCommand = await connection.CreateCommandAsync($@" - SELECT COUNT(*) - FROM ({m_missingBlocksQuery}) - "); + blocklist.m_missingBlocksCommand = + await connection.CreateCommandAsync(m_missingBlocksQuery) + .ConfigureAwait(false); + + blocklist.m_missingBlocksCountCommand = + await connection.CreateCommandAsync($@" + SELECT COUNT(*) + FROM ({m_missingBlocksQuery}) + ") + .ConfigureAwait(false); return blocklist; } @@ -472,25 +489,29 @@ namespace Duplicati.Library.Main.Database /// public async Task SetBlockRestored(string hash, long size, long targetVolumeId) { - var restored = await m_insertCommand.SetTransaction(m_rtr) + var restored = await m_insertCommand + .SetTransaction(m_rtr) .SetParameterValue("@NewRestoredValue", 1) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@PreviousRestoredValue", 0) - .ExecuteNonQueryAsync() == 1; + .ExecuteNonQueryAsync() + .ConfigureAwait(false) == 1; if (restored) { await m_copyIntoDuplicatedBlocks.SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); var c = await m_assignBlocksToNewVolume.SetTransaction(m_rtr) .SetParameterValue("@TargetVolumeId", targetVolumeId) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != 1) throw new Exception($"Unexpected number of updated blocks: {c} != 1"); @@ -528,7 +549,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); var size = rd.ConvertValueToInt64(1); @@ -568,7 +589,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); var size = rd.ConvertValueToInt64(1); @@ -621,7 +642,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@HashesPerBlock", hashesPerBlock) .SetParameterValue("@Restored", 0) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); cmd.SetCommandAndParameters($@" SELECT @@ -638,7 +660,7 @@ namespace Duplicati.Library.Main.Database ""BlocksetEntryIndex"" "); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); var blocksetId = rd.ConvertValueToInt64(1); @@ -663,7 +685,8 @@ namespace Duplicati.Library.Main.Database { await cmd.ExecuteNonQueryAsync($@" DROP TABLE IF EXISTS ""{blocklistTableName}"" - "); + ") + .ConfigureAwait(false); } catch { } } @@ -675,7 +698,8 @@ namespace Duplicati.Library.Main.Database return await m_missingBlocksCountCommand .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); } /// @@ -688,8 +712,11 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); - await foreach (var rd in m_missingBlocksCommand.ExecuteReaderEnumerableAsync()) - yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + await foreach (var rd in m_missingBlocksCommand.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + yield return ( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1) + ); } /// @@ -721,7 +748,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@SourceVolumeId", sourceVolumeId) .SetParameterValue("@Restored", 1); - var moved = await cmd.ExecuteNonQueryAsync(); + var moved = await cmd.ExecuteNonQueryAsync() + .ConfigureAwait(false); // Then update the blocks table to point to the new volume var updated = await cmd.SetCommandAndParameters($@" @@ -740,7 +768,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TargetVolumeId", targetVolumeId) .SetParameterValue("@SourceVolumeId", sourceVolumeId) .SetParameterValue("@Restored", 1) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (updated != moved) throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); @@ -805,7 +834,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", @@ -835,7 +864,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Restored", 0) .SetParameterValue("@Name", m_volumename); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", @@ -862,9 +891,12 @@ namespace Duplicati.Library.Main.Database if (m_tablename != null) { - using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}"""); + using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + .ConfigureAwait(false); + await cmd.SetTransaction(m_rtr) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } catch { } @@ -876,7 +908,8 @@ namespace Duplicati.Library.Main.Database public async Task CreateBlockList(string volumename) { - return await MissingBlockList.CreateMissingBlockList(volumename, m_connection, m_rtr); + return await MissingBlockList.CreateMissingBlockList(volumename, m_connection, m_rtr) + .ConfigureAwait(false); } public async Task FixDuplicateMetahash() @@ -899,7 +932,8 @@ namespace Duplicati.Library.Main.Database ) "; - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + .ConfigureAwait(false); if (x > 1) { @@ -910,7 +944,8 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File"" - "); + ") + .ConfigureAwait(false); var sql = @" SELECT @@ -945,13 +980,16 @@ namespace Duplicati.Library.Main.Database AND ""ID"" != @MetadataId "); - using var rd = await cmd.ExecuteReaderAsync(sql); + using var rd = await cmd.ExecuteReaderAsync(sql) + .ConfigureAwait(false); - while (await rd.ReadAsync()) + while (await rd.ReadAsync().ConfigureAwait(false)) { - await c2.SetParameterValue("@MetadataId", rd.GetValue(0)) + await c2 + .SetParameterValue("@MetadataId", rd.GetValue(0)) .SetParameterValue("@BlocksetId", rd.GetValue(1)) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } @@ -1000,13 +1038,15 @@ namespace Duplicati.Library.Main.Database AND ""ID"" != @FileId "); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql).ConfigureAwait(false)) { - await c2.SetParameterValue("@FileId", rd.GetValue(0)) + await c2 + .SetParameterValue("@FileId", rd.GetValue(0)) .SetParameterValue("@Path", rd.GetValue(1)) .SetParameterValue("@BlocksetId", rd.GetValue(2)) .SetParameterValue("@MetadataId", rd.GetValue(3)) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } @@ -1016,7 +1056,8 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""{tablename}"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{tablename}-Ix"" @@ -1024,7 +1065,8 @@ namespace Duplicati.Library.Main.Database ""ID"", ""MetadataID"" ) - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" UPDATE ""FileLookup"" @@ -1033,17 +1075,21 @@ namespace Duplicati.Library.Main.Database FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"" ) - "); + ") + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{tablename}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{tablename}"" ") + .ConfigureAwait(false); + + x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + .ConfigureAwait(false); - x = await cmd.ExecuteScalarInt64Async(sql_count, 0); if (x > 1) throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } } @@ -1070,7 +1116,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Duplicates"" > 1 "; - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + .ConfigureAwait(false); if (x > 0) { @@ -1128,25 +1175,28 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(sql); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { - await c2.SetParameterValue("@FileId", rd.GetValue(0)) + await c2 + .SetParameterValue("@FileId", rd.GetValue(0)) .SetParameterValue("@PrefixId", rd.GetValue(1)) .SetParameterValue("@Path", rd.GetValue(2)) .SetParameterValue("@BlocksetId", rd.GetValue(3)) .SetParameterValue("@MetadataId", rd.GetValue(4)) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } - x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + .ConfigureAwait(false); if (x > 1) throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } } @@ -1196,7 +1246,9 @@ namespace Duplicati.Library.Main.Database FROM ({sql}) "; - var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); + var itemswithnoblocklisthash = await cmd + .ExecuteScalarInt64Async(countsql, 0) + .ConfigureAwait(false); if (itemswithnoblocklisthash != 0) { @@ -1265,7 +1317,7 @@ namespace Duplicati.Library.Main.Database WHERE ""ID"" = @DeletedBlockId; "); - await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql)) + await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql).ConfigureAwait(false)) { var blocksetid = e.ConvertValueToInt64(0); var ix = 0L; @@ -1276,7 +1328,8 @@ namespace Duplicati.Library.Main.Database WHERE ""BlocksetID"" = @BlocksetId ") .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); c2.SetCommandAndParameters(@" SELECT ""A"".""Hash"" @@ -1290,7 +1343,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@BlocksetId", blocksetid); - await foreach (var h in c2.ExecuteReaderEnumerableAsync()) + await foreach (var h in c2.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); if (blocklistbuffer.Length - blocklistoffset < tmp.Length) @@ -1301,7 +1354,8 @@ namespace Duplicati.Library.Main.Database var existingBlockId = await c4 .SetParameterValue("@Hash", blkey) .SetParameterValue("@Size", blocklistoffset) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (existingBlockId <= 0) { @@ -1311,7 +1365,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (deletedBlockId <= 0) throw new Exception($"Missing block for blocklisthash: {blkey}"); @@ -1319,7 +1374,8 @@ namespace Duplicati.Library.Main.Database { var rc = await c6 .SetParameterValue("@DeletedBlockId", deletedBlockId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (rc != 2) throw new Exception($"Unexpected update count: {rc}"); @@ -1330,7 +1386,8 @@ namespace Duplicati.Library.Main.Database await c3.SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", blkey) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); ix++; blocklistoffset = 0; @@ -1349,7 +1406,8 @@ namespace Duplicati.Library.Main.Database var existingBlockId = await c4 .SetParameterValue("@Hash", blkeyfinal) .SetParameterValue("@Size", blocklistoffset) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (existingBlockId <= 0) { @@ -1359,7 +1417,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (deletedBlockId == 0) throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); @@ -1367,7 +1426,8 @@ namespace Duplicati.Library.Main.Database { var rc = await c6 .SetParameterValue("@DeletedBlockId", deletedBlockId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (rc != 2) throw new Exception($"Unexpected update count: {rc}"); } @@ -1378,19 +1438,22 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", blkeyfinal) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } } - itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async(countsql, 0); + itemswithnoblocklisthash = await cmd + .ExecuteScalarInt64Async(countsql, 0) + .ConfigureAwait(false); if (itemswithnoblocklisthash != 0) throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } } @@ -1419,7 +1482,8 @@ namespace Duplicati.Library.Main.Database FROM ({dup_sql}) "; - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0); + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + .ConfigureAwait(false); if (x > 0) { @@ -1433,7 +1497,8 @@ namespace Duplicati.Library.Main.Database ""Index"" FROM ""BlocklistHash"" ) - ", 0); + ", 0) + .ConfigureAwait(false); using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) { @@ -1449,21 +1514,23 @@ namespace Duplicati.Library.Main.Database ) "); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql).ConfigureAwait(false)) { var expected = rd.GetInt32(2) - 1; var actual = await c2 .SetParameterValue("@BlocksetId", rd.GetValue(0)) .SetParameterValue("@Index", rd.GetValue(1)) .SetParameterValue("@Limit", expected) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (actual != expected) throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); } } - x = await cmd.ExecuteScalarInt64Async(sql_count); + x = await cmd.ExecuteScalarInt64Async(sql_count) + .ConfigureAwait(false); if (x > 1) throw new Exception("Repair failed, there are still duplicate file entries!"); @@ -1471,14 +1538,16 @@ namespace Duplicati.Library.Main.Database var real_count = await cmd.ExecuteScalarInt64Async(@" SELECT Count(*) FROM ""BlocklistHash"" - ", 0); + ", 0) + .ConfigureAwait(false); if (real_count != unique_count) throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); try { - await VerifyConsistency(blocksize, hashsize, true); + await VerifyConsistency(blocksize, hashsize, true) + .ConfigureAwait(false); } catch (Exception ex) { @@ -1487,7 +1556,7 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } } @@ -1503,7 +1572,8 @@ namespace Duplicati.Library.Main.Database ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); cmd.SetCommandAndParameters($@" INSERT INTO ""{tablename}"" ( @@ -1521,7 +1591,8 @@ namespace Duplicati.Library.Main.Database await cmd .SetParameterValue("@Hash", kp.Key) .SetParameterValue("@Size", kp.Value) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } var id = await cmd.SetCommandAndParameters(@" @@ -1530,7 +1601,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name ") .SetParameterValue("@Name", filename) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); var aliens = await cmd.SetCommandAndParameters($@" SELECT COUNT(*) @@ -1544,14 +1616,16 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" != @VolumeId ") .SetParameterValue("@VolumeId", id) - .ExecuteScalarInt64Async(0); + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); if (aliens != 0) throw new Exception($"Not all blocks were found in {filename}"); } finally { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ") + .ConfigureAwait(false); } } @@ -1596,7 +1670,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", length); - await foreach (var r in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var r in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { if (!en.MoveNext()) throw new Exception($"Too few entries in source blocklist with hash {hash}"); @@ -1628,7 +1702,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Deleted.ToString() ]); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? ""; } @@ -1655,7 +1729,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Deleted.ToString() ]); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) yield return ( rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), @@ -1686,7 +1760,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Verified.ToString() ]); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 69f6c1592..7fc94c784 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -63,7 +63,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalRestoreDatabase(); - dbnew = (LocalRestoreDatabase)await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew); + dbnew = (LocalRestoreDatabase) + await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew) + .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; @@ -73,7 +75,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalRestoreDatabase(); - return (LocalRestoreDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + return (LocalRestoreDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew) + .ConfigureAwait(false); } /// @@ -101,7 +105,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr); // How to handle METADATA? - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( ""FileId"" INTEGER PRIMARY KEY, @@ -110,9 +116,12 @@ namespace Duplicati.Library.Main.Database ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ") + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" "); await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( ""TotalFiles"" INTEGER NOT NULL, @@ -123,16 +132,20 @@ namespace Duplicati.Library.Main.Database ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); if (createFilesNewlyDoneTracker) { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( ""ID"" INTEGER PRIMARY KEY ) - "); + ") + .ConfigureAwait(false); } try @@ -181,7 +194,8 @@ namespace Duplicati.Library.Main.Database ON ""B"".""FileID"" = ""F"".""ID"" WHERE ""B"".""Metadata"" IS NOT 1 GROUP BY ""F"".""ID"" - "); + ") + .ConfigureAwait(false); // Will result in a single line (no support to also track metadata) await cmd.ExecuteNonQueryAsync($@" @@ -223,7 +237,8 @@ namespace Duplicati.Library.Main.Database IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) FROM ""{m_fileprogtable}"" ""P"" - "); + ") + .ConfigureAwait(false); // Finally we create TRIGGERs to keep all our statistics up to date. // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic. @@ -248,7 +263,8 @@ namespace Duplicati.Library.Main.Database + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" ; END - "); + ") + .ConfigureAwait(false); // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) await cmd.ExecuteNonQueryAsync($@" @@ -293,8 +309,8 @@ namespace Duplicati.Library.Main.Database + NEW.""SizeRestored"" - OLD.""SizeRestored"" ; END - "); - + ") + .ConfigureAwait(false); if (createFilesNewlyDoneTracker) { @@ -310,7 +326,8 @@ namespace Duplicati.Library.Main.Database INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") VALUES (NEW.""FileId""); END - "); + ") + .ConfigureAwait(false); } } @@ -323,7 +340,7 @@ namespace Duplicati.Library.Main.Database } finally { - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } } @@ -335,7 +352,11 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); - var filesetIds = await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToListAsync(); + var filesetIds = + await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions) + .ToListAsync() + .ConfigureAwait(false); + while (filesetIds.Count > 0) { var filesetId = filesetIds[0]; @@ -347,20 +368,25 @@ namespace Duplicati.Library.Main.Database WHERE ""ID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId); - m_restoreTime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); + m_restoreTime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0).ConfigureAwait(false)); var ix = await FilesetTimes() .Select((value, index) => new { value.Key, index }) .Where(n => n.Key == filesetId) .Select(pair => pair.index + 1) - .FirstOrDefaultAsync() - 1; + .FirstOrDefaultAsync() + .ConfigureAwait(false) - 1; Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime); cmd.Parameters.Clear(); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" "); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ") + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{m_tempfiletable}"" ( ""ID"" INTEGER PRIMARY KEY, @@ -372,7 +398,9 @@ namespace Duplicati.Library.Main.Database ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN ) - "); + ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{m_tempblocktable}"" ( ""ID"" INTEGER PRIMARY KEY, @@ -385,7 +413,8 @@ namespace Duplicati.Library.Main.Database ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); // TODO: Optimize to use the path prefix @@ -414,12 +443,14 @@ namespace Duplicati.Library.Main.Database AND ""FilesetEntry"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple) { using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset")) - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); + cmd.SetTransaction(m_rtr); // If we get a list of filenames, the lookup table is faster // unfortunately we cannot do this if the filesystem is case sensitive as @@ -430,16 +461,21 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{m_filenamestable}"" ( ""Path"" TEXT NOT NULL ) - "); + ") + .ConfigureAwait(false); + cmd.SetCommandAndParameters($@" INSERT INTO ""{m_filenamestable}"" (""Path"") VALUES (@Path) - "); + ") + .ConfigureAwait(false); foreach (var s in p) { - await cmd.SetParameterValue("@Path", s) - .ExecuteNonQueryAsync(); + await cmd + .SetParameterValue("@Path", s) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } var c = await cmd.SetCommandAndParameters($@" @@ -468,7 +504,8 @@ namespace Duplicati.Library.Main.Database ) ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (c != p.Length && c != 0) { @@ -483,8 +520,8 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tempfiletable}"" ) "); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) sb.AppendLine(rd.ConvertValueToString(0)); cmd.SetCommandAndParameters(@" @@ -493,16 +530,17 @@ namespace Duplicati.Library.Main.Database WHERE ""ID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId); - var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0)); + var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0).ConfigureAwait(false)); Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); cmd.Parameters.Clear(); } - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" "); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ") + .ConfigureAwait(false); using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset")) - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); cmd.SetTransaction(m_rtr); } else @@ -541,18 +579,21 @@ namespace Duplicati.Library.Main.Database 0 ) "); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync() + .ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) { rd.GetValues(values); var path = values[1] as string; if (path != null && FilterExpression.Matches(filter, path.ToString()!)) { - await cmd2.SetParameterValue("@ID", values[0]) + await cmd2 + .SetParameterValue("@ID", values[0]) .SetParameterValue("@Path", values[1]) .SetParameterValue("@BlocksetID", values[2]) .SetParameterValue("@MetadataID", values[3]) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } } @@ -561,15 +602,20 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_ID"" ON ""{m_tempfiletable}"" (""ID"") - "); + ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_TargetPath"" ON ""{m_tempfiletable}"" (""TargetPath"") - "); + ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_Path"" ON ""{m_tempfiletable}"" (""Path"") - "); + ") + .ConfigureAwait(false); cmd.SetCommandAndParameters($@" SELECT @@ -580,12 +626,12 @@ namespace Duplicati.Library.Main.Database ""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" "); - using (var rd = await cmd.ExecuteReaderAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) { var filecount = 0L; var filesize = 0L; - if (await rd.ReadAsync()) + if (await rd.ReadAsync().ConfigureAwait(false)) { filecount = rd.ConvertValueToInt64(0, 0); filesize = rd.ConvertValueToInt64(1, 0); @@ -599,7 +645,7 @@ namespace Duplicati.Library.Main.Database } } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } return new Tuple(0, 0); @@ -615,7 +661,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - var v0 = await cmd.ExecuteScalarAsync(); + var v0 = await cmd.ExecuteScalarAsync().ConfigureAwait(false); if (v0 == null || v0 == DBNull.Value) return null; @@ -632,7 +678,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - var v0 = await cmd.ExecuteScalarAsync(); + var v0 = await cmd.ExecuteScalarAsync().ConfigureAwait(false); var maxpath = ""; if (v0 != null && v0 != DBNull.Value) maxpath = v0.ToString()!; @@ -642,7 +688,9 @@ namespace Duplicati.Library.Main.Database var filecount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{m_tempfiletable}"" - ", -1); + ", -1) + .ConfigureAwait(false); + var foundfiles = -1L; //TODO: Handle FS case-sensitive? @@ -655,9 +703,11 @@ namespace Duplicati.Library.Main.Database while (filecount != foundfiles && maxpath.Length > 0) { var mp = Util.AppendDirSeparator(maxpath, dirsep); - foundfiles = await cmd.SetParameterValue("@PrefixLength", mp.Length) + foundfiles = await cmd + .SetParameterValue("@PrefixLength", mp.Length) .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64Async(-1); + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); if (filecount != foundfiles) { @@ -675,7 +725,10 @@ namespace Duplicati.Library.Main.Database public async Task SetTargetPaths(string largest_prefix, string destination) { - var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? await GetFirstPath() : largest_prefix); + var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? + await GetFirstPath().ConfigureAwait(false) + : + largest_prefix); using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); @@ -696,7 +749,9 @@ namespace Duplicati.Library.Main.Database THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END - "); + ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = @@ -705,7 +760,8 @@ namespace Duplicati.Library.Main.Database THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END - "); + ") + .ConfigureAwait(false); } else if (OperatingSystem.IsWindows() && dirsep == "/") @@ -721,7 +777,8 @@ namespace Duplicati.Library.Main.Database END ") .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } else { @@ -729,7 +786,8 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = ""Path"" - "); + ") + .ConfigureAwait(false); } } else @@ -747,7 +805,8 @@ namespace Duplicati.Library.Main.Database THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END - "); + ") + .ConfigureAwait(false); // For UNC paths, we use \\server\folder -> / / await cmd.ExecuteNonQueryAsync($@" @@ -758,7 +817,8 @@ namespace Duplicati.Library.Main.Database THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END - "); + ") + .ConfigureAwait(false); } else { @@ -768,7 +828,8 @@ namespace Duplicati.Library.Main.Database SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength) ") .SetParameterValue("@PrefixLength", largest_prefix.Length + 1) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } @@ -778,13 +839,15 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/') - "); + ") + .ConfigureAwait(false); else if (OperatingSystem.IsWindows() && dirsep == "/") // For Linux paths on Windows await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\') - "); + ") + .ConfigureAwait(false); if (!string.IsNullOrEmpty(destination)) { @@ -795,10 +858,11 @@ namespace Duplicati.Library.Main.Database SET ""TargetPath"" = @Destination || ""TargetPath"" ") .SetParameterValue("@Destination", Util.AppendDirSeparator(destination)) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } public async Task FindMissingBlocks(bool skipMetadata) @@ -832,7 +896,8 @@ namespace Duplicati.Library.Main.Database WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - "); + ") + .ConfigureAwait(false); var p2 = 0; if (!skipMetadata) @@ -865,26 +930,32 @@ namespace Duplicati.Library.Main.Database ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - "); + ") + .ConfigureAwait(false); - //creating indexes after insertion is much faster + // Creating indexes after insertion is much faster await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" ON ""{m_tempblocktable}"" (""Hash"", ""Size"") - "); - // better suited to speed up commit on UpdateBlocks + ") + .ConfigureAwait(false); + + // Better suited to speed up commit on UpdateBlocks await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" ON ""{m_tempblocktable}"" (""FileId"", ""Index"") - "); + ") + .ConfigureAwait(false); var size = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_tempblocktable}"" - ", 0); + ", 0) + .ConfigureAwait(false); + Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } public async Task UpdateTargetPath(long ID, string newname) @@ -894,12 +965,14 @@ namespace Duplicati.Library.Main.Database SET ""TargetPath"" = @TargetPath WHERE ""ID"" = @ID "); - await cmd.SetTransaction(m_rtr) + await cmd + .SetTransaction(m_rtr) .SetParameterValue("@TargetPath", newname) .SetParameterValue("@ID", ID) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } public interface IExistingFileBlock @@ -996,7 +1069,7 @@ namespace Duplicati.Library.Main.Database while (HasMore && p == TargetPath) { yield return new ExistingFileBlock(m_reader); - HasMore = await m_reader.ReadAsync(); + HasMore = await m_reader.ReadAsync().ConfigureAwait(false); } } @@ -1025,8 +1098,8 @@ namespace Duplicati.Library.Main.Database ""BlocksetEntry"".""Index"" ") .SetTransaction(db.Transaction); - using var rd = await cmd.ExecuteReaderAsync(); - if (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) { var more = true; while (more) @@ -1037,7 +1110,7 @@ namespace Duplicati.Library.Main.Database more = f.HasMore; while (more && current == f.TargetPath) - more = await rd.ReadAsync(); + more = await rd.ReadAsync().ConfigureAwait(false); } } } @@ -1086,7 +1159,7 @@ namespace Duplicati.Library.Main.Database while (HasMore && p == TargetPath && h == Hash && s == Size) { yield return new BlockSource(m_reader); - HasMore = await m_reader.ReadAsync(); + HasMore = await m_reader.ReadAsync().ConfigureAwait(false); } } @@ -1113,7 +1186,7 @@ namespace Duplicati.Library.Main.Database HasMore = c.HasMore; while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) - HasMore = await m_reader.ReadAsync(); + HasMore = await m_reader.ReadAsync().ConfigureAwait(false); } } @@ -1151,8 +1224,8 @@ namespace Duplicati.Library.Main.Database ""B"".""Index"" ") .SetTransaction(db.Transaction); - using var rd = await cmd.ExecuteReaderAsync(); - var more = await rd.ReadAsync(); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + var more = await rd.ReadAsync().ConfigureAwait(false); while (more) { @@ -1162,7 +1235,7 @@ namespace Duplicati.Library.Main.Database more = f.HasMore; while (more && current == f.TargetPath) - more = await rd.ReadAsync(); + more = await rd.ReadAsync().ConfigureAwait(false); } } } @@ -1206,8 +1279,8 @@ namespace Duplicati.Library.Main.Database // Now it is likely to restore all files from front to back. Large files will always be done last. // One could also use like the average block number in a volume, that needs to be measured. - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) { yield return new RemoteVolume( rd.ConvertValueToString(0), @@ -1258,7 +1331,8 @@ namespace Duplicati.Library.Main.Database ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL ) - "); + ") + .ConfigureAwait(false); c.SetCommandAndParameters($@" INSERT INTO ""{fam.m_tmptable}"" ( @@ -1273,12 +1347,13 @@ namespace Duplicati.Library.Main.Database { await c.SetParameterValue("@Hash", s.Key) .SetParameterValue("@Size", s.Value) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks - await db.Transaction.CommitAsync(); + await db.Transaction.CommitAsync().ConfigureAwait(false); return fam; } @@ -1294,8 +1369,8 @@ namespace Duplicati.Library.Main.Database { using var c = m_db.Connection.CreateCommand(@$"DROP TABLE IF EXISTS ""{m_tmptable}""") .SetTransaction(m_db.Transaction); - await c.ExecuteNonQueryAsync(); - await m_db.Transaction.CommitAsync(); + await c.ExecuteNonQueryAsync().ConfigureAwait(false); + await m_db.Transaction.CommitAsync().ConfigureAwait(false); } } @@ -1324,7 +1399,7 @@ namespace Duplicati.Library.Main.Database while (HasMore && p == Path) { yield return new PatchBlock(m_reader); - HasMore = await m_reader.ReadAsync(); + HasMore = await m_reader.ReadAsync().ConfigureAwait(false); } } } @@ -1361,8 +1436,8 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_db.Transaction); - using var rd = await cmd.ExecuteReaderAsync(); - if (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) { var more = true; while (more) @@ -1373,7 +1448,7 @@ namespace Duplicati.Library.Main.Database more = f.HasMore; while (more && current == f.Path) - more = await rd.ReadAsync(); + more = await rd.ReadAsync().ConfigureAwait(false); } } } @@ -1410,8 +1485,8 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_db.Transaction); - using var rd = await cmd.ExecuteReaderAsync(); - if (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) { var more = true; while (more) @@ -1422,7 +1497,7 @@ namespace Duplicati.Library.Main.Database more = f.HasMore; while (more && current == f.Path) - more = await rd.ReadAsync(); + more = await rd.ReadAsync().ConfigureAwait(false); } } } @@ -1432,7 +1507,8 @@ namespace Duplicati.Library.Main.Database { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); - return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume); + return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume) + .ConfigureAwait(false); } /// @@ -1443,10 +1519,14 @@ namespace Duplicati.Library.Main.Database { if (!m_connection_pool.TryTake(out var entry)) { - var connection = await SQLiteLoader.LoadConnectionAsync(); + var connection = await SQLiteLoader.LoadConnectionAsync() + .ConfigureAwait(false); + connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared"; - await connection.OpenAsync(); - await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize); + await connection.OpenAsync().ConfigureAwait(false); + await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize) + .ConfigureAwait(false); + var transaction = new ReusableTransaction(connection); return (connection, transaction); @@ -1487,8 +1567,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Verified", !onlyNonVerified); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new FileToRestore( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); } @@ -1515,9 +1595,16 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Length"" DESC ") .SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) - yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new FileRequest( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToString(3), + rd.ConvertValueToInt64(4), + rd.ConvertValueToInt64(5) + ); } /// @@ -1541,9 +1628,10 @@ namespace Duplicati.Library.Main.Database F.BlocksetID = {FOLDER_BLOCKSET_ID} AND F.MetadataID IS NOT NULL AND F.MetadataID >= 0 - "); + ") + .ConfigureAwait(false); - while (await rd.ReadAsync()) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new FileRequest( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), @@ -1574,6 +1662,7 @@ namespace Duplicati.Library.Main.Database INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "; + using var cmd = Connection.CreateCommand($@" SELECT ""Block"".""ID"", @@ -1586,8 +1675,9 @@ namespace Duplicati.Library.Main.Database {metadata_query} ") .SetTransaction(m_rtr); - using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) + + using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await reader.ReadAsync().ConfigureAwait(false)) yield return ( reader.ConvertValueToInt64(0), reader.ConvertValueToInt64(1) @@ -1601,7 +1691,9 @@ namespace Duplicati.Library.Main.Database /// A list of needed to restore the given file. public async IAsyncEnumerable GetBlocksFromFile(long blocksetID) { - var (connection, transaction) = await GetConnectionFromPool(); + var (connection, transaction) = await GetConnectionFromPool() + .ConfigureAwait(false); + try { @@ -1620,8 +1712,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(transaction) .SetParameterValue("@BlocksetID", blocksetID); - using var reader = await cmd.ExecuteReaderAsync(); - for (long i = 0; await reader.ReadAsync(); i++) + using var reader = await cmd.ExecuteReaderAsync() + .ConfigureAwait(false); + for (long i = 0; await reader.ReadAsync().ConfigureAwait(false); i++) yield return new BlockRequest( reader.ConvertValueToInt64(0), i, @@ -1645,7 +1738,8 @@ namespace Duplicati.Library.Main.Database /// A list of needed to restore the metadata of the given file. public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID) { - var (connection, transaction) = await GetConnectionFromPool(); + var (connection, transaction) = await GetConnectionFromPool() + .ConfigureAwait(false); try { using var cmd = connection.CreateCommand($@" @@ -1666,10 +1760,18 @@ namespace Duplicati.Library.Main.Database .SetTransaction(transaction) .SetParameterValue("@FileID", fileID); - using var reader = await cmd.ExecuteReaderAsync(); - for (long i = 0; await reader.ReadAsync(); i++) + using var reader = await cmd.ExecuteReaderAsync() + .ConfigureAwait(false); + for (long i = 0; await reader.ReadAsync().ConfigureAwait(false); i++) { - yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); + yield return new BlockRequest( + reader.ConvertValueToInt64(0), + i, + reader.ConvertValueToString(1), + reader.ConvertValueToInt64(2), + reader.ConvertValueToInt64(3), + false + ); } } finally @@ -1697,9 +1799,14 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@VolumeID", VolumeID); - using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - yield return (reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), reader.ConvertValueToString(2) ?? ""); + using var reader = await cmd.ExecuteReaderAsync() + .ConfigureAwait(false); + while (await reader.ReadAsync().ConfigureAwait(false)) + yield return ( + reader.ConvertValueToString(0) ?? "", + reader.ConvertValueToInt64(1), + reader.ConvertValueToString(2) ?? "" + ); } public async Task DropRestoreTable() @@ -1710,7 +1817,8 @@ namespace Duplicati.Library.Main.Database if (m_tempfiletable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""") + .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_tempfiletable = null; } @@ -1718,7 +1826,8 @@ namespace Duplicati.Library.Main.Database if (m_tempblocktable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""") + .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_tempblocktable = null; } @@ -1726,7 +1835,8 @@ namespace Duplicati.Library.Main.Database if (m_latestblocktable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""") + .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_latestblocktable = null; } @@ -1734,7 +1844,8 @@ namespace Duplicati.Library.Main.Database if (m_fileprogtable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""") + .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_fileprogtable = null; } @@ -1742,7 +1853,8 @@ namespace Duplicati.Library.Main.Database if (m_totalprogtable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""") + .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_totalprogtable = null; } @@ -1750,12 +1862,13 @@ namespace Duplicati.Library.Main.Database if (m_filesnewlydonetable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""") + .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_filesnewlydonetable = null; } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } public interface IBlockMarker : IDisposable @@ -1811,13 +1924,15 @@ namespace Duplicati.Library.Main.Database AND ""Size"" = @Size AND ""Metadata"" = @Metadata AND ""Restored"" = 0 - "), + ") + .ConfigureAwait(false), m_resetfileCommand = await db.Connection.CreateCommandAsync($@" UPDATE ""{blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = @TargetFileId - "), + ") + .ConfigureAwait(false), m_updateAsRestoredCommand = await db.Connection.CreateCommandAsync($@" UPDATE ""{blocktablename}"" @@ -1825,13 +1940,15 @@ namespace Duplicati.Library.Main.Database WHERE ""FileID"" = @TargetFileId AND ""Metadata"" <= @Metadata - "), + ") + .ConfigureAwait(false), m_updateFileAsDataVerifiedCommand = await db.Connection.CreateCommandAsync($@" UPDATE ""{filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = @TargetFileId - "), + ") + .ConfigureAwait(false), m_statUpdateCommand = statstablename == null ? // very slow fallback if stats tables were not created @@ -1842,6 +1959,7 @@ namespace Duplicati.Library.Main.Database FROM ""{blocktablename}"" WHERE ""Restored"" = 1 ") + .ConfigureAwait(false) : // Fields in Stats: TotalFiles, TotalBlocks, TotalSize // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored @@ -1851,6 +1969,7 @@ namespace Duplicati.Library.Main.Database SUM(""SizeRestored"") FROM ""{statstablename}"" ") + .ConfigureAwait(false) }; return dbm; @@ -1864,7 +1983,9 @@ namespace Duplicati.Library.Main.Database m_hasUpdates = false; using var rd = await m_statUpdateCommand .SetTransaction(m_db.Transaction) - .ExecuteReaderAsync(); + .ExecuteReaderAsync() + .ConfigureAwait(false); + var filesprocessed = 0L; var processedsize = 0L; @@ -1884,7 +2005,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_db.Transaction) .SetParameterValue("@TargetFileId", targetfileid); - var r = await m_resetfileCommand.ExecuteNonQueryAsync(); + var r = await m_resetfileCommand.ExecuteNonQueryAsync() + .ConfigureAwait(false); if (r <= 0) throw new Exception("Unexpected reset result"); } @@ -1897,7 +2019,9 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TargetFileId", targetfileid) .SetParameterValue("@Metadata", includeMetadata ? 1 : 0); - var r = await m_updateAsRestoredCommand.ExecuteNonQueryAsync(); + var r = await m_updateAsRestoredCommand + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (r <= 0) throw new Exception("Unexpected reset result"); } @@ -1909,7 +2033,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_db.Transaction) .SetParameterValue("@TargetFileId", targetfileid); - var r = await m_updateFileAsDataVerifiedCommand.ExecuteNonQueryAsync(); + var r = await m_updateFileAsDataVerifiedCommand + .ExecuteNonQueryAsync() + .ConfigureAwait(false); if (r != 1) throw new Exception("Unexpected result when marking file as verified."); } @@ -1925,7 +2051,10 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Size", size) .SetParameterValue("@Metadata", metadata); - var r = await m_insertblockCommand.ExecuteNonQueryAsync(); + var r = await m_insertblockCommand + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + if (r != 1) throw new Exception("Unexpected result when marking block."); } @@ -1933,7 +2062,7 @@ namespace Duplicati.Library.Main.Database public async Task CommitAsync() { using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) - await m_db.Transaction.CommitAsync(); + await m_db.Transaction.CommitAsync().ConfigureAwait(false); } public void Dispose() @@ -1958,7 +2087,8 @@ namespace Duplicati.Library.Main.Database throw new InvalidOperationException("No temporary file table set up for this restore."); if (string.IsNullOrWhiteSpace(m_totalprogtable)) throw new InvalidOperationException("No progress table set up for this restore."); - return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable); + return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable) + .ConfigureAwait(false); } public override void Dispose() @@ -1968,18 +2098,18 @@ namespace Duplicati.Library.Main.Database public override async Task DisposeAsync() { - await DisposePoolAsync(); - await DropRestoreTable(); - await base.DisposeAsync(); + await DisposePoolAsync().ConfigureAwait(false); + await DropRestoreTable().ConfigureAwait(false); + await base.DisposeAsync().ConfigureAwait(false); } public async Task DisposePoolAsync() { foreach (var (connection, transaction) in m_connection_pool) { - await transaction.DisposeAsync(); - connection.Close(); - await connection.DisposeAsync(); + await transaction.DisposeAsync().ConfigureAwait(false); + await connection.CloseAsync().ConfigureAwait(false); + await connection.DisposeAsync().ConfigureAwait(false); } m_connection_pool.Clear(); } @@ -1994,8 +2124,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); - using var rd = await cmd.ExecuteReaderAsync(); - while (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? ""; } @@ -2043,7 +2173,7 @@ namespace Duplicati.Library.Main.Database do { yield return new BlockEntry(m_rd, m_blocksize); - } while ((MoreData = await m_rd.ReadAsync()) && tid == TargetFileID); + } while ((MoreData = await m_rd.ReadAsync().ConfigureAwait(false)) && tid == TargetFileID); } } @@ -2063,15 +2193,17 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"" "); - using (var rd = await cmdReader.ExecuteReaderAsync()) + using (var rd = await cmdReader.ExecuteReaderAsync().ConfigureAwait(false)) { - while (await rd.ReadAsync()) + while (await rd.ReadAsync().ConfigureAwait(false)) { var sourcepath = rd.ConvertValueToString(0); if (SystemIO.IO_OS.FileExists(sourcepath)) { - await cmd.SetParameterValue("@Path", sourcepath) - .ExecuteNonQueryAsync(); + await cmd + .SetParameterValue("@Path", sourcepath) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } else { @@ -2084,9 +2216,10 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"") - "); + ") + .ConfigureAwait(false); - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; @@ -2125,12 +2258,17 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" "); - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}"); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ") + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_latestblocktable}_path"" ON ""{m_latestblocktable}"" (""Path"") - "); + ") + .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" @@ -2139,7 +2277,8 @@ namespace Duplicati.Library.Main.Database FROM ""{m_latestblocktable}"" WHERE Path = ""{m_tempfiletable}"".Path ) - "); + ") + .ConfigureAwait(false); } var sources = $@" @@ -2168,8 +2307,9 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync(sources); - if (await rd.ReadAsync()) + using var rd = await cmd.ExecuteReaderAsync(sources) + .ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) { bool more; do @@ -2180,13 +2320,13 @@ namespace Duplicati.Library.Main.Database more = n.MoreData; while (more && n.TargetFileID == tid) - more = await rd.ReadAsync(); + more = await rd.ReadAsync().ConfigureAwait(false); } while (more); } } - await m_rtr.CommitAsync(); + await m_rtr.CommitAsync().ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 4a63e5ead..ea909caa6 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -36,7 +36,10 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalTestDatabase(); - dbnew = (LocalTestDatabase)await CreateLocalDatabaseAsync(path, "Test", true, pagecachesize, dbnew); + dbnew = (LocalTestDatabase) + await CreateLocalDatabaseAsync(path, "Test", true, pagecachesize, dbnew) + .ConfigureAwait(false); + dbnew.ShouldCloseConnection = true; return dbnew; @@ -46,7 +49,9 @@ namespace Duplicati.Library.Main.Database { dbnew ??= new LocalTestDatabase(); - return (LocalTestDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew); + return (LocalTestDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew) + .ConfigureAwait(false); } public async Task UpdateVerificationCount(string name) @@ -67,7 +72,8 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name ") .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } private record RemoteVolume : IRemoteVolume @@ -142,7 +148,8 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable SelectTestTargets(long samples, Options options) { - var tp = await GetFilelistWhereClause(options.Time, options.Version); + var tp = await GetFilelistWhereClause(options.Time, options.Version) + .ConfigureAwait(false); samples = Math.Max(1, samples); using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -249,8 +256,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) files.Add(new RemoteVolume(rd)); foreach (var f in FilterByVerificationCount(files, samples, max)) @@ -278,8 +285,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) files.Add(new RemoteVolume(rd)); foreach (var f in FilterByVerificationCount(files, samples, max)) @@ -313,14 +320,17 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat} - "); + ") + .ConfigureAwait(false); + bl.m_tablename = tablename; } bl.m_insertCommand = await bl.m_db.Connection.CreateCommandAsync($@" INSERT INTO ""{bl.m_tablename}"" {insertCommand} - "); + ") + .ConfigureAwait(false); return bl; } @@ -336,12 +346,13 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction)) - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}"""); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + .ConfigureAwait(false); } catch { } finally { m_tablename = null!; } - await m_insertCommand.DisposeAsync(); + await m_insertCommand.DisposeAsync().ConfigureAwait(false); } } @@ -392,7 +403,9 @@ namespace Duplicati.Library.Main.Database public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Filelist(); - return (Filelist)await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + return (Filelist) + await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + .ConfigureAwait(false); } public async Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time) @@ -404,7 +417,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Metasize", metasize) .SetParameterValue("@Metahash", metahash) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } public async IAsyncEnumerable> Compare() @@ -493,7 +507,8 @@ namespace Duplicati.Library.Main.Database await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); cmd .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") @@ -501,8 +516,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( (Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "" @@ -513,7 +528,9 @@ namespace Duplicati.Library.Main.Database { try { - await cmd.ExecuteNonQueryAsync(drop); + await cmd + .ExecuteNonQueryAsync(drop) + .ConfigureAwait(false); } catch { } } @@ -563,16 +580,20 @@ namespace Duplicati.Library.Main.Database public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Indexlist(); - return (Indexlist)await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + return (Indexlist) + await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + .ConfigureAwait(false); } public async Task AddBlockLink(string filename, string hash, long length) { - await m_insertCommand.SetTransaction(m_db.Transaction) + await m_insertCommand + .SetTransaction(m_db.Transaction) .SetParameterValue("@Name", filename) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", length) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } public async IAsyncEnumerable> Compare() @@ -640,7 +661,8 @@ namespace Duplicati.Library.Main.Database await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); cmd .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") @@ -648,8 +670,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } @@ -657,7 +679,9 @@ namespace Duplicati.Library.Main.Database { try { - await cmd.ExecuteNonQueryAsync(drop); + await cmd + .ExecuteNonQueryAsync(drop) + .ConfigureAwait(false); } catch { } } @@ -694,7 +718,9 @@ namespace Duplicati.Library.Main.Database public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Blocklist(); - return (Blocklist)await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + return (Blocklist) + await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + .ConfigureAwait(false); } public async Task AddBlock(string hash, long size) @@ -703,7 +729,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } public async IAsyncEnumerable> Compare() @@ -805,7 +832,8 @@ namespace Duplicati.Library.Main.Database await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); cmd .SetCommandAndParameters($@" @@ -817,8 +845,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } @@ -826,7 +854,9 @@ namespace Duplicati.Library.Main.Database { try { - await cmd.ExecuteNonQueryAsync(drop); + await cmd + .ExecuteNonQueryAsync(drop) + .ConfigureAwait(false); } catch { } } @@ -851,7 +881,9 @@ namespace Duplicati.Library.Main.Database public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new BlocklistHashList(); - return (BlocklistHashList)await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND); + return (BlocklistHashList) + await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + .ConfigureAwait(false); } public async Task AddBlockHash(string hash, long size) @@ -860,7 +892,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } public async IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize) @@ -971,7 +1004,8 @@ namespace Duplicati.Library.Main.Database await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync(); + .ExecuteNonQueryAsync() + .ConfigureAwait(false); // Compare against actual values inserted into temp table cmd @@ -980,8 +1014,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using (var rd = await cmd.ExecuteReaderAsync()) - while (await rd.ReadAsync()) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); @@ -990,7 +1024,9 @@ namespace Duplicati.Library.Main.Database { try { - await cmd.ExecuteNonQueryAsync(drop); + await cmd + .ExecuteNonQueryAsync(drop) + .ConfigureAwait(false); } catch { } } @@ -1000,22 +1036,23 @@ namespace Duplicati.Library.Main.Database public async Task CreateFilelist(string name) { - return await Filelist.CreateAsync(this, name); + return await Filelist.CreateAsync(this, name).ConfigureAwait(false); } public async Task CreateIndexlist(string name) { - return await Indexlist.CreateAsync(this, name); + return await Indexlist.CreateAsync(this, name).ConfigureAwait(false); } public async Task CreateBlocklist(string name) { - return await Blocklist.CreateAsync(this, name); + return await Blocklist.CreateAsync(this, name).ConfigureAwait(false); } public async Task CreateBlocklistHashList(string name) { - return await BlocklistHashList.CreateAsync(this, name); + return await BlocklistHashList.CreateAsync(this, name) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index c8bb256d6..3f2ead105 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -77,8 +77,8 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran message ??= "Unnamed commit"; using (var timer = new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) - await m_transaction.CommitAsync(); - await m_transaction.DisposeAsync(); + await m_transaction.CommitAsync().ConfigureAwait(false); + await m_transaction.DisposeAsync().ConfigureAwait(false); if (restart) m_transaction = m_con.BeginTransaction(deferred: true); @@ -106,7 +106,7 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran try { using (var timer = new Logging.Timer(LOGTAG, "Dispose", "Rollback during transaction dispose")) - await m_transaction.RollbackAsync(); + await m_transaction.RollbackAsync().ConfigureAwait(false); } catch (Exception ex) { @@ -116,7 +116,7 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran finally { m_disposed = true; - await m_transaction.DisposeAsync(); + await m_transaction.DisposeAsync().ConfigureAwait(false); } } } @@ -127,8 +127,8 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran throw new InvalidOperationException("Transaction is already disposed"); using (var timer = new Logging.Timer(LOGTAG, message, $"RollbackTransaction: {message}")) - await m_transaction.RollbackAsync(); - await m_transaction.DisposeAsync(); + await m_transaction.RollbackAsync().ConfigureAwait(false); + await m_transaction.DisposeAsync().ConfigureAwait(false); if (restart) { diff --git a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs index 9f51192c2..191f7d7db 100644 --- a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs +++ b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs @@ -86,28 +86,32 @@ internal class TemporaryDbValueList : IDisposable internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) { - return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "INTEGER")); + return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "INTEGER")) + .ConfigureAwait(false); } internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) { - return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "TEXT")); + return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "TEXT")) + .ConfigureAwait(false); } internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values) { - return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "INTEGER")); + return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "INTEGER")) + .ConfigureAwait(false); } internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values) { - return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "TEXT")); + return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "TEXT")) + .ConfigureAwait(false); } private static async Task DoCreateAsync(TemporaryDbValueList valueList) { if (valueList._values.Count() > LocalDatabase.CHUNK_SIZE) - await valueList.ForceToTable(); + await valueList.ForceToTable().ConfigureAwait(false); return valueList; } @@ -131,7 +135,7 @@ internal class TemporaryDbValueList : IDisposable if (_disposed) throw new ObjectDisposedException(nameof(TemporaryDbValueList)); - await ForceToTable(); + await ForceToTable().ConfigureAwait(false); return $@" SELECT ""Value"" @@ -153,7 +157,9 @@ internal class TemporaryDbValueList : IDisposable _cmd = _con.CreateCommand(_rtr); await _cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType}) - "); + ") + .ConfigureAwait(false); + _isTable = true; foreach (var slice in _values.Chunk(LocalDatabase.CHUNK_SIZE)) { @@ -167,7 +173,7 @@ internal class TemporaryDbValueList : IDisposable for (int i = 0; i < slice.Length; i++) _cmd.SetParameterValue(parameterNames[i], slice[i]); - await _cmd.ExecuteNonQueryAsync(); + await _cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } } @@ -194,8 +200,9 @@ internal class TemporaryDbValueList : IDisposable { if (_cmd != null) { - await _cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{_tableName}"""); - await _cmd.DisposeAsync(); + await _cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{_tableName}""") + .ConfigureAwait(false); + await _cmd.DisposeAsync().ConfigureAwait(false); } _cmd = null; } diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index c0dc09841..f21a60bb1 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -56,47 +56,83 @@ namespace Duplicati.Library.Main.Operation.Backup public Task FindBlockIDAsync(string hash, long size) { - return RunOnMain(async () => await m_database.FindBlockID(hash, size)); + return RunOnMain(async () => + await m_database + .FindBlockID(hash, size) + .ConfigureAwait(false) + ); } public Task AddBlockAsync(string hash, long size, long volumeid) { - return RunOnMain(async () => await m_database.AddBlock(hash, size, volumeid)); + return RunOnMain(async () => + await m_database + .AddBlock(hash, size, volumeid) + .ConfigureAwait(false) + ); } public Task GetFileHashAsync(long fileid) { - return RunOnMain(async () => await m_database.GetFileHash(fileid)); + return RunOnMain(async () => + await m_database + .GetFileHash(fileid) + .ConfigureAwait(false) + ); } public Task<(bool, long)> AddMetadatasetAsync(string hash, long size, long blocksetid) { - return RunOnMain(async () => await m_database.AddMetadataset(hash, size, blocksetid)); + return RunOnMain(async () => + await m_database + .AddMetadataset(hash, size, blocksetid) + .ConfigureAwait(false) + ); } public Task<(bool, long)> GetMetadataIDAsync(string hash, long size) { - return RunOnMain(async () => await m_database.GetMetadatasetID(hash, size)); + return RunOnMain(async () => + await m_database + .GetMetadatasetID(hash, size) + .ConfigureAwait(false) + ); } public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified) { - return RunOnMain(async () => await m_database.AddDirectoryEntry(filename, metadataid, lastModified)); + return RunOnMain(async () => + await m_database + .AddDirectoryEntry(filename, metadataid, lastModified) + .ConfigureAwait(false) + ); } public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified) { - return RunOnMain(async () => await m_database.AddSymlinkEntry(filename, metadataid, lastModified)); + return RunOnMain(async () => + await m_database + .AddSymlinkEntry(filename, metadataid, lastModified) + .ConfigureAwait(false) + ); } public Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFileAsync(long fileid) { - return RunOnMain(async () => await m_database.GetMetadataHashAndSizeForFile(fileid)); + return RunOnMain(async () => + await m_database + .GetMetadataHashAndSizeForFile(fileid) + .ConfigureAwait(false) + ); } public Task<(long, DateTime, long)> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength) { - return RunOnMain(async () => await m_database.GetFileLastModified(prefixid, path, lastfilesetid, includeLength)); + return RunOnMain(async () => + await m_database + .GetFileLastModified(prefixid, path, lastfilesetid, includeLength) + .ConfigureAwait(false) + ); } public Task GetFileEntryAsync(long prefixid, string path, long lastfilesetid) @@ -104,10 +140,13 @@ namespace Duplicati.Library.Main.Operation.Backup return RunOnMain(async () => { var (id, oldModified, lastFileSize, oldMetahash, oldMetasize) = - await m_database.GetFileEntry(prefixid, path, lastfilesetid); + await m_database.GetFileEntry(prefixid, path, lastfilesetid) + .ConfigureAwait(false); + return id < 0 ? - null : + null + : new FileEntryData() { id = id, @@ -123,7 +162,9 @@ namespace Duplicati.Library.Main.Operation.Backup { return RunOnMain(async () => { - var (_, blocksetid) = await m_database.AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes); + var (_, blocksetid) = await m_database + .AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes) + .ConfigureAwait(false); return blocksetid; }); @@ -131,33 +172,57 @@ namespace Duplicati.Library.Main.Operation.Backup public Task GetOrCreatePathPrefix(string prefix) { - return RunOnMain(async () => await m_database.GetOrCreatePathPrefix(prefix)); + return RunOnMain(async () => + await m_database + .GetOrCreatePathPrefix(prefix) + .ConfigureAwait(false) + ); } public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid) { - return RunOnMain(async () => await m_database.AddFile(prefixid, filename, lastmodified, blocksetid, metadataid)); + return RunOnMain(async () => + await m_database + .AddFile(prefixid, filename, lastmodified, blocksetid, metadataid) + .ConfigureAwait(false) + ); } public Task AddUnmodifiedAsync(long fileid, DateTime lastModified) { - return RunOnMain(async () => await m_database.AddKnownFile(fileid, lastModified)); + return RunOnMain(async () => + await m_database + .AddKnownFile(fileid, lastModified) + .ConfigureAwait(false) + ); } public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid) { - return RunOnMain(async () => await m_database.MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid)); + return RunOnMain(async () => + await m_database + .MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid) + .ConfigureAwait(false) + ); } public Task SafeDeleteRemoteVolumeAsync(string remotename) { - return RunOnMain(async () => await m_database.SafeDeleteRemoteVolume(remotename)); + return RunOnMain(async () => + await m_database + .SafeDeleteRemoteVolume(remotename) + .ConfigureAwait(false) + ); } public Task GetBlocklistHashesAsync(string remotename) { - return RunOnMain(async () => await m_database.GetBlocklistHashes(remotename)); + return RunOnMain(async () => + await m_database + .GetBlocklistHashes(remotename) + .ConfigureAwait(false) + ); } public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume) @@ -167,29 +232,46 @@ namespace Duplicati.Library.Main.Operation.Backup return RunOnMain(async () => { - await m_database.AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID); + await m_database + .AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID) + .ConfigureAwait(false); + indexvolume.StartVolume(blockvolume.RemoteFilename); - await foreach (var b in m_database.GetBlocks(blockvolume.VolumeID)) + await foreach (var b in m_database.GetBlocks(blockvolume.VolumeID).ConfigureAwait(false)) indexvolume.AddBlock(b.Hash, b.Size); - await m_database.UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await m_database + .UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .ConfigureAwait(false); }); } public Task AppendFilesFromPreviousSetAsync() { - return RunOnMain(async () => await m_database.AppendFilesFromPreviousSet()); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSet() + .ConfigureAwait(false) + ); } public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist) { - return RunOnMain(async () => await m_database.AppendFilesFromPreviousSet(deletedFilelist)); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSet(deletedFilelist) + .ConfigureAwait(false) + ); } public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp) { - return RunOnMain(async () => await m_database.AppendFilesFromPreviousSet(deletedFilelist, filesetid, prevId, timestamp)); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSet(deletedFilelist, filesetid, prevId, timestamp) + .ConfigureAwait(false) + ); } /// @@ -200,7 +282,11 @@ namespace Duplicati.Library.Main.Operation.Backup /// Optional exclusion predicate (true = exclude file) public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate) { - return RunOnMain(async () => await m_database.AppendFilesFromPreviousSetWithPredicate(exclusionPredicate)); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSetWithPredicate(exclusionPredicate) + .ConfigureAwait(false) + ); } /// @@ -214,47 +300,84 @@ namespace Duplicati.Library.Main.Operation.Backup /// If prevFileSetId == -1, used to locate previous fileset public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) { - return RunOnMain(async () => await m_database.AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, fileSetId, prevFileSetId, timestamp)); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, fileSetId, prevFileSetId, timestamp) + .ConfigureAwait(false) + ); } public Task[]> GetIncompleteFilesetsAsync() { - return RunOnMain(async () => await m_database.GetIncompleteFilesets().OrderBy(x => x.Value).ToArrayAsync()); + return RunOnMain(async () => + await m_database + .GetIncompleteFilesets() + .OrderBy(x => x.Value) + .ToArrayAsync() + .ConfigureAwait(false) + ); } public Task[]> GetFilesetTimesAsync() { - return RunOnMain(async () => await m_database.FilesetTimes().ToArrayAsync()); + return RunOnMain(async () => + await m_database + .FilesetTimes() + .ToArrayAsync() + .ConfigureAwait(false) + ); } public Task CreateFilesetAsync(long volumeID, DateTime fileTime) { return RunOnMain(async () => { - var fs = await m_database.CreateFileset(volumeID, fileTime); - await m_database.Transaction.CommitAsync(); + var fs = await m_database + .CreateFileset(volumeID, fileTime) + .ConfigureAwait(false); + + await m_database.Transaction + .CommitAsync() + .ConfigureAwait(false); + return fs; }); } public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid) { - return RunOnMain(async () => await m_database.LinkFilesetToVolume(filesetid, volumeid)); + return RunOnMain(async () => + await m_database + .LinkFilesetToVolume(filesetid, volumeid) + .ConfigureAwait(false) + ); } public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid) { - return RunOnMain(async () => await m_database.WriteFileset(fsw, filesetid)); + return RunOnMain(async () => + await m_database + .WriteFileset(fsw, filesetid) + .ConfigureAwait(false) + ); } public Task PushTimestampChangesToPreviousVersionAsync(long filesetid) { - return RunOnMain(async () => await m_database.PushTimestampChangesToPreviousVersion(filesetid)); + return RunOnMain(async () => + await m_database + .PushTimestampChangesToPreviousVersion(filesetid) + .ConfigureAwait(false) + ); } public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid) { - return RunOnMain(async () => await m_database.UpdateFullBackupStateInFileset(filesetid, true)); + return RunOnMain(async () => + await m_database + .UpdateFullBackupStateInFileset(filesetid, true) + .ConfigureAwait(false) + ); } public Task> GetMissingIndexFilesAsync() @@ -264,37 +387,65 @@ namespace Duplicati.Library.Main.Operation.Backup public Task UpdateChangeStatisticsAsync(BackupResults result) { - return RunOnMain(async () => await m_database.UpdateChangeStatistics(result)); + return RunOnMain(async () => + await m_database + .UpdateChangeStatistics(result) + .ConfigureAwait(false) + ); } public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists) { - return RunOnMain(async () => await m_database.VerifyConsistency(blocksize, blockhashSize, verifyfilelists)); + return RunOnMain(async () => + await m_database + .VerifyConsistency(blocksize, blockhashSize, verifyfilelists) + .ConfigureAwait(false) + ); } public Task RemoveRemoteVolumeAsync(string remoteFilename) { - return RunOnMain(async () => await m_database.RemoveRemoteVolume(remoteFilename)); + return RunOnMain(async () => + await m_database + .RemoveRemoteVolume(remoteFilename) + .ConfigureAwait(false) + ); } public Task GetRemoteVolumeFromFilesetIDAsync(long filesetID) { - return RunOnMain(async () => await m_database.GetRemoteVolumeFromFilesetID(filesetID)); + return RunOnMain(async () => + await m_database + .GetRemoteVolumeFromFilesetID(filesetID) + .ConfigureAwait(false) + ); } public Task CreateChangeJournalDataAsync(IEnumerable journalData) { - return RunOnMain(async () => await m_database.CreateChangeJournalData(journalData)); + return RunOnMain(async () => + await m_database + .CreateChangeJournalData(journalData) + .ConfigureAwait(false) + ); } public Task UpdateChangeJournalDataAsync(IEnumerable journalData, long lastfilesetid) { - return RunOnMain(async () => await m_database.UpdateChangeJournalData(journalData, lastfilesetid)); + return RunOnMain(async () => + await m_database + .UpdateChangeJournalData(journalData, lastfilesetid) + .ConfigureAwait(false) + ); } public Task IsBlocklistHashKnownAsync(string hash) { - return RunOnMain(async () => await m_database.IsBlocklistHashKnown(hash)); + return RunOnMain(async () => + await m_database + .IsBlocklistHashKnown(hash) + .ConfigureAwait(false) + ); } } diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 2412fbfce..34e598b5b 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -114,7 +114,8 @@ namespace Duplicati.Library.Main.Operation.Backup if (newBlock) { - await blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint); + await blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint) + .ConfigureAwait(false); if (indexvolume != null) indexvolume.AddBlock(b.HashKey, b.Size); @@ -163,7 +164,9 @@ namespace Duplicati.Library.Main.Operation.Backup { if (blockvolume.SourceSize > 0 || blocklistHashesAdded > 0) { - await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume)); + await self.SpillPickup + .WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume)) + .ConfigureAwait(false); } else { diff --git a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs index 387b8431b..f8a204f9a 100644 --- a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs +++ b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs @@ -35,7 +35,7 @@ internal static class RecreateMissingIndexFiles { if (options.IndexfilePolicy != Options.IndexFileStrategy.None) { - await foreach (var blockfile in await database.GetMissingIndexFilesAsync().ConfigureAwait(false)) + await foreach (var blockfile in (await database.GetMissingIndexFilesAsync().ConfigureAwait(false)).ConfigureAwait(false)) { if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; diff --git a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs index 922c664c3..ee4c38c0e 100644 --- a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Operation.Backup target = await GetNextTarget(source).ConfigureAwait(false); var len = rd.ReadBlock(file.Key, buffer); - await target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default); + await target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default).ConfigureAwait(false); await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false); if (target.IndexVolume != null) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index c98543e05..be52b81e0 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -300,12 +300,14 @@ namespace Duplicati.Library.Main.Operation { try { - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options); + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + .ConfigureAwait(false); var lastFilestTime = await database.FilesetTimes() .Select(x => x.Value) .Append(DateTime.UnixEpoch) .Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds) - .MaxAsync(); + .MaxAsync() + .ConfigureAwait(false); var currentStamp = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(database.OperationTimestamp); if (lastFilestTime > currentStamp) @@ -317,31 +319,41 @@ namespace Duplicati.Library.Main.Operation Log.WriteVerboseMessage(LOGTAG, "DatabaseTimestampError", "The database has a timestamp of {0}, but the last fileset has a timestamp of {1}. Sleeping for {2} seconds to allow the clock to catch up.", database.OperationTimestamp.ToLocalTime(), Library.Utility.Utility.EPOCH.AddSeconds(lastFilestTime).ToLocalTime(), sleepSeconds); Thread.Sleep(TimeSpan.FromSeconds(sleepSeconds)); database.Dispose(); - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options); + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + .ConfigureAwait(false); } result.Dryrun = options.Dryrun; // Check the database integrity - await Utility.UpdateOptionsFromDb(database, options); - await Utility.VerifyOptionsAndUpdateDatabase(database, options); + await Utility.UpdateOptionsFromDb(database, options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(database, options) + .ConfigureAwait(false); + + var probe_path = await database + .GetFirstPath() + .ConfigureAwait(false); - var probe_path = await database.GetFirstPath(); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); - if (await database.PartiallyRecreated()) + if (await database.PartiallyRecreated().ConfigureAwait(false)) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (await database.RepairInProgress()) + if (await database.RepairInProgress().ConfigureAwait(false)) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress"); // Make sure the database is sane - await database.VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks); + await database + .VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks) + .ConfigureAwait(false); // Guard the last filelist if (!options.DisableSyntheticFilelist) - lastTempFilelist = await database.GetLastIncompleteFilesetVolume(); + lastTempFilelist = await database + .GetLastIncompleteFilesetVolume() + .ConfigureAwait(false); try { @@ -353,7 +365,7 @@ namespace Duplicati.Library.Main.Operation else { await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); - var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name); + var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name).ConfigureAwait(false); if (!string.IsNullOrWhiteSpace(updatedLastTemp.Name) && updatedLastTemp.State == RemoteVolumeState.Deleting) { // The last temporary filelist was emptied, and scheduled for deletion, so we need to delete it @@ -368,7 +380,9 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(database, result.TaskControl.ProgressToken).ConfigureAwait(false); // In case the backend delete failed, manually set the state to deleted - await database.UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null); + await database + .UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null) + .ConfigureAwait(false); // The last temporary filelist is no more lastTempFilelist = default; } @@ -388,7 +402,8 @@ namespace Duplicati.Library.Main.Operation .ConfigureAwait(false); // Re-open the database and backend manager - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options); + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + .ConfigureAwait(false); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); @@ -398,7 +413,7 @@ namespace Duplicati.Library.Main.Operation } // Return the updated version of the fileset, in case it was actually complete - return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume()); + return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume().ConfigureAwait(false)); } catch { @@ -457,7 +472,9 @@ namespace Duplicati.Library.Main.Operation if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1) { - await database.AppendFilesFromPreviousSetAsync(options.DeletedFilelist); + await database + .AppendFilesFromPreviousSetAsync(options.DeletedFilelist) + .ConfigureAwait(false); } else if (journalService != null) { @@ -478,7 +495,8 @@ namespace Duplicati.Library.Main.Operation stats.AddExaminedFile(fileSize); return false; - }); + }) + .ConfigureAwait(false); // store journal data in database, unless job is being canceled if (!result.PartialBackup) @@ -487,10 +505,14 @@ namespace Duplicati.Library.Main.Operation if (data.Any()) { // always record change journal data for current fileset (entry may be dropped later if nothing is uploaded) - await database.CreateChangeJournalDataAsync(data); + await database + .CreateChangeJournalDataAsync(data) + .ConfigureAwait(false); // update the previous fileset's change journal entry to resume at this point in case nothing was backed up - await database.UpdateChangeJournalDataAsync(data, lastfilesetid); + await database + .UpdateChangeJournalDataAsync(data, lastfilesetid) + .ConfigureAwait(false); } } } @@ -501,7 +523,9 @@ namespace Duplicati.Library.Main.Operation } else { - await database.UpdateFilesetAndMarkAsFullBackupAsync(filesetid); + await database + .UpdateFilesetAndMarkAsFullBackupAsync(filesetid) + .ConfigureAwait(false); } result.OperationProgressUpdater.UpdatefileCount(result.ExaminedFiles, result.SizeOfExaminedFiles, true); @@ -516,7 +540,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); - using (var db = await LocalDeleteDatabase.CreateAsync(m_database)) + using (var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false)) await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) .DoRunAsync(db, true, currentIsSmall, backendManager) .ConfigureAwait(false); @@ -526,7 +550,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); - using (var db = await LocalDeleteDatabase.CreateAsync(m_database)) + using (var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false)) await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) .DoCompactAsync(db, true, backendManager) .ConfigureAwait(false); @@ -538,14 +562,19 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(m_database, m_taskReader.ProgressToken); + + await backendManager + .WaitForEmptyAsync(m_database, m_taskReader.ProgressToken) + .ConfigureAwait(false); // Calculate the number of samples to test, using the largest number of file of a given type - long remoteVolumeCount = await m_database.GetRemoteVolumes() + long remoteVolumeCount = await m_database + .GetRemoteVolumes() .Where(x => x.State == RemoteVolumeState.Verified) .GroupBy(x => x.Type) .SelectAwait(async x => await x.LongCountAsync()) - .MaxAsync(); + .MaxAsync() + .ConfigureAwait(false); long samplesToTest = Math.Max(m_options.BackupTestSampleCount, (long)Math.Round(remoteVolumeCount * (m_options.BackupTestPercentage / 100m), MidpointRounding.AwayFromZero)); if (samplesToTest > 0 && remoteVolumeCount > 0) @@ -553,7 +582,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest); m_result.TestResults = new TestResults(m_result); - using (var testdb = await LocalTestDatabase.CreateAsync(m_database)) + using (var testdb = await LocalTestDatabase.CreateAsync(m_database).ConfigureAwait(false)) await new TestHandler(m_options, (TestResults)m_result.TestResults) .DoRunAsync(samplesToTest, testdb, backendManager) .ConfigureAwait(false); @@ -567,15 +596,37 @@ namespace Duplicati.Library.Main.Operation { if (result.BackendWriter != null) { - result.BackendWriter.KnownFileCount = await database.GetRemoteVolumes().CountAsync(); - result.BackendWriter.KnownFilesets = await database.GetRemoteVolumes().CountAsync(x => x.Type == RemoteVolumeType.Files); - result.BackendWriter.KnownFileSize = await database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).SumAsync(); + result.BackendWriter.KnownFileCount = await database + .GetRemoteVolumes() + .CountAsync() + .ConfigureAwait(false); + + result.BackendWriter.KnownFilesets = await database + .GetRemoteVolumes() + .CountAsync(x => x.Type == RemoteVolumeType.Files) + .ConfigureAwait(false); + + result.BackendWriter.KnownFileSize = await database + .GetRemoteVolumes() + .Select(x => Math.Max(0, x.Size)) + .SumAsync() + .ConfigureAwait(false); result.BackendWriter.UnknownFileCount = 0; result.BackendWriter.UnknownFileSize = 0; - result.BackendWriter.BackupListCount = await database.FilesetTimes().CountAsync(); - result.BackendWriter.LastBackupDate = (await database.FilesetTimes().FirstOrDefaultAsync()).Value.ToLocalTime(); + result.BackendWriter.BackupListCount = await database + .FilesetTimes() + .CountAsync() + .ConfigureAwait(false); + + result.BackendWriter.LastBackupDate = ( + await database + .FilesetTimes() + .FirstOrDefaultAsync() + .ConfigureAwait(false) + ) + .Value.ToLocalTime(); if (!options.QuotaDisable) { @@ -638,7 +689,9 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin); // Do a remote verification, unless disabled - var (database, lastTempFilelist) = await PreBackupVerify(m_options, m_result, backendManager); + var (database, lastTempFilelist) = await PreBackupVerify(m_options, m_result, backendManager) + .ConfigureAwait(false); + var lastTempVolumeIncomplete = false; if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name)) @@ -657,7 +710,9 @@ namespace Duplicati.Library.Main.Operation { // If we crash now, there is a chance we leave behind partial remote files if (!m_options.Dryrun) - await database.TerminatedWithActiveUploads(true); + await database + .TerminatedWithActiveUploads(true) + .ConfigureAwait(false); using (m_database = database) using (var db = new Backup.BackupDatabase(m_database, m_options)) @@ -671,37 +726,51 @@ namespace Duplicati.Library.Main.Operation try { // If the previous backup was interrupted, send a synthetic list - await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist); + await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist) + .ConfigureAwait(false); // Grab the previous backup ID, if any - var prevfileset = await m_database.FilesetTimes().FirstOrDefaultAsync(); + var prevfileset = await m_database + .FilesetTimes() + .FirstOrDefaultAsync() + .ConfigureAwait(false); + if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime()) throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime())); var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key; // Rebuild any index files that are missing - await Backup.RecreateMissingIndexFiles.Run(db, backendManager, m_options, m_result.TaskControl); + await Backup.RecreateMissingIndexFiles.Run(db, backendManager, m_options, m_result.TaskControl) + .ConfigureAwait(false); // Prepare the operation by registering the filelist m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles); var repcnt = 0; - while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0) + while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename).ConfigureAwait(false) >= 0) filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++)); - if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0) + if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename).ConfigureAwait(false) >= 0) throw new Exception("Unable to generate a unique fileset name"); - var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); - filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time); + var filesetvolumeid = await db + .RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary) + .ConfigureAwait(false); + + filesetid = await db + .CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time) + .ConfigureAwait(false); var journalService = GetJournalService(source, filter, lastfilesetid); // Start parallel scan, or use the database if (m_options.DisableFileScanner) { - var d = await m_database.GetLastBackupFileCountAndSize(); + var d = await m_database + .GetLastBackupFileCountAndSize() + .ConfigureAwait(false); + m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true); } else @@ -728,47 +797,67 @@ namespace Duplicati.Library.Main.Operation // Ensure the database is in a sane state after adding data using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency")) - await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false); + await db + .VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false) + .ConfigureAwait(false); - await FlushBackend(db, m_result, backendManager).ConfigureAwait(false); + await FlushBackend(db, m_result, backendManager) + .ConfigureAwait(false); // Send the actual filelist - await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl, lastTempVolumeIncomplete); + await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl, lastTempVolumeIncomplete) + .ConfigureAwait(false); // Wait for upload completion m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); - await FlushBackend(db, m_result, backendManager).ConfigureAwait(false); + await FlushBackend(db, m_result, backendManager) + .ConfigureAwait(false); if (!m_options.Dryrun) - await database.TerminatedWithActiveUploads(false); + await database + .TerminatedWithActiveUploads(false) + .ConfigureAwait(false); // Make sure we have the database up-to-date - await db.CommitTransactionAsync("CommitAfterUpload"); + await db.CommitTransactionAsync("CommitAfterUpload") + .ConfigureAwait(false); // If this throws, we should roll back the transaction if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - var lastVolumeSize = await m_database.GetLastWrittenDBlockVolumeSize(); - await CompactIfRequired(backendManager, lastVolumeSize); + var lastVolumeSize = await m_database + .GetLastWrittenDBlockVolumeSize() + .ConfigureAwait(false); + + await CompactIfRequired(backendManager, lastVolumeSize) + .ConfigureAwait(false); } if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database) + .ConfigureAwait(false); } if (!m_options.Dryrun) { - await database.Transaction.CommitAsync("CommitFinalizingBackup"); + await database.Transaction + .CommitAsync("CommitFinalizingBackup") + .ConfigureAwait(false); + if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { if (m_options.NoBackendverification) - await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); + await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken) + .ConfigureAwait(false); else - await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager).ConfigureAwait(false); + await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager) + .ConfigureAwait(false); } - await database.Transaction.CommitAsync("CommitAfterVerification"); + await database.Transaction + .CommitAsync("CommitAfterVerification") + .ConfigureAwait(false); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); diff --git a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs index 364b4bd00..fba1a951b 100644 --- a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs +++ b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs @@ -51,12 +51,20 @@ namespace Duplicati.Library.Main.Operation.Common public Task RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state) { - return RunOnMain(async () => await m_db.RegisterRemoteVolume(name, type, state)); + return RunOnMain(async () => + await m_db + .RegisterRemoteVolume(name, type, state) + .ConfigureAwait(false) + ); } public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan), bool? setArchived = null) { - return RunOnMain(async () => await m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived)); + return RunOnMain(async () => + await m_db + .UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived) + .ConfigureAwait(false) + ); } public async Task FlushBackendMessagesAndCommitAsync(IBackendManager backendManager) @@ -74,58 +82,96 @@ namespace Duplicati.Library.Main.Operation.Common { if (m_options.Dryrun) { - await m_db.Transaction.RollBackAsync(); + await m_db.Transaction + .RollBackAsync() + .ConfigureAwait(false); } else { using (new Logging.Timer(LOGTAG, "CommitTransactionAsync", message)) - await m_db.Transaction.CommitAsync(message, restart); + await m_db.Transaction + .CommitAsync(message, restart) + .ConfigureAwait(false); } }); } public Task RollbackTransactionAsync() { - return RunOnMain(async () => await m_db.Transaction.RollBackAsync()); + return RunOnMain(async () => + await m_db.Transaction + .RollBackAsync() + .ConfigureAwait(false) + ); } public Task RenameRemoteFileAsync(string oldname, string newname) { - return RunOnMain(async () => await m_db.RenameRemoteFile(oldname, newname)); + return RunOnMain(async () => + await m_db + .RenameRemoteFile(oldname, newname) + .ConfigureAwait(false) + ); } public Task LogRemoteOperationAsync(string operation, string path, string data) { - return RunOnMain(async () => await m_db.LogRemoteOperation(operation, path, data)); + return RunOnMain(async () => + await m_db + .LogRemoteOperation(operation, path, data) + .ConfigureAwait(false) + ); } public Task GetBlocksAsync(long volumeid) { // TODO: Figure out how to return the enumerable, while keeping the lock // and not creating the entire result in memory - return RunOnMain(async () => await m_db.GetBlocks(volumeid).ToArrayAsync()); + return RunOnMain(async () => + await m_db + .GetBlocks(volumeid) + .ToArrayAsync() + .ConfigureAwait(false) + ); } public Task GetVolumeInfoAsync(string remotename) { - return RunOnMain(async () => await m_db.GetRemoteVolume(remotename)); + return RunOnMain(async () => + await m_db + .GetRemoteVolume(remotename) + .ConfigureAwait(false) + ); } public Task<(string Hash, byte[] Buffer, int Size)[]> GetBlocklistsAsync(long volumeid, int blocksize, int hashsize) { // TODO: Figure out how to return the enumerable, while keeping the lock // and not creating the entire result in memory - return RunOnMain(async () => await m_db.GetBlocklists(volumeid, blocksize, hashsize).ToArrayAsync()); + return RunOnMain(async () => + await m_db + .GetBlocklists(volumeid, blocksize, hashsize) + .ToArrayAsync() + .ConfigureAwait(false) + ); } public Task GetRemoteVolumeIDAsync(string remotename) { - return RunOnMain(async () => await m_db.GetRemoteVolumeID(remotename)); + return RunOnMain(async () => + await m_db + .GetRemoteVolumeID(remotename) + .ConfigureAwait(false) + ); } public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID) { - return RunOnMain(async () => await m_db.AddIndexBlockLink(indexVolumeID, blockVolumeID)); + return RunOnMain(async () => + await m_db + .AddIndexBlockLink(indexVolumeID, blockVolumeID) + .ConfigureAwait(false) + ); } diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 2c1812160..e7c642410 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -52,10 +52,12 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache)) + using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache).ConfigureAwait(false)) { - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); var changed = await DoCompactAsync(db, false, backendManager).ConfigureAwait(false); @@ -64,10 +66,13 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { - await db.Transaction.CommitAsync("CommitCompact"); + await db.Transaction + .CommitAsync("CommitCompact") + .ConfigureAwait(false); + if (changed) { - await db.WriteResults(m_result); + await db.WriteResults(m_result).ConfigureAwait(false); if (m_options.AutoVacuum) { m_result.VacuumResults = new VacuumResults(m_result); @@ -80,7 +85,10 @@ namespace Duplicati.Library.Main.Operation internal async Task DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, IBackendManager backendManager) { - var report = await db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount); + var report = await db + .GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount) + .ConfigureAwait(false); + report.ReportCompactData(); if (report.ShouldReclaim || report.ShouldCompact) @@ -90,19 +98,30 @@ namespace Duplicati.Library.Main.Operation await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); var newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = await db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); + newvol.VolumeID = await db + .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary) + .ConfigureAwait(false); IndexVolumeWriter newvolindex = null; if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = await db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); - await db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID); + newvolindex.VolumeID = await db + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .ConfigureAwait(false); + + await db + .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID) + .ConfigureAwait(false); } var blocksInVolume = 0L; var buffer = new byte[m_options.Blocksize]; - var remoteList = await db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArrayAsync(); + var remoteList = await db + .GetRemoteVolumes() + .Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified) + .ToArrayAsync() + .ConfigureAwait(false); //These are for bookkeeping var uploadedVolumes = new List>(); @@ -132,7 +151,9 @@ namespace Duplicati.Library.Main.Operation { // If we crash now, we may leave partial files if (!m_options.Dryrun) - await db.TerminatedWithActiveUploads(true); + await db + .TerminatedWithActiveUploads(true) + .ConfigureAwait(false); newvolindex?.StartVolume(newvol.RemoteFilename); List volumesToDownload = []; @@ -146,7 +167,7 @@ namespace Duplicati.Library.Main.Operation .ToList(); } - using (var q = await db.CreateBlockQueryHelper()) + using (var q = await db.CreateBlockQueryHelper().ConfigureAwait(false)) { await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { @@ -160,38 +181,56 @@ namespace Duplicati.Library.Main.Operation } downloadedVolumes.Add(new KeyValuePair(entry.Name, entry.Size)); - var volumeid = await db.GetRemoteVolumeID(entry.Name); + var volumeid = await db + .GetRemoteVolumeID(entry.Name) + .ConfigureAwait(false); + var inst = VolumeBase.ParseFilename(entry.Name); using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options)) { foreach (var e in f.Blocks) { - if (await q.UseBlock(e.Key, e.Value, volumeid)) + if (await q.UseBlock(e.Key, e.Value, volumeid).ConfigureAwait(false)) { //TODO: How do we get the compression hint? Reverse query for filename in db? var s = f.ReadBlock(e.Key, buffer); if (s != e.Value) throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value)); - await newvol.AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible); + await newvol + .AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible) + .ConfigureAwait(false); + if (newvolindex != null) newvolindex.AddBlock(e.Key, e.Value); - await db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID); + await db + .RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID) + .ConfigureAwait(false); + blocksInVolume++; if (newvol.Filesize > (m_options.VolumeSize - m_options.Blocksize)) { - await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes); + await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes) + .ConfigureAwait(false); newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = await db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); + newvol.VolumeID = await db + .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary) + .ConfigureAwait(false); if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = await db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); - await db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID); + newvolindex.VolumeID = await db + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .ConfigureAwait(false); + + await db + .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID) + .ConfigureAwait(false); + newvolindex.StartVolume(newvol.RemoteFilename); } @@ -202,7 +241,9 @@ namespace Duplicati.Library.Main.Operation // Commit as we have uploaded a volume if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitCompact"); + await db.Transaction + .CommitAsync("CommitCompact") + .ConfigureAwait(false); if (deleteableVolumes.Any()) { @@ -226,10 +267,16 @@ namespace Duplicati.Library.Main.Operation } else { - await db.RemoveRemoteVolume(newvol.RemoteFilename); + await db + .RemoveRemoteVolume(newvol.RemoteFilename) + .ConfigureAwait(false); + if (newvolindex != null) { - await db.RemoveRemoteVolume(newvolindex.RemoteFilename); + await db + .RemoveRemoteVolume(newvolindex.RemoteFilename) + .ConfigureAwait(false); + newvolindex.FinishVolume(null, 0); } } @@ -237,7 +284,9 @@ namespace Duplicati.Library.Main.Operation // The remainder of the operation cannot leave partial files if (!m_options.Dryrun) - await db.TerminatedWithActiveUploads(false); + await db + .TerminatedWithActiveUploads(false) + .ConfigureAwait(false); } else { @@ -305,27 +354,38 @@ namespace Duplicati.Library.Main.Operation private async IAsyncEnumerable> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable deleteableVolumes, [EnumeratorCancellation] CancellationToken cancellationToken) { // Find volumes that can be deleted - var remoteFilesToRemove = await db.ReOrderDeleteableVolumes(deleteableVolumes).ToListAsync(); + var remoteFilesToRemove = await db + .ReOrderDeleteableVolumes(deleteableVolumes) + .ToListAsync() + .ConfigureAwait(false); // Make sure we do not re-assign blocks to any of the volumes we are about to delete var toRemoveVolumeIds = await db .GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name)) .Select(x => x.Value) .Distinct() - .ToListAsync(); + .ToListAsync() + .ConfigureAwait(false); // Mark all volumes and relevant index files as disposable foreach (var f in remoteFilesToRemove) { - await db.PrepareForDelete(f.Name, toRemoveVolumeIds); - await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash); + await db + .PrepareForDelete(f.Name, toRemoveVolumeIds) + .ConfigureAwait(false); + + await db + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .ConfigureAwait(false); } // Before we commit the current state, make sure the backend has caught up await backend.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitDelete"); + await db.Transaction + .CommitAsync("CommitDelete") + .ConfigureAwait(false); await foreach (var d in PerformDelete(backend, remoteFilesToRemove, cancellationToken).ConfigureAwait(false)) yield return d; @@ -337,7 +397,7 @@ namespace Duplicati.Library.Main.Operation if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { - await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) + await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize).ConfigureAwait(false)) newvolindex.WriteBlocklist(blocklist.Hash, blocklist.Buffer, 0, blocklist.Size); }; @@ -348,7 +408,9 @@ namespace Duplicati.Library.Main.Operation // We can handle at most one in-flight upload at a time, // because the transaction is not thread-safe, and shared with the upload await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - await db.UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await db + .UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .ConfigureAwait(false); // TODO: The upload here does not flush the database messages, // and this can leave the database in a state where it does not know of the remote file @@ -356,7 +418,10 @@ namespace Duplicati.Library.Main.Operation // Once fixed, we can perhaps let he backend manager simply call the database directly if (!m_options.Dryrun) { - await db.Transaction.CommitAsync("CommitUpload"); + await db.Transaction + .CommitAsync("CommitUpload") + .ConfigureAwait(false); + await backendManager.PutAsync(newvol, newvolindex, indexVolumeFinished, false, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } else diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index bf5df7633..deca98811 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -68,11 +68,11 @@ namespace Duplicati.Library.Main.Operation using (var tmp = new TempFile()) { File.Copy(m_options.Dbpath, tmp, true); - using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache)) + using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache).ConfigureAwait(false)) { - await db.Fix(); + await db.Fix().ConfigureAwait(false); if (m_options.AutoVacuum) - await db.Vacuum(); + await db.Vacuum().ConfigureAwait(false); } // Apply zip64 option if not already set diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 6516fbd69..c9392cdd7 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -51,15 +51,20 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache)) + using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache).ConfigureAwait(false)) { - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("ComitDelete"); + await db.Transaction + .CommitAsync("ComitDelete") + .ConfigureAwait(false); } } @@ -70,7 +75,11 @@ namespace Duplicati.Library.Main.Operation // We collapse the async enumerablo into a array to avoid multiple // enumerations (and thus multiple database queries) - var filesets = await db.FilesetsWithBackupVersion().ToArrayAsync(); + var filesets = await db + .FilesetsWithBackupVersion() + .ToArrayAsync() + .ConfigureAwait(false); + List versionsToDelete = [ .. new SpecificVersionsRemover(m_options).GetFilesetsToDelete(filesets), @@ -98,12 +107,24 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - var lst = await db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray()).ToArrayAsync(); + var lst = await db + .DropFilesetsFromTable( + versionsToDelete + .Select(x => x.Time) + .ToArray() + ) + .ToArrayAsync() + .ConfigureAwait(false); + foreach (var f in lst) - await db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null); + await db + .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null) + .ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitBeforeDelete"); + await db.Transaction + .CommitAsync("CommitBeforeDelete") + .ConfigureAwait(false); foreach (var f in lst) { diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index c4d4d07d6..4b0455e61 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -69,7 +69,7 @@ namespace Duplicati.Library.Main.Operation public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) { var locallist = database.GetRemoteVolumes(); - await foreach (var i in locallist) + await foreach (var i in locallist.ConfigureAwait(false)) { switch (i.State) { @@ -117,8 +117,14 @@ namespace Duplicati.Library.Main.Operation { if (!options.NoBackendverification) { - LocalBackupDatabase backupDatabase = await LocalBackupDatabase.CreateAsync(database, options); - IEnumerable protectedFiles = await backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly); + LocalBackupDatabase backupDatabase = + await LocalBackupDatabase.CreateAsync(database, options) + .ConfigureAwait(false); + + IEnumerable protectedFiles = await backupDatabase + .GetTemporaryFilelistVolumeNames(latestVolumesOnly) + .ConfigureAwait(false); + await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); } } @@ -250,7 +256,7 @@ namespace Duplicati.Library.Main.Operation public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) { // If the last operation completed, no cleanup should be required - if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads()) + if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads().ConfigureAwait(false)) verifyMode = VerifyMode.VerifyStrict; // Force cleanup should set the mode to cleanup @@ -288,7 +294,10 @@ namespace Duplicati.Library.Main.Operation log.KnownFileSize = knownFileSize; log.UnknownFileCount = unknownlist.Count; log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum(); - log.BackupListCount = await database.FilesetTimes().CountAsync(); + log.BackupListCount = await database + .FilesetTimes() + .CountAsync() + .ConfigureAwait(false); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); await CheckQuota(backendManager, options, log, knownFileSize).ConfigureAwait(false); @@ -302,16 +311,18 @@ namespace Duplicati.Library.Main.Operation var missingUploadingVolumes = new HashSet(); var protectedVolumes = new HashSet(); - await foreach (var e in database.DuplicateRemoteVolumes()) + await foreach (var e in database.DuplicateRemoteVolumes().ConfigureAwait(false)) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) - await database.UnlinkRemoteVolume(e.Key, e.Value); + await database + .UnlinkRemoteVolume(e.Key, e.Value) + .ConfigureAwait(false); else throw new RemoteListVerificationException(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()), "AmbiguousStateRemoteFiles"); } var locallist = database.GetRemoteVolumes(); - await foreach (var i in locallist) + await foreach (var i in locallist.ConfigureAwait(false)) { var remoteFound = lookup.TryGetValue(i.Name, out var r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); @@ -319,9 +330,13 @@ namespace Duplicati.Library.Main.Operation if (archived && r.File.Size == 0) correctSize = true; if (archived && i.ArchiveTime <= DateTime.UnixEpoch) - await database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true); + await database + .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true) + .ConfigureAwait(false); else if (!archived && i.ArchiveTime > DateTime.UnixEpoch) - await database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false); + await database + .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false) + .ConfigureAwait(false); lookup.Remove(i.Name); @@ -376,7 +391,9 @@ namespace Duplicati.Library.Main.Operation if (remoteFound && correctSize && r.File.Size >= 0) { Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); - await database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); + await database + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash) + .ConfigureAwait(false); } else if (!remoteFound) { @@ -384,7 +401,9 @@ namespace Duplicati.Library.Main.Operation { protectedVolumes.Add(i.Name); Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); - await database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null); + await database + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null) + .ConfigureAwait(false); } else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) { @@ -395,7 +414,9 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); missingUploadingVolumes.Add(i.Name); - await database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null); + await database + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null) + .ConfigureAwait(false); } } else @@ -433,7 +454,9 @@ namespace Duplicati.Library.Main.Operation if (!remoteFound) missing.Add(i); else if (correctSize) - await database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); + await database + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash) + .ConfigureAwait(false); else missingHash.Add(new Tuple(r.File.Size, i)); @@ -460,12 +483,20 @@ namespace Duplicati.Library.Main.Operation throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); else if (verifyMode == VerifyMode.VerifyAndClean || verifyMode == VerifyMode.VerifyStrict) { - await database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes)); + await database + .RemoveRemoteVolumes( + missingUploadingVolumes + .Concat(temporaryAndDeletedVolumes) + ) + .ConfigureAwait(false); + // Clear the flag after we have cleaned up if (!options.Dryrun && protectedVolumes.Count == 0) - await database.TerminatedWithActiveUploads(false); + await database + .TerminatedWithActiveUploads(false) + .ConfigureAwait(false); } - else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads()) + else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads().ConfigureAwait(false)) { Logging.Log.WriteWarningMessage(LOGTAG, "ActiveUploadsDetected", null, "Active uploads detected, but no cleanup was performed. Run the \"repair\" command to clean up the incomplete files"); } diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 1586e3df5..10920ee99 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -44,24 +44,42 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) { if (callback == null) { m_result.SetResult( - await db.GetFilesets(args).OrderByDescending(x => x.Time).ToArrayAsync(), - await db.GetFiles(args).ToArrayAsync(), - await db.GetLogLines(args).ToArrayAsync(), - await db.GetVolumes(args).ToArrayAsync() + await db.GetFilesets(args) + .OrderByDescending(x => x.Time) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetFiles(args) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetLogLines(args) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetVolumes(args) + .ToArrayAsync() + .ConfigureAwait(false) ); } else { m_result.SetResult( - await db.GetFilesets(args).OrderByDescending(x => x.Time).ToArrayAsync(), - await db.GetFiles(args).ToArrayAsync(), - await db.GetLogLines(args).ToArrayAsync(), - await db.GetVolumes(args).ToArrayAsync() + await db.GetFilesets(args) + .OrderByDescending(x => x.Time) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetFiles(args) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetLogLines(args) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetVolumes(args) + .ToArrayAsync() + .ConfigureAwait(false) ); callback(m_result); diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 732ba47ac..2b7ca9224 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -52,10 +52,12 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) { - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); await db.Transaction.RollBackAsync().ConfigureAwait(false); } @@ -64,11 +66,14 @@ namespace Duplicati.Library.Main.Operation public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[]?, List? Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, Options options) { List? missing = null; - var brokensets = await db.GetBrokenFilesets(options.Time, options.Version).ToArrayAsync(); + var brokensets = await db + .GetBrokenFilesets(options.Time, options.Version) + .ToArrayAsync() + .ConfigureAwait(false); if (brokensets.Length == 0) { - if (await db.RepairInProgress()) + if (await db.RepairInProgress().ConfigureAwait(false)) throw new UserInformationException("Cannot continue because the database is marked as being under repair, but does not have broken files.", "CannotListOnDatabaseInRepair"); Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); @@ -86,18 +91,27 @@ namespace Duplicati.Library.Main.Operation // Mark all volumes as disposable foreach (var f in missing) - await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash); + await db + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count); // Drop all content from tables - await db.RemoveMissingBlocks(missing.Select(x => x.Name)); + await db + .RemoveMissingBlocks(missing.Select(x => x.Name)) + .ConfigureAwait(false); // Mark all orphaned index files as disposable after removing the missing block files - await foreach (var f in db.GetOrphanedIndexFiles()) - await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash); + await foreach (var f in db.GetOrphanedIndexFiles().ConfigureAwait(false)) + await db + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .ConfigureAwait(false); - brokensets = await db.GetBrokenFilesets(options.Time, options.Version).ToArrayAsync(); + brokensets = await db + .GetBrokenFilesets(options.Time, options.Version) + .ToArrayAsync() + .ConfigureAwait(false); } return (brokensets, missing); @@ -108,7 +122,7 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles"); - if (await db.PartiallyRecreated()) + if (await db.PartiallyRecreated().ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase"); (var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); @@ -129,7 +143,10 @@ namespace Duplicati.Library.Main.Operation return; } - var fstimes = await db.FilesetTimes().ToListAsync(); + var fstimes = await db + .FilesetTimes() + .ToListAsync() + .ConfigureAwait(false); var brokenfilesets = brokensets.Select(x => new @@ -149,14 +166,18 @@ namespace Duplicati.Library.Main.Operation x.Version, x.Timestamp, callbackhandler == null && !m_options.ListSetsOnly - ? await db.GetBrokenFilenames(x.FilesetID).ToArrayAsync() + ? await db + .GetBrokenFilenames(x.FilesetID) + .ToArrayAsync() + .ConfigureAwait(false) : new MockList>((int)x.BrokenCount) - ))); + ))) + .ConfigureAwait(false); if (callbackhandler != null) foreach (var bs in brokenfilesets) - await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID)) + await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID).ConfigureAwait(false)) if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2)) break; } diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index f056c3e76..e83d07fa2 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -80,12 +80,16 @@ namespace Duplicati.Library.Main.Operation DateTime compareVersionTime; using (var tmpdb = useLocalDb ? null : new TempFile()) - using (var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache)) - using (var storageKeeper = await db.CreateStorageHelper()) + using (var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache).ConfigureAwait(false)) + using (var storageKeeper = await db.CreateStorageHelper().ConfigureAwait(false)) { if (useLocalDb) { - var dbtimes = await db.FilesetTimes().ToListAsync(); + var dbtimes = await db + .FilesetTimes() + .ToListAsync() + .ConfigureAwait(false); + if (dbtimes.Count < 2) throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", dbtimes.Count), "NeedTwoBackupsToStartDiff"); @@ -97,8 +101,12 @@ namespace Duplicati.Library.Main.Operation times.Remove(bt); SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId); - await storageKeeper.AddFromDb(baseVersionId, false, filter); - await storageKeeper.AddFromDb(compareVersionId, true, filter); + await storageKeeper + .AddFromDb(baseVersionId, false, filter) + .ConfigureAwait(false); + await storageKeeper + .AddFromDb(compareVersionId, true, filter) + .ConfigureAwait(false); } else { @@ -143,7 +151,9 @@ namespace Duplicati.Library.Main.Operation using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options)) foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) - await storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false); + await storageKeeper + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false) + .ConfigureAwait(false); if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -152,11 +162,18 @@ namespace Duplicati.Library.Main.Operation using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options)) foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) - await storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true); + await storageKeeper + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true) + .ConfigureAwait(false); } - var changes = await storageKeeper.CreateChangeCountReport(); - var sizes = await storageKeeper.CreateChangeSizeReport(); + var changes = await storageKeeper + .CreateChangeCountReport() + .ConfigureAwait(false); + + var sizes = await storageKeeper + .CreateChangeSizeReport() + .ConfigureAwait(false); var lst = (m_options.FullResult || callback != null) ? (from n in storageKeeper.CreateChangedFileReport() @@ -168,7 +185,7 @@ namespace Duplicati.Library.Main.Operation changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles, changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles, sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize, - (lst == null || callback == null) ? null : await lst.ToArrayAsync() + (lst == null || callback == null) ? null : await lst.ToArrayAsync().ConfigureAwait(false) ); if (callback != null) diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index f43e75d94..96a311e70 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.Operation { var cancellationToken = CancellationToken.None; using (var tmpdb = new TempFile()) - using (var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache)) + using (var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false)) { var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); @@ -62,7 +62,9 @@ namespace Duplicati.Library.Main.Operation return; var file = fileversion.Value.File; - var entry = await db.GetRemoteVolume(file.Name); + var entry = await db + .GetRemoteVolume(file.Name) + .ConfigureAwait(false); var files = new List(); using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false)) diff --git a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs index af107ad7b..05447a7ef 100644 --- a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs @@ -47,11 +47,17 @@ internal static class ListFileVersionsHandler if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache); + using var db = + await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) { - filesetIds = await db.GetFilesetIDs(options.Time, options.Version).ToArrayAsync(); + filesetIds = await db + .GetFilesetIDs(options.Time, options.Version) + .ToArrayAsync() + .ConfigureAwait(false); + if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); } @@ -60,6 +66,8 @@ internal static class ListFileVersionsHandler throw new UserInformationException("No path specified", "NoPathSpecified"); paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); - result.FileVersions = await db.ListFileVersions(paths, filesetIds, offset, limit); + result.FileVersions = await db + .ListFileVersions(paths, filesetIds, offset, limit) + .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 30a3341b6..4e4487aef 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -55,15 +55,17 @@ namespace Duplicati.Library.Main.Operation //Use a speedy local query if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) - using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) { - using (var filesets = await db.SelectFileSets(m_options.Time, m_options.Version)) + using (var filesets = await db.SelectFileSets(m_options.Time, m_options.Version).ConfigureAwait(false)) { if (!filter.Empty) { if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) { - await filesets.TakeFirst(); + await filesets + .TakeFirst() + .ConfigureAwait(false); } } @@ -91,7 +93,8 @@ namespace Duplicati.Library.Main.Operation await filesets .QuickSets() .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) - .ToArrayAsync(), + .ToArrayAsync() + .ConfigureAwait(false), null ); } @@ -101,7 +104,8 @@ namespace Duplicati.Library.Main.Operation await filesets .Sets() .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) - .ToArrayAsync(), + .ToArrayAsync() + .ConfigureAwait(false), files == null ? null : @@ -135,13 +139,14 @@ namespace Duplicati.Library.Main.Operation // Otherwise, grab info from remote location using (var tmpdb = new TempFile()) - using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache)) + using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache).ConfigureAwait(false)) { var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) throw new UserInformationException("No filesets found on remote target", "EmptyRemoteFolder"); - var numberSeq = await CreateResultSequence(filteredList, backendManager, m_options, cancellationToken); + var numberSeq = await CreateResultSequence(filteredList, backendManager, m_options, cancellationToken) + .ConfigureAwait(false); if (filter.Empty) { m_result.SetResult(numberSeq, null); diff --git a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs index d5e74b1bc..32e28cb6d 100644 --- a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs @@ -49,8 +49,14 @@ internal static class ListFilesetsHandler // Use a speedy local query if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb) { - using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache); - result.Filesets = await db.ListFilesetsExtended().ToArrayAsync(); + using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + .ConfigureAwait(false); + + result.Filesets = await db + .ListFilesetsExtended() + .ToArrayAsync() + .ConfigureAwait(false); + return; } diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index 7d90dfec3..9562cb98e 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -46,8 +46,14 @@ internal static class ListFolderHandler if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache); - var filesetIds = await db.GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true).ToArrayAsync(); + using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + .ConfigureAwait(false); + + var filesetIds = await db + .GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true) + .ToArrayAsync() + .ConfigureAwait(false); + if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); if (filesetIds.Length > 1) @@ -57,11 +63,25 @@ internal static class ListFolderHandler { if (folders != null && folders.Length > 1) throw new UserInformationException("When no folder is specified, only one folder can be listed", "MultipleFoldersFound"); - result.Entries = await db.ListFilesetEntries(db.GetRootPrefixes(filesetIds[0]).ToEnumerable(), filesetIds[0], offset, limit); + result.Entries = await db + .ListFilesetEntries( + db.GetRootPrefixes(filesetIds[0]).ToEnumerable(), + filesetIds[0], + offset, + limit + ) + .ConfigureAwait(false); } else { - result.Entries = await db.ListFolder(db.GetPrefixIds(folders).ToEnumerable(), filesetIds[0], offset, limit); + result.Entries = await db + .ListFolder( + db.GetPrefixIds(folders).ToEnumerable(), + filesetIds[0], + offset, + limit + ) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 69ed321ac..9b68834b9 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -54,13 +54,15 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - using (var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) { - if (await db.PartiallyRecreated()) + if (await db.PartiallyRecreated().ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); if (sets == null) @@ -82,7 +84,10 @@ namespace Duplicati.Library.Main.Operation var pgoffset = 0.0f; var pgspan = 0.95f / sets.Length; - var filesets = await db.FilesetTimes().ToListAsync(); + var filesets = await db + .FilesetTimes() + .ToListAsync() + .ConfigureAwait(false); var compare_list = sets.Select(async x => new { @@ -90,7 +95,9 @@ namespace Duplicati.Library.Main.Operation Timestamp = x.FilesetTime, RemoveCount = x.RemoveCount, Version = filesets.FindIndex(y => y.Key == x.FilesetID), - SetCount = await db.GetFilesetFileCount(x.FilesetID) + SetCount = await db + .GetFilesetFileCount(x.FilesetID) + .ConfigureAwait(false) }) .Select(x => x.Result) .ToArray(); @@ -99,7 +106,13 @@ namespace Duplicati.Library.Main.Operation if (!m_options.DisableReplaceMissingMetadata) { var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); - replacementMetadataBlocksetId = await db.GetEmptyMetadataBlocksetId((missing ?? []).Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length); + replacementMetadataBlocksetId = await db + .GetEmptyMetadataBlocksetId( + (missing ?? []).Select(x => x.ID), + emptymetadata.FileHash, + emptymetadata.Blob.Length + ) + .ConfigureAwait(false); if (replacementMetadataBlocksetId < 0) throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); } @@ -107,11 +120,11 @@ namespace Duplicati.Library.Main.Operation var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); - if (fully_emptied.Length == await db.FilesetTimes().CountAsync()) + if (fully_emptied.Length == await db.FilesetTimes().CountAsync().ConfigureAwait(false)) throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); if (!m_options.Dryrun) - await db.Transaction.CommitAsync(); + await db.Transaction.CommitAsync().ConfigureAwait(false); if (fully_emptied.Length != 0) @@ -122,7 +135,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); m_result.DeleteResults = new DeleteResults(m_result); - using (var rmdb = await LocalDeleteDatabase.CreateAsync(db)) + using (var rmdb = await LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) { var opts = new Options(new Dictionary(m_options.RawOptions)); opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); @@ -133,7 +146,9 @@ namespace Duplicati.Library.Main.Operation .DoRunAsync(rmdb, true, false, backendManager).ConfigureAwait(false); if (!m_options.Dryrun) - await rmdb.Transaction.CommitAsync("CommitDelete"); + await rmdb.Transaction + .CommitAsync("CommitDelete") + .ConfigureAwait(false); } pgoffset += (pgspan * fully_emptied.Length); @@ -149,10 +164,13 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); var opts = new Options(new Dictionary(m_options.RawOptions)); - using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db)) + using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db).ConfigureAwait(false)) { // Recompute the version number after we deleted the versions before - filesets = await pgdb.FilesetTimes().ToListAsync(); + filesets = await pgdb + .FilesetTimes() + .ToListAsync() + .ConfigureAwait(false); var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); if (thisversion < 0) throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); @@ -184,17 +202,21 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0.95f); - if (!m_options.Dryrun && await db.RepairInProgress()) + if (!m_options.Dryrun && await db.RepairInProgress().ConfigureAwait(false)) { Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); - await db.RepairInProgress(false); + await db.RepairInProgress(false).ConfigureAwait(false); } else { - await db.Transaction.RollBackAsync(); - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); + await db.Transaction.RollBackAsync().ConfigureAwait(false); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); } m_result.OperationProgressUpdater.UpdateProgress(1.0f); diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index b743c7d31..ccec9dc13 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -52,10 +52,12 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) { await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); } } @@ -69,29 +71,33 @@ namespace Duplicati.Library.Main.Operation var doCompactStep = !m_options.NoAutoCompact && filtercommand == null; - if (await db.PartiallyRecreated()) + if (await db.PartiallyRecreated().ConfigureAwait(false)) throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase"); - if (await db.RepairInProgress() && filtercommand == null) + if (await db.RepairInProgress().ConfigureAwait(false) && filtercommand == null) throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase"); var versions = await db .GetFilesetIDs(m_options.Time, m_options.Version) .OrderByDescending(x => x) - .ToArrayAsync(); + .ToArrayAsync() + .ConfigureAwait(false); if (versions.Length <= 0) throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion"); - var orphans = await db.CountOrphanFiles(); + var orphans = await db.CountOrphanFiles().ConfigureAwait(false); if (orphans != 0) throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans"); - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); if (filtercommand == null) { - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false) + .ConfigureAwait(false); if (m_options.NoBackendverification) await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); @@ -102,7 +108,8 @@ namespace Duplicati.Library.Main.Operation var filesets = await db .FilesetTimes() .OrderByDescending(x => x.Value) - .ToArrayAsync(); + .ToArrayAsync() + .ConfigureAwait(false); var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan; var currentprogress = pgoffset; @@ -113,7 +120,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, it is possible that the remote storage contains partial files if (!m_options.Dryrun) - await db.TerminatedWithActiveUploads(true); + await db.TerminatedWithActiveUploads(true).ConfigureAwait(false); // Reverse makes sure we re-write the old versions first foreach (var versionid in versions.Reverse()) @@ -125,24 +132,29 @@ namespace Duplicati.Library.Main.Operation if (ix < 0 || tsOriginal.Ticks == 0) throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); - var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal); - var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key); + var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal) + .ConfigureAwait(false); + var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key) + .ConfigureAwait(false); if (ix != 0 && filesets[ix - 1].Value <= ts) throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); - using (var tempset = await db.CreateTemporaryFileset(versionid)) + using (var tempset = await db.CreateTemporaryFileset(versionid).ConfigureAwait(false)) { if (filtercommand == null) - await tempset.ApplyFilter(filter); + await tempset.ApplyFilter(filter).ConfigureAwait(false); else - await tempset.ApplyFilter(filtercommand); + await tempset.ApplyFilter(filtercommand) + .ConfigureAwait(false); if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) { Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); currentprogress += versionprogress; - await db.Transaction.RollBackAsync(); + await db.Transaction + .RollBackAsync() + .ConfigureAwait(false); continue; } else @@ -150,14 +162,20 @@ namespace Duplicati.Library.Main.Operation using (var tf = new TempFile()) using (var vol = new FilesetVolumeWriter(m_options, ts)) { - var isOriginalFilesetFullBackup = await db.IsFilesetFullBackup(tsOriginal); - var newids = await tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); + var isOriginalFilesetFullBackup = await db + .IsFilesetFullBackup(tsOriginal) + .ConfigureAwait(false); + var newids = await tempset + .ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup) + .ConfigureAwait(false); vol.VolumeID = newids.Item1; vol.CreateFilesetFile(isOriginalFilesetFullBackup); Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); - await db.WriteFileset(vol, newids.Item2); + await db + .WriteFileset(vol, newids.Item2) + .ConfigureAwait(false); m_result.RemovedFileSize += tempset.RemovedFileSize; m_result.RemovedFileCount += tempset.RemovedFileCount; @@ -169,7 +187,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.Dryrun || m_options.FullResult) { - await foreach (var fe in tempset.ListAllDeletedFiles()) + await foreach (var fe in tempset.ListAllDeletedFiles().ConfigureAwait(false)) { var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); @@ -189,16 +207,27 @@ namespace Duplicati.Library.Main.Operation if (m_options.Dryrun) { Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount); - await db.Transaction.RollBackAsync(); + await db.Transaction + .RollBackAsync() + .ConfigureAwait(false); } else { - await db.UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - var lst = await db.DropFilesetsFromTable(new[] { tsOriginal }).ToArrayAsync(); + await db + .UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .ConfigureAwait(false); + var lst = await db + .DropFilesetsFromTable(new[] { tsOriginal }) + .ToArrayAsync() + .ConfigureAwait(false); foreach (var f in lst) - await db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null); + await db + .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null) + .ConfigureAwait(false); - await db.Transaction.CommitAsync(); + await db.Transaction + .CommitAsync() + .ConfigureAwait(false); await backendManager.PutAsync(vol, null, null, true, async () => { @@ -207,7 +236,9 @@ namespace Duplicati.Library.Main.Operation // Microsoft.Data.Sqlite backend way of not // passing down a transaction, but maybe is // not needed anymore. - await db.Transaction.CommitAsync(); + await db.Transaction + .CommitAsync() + .ConfigureAwait(false); }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); @@ -221,7 +252,9 @@ namespace Duplicati.Library.Main.Operation } if (!m_options.Dryrun) - await db.TerminatedWithActiveUploads(false); + await db + .TerminatedWithActiveUploads(false) + .ConfigureAwait(false); if (doCompactStep) { @@ -234,7 +267,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); m_result.CompactResults = new CompactResults(m_result); - using (var cdb = await Database.LocalDeleteDatabase.CreateAsync(db)) + using (var cdb = await Database.LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) { await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) .DoCompactAsync(cdb, true, backendManager) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 77cd1a001..a230b3438 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -64,12 +64,16 @@ namespace Duplicati.Library.Main.Operation if (File.Exists(path)) throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); - using var db = await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache); + using var db = + await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache) + .ConfigureAwait(false); await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); // Ensure database is consistent after the recreate - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); } /// @@ -83,8 +87,10 @@ namespace Duplicati.Library.Main.Operation if (!m_options.RepairOnlyPaths) throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); - using var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache); - if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version)).Any()) + using var db = + await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache) + .ConfigureAwait(false); + if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version).ConfigureAwait(false)).Any()) { if (m_options.IgnoreUpdateIfVersionExists) { @@ -96,14 +102,18 @@ namespace Duplicati.Library.Main.Operation } // Mark as incomplete - await db.PartiallyRecreated(true); + await db.PartiallyRecreated(true).ConfigureAwait(false); - var preexistingOptionsInDatabase = await Utility.ContainsOptionsForVerification(db); - await Utility.UpdateOptionsFromDb(db, m_options); + var preexistingOptionsInDatabase = + await Utility.ContainsOptionsForVerification(db) + .ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); // Make sure the options have not changed between calls, unless there are no previous options if (preexistingOptionsInDatabase) - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); } @@ -121,8 +131,10 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running); //We build a local database in steps. - using var restoredb = await LocalRecreateDatabase.CreateAsync(dbparent, m_options); - await restoredb.RepairInProgress(true); + using var restoredb = + await LocalRecreateDatabase.CreateAsync(dbparent, m_options) + .ConfigureAwait(false); + await restoredb.RepairInProgress(true).ConfigureAwait(false); var expRecreateDb = false; // experimental recreate db code flag var volumeIds = new Dictionary(); @@ -189,7 +201,11 @@ namespace Duplicati.Library.Main.Operation // If we are updating, all files should be accounted for foreach (var fl in remotefiles) - volumeIds[fl.File.Name] = updating ? await restoredb.GetRemoteVolumeID(fl.File.Name) : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded); + volumeIds[fl.File.Name] = updating + ? await restoredb.GetRemoteVolumeID(fl.File.Name) + .ConfigureAwait(false) + : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded) + .ConfigureAwait(false); var hasUpdatedOptions = false; @@ -204,7 +220,9 @@ namespace Duplicati.Library.Main.Operation { foreach (var n in filelists) if (volumeIds[n.File.Name] == -1) - volumeIds[n.File.Name] = await restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0)); + volumeIds[n.File.Name] = await restoredb + .RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0)) + .ConfigureAwait(false); } var isFirstFilelist = true; @@ -218,7 +236,9 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); return; } @@ -238,7 +258,9 @@ namespace Duplicati.Library.Main.Operation isFirstFilelist = false; if (!string.IsNullOrWhiteSpace(hash) && size > 0) - await restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash); + await restoredb + .UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash) + .ConfigureAwait(false); var parsed = VolumeBase.ParseFilename(entry.Name); @@ -254,9 +276,12 @@ namespace Duplicati.Library.Main.Operation } // Create timestamped operations based on the file timestamp - var filesetid = await restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time); + var filesetid = await restoredb + .CreateFileset(volumeIds[entry.Name], parsed.Time) + .ConfigureAwait(false); - await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter); + await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter) + .ConfigureAwait(false); } } catch (Exception ex) @@ -266,7 +291,9 @@ namespace Duplicati.Library.Main.Operation { m_result.EndTime = DateTime.UtcNow; // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); throw; } @@ -274,14 +301,18 @@ namespace Duplicati.Library.Main.Operation { m_result.EndTime = DateTime.UtcNow; // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); throw; } if (m_options.UnittestMode) { // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); throw; } } @@ -289,10 +320,13 @@ namespace Duplicati.Library.Main.Operation //Make sure we write the config if it has been read from a manifest if (hasUpdatedOptions) - await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options); + await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options) + .ConfigureAwait(false); using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) - await restoredb.Transaction.CommitAsync(); + await restoredb.Transaction + .CommitAsync() + .ConfigureAwait(false); // do we stop after just handling the dlist files ? // (if yes, we never will be able to do a backup !) @@ -323,7 +357,9 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); return; } @@ -334,18 +370,24 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) { if (!string.IsNullOrWhiteSpace(hash) && size > 0) - await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash); + await restoredb + .UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash) + .ConfigureAwait(false); using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) { foreach (var a in svr.Volumes) { var filename = a.Filename; - var volumeID = await restoredb.GetRemoteVolumeID(filename); + var volumeID = await restoredb + .GetRemoteVolumeID(filename) + .ConfigureAwait(false); // No such file if (volumeID < 0) - (volumeID, filename) = await ProbeForMatchingFilename(filename, restoredb); + (volumeID, filename) = + await ProbeForMatchingFilename(filename, restoredb) + .ConfigureAwait(false); var missing = false; // Still broken, register a missing item @@ -356,18 +398,27 @@ namespace Duplicati.Library.Main.Operation throw new Exception(string.Format("Unable to parse filename: {0}", filename)); Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); missing = true; - volumeID = await restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary); + volumeID = await restoredb + .RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary) + .ConfigureAwait(false); } bool anyChange = false; //Add all block/volume mappings foreach (var b in a.Blocks) { - anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID)).Item1; + anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID).ConfigureAwait(false)).Item1; } - await restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash); - await restoredb.AddIndexBlockLink(await restoredb.GetRemoteVolumeID(name), volumeID); + await restoredb + .UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash) + .ConfigureAwait(false); + await restoredb + .AddIndexBlockLink( + await restoredb.GetRemoteVolumeID(name).ConfigureAwait(false), + volumeID + ) + .ConfigureAwait(false); } //If there are blocklists in the index file, add them to the temp blocklist hashes table @@ -381,7 +432,9 @@ namespace Duplicati.Library.Main.Operation // We need to instantiate the list to ensure the verification is // done before we add it to the database, since we do not have nested transactions var list = b.Blocklist.ToList(); - await restoredb.AddTempBlockListHash(b.Hash, list); + await restoredb + .AddTempBlockListHash(b.Hash, list) + .ConfigureAwait(false); } catch (System.IO.InvalidDataException e) { @@ -404,35 +457,45 @@ namespace Duplicati.Library.Main.Operation { m_result.EndTime = DateTime.UtcNow; // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); throw; } if (m_options.UnittestMode) { // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); throw; } } } using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) - await restoredb.Transaction.CommitAsync(); + await restoredb.Transaction + .CommitAsync() + .ConfigureAwait(false); // TODO: In some cases, we can avoid downloading all index files, // if we are lucky and pick the right ones } - await restoredb.CleanupMissingVolumes(); + await restoredb.CleanupMissingVolumes().ConfigureAwait(false); // Update the real tables from the temp tables if (expRecreateDb) // add missing blocks and blocksetentry data (at this point // we have not yet anything in the blocksetentry table) - await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize); + await restoredb + .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize) + .ConfigureAwait(false); else - await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize); + await restoredb + .FindMissingBlocklistHashes(hashsize, m_options.Blocksize) + .ConfigureAwait(false); // We have now grabbed as much information as possible, // if we are still missing data, we must now fetch block files @@ -440,7 +503,11 @@ namespace Duplicati.Library.Main.Operation for (var i = 0; i < 3; i++) { // Grab the list matching the pass type - var lst = await restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse).ToListAsync(); + var lst = await restoredb + .GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse) + .ToListAsync() + .ConfigureAwait(false); + if (lst.Count == 0) continue; @@ -474,7 +541,9 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); return; } @@ -482,23 +551,27 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f)); Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); - var volumeid = await restoredb.GetRemoteVolumeID(name); + var volumeid = await restoredb + .GetRemoteVolumeID(name) + .ConfigureAwait(false); - await restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash); + await restoredb + .UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash) + .ConfigureAwait(false); bool anyChange = false; // Update the block table so we know about the block/volume map foreach (var h in rd.Blocks) { - anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid)).Item1; + anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid).ConfigureAwait(false)).Item1; } // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes // to the needed list blocks in the volume, so grab them from the database // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table - await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) + await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid).ConfigureAwait(false)) { - if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize))) + if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize)).ConfigureAwait(false)) { anyChange = true; } @@ -516,13 +589,19 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name); } if (expRecreateDb) - await restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false); + await restoredb + .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false) + .ConfigureAwait(false); else - await restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize); + await restoredb + .FindMissingBlocklistHashes(hashsize, m_options.Blocksize) + .ConfigureAwait(false); } using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) - await restoredb.Transaction.CommitAsync(); + await restoredb.Transaction + .CommitAsync() + .ConfigureAwait(false); //At this point we can patch files with data from the block volume if (blockprocessor != null) @@ -535,7 +614,9 @@ namespace Duplicati.Library.Main.Operation if (m_options.UnittestMode) { // Implicit rollback - await restoredb.Transaction.RollBackAsync(); + await restoredb.Transaction + .RollBackAsync() + .ConfigureAwait(false); throw; } } @@ -550,10 +631,10 @@ namespace Duplicati.Library.Main.Operation // All blocks are collected and added into the Block table // Find out which blocks are deleted and move them into DeletedBlock, // so that compact can calculate the unused space - await restoredb.CleanupDeletedBlocks(); + await restoredb.CleanupDeletedBlocks().ConfigureAwait(false); } - await restoredb.CleanupMissingVolumes(); + await restoredb.CleanupMissingVolumes().ConfigureAwait(false); if (m_options.RepairOnlyPaths) { @@ -568,18 +649,24 @@ namespace Duplicati.Library.Main.Operation // except to continue a backup m_result.EndTime = DateTime.UtcNow; - using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb)) + using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb).ConfigureAwait(false)) { - var broken = await lbfdb.GetBrokenFilesets(new DateTime(0), null).CountAsync(); + var broken = await lbfdb + .GetBrokenFilesets(new DateTime(0), null) + .CountAsync() + .ConfigureAwait(false); + if (broken != 0) throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"), "DatabaseIsBrokenConsiderPurge"); } - await restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true); + await restoredb + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); - await restoredb.RepairInProgress(false); + await restoredb.RepairInProgress(false).ConfigureAwait(false); } m_result.EndTime = DateTime.UtcNow; @@ -594,10 +681,12 @@ namespace Duplicati.Library.Main.Operation var filesetData = VolumeReaderBase.GetFilesetData(compressor, options); // update fileset using filesetData - await restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup); + await restoredb + .UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup) + .ConfigureAwait(false); // clear any existing fileset entries - await restoredb.ClearFilesetEntries(filesetid); + await restoredb.ClearFilesetEntries(filesetid).ConfigureAwait(false); using (var filelistreader = new FilesetVolumeReader(compressor, options)) foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) @@ -610,37 +699,58 @@ namespace Duplicati.Library.Main.Operation var metadataid = long.MinValue; var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path); - var prefixid = await restoredb.GetOrCreatePathPrefix(split.Key); + var prefixid = await restoredb + .GetOrCreatePathPrefix(split.Key) + .ConfigureAwait(false); switch (fe.Type) { case FilelistEntryType.Folder: - metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes); - await restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid); + metadataid = await restoredb + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .ConfigureAwait(false); + await restoredb + .AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid) + .ConfigureAwait(false); + break; case FilelistEntryType.File: var expectedblocks = (fe.Size + blocksize - 1) / blocksize; var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block; if (expectedblocks <= 1) expectedblocklisthashes = 0; - var blocksetid = await restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes); - metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes); - await restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid); + var blocksetid = await restoredb + .AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes) + .ConfigureAwait(false); + metadataid = await restoredb + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .ConfigureAwait(false); + await restoredb + .AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid) + .ConfigureAwait(false); if (fe.Size <= blocksize) { if (!string.IsNullOrWhiteSpace(fe.Blockhash)) - await restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize); + await restoredb + .AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize) + .ConfigureAwait(false); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) - await restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size); + await restoredb + .AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size) + .ConfigureAwait(false); else if (fe.Size > 0) Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); } break; case FilelistEntryType.Symlink: - metadataid = await restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes); - await restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid); + metadataid = await restoredb + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .ConfigureAwait(false); + await restoredb + .AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid) + .ConfigureAwait(false); break; default: Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path); @@ -650,9 +760,13 @@ namespace Duplicati.Library.Main.Operation if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink)) { if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) - await restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize); + await restoredb + .AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize) + .ConfigureAwait(false); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) - await restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize); + await restoredb + .AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize) + .ConfigureAwait(false); else Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path); } @@ -663,7 +777,7 @@ namespace Duplicati.Library.Main.Operation } } - await restoredb.Transaction.CommitAsync(); + await restoredb.Transaction.CommitAsync().ConfigureAwait(false); } /// @@ -681,7 +795,10 @@ namespace Duplicati.Library.Main.Operation foreach (var encmodule in Library.DynamicLoader.EncryptionLoader.Keys.Union([""])) { var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule); - var tvid = await restoredb.GetRemoteVolumeID(testfilename); + var tvid = await restoredb + .GetRemoteVolumeID(testfilename) + .ConfigureAwait(false); + if (tvid >= 0) { Logging.Log.WriteWarningMessage(LOGTAG, "RewritingFilenameMapping", null, "Unable to find volume {0}, but mapping to matching file {1}", filename, testfilename); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 23e7f0b2b..7353805a0 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -57,7 +57,7 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) { await RunRepairLocalAsync(backendManager, filter).ConfigureAwait(false); - await RunRepairCommon(); + await RunRepairCommon().ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -65,8 +65,13 @@ namespace Duplicati.Library.Main.Operation long knownRemotes = -1; try { - using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); - knownRemotes = await db.GetRemoteVolumes().CountAsync(); + using var db = + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + .ConfigureAwait(false); + knownRemotes = await db + .GetRemoteVolumes() + .CountAsync() + .ConfigureAwait(false); } catch (Exception ex) { @@ -91,11 +96,11 @@ namespace Duplicati.Library.Main.Operation } await RunRepairLocalAsync(backendManager, filter).ConfigureAwait(false); - await RunRepairCommon(); + await RunRepairCommon().ConfigureAwait(false); } else { - await RunRepairCommon(); + await RunRepairCommon().ConfigureAwait(false); await RunRepairBrokenFilesets(backendManager).ConfigureAwait(false); await RunRepairRemoteAsync(backendManager, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } @@ -128,45 +133,72 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); + using var db = + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + .ConfigureAwait(false); try { - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); - if (await db.PartiallyRecreated()) + if (await db.PartiallyRecreated().ConfigureAwait(false)) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (await db.RepairInProgress()) + if (await db.RepairInProgress().ConfigureAwait(false)) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); // Ensure the database is consistent before we start fixing the remote - await db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true); + await db + .VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist - var lastTempFilelist = await db.GetLastIncompleteFilesetVolume(); + var lastTempFilelist = await db + .GetLastIncompleteFilesetVolume() + .ConfigureAwait(false); var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); - await db.Transaction.CommitAsync("CommitRemoteListAnalysisTransaction"); + await db.Transaction + .CommitAsync("CommitRemoteListAnalysisTransaction") + .ConfigureAwait(false); var buffer = new byte[m_options.Blocksize]; var hashsize = m_options.BlockhashSize; - var missingRemoteFilesets = await db.MissingRemoteFilesets().ToListAsync(cancellationToken: cancellationToken); - var missingLocalFilesets = await db.MissingLocalFilesets().ToListAsync(cancellationToken: cancellationToken); - var emptyIndexFiles = await db.EmptyIndexFiles().ToListAsync(cancellationToken: cancellationToken); + var missingRemoteFilesets = await db + .MissingRemoteFilesets() + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + var missingLocalFilesets = await db + .MissingLocalFilesets() + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + var emptyIndexFiles = await db + .EmptyIndexFiles() + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); var progress = 0; - var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; + var targetProgess = + tp.ExtraVolumes.Count() + + tp.MissingVolumes.Count() + + tp.VerificationRequiredVolumes.Count() + + missingRemoteFilesets.Count + + missingLocalFilesets.Count + + emptyIndexFiles.Count; // Find the most recent timestamp from either a fileset or a remote volume - var mostRecentLocal = await db.GetRemoteVolumes() + var mostRecentLocal = await db + .GetRemoteVolumes() .Where(x => x.Type == RemoteVolumeType.Files) .Select(x => VolumeBase.ParseFilename(x.Name).Time.ToLocalTime()) .Concat(db.FilesetTimes().Select(x => x.Value.ToLocalTime())) .Append(DateTime.MinValue) - .MaxAsync(); + .MaxAsync() + .ConfigureAwait(false); var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); if (mostRecentLocal < DateTime.UnixEpoch) @@ -198,7 +230,9 @@ namespace Duplicati.Library.Main.Operation { if (tp.VerificationRequiredVolumes.Any()) { - using var testdb = await LocalTestDatabase.CreateAsync(db); + using var testdb = + await LocalTestDatabase.CreateAsync(db) + .ConfigureAwait(false); foreach (var n in tp.VerificationRequiredVolumes) try @@ -207,9 +241,13 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(testdb, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await testdb.Transaction.CommitAsync("CommitEarlyExit", false); + await testdb.Transaction + .CommitAsync("CommitEarlyExit", false) + .ConfigureAwait(false); else - await testdb.Transaction.RollBackAsync(); + await testdb.Transaction + .RollBackAsync() + .ConfigureAwait(false); return; } @@ -219,7 +257,9 @@ namespace Duplicati.Library.Main.Operation KeyValuePair>> res; var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); using (tf) - res = await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1); + res = + await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1) + .ConfigureAwait(false); if (res.Value.Any()) throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); @@ -227,7 +267,9 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); - await db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash); + await db + .UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash) + .ConfigureAwait(false); } } @@ -238,7 +280,9 @@ namespace Duplicati.Library.Main.Operation throw; } - await db.Transaction.CommitAsync("CommitVerificationTransaction"); + await db.Transaction + .CommitAsync("CommitVerificationTransaction") + .ConfigureAwait(false); } // TODO: It is actually possible to use the extra files if we parse them @@ -249,9 +293,13 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitEarlyExit", false); + await db.Transaction + .CommitAsync("CommitEarlyExit", false) + .ConfigureAwait(false); else - await db.Transaction.RollBackAsync(); + await db.Transaction + .RollBackAsync() + .ConfigureAwait(false); return; } @@ -270,7 +318,9 @@ namespace Duplicati.Library.Main.Operation { foreach (var rv in ifr.Volumes) { - var entry = await db.GetRemoteVolume(rv.Filename); + var entry = await db + .GetRemoteVolume(rv.Filename) + .ConfigureAwait(false); if (entry.ID < 0) throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); @@ -280,31 +330,47 @@ namespace Duplicati.Library.Main.Operation if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); - await db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks); + await db + .CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks) + .ConfigureAwait(false); } var blocksize = m_options.Blocksize; foreach (var ixb in ifr.BlockLists) - await db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize); + await db + .CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize) + .ConfigureAwait(false); // Register the new index file and link it to the block files - var selfid = await db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0)); + var selfid = await db + .RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0)) + .ConfigureAwait(false); foreach (var rv in ifr.Volumes) { // Guard against unknown block files - long id = await db.GetRemoteVolumeID(rv.Filename); + long id = await db + .GetRemoteVolumeID(rv.Filename) + .ConfigureAwait(false); if (id == -1) Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); else - await db.AddIndexBlockLink(selfid, id); + await db + .AddIndexBlockLink(selfid, id) + .ConfigureAwait(false); } + if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitIndexFileTransaction"); + await db.Transaction + .CommitAsync("CommitIndexFileTransaction") + .ConfigureAwait(false); } // All checks fine, we accept the new index file Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); - await db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash); + await db + .UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash) + .ConfigureAwait(false); + continue; } catch (Exception rex) @@ -315,7 +381,9 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { - await db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); + await db + .RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting) + .ConfigureAwait(false); await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); } else @@ -339,15 +407,21 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitEarlyExit", false); + await db.Transaction + .CommitAsync("CommitEarlyExit", false) + .ConfigureAwait(false); else - await db.Transaction.RollBackAsync(); + await db.Transaction + .RollBackAsync() + .ConfigureAwait(false); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); + var fileTime = + await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp) + .ConfigureAwait(false); var fsw = new FilesetVolumeWriter(m_options, fileTime); Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); @@ -357,12 +431,20 @@ namespace Duplicati.Library.Main.Operation fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); fsw.CreateFilesetFile(isfull); - await db.WriteFileset(fsw, filesetId); + await db + .WriteFileset(fsw, filesetId) + .ConfigureAwait(false); fsw.Close(); - fsw.VolumeID = await db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); - await db.LinkFilesetToVolume(filesetId, fsw.VolumeID); + fsw.VolumeID = await db + .RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary) + .ConfigureAwait(false); + + await db + .LinkFilesetToVolume(filesetId, fsw.VolumeID) + .ConfigureAwait(false); + if (m_options.Dryrun) { fsw.Dispose(); @@ -377,33 +459,45 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitFilesetTransaction"); + await db.Transaction + .CommitAsync("CommitFilesetTransaction") + .ConfigureAwait(false); } foreach (var volumename in missingLocalFilesets) { - var remoteVolume = await db.GetRemoteVolume(volumename); + var remoteVolume = await db + .GetRemoteVolume(volumename) + .ConfigureAwait(false); using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) { var parsed = VolumeBase.ParseFilename(remoteVolume.Name); using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) - using (var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options)) + using (var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false)) { if (compressor == null) throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - var filesetid = await db.CreateFileset(remoteVolume.ID, parsed.Time); + var filesetid = await db + .CreateFileset(remoteVolume.ID, parsed.Time) + .ConfigureAwait(false); + + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()) + .ConfigureAwait(false); - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()); if (!m_options.Dryrun) - await recreatedb.Transaction.CommitAsync("CommitRecreateFilesetTransaction"); + await recreatedb.Transaction + .CommitAsync("CommitRecreateFilesetTransaction") + .ConfigureAwait(false); } } } if (!m_options.Dryrun && tp.MissingVolumes.Any()) - await db.TerminatedWithActiveUploads(true); + await db + .TerminatedWithActiveUploads(true) + .ConfigureAwait(false); if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); @@ -429,7 +523,9 @@ namespace Duplicati.Library.Main.Operation try { var timestamp = VolumeBase.ParseFilename(n.Name).Time; - var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); + var fileTime = + await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp) + .ConfigureAwait(false); var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); await RunRepairDlist(backendManager, db, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); @@ -456,7 +552,7 @@ namespace Duplicati.Library.Main.Operation try { // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = (await db.GetRemoteVolume(n.Name)).State; + var currentState = (await db.GetRemoteVolume(n.Name).ConfigureAwait(false)).State; if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) continue; @@ -484,7 +580,9 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); // Implicit rollback - await db.Transaction.RollBackAsync(); + await db.Transaction + .RollBackAsync() + .ConfigureAwait(false); return; } @@ -509,7 +607,9 @@ namespace Duplicati.Library.Main.Operation // passing down a transaction to // FlushPendingMessagesAsync, but maybe it // is not needed anymore. - await db.Transaction.CommitAsync(); + await db.Transaction + .CommitAsync() + .ConfigureAwait(false); } } } @@ -531,18 +631,24 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) { - await db.TerminatedWithActiveUploads(false); - await db.Transaction.CommitAsync("CommitRepairTransaction"); + await db + .TerminatedWithActiveUploads(false) + .ConfigureAwait(false); + await db.Transaction + .CommitAsync("CommitRepairTransaction") + .ConfigureAwait(false); } else { - await db.Transaction.RollBackAsync("RollbackRepairTransaction"); + await db.Transaction + .RollBackAsync("RollbackRepairTransaction") + .ConfigureAwait(false); } } catch (Exception) { // Implicit rollback - await db.Transaction.RollBackAsync(); + await db.Transaction.RollBackAsync().ConfigureAwait(false); throw; } } @@ -560,38 +666,59 @@ namespace Duplicati.Library.Main.Operation /// A task representing the asynchronous operation private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) { - volumeWriter.VolumeID = await db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero); - (var prevFilesetId, var _, var isPrevFull) = await db.GetFilesetFromRemotename(originalVolume.Name); + volumeWriter.VolumeID = await db + .RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .ConfigureAwait(false); + (var prevFilesetId, var _, var isPrevFull) = await db + .GetFilesetFromRemotename(originalVolume.Name) + .ConfigureAwait(false); if (!string.IsNullOrEmpty(m_options.ControlFiles)) foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) volumeWriter.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); volumeWriter.CreateFilesetFile(isPrevFull); - var newFilesetID = await db.CreateFileset(volumeWriter.VolumeID, filesetTime); - await db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull); + var newFilesetID = await db + .CreateFileset(volumeWriter.VolumeID, filesetTime) + .ConfigureAwait(false); + await db + .UpdateFullBackupStateInFileset(newFilesetID, isPrevFull) + .ConfigureAwait(false); + await db + .LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID) + .ConfigureAwait(false); + await db + .MoveFilesFromFileset(newFilesetID, prevFilesetId) + .ConfigureAwait(false); + await db + .WriteFileset(volumeWriter, newFilesetID) + .ConfigureAwait(false); - await db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID); - await db.MoveFilesFromFileset(newFilesetID, prevFilesetId); - - await db.WriteFileset(volumeWriter, newFilesetID); volumeWriter.Close(); - await db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - await db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null); + await db + .UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .ConfigureAwait(false); + await db + .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null) + .ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - await db.Transaction.CommitAsync("CommitPriorToFilesetUpload"); + await db.Transaction + .CommitAsync("CommitPriorToFilesetUpload") + .ConfigureAwait(false); await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitFilesetTransaction"); + await db.Transaction + .CommitAsync("CommitFilesetTransaction") + .ConfigureAwait(false); } /// @@ -609,23 +736,27 @@ namespace Duplicati.Library.Main.Operation public static async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, IndexVolumeWriter indexWriter, IRemoteVolume originalVolume, Options options, CancellationToken cancellationToken) { - indexWriter.VolumeID = await db.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero); + indexWriter.VolumeID = await db + .RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero) + .ConfigureAwait(false); var blockvolumeids = new List(); using var h = HashFactory.CreateHasher(options.BlockHashAlgorithm); - await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name)) + await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name).ConfigureAwait(false)) { indexWriter.StartVolume(blockvolume.Name); - var volumeid = await db.GetRemoteVolumeID(blockvolume.Name); + var volumeid = await db + .GetRemoteVolumeID(blockvolume.Name) + .ConfigureAwait(false); - await foreach (var b in db.GetBlocks(volumeid)) + await foreach (var b in db.GetBlocks(volumeid).ConfigureAwait(false)) indexWriter.AddBlock(b.Hash, b.Size); indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size); blockvolumeids.Add(volumeid); if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) - await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize)) + await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize).ConfigureAwait(false)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Buffer, 0, b.Size)); if (bh != b.Hash) @@ -636,7 +767,9 @@ namespace Duplicati.Library.Main.Operation } foreach (var blockvolumeid in blockvolumeids) - await db.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid); + await db + .AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid) + .ConfigureAwait(false); indexWriter.Close(); @@ -644,14 +777,18 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(indexWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - await db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null); + await db + .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null) + .ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!options.Dryrun) - await db.Transaction.CommitAsync("CommitRepairTransaction"); + await db.Transaction + .CommitAsync("CommitRepairTransaction") + .ConfigureAwait(false); } /// @@ -705,7 +842,9 @@ namespace Duplicati.Library.Main.Operation public static async Task CreateAsync(Options options, LocalDatabase db) { var ipdv = new InProgressDblockVolumes(options, db); - ipdv.m_activeWriter.VolumeID = await db.RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero); + ipdv.m_activeWriter.VolumeID = await db + .RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .ConfigureAwait(false); return ipdv; } @@ -745,10 +884,12 @@ namespace Duplicati.Library.Main.Operation /// The compression hint for the block public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) { - await m_activeWriter.AddBlock(hash, buffer, offset, size, hint); + await m_activeWriter + .AddBlock(hash, buffer, offset, size, hint) + .ConfigureAwait(false); m_anyData = true; if (IsFull) - await StartNewVolume(); + await StartNewVolume().ConfigureAwait(false); } /// @@ -762,7 +903,9 @@ namespace Duplicati.Library.Main.Operation m_completedWriters.Add(m_activeWriter); m_activeWriter = new BlockVolumeWriter(m_options); - m_activeWriter.VolumeID = await m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero); + m_activeWriter.VolumeID = await m_database + .RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .ConfigureAwait(false); m_anyData = false; } } @@ -779,7 +922,9 @@ namespace Duplicati.Library.Main.Operation /// >A task representing the asynchronous operation private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) { - var currentVolume = await InProgressDblockVolumes.CreateAsync(m_options, db); + var currentVolume = + await InProgressDblockVolumes.CreateAsync(m_options, db) + .ConfigureAwait(false); foreach (var n in missingDblockFiles) { @@ -789,7 +934,9 @@ namespace Duplicati.Library.Main.Operation { await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("CommitEarlyExit", false); + await db.Transaction + .CommitAsync("CommitEarlyExit", false) + .ConfigureAwait(false); return; } @@ -809,12 +956,14 @@ namespace Duplicati.Library.Main.Operation // If we have any data in the volume, complete it so it will be uploaded if (currentVolume.AnyData) - await currentVolume.StartNewVolume(); + await currentVolume.StartNewVolume().ConfigureAwait(false); // Complete any pending uploads await UploadCompletedBlockVolumes(backendManager, currentVolume, db, cancellationToken).ConfigureAwait(false); // After reset, we need to remove the temporary volume from the database - await db.RemoveRemoteVolume(currentVolume.RemoteFilename); + await db + .RemoveRemoteVolume(currentVolume.RemoteFilename) + .ConfigureAwait(false); } /// @@ -830,7 +979,9 @@ namespace Duplicati.Library.Main.Operation { foreach (var completedVolume in activeVolume.CompletedWriters) { - await db.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await db + .UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .ConfigureAwait(false); // Create a new index file that points to the new volume IndexVolumeWriter newvolindex = null; @@ -838,16 +989,20 @@ namespace Duplicati.Library.Main.Operation if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = await db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); + newvolindex.VolumeID = await db + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .ConfigureAwait(false); newvolindex.StartVolume(completedVolume.RemoteFilename); - await foreach (var b in db.GetBlocks(completedVolume.VolumeID)) + await foreach (var b in db.GetBlocks(completedVolume.VolumeID).ConfigureAwait(false)) newvolindex.AddBlock(b.Hash, b.Size); - await db.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID); + await db + .AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID) + .ConfigureAwait(false); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { - await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) + await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize).ConfigureAwait(false)) newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); }; } @@ -858,7 +1013,9 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReplaceBlockFile", "would upload new block file {0}", completedVolume.RemoteFilename); else { - await db.Transaction.CommitAsync("PostRepairPreUploadBlockVolume"); + await db.Transaction + .CommitAsync("PostRepairPreUploadBlockVolume") + .ConfigureAwait(false); await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); } } @@ -868,11 +1025,15 @@ namespace Duplicati.Library.Main.Operation // Prepare for deleting the old stuff foreach (var vol in activeVolume.VolumesToDelete) - await db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null); + await db + .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null) + .ConfigureAwait(false); // Persist desired state prior to deleting the old files if (!m_options.Dryrun) - await db.Transaction.CommitAsync("PostRepairBlockVolume"); + await db.Transaction + .CommitAsync("PostRepairBlockVolume") + .ConfigureAwait(false); // Delete the old files foreach (var vol in activeVolume.VolumesToDelete) @@ -884,7 +1045,9 @@ namespace Duplicati.Library.Main.Operation // All done, the new dblocks are now in place await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - await db.Transaction.CommitAsync("PostRepairBlockVolume"); + await db.Transaction + .CommitAsync("PostRepairBlockVolume") + .ConfigureAwait(false); activeVolume.CompletedWriters.Clear(); activeVolume.VolumesToDelete.Clear(); @@ -905,15 +1068,19 @@ namespace Duplicati.Library.Main.Operation // The dblock files are the most complex to recreate // as data can be either file contents, metadata or blocklist hashes // We attempt to recover all three source parts in the steps below - using var mbl = await db.CreateBlockList(originalVolume.Name); + using var mbl = await db + .CreateBlockList(originalVolume.Name) + .ConfigureAwait(false); - var originalMissingBlockCount = await mbl.GetMissingBlockCount(); + var originalMissingBlockCount = await mbl + .GetMissingBlockCount() + .ConfigureAwait(false); var recoveredSourceBlocks = 0L; // First we grab all known blocks from local files string lastRestoredHash = null; long lastRestoredSize = -1; - await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) + await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize).ConfigureAwait(false)) { if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) continue; @@ -949,10 +1116,12 @@ namespace Duplicati.Library.Main.Operation // Found it, no need to look again lastRestoredHash = block.Hash; lastRestoredSize = block.Size; - if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID).ConfigureAwait(false)) { recoveredSourceBlocks++; - await pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); + await pendingVolume + .AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default) + .ConfigureAwait(false); } } catch (Exception ex) @@ -962,7 +1131,7 @@ namespace Duplicati.Library.Main.Operation } // Then grab all blocks with metadata - await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks()) + await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks().ConfigureAwait(false)) { if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) continue; @@ -998,10 +1167,12 @@ namespace Duplicati.Library.Main.Operation // Found it, no need to look again lastRestoredHash = block.Hash; lastRestoredSize = block.Size; - if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID).ConfigureAwait(false)) { recoveredSourceBlocks++; - await pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); + await pendingVolume + .AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default) + .ConfigureAwait(false); } } catch (Exception ex) @@ -1035,8 +1206,10 @@ namespace Duplicati.Library.Main.Operation if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) { - if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) - await pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); + if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID).ConfigureAwait(false)) + await pendingVolume + .AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default) + .ConfigureAwait(false); } else { @@ -1046,12 +1219,12 @@ namespace Duplicati.Library.Main.Operation blocklistHash.SetLength(0); } - await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) + await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock).ConfigureAwait(false)) { lastBlocklist ??= blocklist; if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) - await EmitBlockListBlock(); + await EmitBlockListBlock().ConfigureAwait(false); var data = Convert.FromBase64String(blocklist.Hash); blocklistHash.Write(data, 0, data.Length); @@ -1060,10 +1233,10 @@ namespace Duplicati.Library.Main.Operation // Handle any trailing blocklist hash if (blocklistHash.Length > 0) - await EmitBlockListBlock(); + await EmitBlockListBlock().ConfigureAwait(false); //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources().ToListAsync(cancellationToken: cancellationToken), cancellationToken).ConfigureAwait(false)) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources().ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false)) { try { @@ -1071,9 +1244,11 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) foreach (var b in f.Blocks) - if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) + if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID).ConfigureAwait(false)) if (f.ReadBlock(b.Key, buffer) == b.Value) - await pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); + await pendingVolume + .AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default) + .ConfigureAwait(false); } catch (Exception ex) { @@ -1082,12 +1257,14 @@ namespace Duplicati.Library.Main.Operation } } - var missingBlocks = await mbl.GetMissingBlockCount(); + var missingBlocks = await mbl + .GetMissingBlockCount() + .ConfigureAwait(false); var recoveredBlocks = originalMissingBlockCount - missingBlocks; if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) { Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); - await foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) + await foreach (var f in mbl.GetFilesetsUsingMissingBlocks().ConfigureAwait(false)) Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); @@ -1107,15 +1284,23 @@ namespace Duplicati.Library.Main.Operation if (missingBlocks == 0) { pendingVolume.VolumesToDelete.Add(originalVolume); - var oldIndexFiles = await db.GetIndexFilesReferencingBlockFile(originalVolume.ID).ToListAsync(cancellationToken: cancellationToken); + var oldIndexFiles = await db + .GetIndexFilesReferencingBlockFile(originalVolume.ID) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); // Find all index files that point to the old volume foreach (var oldIndexFile in oldIndexFiles) { - var oldVolume = await db.GetRemoteVolume(oldIndexFile); + var oldVolume = await db + .GetRemoteVolume(oldIndexFile) + .ConfigureAwait(false); if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) { - var blockVolumesReferenced = await db.GetBlockVolumesFromIndexName(oldIndexFile).ToListAsync(cancellationToken: cancellationToken); + var blockVolumesReferenced = await db + .GetBlockVolumesFromIndexName(oldIndexFile) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); else @@ -1135,9 +1320,15 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); + using var db = + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + .ConfigureAwait(false); + + var sets = await db + .GetFilesetsWithMissingFiles() + .ToListAsync() + .ConfigureAwait(false); - var sets = await db.GetFilesetsWithMissingFiles().ToListAsync(); if (sets.Count == 0) return; @@ -1147,7 +1338,9 @@ namespace Duplicati.Library.Main.Operation { ix++; Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); - var volume = await db.GetRemoteVolumeFromFilesetID(entry.Key); + var volume = await db + .GetRemoteVolumeFromFilesetID(entry.Key) + .ConfigureAwait(false); var parsed = VolumeBase.ParseFilename(volume.Name); using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); @@ -1156,13 +1349,16 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); // Clear out the old fileset - await db.DeleteFilesetEntries(entry.Key); - using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options)) + await db.DeleteFilesetEntries(entry.Key).ConfigureAwait(false); + using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false)) { - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression()); + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression()) + .ConfigureAwait(false); } - await db.Transaction.CommitAsync("PostRepairFileset"); + await db.Transaction + .CommitAsync("PostRepairFileset") + .ConfigureAwait(false); } } @@ -1173,17 +1369,24 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache); + using var db = + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + .ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); - if (await db.RepairInProgress() || await db.PartiallyRecreated()) + if (await db.RepairInProgress().ConfigureAwait(false) || await db.PartiallyRecreated().ConfigureAwait(false)) Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); - await db.FixDuplicateMetahash(); - await db.FixDuplicateFileentries(); - await db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize); - await db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize); + await db.FixDuplicateMetahash().ConfigureAwait(false); + await db.FixDuplicateFileentries().ConfigureAwait(false); + await db + .FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize) + .ConfigureAwait(false); + await db + .FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 2880f34f0..be179ef37 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -310,7 +310,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_get_wait?.Stop(); } - return await new_tcs.Task; + return await new_tcs.Task.ConfigureAwait(false); } /// diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index ff8c901bc..1a7e3fbd1 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -67,7 +67,11 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_get_files?.Start(); // The enumerables are cast to arrays to force the query to be executed and release the database lock. - var files = await db.GetFilesAndSymlinksToRestore().ToArrayAsync(); + var files = await db + .GetFilesAndSymlinksToRestore() + .ToArrayAsync() + .ConfigureAwait(false); + result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); sw_get_files?.Stop(); @@ -80,7 +84,10 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_get_folders?.Start(); // The enumerables are cast to arrays to force the query to be executed and release the database lock. - var folders = await db.GetFolderMetadataToRestore().ToArrayAsync(); + var folders = await db + .GetFolderMetadataToRestore() + .ToArrayAsync() + .ConfigureAwait(false); sw_get_folders?.Stop(); sw_write_folder?.Start(); diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 52bd07367..79454c9ac 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -115,11 +115,13 @@ namespace Duplicati.Library.Main.Operation.Restore if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted) file_processor_continue.SetResult(); else - await RendezvousBeforeProcessingFolderMetadata(); + await RendezvousBeforeProcessingFolderMetadata() + .ConfigureAwait(false); decremented = true; } - await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp); + await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp) + .ConfigureAwait(false); continue; } @@ -127,7 +129,10 @@ namespace Duplicati.Library.Main.Operation.Restore // Get information about the blocks for the file // TODO rather than keeping all of the blocks in memory, we could do a single pass over the blocks using a cursor, only keeping the relevant block requests in memory. Maybe even only a single block request at a time. sw_block?.Start(); - var blocks = await db.GetBlocksFromFile(file.BlocksetID).ToArrayAsync(); + var blocks = await db + .GetBlocksFromFile(file.BlocksetID) + .ToArrayAsync() + .ConfigureAwait(false); sw_block?.Stop(); sw_work_verify_target?.Start(); @@ -367,12 +372,17 @@ namespace Duplicati.Library.Main.Operation.Restore else { // Not a missing block, so read from the file - sw_work_read?.Start(); - var read = await fs?.ReadAsync(buffer, 0, buffer.Length); - sw_work_read?.Stop(); - sw_work_hash?.Start(); - filehasher.TransformBlock(buffer, 0, read, buffer, 0); - sw_work_hash?.Stop(); + if (fs != null) + { + sw_work_read?.Start(); + var read = await fs + .ReadAsync(buffer, 0, buffer.Length) + .ConfigureAwait(false); + sw_work_read?.Stop(); + sw_work_hash?.Start(); + filehasher.TransformBlock(buffer, 0, read, buffer, 0); + sw_work_hash?.Stop(); + } } } @@ -604,7 +614,7 @@ namespace Duplicati.Library.Main.Operation.Restore using var ms = new MemoryStream(); - await foreach (var block in blocks) + await foreach (var block in blocks.ConfigureAwait(false)) { sw_work?.Stop(); sw_req?.Start(); @@ -790,7 +800,7 @@ namespace Duplicati.Library.Main.Operation.Restore int j = 0; for (long i = 0; i < total_blocks; i++) { - int read; + int read = 0; if (j < blocks.Count && blocks[j].BlockOffset == i) { // The current block is a missing block @@ -819,18 +829,23 @@ namespace Duplicati.Library.Main.Operation.Restore { if (!options.Dryrun) { - try + if (f_target != null) { - f_target?.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin); - await f_target?.WriteAsync(buffer, 0, read); - } - catch (Exception) - { - lock (results) + try { - results.BrokenLocalFiles.Add(file.TargetPath); + f_target.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin); + await f_target + .WriteAsync(buffer, 0, read) + .ConfigureAwait(false); + } + catch (Exception) + { + lock (results) + { + results.BrokenLocalFiles.Add(file.TargetPath); + } + throw; } - throw; } } bytes_read += read; @@ -863,18 +878,23 @@ namespace Duplicati.Library.Main.Operation.Restore else { // The current block is not a missing block - read from the target file. - try + if (f_target != null) { - f_target?.Seek(i * options.Blocksize, SeekOrigin.Begin); - read = await f_target?.ReadAsync(buffer, 0, options.Blocksize); - } - catch (Exception) - { - lock (results) + try { - results.BrokenLocalFiles.Add(file.TargetPath); + f_target.Seek(i * options.Blocksize, SeekOrigin.Begin); + read = await f_target + .ReadAsync(buffer, 0, options.Blocksize) + .ConfigureAwait(false); + } + catch (Exception) + { + lock (results) + { + results.BrokenLocalFiles.Add(file.TargetPath); + } + throw; } - throw; } } filehasher.TransformBlock(buffer, 0, read, buffer, 0); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index dccbd2828..251f2f48c 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -75,7 +75,10 @@ namespace Duplicati.Library.Main.Operation.Restore // Trigger the download. sw_wait?.Start(); TempFile f; - var (volume_name, size, hash) = await db.GetVolumeInfo(volume_id).FirstAsync(); + var (volume_name, size, hash) = await db + .GetVolumeInfo(volume_id) + .FirstAsync() + .ConfigureAwait(false); try { f = await backend.GetDirectAsync(volume_name, hash, size, CancellationToken.None).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index 9600239cf..6621b6702 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Operation Directory.CreateDirectory(m_options.Restorepath); using (var tmpdb = new TempFile()) - using (var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache)) + using (var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false)) { var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); @@ -68,7 +68,9 @@ namespace Duplicati.Library.Main.Operation } var file = fileversion.Value.File; - var entry = await db.GetRemoteVolume(file.Name); + var entry = await db + .GetRemoteVolume(file.Name) + .ConfigureAwait(false); var res = new List(); using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index b24c1dc73..3babe6ddd 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -115,14 +115,16 @@ namespace Duplicati.Library.Main.Operation { if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) { - db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache); + db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache) + .ConfigureAwait(false); } else { Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); tmpdb = new TempFile(); RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); - db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache); + db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache) + .ConfigureAwait(false); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) await new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) @@ -157,10 +159,10 @@ namespace Duplicati.Library.Main.Operation var fullblockverification = options.FullBlockVerification; using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var blockmarker = await database.CreateBlockMarkerAsync()) - using (var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize)) + using (var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false)) + using (var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize).ConfigureAwait(false)) { - await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks()) + await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks().ConfigureAwait(false)) { var targetpath = restorelist.Path; @@ -184,7 +186,7 @@ namespace Duplicati.Library.Main.Operation // TODO: Much faster if we iterate the volume and checks what blocks are used, // because the compressors usually like sequential reading using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - await foreach (var targetblock in restorelist.Blocks()) + await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) { file.Position = targetblock.Offset; var size = blocks.ReadBlock(targetblock.Key, blockbuffer); @@ -203,7 +205,9 @@ namespace Duplicati.Library.Main.Operation if (valid) { file.Write(blockbuffer, 0, size); - await blockmarker.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false); + await blockmarker + .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false) + .ConfigureAwait(false); } } else @@ -213,7 +217,9 @@ namespace Duplicati.Library.Main.Operation } if ((++updateCounter) % 20 == 0) - await blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); } catch (Exception ex) { @@ -226,7 +232,7 @@ namespace Duplicati.Library.Main.Operation if (!options.SkipMetadata) { - await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks()) + await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks().ConfigureAwait(false)) { var targetpath = restoremetadata.Path; Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); @@ -236,14 +242,16 @@ namespace Duplicati.Library.Main.Operation // TODO: When we support multi-block metadata this needs to deal with it using (var ms = new System.IO.MemoryStream()) { - await foreach (var targetblock in restoremetadata.Blocks()) + await foreach (var targetblock in restoremetadata.Blocks().ConfigureAwait(false)) { ms.Position = targetblock.Offset; var size = blocks.ReadBlock(targetblock.Key, blockbuffer); if (targetblock.Size == size) { ms.Write(blockbuffer, 0, size); - await blockmarker.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true); + await blockmarker + .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true) + .ConfigureAwait(false); } } @@ -260,8 +268,10 @@ namespace Duplicati.Library.Main.Operation } } } - await blockmarker.UpdateProcessed(result.OperationProgressUpdater); - await blockmarker.CommitAsync(); + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + await blockmarker.CommitAsync().ConfigureAwait(false); } } @@ -308,8 +318,10 @@ namespace Duplicati.Library.Main.Operation private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { // Perform initial setup - await Utility.UpdateOptionsFromDb(database, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(database, m_options); + await Utility.UpdateOptionsFromDb(database, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options) + .ConfigureAwait(false); // Verify the backend if necessary if (!m_options.NoBackendverification) @@ -321,13 +333,16 @@ namespace Duplicati.Library.Main.Operation // Prepare the block and file list and create the directory structure m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - await PrepareBlockAndFileList(database, m_options, filter, m_result); + await PrepareBlockAndFileList(database, m_options, filter, m_result) + .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); // At this point, there should be no more writes to the database, so we have to unlock the database: - await database.Transaction.CommitAsync("CommitBeforeRestore", restart: true).ConfigureAwait(false); + await database.Transaction + .CommitAsync("CommitBeforeRestore") + .ConfigureAwait(false); using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); // Create the channels between BlockManager and FileProcessor @@ -377,7 +392,9 @@ namespace Duplicati.Library.Main.Operation kill_updater.Cancel(); } - await database.Transaction.CommitAsync("CommitAfterRestore", restart: true).ConfigureAwait(false); + await database.Transaction + .CommitAsync("CommitAfterRestore") + .ConfigureAwait(false); await database.DisposePoolAsync().ConfigureAwait(false); @@ -407,7 +424,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - await database.DropRestoreTable(); + await database.DropRestoreTable().ConfigureAwait(false); await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Report that the restore is complete @@ -428,8 +445,10 @@ namespace Duplicati.Library.Main.Operation //using (var database = new LocalRestoreDatabase(dbparent)) using (var metadatastorage = new RestoreHandlerMetadataStorage()) { - await Utility.UpdateOptionsFromDb(database, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(database, m_options); + await Utility.UpdateOptionsFromDb(database, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options) + .ConfigureAwait(false); m_blockbuffer = new byte[m_options.Blocksize]; if (!m_options.NoBackendverification) @@ -441,12 +460,14 @@ namespace Duplicati.Library.Main.Operation //Figure out what files are to be patched, and what blocks are needed m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - await PrepareBlockAndFileList(database, m_options, filter, m_result); + await PrepareBlockAndFileList(database, m_options, filter, m_result) + .ConfigureAwait(false); //Make the entire output setup m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) - await CreateDirectoryStructure(database, m_options, m_result); + await CreateDirectoryStructure(database, m_options, m_result) + .ConfigureAwait(false); //If we are patching an existing target folder, do not touch stuff that is already updated m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles); @@ -490,7 +511,10 @@ namespace Duplicati.Library.Main.Operation // Fill BLOCKS with remote sources List volumes; using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes")) - volumes = await database.GetMissingVolumes().ToListAsync(); + volumes = await database + .GetMissingVolumes() + .ToListAsync() + .ConfigureAwait(false); if (volumes.Count > 0) { @@ -513,7 +537,8 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) - await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); + await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage) + .ConfigureAwait(false); } catch (Exception ex) { @@ -527,7 +552,7 @@ namespace Duplicati.Library.Main.Operation var fileErrors = 0L; // Restore empty files. They might not have any blocks so don't appear in any volume. - await foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0)) + await foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0).ConfigureAwait(false)) { Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path); @@ -566,7 +591,7 @@ namespace Duplicati.Library.Main.Operation // After all blocks in the files are restored, verify the file hash using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification")) - await foreach (var file in database.GetFilesToRestore(true)) + await foreach (var file in database.GetFilesToRestore(true).ConfigureAwait(false)) { try { @@ -609,7 +634,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - await database.DropRestoreTable(); + await database.DropRestoreTable().ConfigureAwait(false); await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); } @@ -689,10 +714,10 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) { // Fill BLOCKS with data from known local source files - using (var blockmarker = await database.CreateBlockMarkerAsync()) + using (var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false)) { var updateCount = 0L; - await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize)) + await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize).ConfigureAwait(false)) { var targetpath = entry.TargetPath; var targetfileid = entry.TargetFileID; @@ -716,7 +741,7 @@ namespace Duplicati.Library.Main.Operation { using (var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath)) { - await foreach (var block in entry.Blocks()) + await foreach (var block in entry.Blocks().ConfigureAwait(false)) { if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -740,7 +765,8 @@ namespace Duplicati.Library.Main.Operation targetstream.Write(blockbuffer, 0, size); } - await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false); + await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false) + .ConfigureAwait(false); } } } @@ -757,7 +783,9 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) { - await blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; } @@ -786,18 +814,20 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath); } - await blockmarker.UpdateProcessed(result.OperationProgressUpdater); - await blockmarker.CommitAsync(); + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + await blockmarker.CommitAsync().ConfigureAwait(false); } } private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) { // Fill BLOCKS with data from known local source files - using (var blockmarker = await database.CreateBlockMarkerAsync()) + using (var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false)) { var updateCount = 0L; - await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize)) + await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize).ConfigureAwait(false)) { var targetpath = restorelist.TargetPath; var targetfileid = restorelist.TargetFileID; @@ -815,9 +845,9 @@ namespace Duplicati.Library.Main.Operation } using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - await foreach (var targetblock in restorelist.Blocks()) + await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) { - await foreach (var source in targetblock.BlockSources()) + await foreach (var source in targetblock.BlockSources().ConfigureAwait(false)) { try { @@ -855,7 +885,9 @@ namespace Duplicati.Library.Main.Operation } } - await blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false); + await blockmarker + .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false) + .ConfigureAwait(false); patched = true; break; } @@ -874,7 +906,9 @@ namespace Duplicati.Library.Main.Operation } if ((++updateCount) % 20 == 0) - await blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); } catch (Exception ex) { @@ -892,8 +926,10 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath)); } - await blockmarker.UpdateProcessed(result.OperationProgressUpdater); - await blockmarker.CommitAsync(); + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + await blockmarker.CommitAsync().ConfigureAwait(false); } } @@ -903,7 +939,9 @@ namespace Duplicati.Library.Main.Operation // Delete all entries from the temp table that are excluded by the filter(s) using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList")) { - var c = await database.PrepareRestoreFilelist(options.Time, options.Version, filter); + var c = await database + .PrepareRestoreFilelist(options.Time, options.Version, filter) + .ConfigureAwait(false); result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); } @@ -911,25 +949,33 @@ namespace Duplicati.Library.Main.Operation if (!string.IsNullOrEmpty(options.Restorepath)) { // Find the largest common prefix - var largest_prefix = options.DontCompressRestorePaths ? "" : await database.GetLargestPrefix(); + var largest_prefix = options.DontCompressRestorePaths + ? "" : + await database.GetLargestPrefix().ConfigureAwait(false); Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath)); // Set the target paths, special care with C:\ and / - await database.SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)); + await database + .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)) + .ConfigureAwait(false); } else { - await database.SetTargetPaths("", ""); + await database.SetTargetPaths("", "").ConfigureAwait(false); } // Create a temporary table BLOCKS that lists all blocks that needs to be recovered using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks")) - await database.FindMissingBlocks(options.SkipMetadata); + await database + .FindMissingBlocks(options.SkipMetadata) + .ConfigureAwait(false); // Create temporary tables and triggers that automatically track progress using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker")) - await database.CreateProgressTracker(false); + await database + .CreateProgressTracker(false) + .ConfigureAwait(false); } @@ -947,7 +993,7 @@ namespace Duplicati.Library.Main.Operation SystemIO.IO_OS.DirectoryCreate(options.Restorepath); } - await foreach (var folder in database.GetTargetFolders()) + await foreach (var folder in database.GetTargetFolders().ConfigureAwait(false)) { try { @@ -978,10 +1024,10 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) { // Scan existing files for existing BLOCKS - using (var blockmarker = await database.CreateBlockMarkerAsync()) + using (var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false)) { var updateCount = 0L; - await foreach (var restorelist in database.GetExistingFilesWithBlocks()) + await foreach (var restorelist in database.GetExistingFilesWithBlocks().ConfigureAwait(false)) { var rename = !options.Overwrite; var targetpath = restorelist.TargetPath; @@ -1034,7 +1080,7 @@ namespace Duplicati.Library.Main.Operation using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) using (var block = new Blockprocessor(file, blockbuffer)) - await foreach (var targetblock in restorelist.Blocks()) + await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) { var size = block.Readblock(); if (size <= 0) @@ -1059,7 +1105,9 @@ namespace Duplicati.Library.Main.Operation if (key == targetblock.Hash) { - await blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, size, false); + await blockmarker + .SetBlockRestored(targetfileid, targetblock.Index, key, size, false) + .ConfigureAwait(false); blockhashmatch = true; } } @@ -1094,7 +1142,9 @@ namespace Duplicati.Library.Main.Operation if (fullfilehashmatch) { //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. - await blockmarker.SetFileDataVerified(targetfileid); + await blockmarker + .SetFileDataVerified(targetfileid) + .ConfigureAwait(false); Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); rename = false; } @@ -1102,13 +1152,17 @@ namespace Duplicati.Library.Main.Operation { // The new file will have none of the correct blocks, // even if the scanned file had some - await blockmarker.SetAllBlocksMissing(targetfileid); + await blockmarker + .SetAllBlocksMissing(targetfileid) + .ConfigureAwait(false); } } if ((++updateCount) % 20 == 0) { - await blockmarker.UpdateProcessed(result.OperationProgressUpdater); + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; } @@ -1155,8 +1209,12 @@ namespace Duplicati.Library.Main.Operation { //TODO: Also needs metadata check to make correct decision. // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. - await blockmarker.SetAllBlocksRestored(targetfileid, false); - await blockmarker.SetFileDataVerified(targetfileid); + await blockmarker + .SetAllBlocksRestored(targetfileid, false) + .ConfigureAwait(false); + await blockmarker + .SetFileDataVerified(targetfileid) + .ConfigureAwait(false); break; } } @@ -1172,13 +1230,19 @@ namespace Duplicati.Library.Main.Operation newname = tr; Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); - await database.UpdateTargetPath(targetfileid, newname); + await database + .UpdateTargetPath(targetfileid, newname) + .ConfigureAwait(false); } } - await blockmarker.UpdateProcessed(result.OperationProgressUpdater); - await blockmarker.CommitAsync(); + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + await blockmarker + .CommitAsync() + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs index 76817b01e..3c55ba2ba 100644 --- a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs +++ b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs @@ -49,11 +49,17 @@ internal static class SearchEntriesHandler if (!System.IO.File.Exists(options.Dbpath)) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache); + using var db = + await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) { - filesetIds = await db.GetFilesetIDs(options.Time, options.Version).ToArrayAsync(); + filesetIds = await db + .GetFilesetIDs(options.Time, options.Version) + .ToArrayAsync() + .ConfigureAwait(false); + if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); } @@ -61,6 +67,8 @@ internal static class SearchEntriesHandler if (paths != null) paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); - result.FileVersions = await db.SearchEntries(paths, filter, filesetIds, offset, limit); + result.FileVersions = await db + .SearchEntries(paths, filter, filesetIds, offset, limit) + .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index cdd539d25..c09284738 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -52,20 +52,31 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)) + using (var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) { - await Utility.UpdateOptionsFromDb(db, m_options); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); + + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks) + .ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); - await db.Transaction.CommitAsync("TestHandlerComplete"); + await db.Transaction + .CommitAsync("TestHandlerComplete") + .ConfigureAwait(false); } } public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend) { - var files = await db.SelectTestTargets(samples, m_options).ToListAsync(); + var files = await db + .SelectTestTargets(samples, m_options) + .ToListAsync() + .ConfigureAwait(false); m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); m_results.OperationProgressUpdater.UpdateProgress(0); @@ -91,7 +102,8 @@ namespace Duplicati.Library.Main.Operation KeyValuePair>> res; using (tf) - res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2); + res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2) + .ConfigureAwait(false); var parsedInfo = VolumeBase.ParseFilename(vol.Name); if (parsedInfo.FileType == RemoteVolumeType.Index) @@ -115,7 +127,9 @@ namespace Duplicati.Library.Main.Operation { if (res.Value == null || !res.Value.Any()) { - var rv = await db.GetRemoteVolume(vol.Name); + var rv = await db + .GetRemoteVolume(vol.Name) + .ConfigureAwait(false); if (rv.ID < 0) { @@ -128,14 +142,18 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", "Successfully captured hash and size for {0}, updating database", vol.Name); - await db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash); + await db + .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash) + .ConfigureAwait(false); } } } } } - await db.UpdateVerificationCount(vol.Name); + await db + .UpdateVerificationCount(vol.Name) + .ConfigureAwait(false); } catch (Exception ex) { @@ -184,7 +202,8 @@ namespace Duplicati.Library.Main.Operation (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false); using (tf) - res = await TestVolumeInternals(db, f, tf, m_options, 1); + res = await TestVolumeInternals(db, f, tf, m_options, 1) + .ConfigureAwait(false); m_results.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(hash) && size > 0) @@ -198,7 +217,9 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", "Successfully captured hash and size for {0}, updating database", f.Name); - await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash); + await db + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash) + .ConfigureAwait(false); } } } @@ -209,7 +230,9 @@ namespace Duplicati.Library.Main.Operation { } } - await db.UpdateVerificationCount(f.Name); + await db + .UpdateVerificationCount(f.Name) + .ConfigureAwait(false); m_results.AddResult(f.Name, []); } catch (Exception ex) @@ -256,13 +279,15 @@ namespace Duplicati.Library.Main.Operation case RemoteVolumeType.Files: //Compare with db and see if all files are accounted for // with correct file hashes and blocklist hashes - using (var fl = await db.CreateFilelist(vol.Name)) + using (var fl = await db.CreateFilelist(vol.Name).ConfigureAwait(false)) { using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options)) foreach (var f in rd.Files) - await fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time); + await fl + .Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time) + .ConfigureAwait(false); - return new KeyValuePair>>(vol.Name, await fl.Compare().ToListAsync()); + return new KeyValuePair>>(vol.Name, await fl.Compare().ToListAsync().ConfigureAwait(false)); } case RemoteVolumeType.Index: @@ -275,46 +300,69 @@ namespace Duplicati.Library.Main.Operation foreach (var v in rd.Volumes) { blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); - using (var bl = await db.CreateBlocklist(v.Filename)) + using (var bl = await db.CreateBlocklist(v.Filename).ConfigureAwait(false)) { foreach (var h in v.Blocks) - await bl.AddBlock(h.Key, h.Value); + await bl + .AddBlock(h.Key, h.Value) + .ConfigureAwait(false); - combined.AddRange(await bl.Compare().ToListAsync()); + combined.AddRange( + await bl + .Compare() + .ToListAsync() + .ConfigureAwait(false) + ); } } if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) { var hashesPerBlock = options.Blocksize / options.BlockhashSize; - using (var bl = await db.CreateBlocklistHashList(vol.Name)) + using (var bl = await db.CreateBlocklistHashList(vol.Name).ConfigureAwait(false)) { foreach (var b in rd.BlockLists) - await bl.AddBlockHash(b.Hash, b.Length); + await bl + .AddBlockHash(b.Hash, b.Length) + .ConfigureAwait(false); - combined.AddRange(await bl.Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize).ToListAsync()); + combined.AddRange( + await bl + .Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize) + .ToListAsync() + .ConfigureAwait(false) + ); } } } // Compare with db and see that all blocklists are listed - using (var il = await db.CreateIndexlist(vol.Name)) + using (var il = await db.CreateIndexlist(vol.Name).ConfigureAwait(false)) { foreach (var t in blocklinks) - await il.AddBlockLink(t.Item1, t.Item2, t.Item3); + await il + .AddBlockLink(t.Item1, t.Item2, t.Item3) + .ConfigureAwait(false); - combined.AddRange(await il.Compare().ToArrayAsync()); + combined.AddRange( + await il + .Compare() + .ToArrayAsync() + .ConfigureAwait(false) + ); } return new KeyValuePair>>(vol.Name, combined); case RemoteVolumeType.Blocks: using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var bl = await db.CreateBlocklist(vol.Name)) + using (var bl = await db.CreateBlocklist(vol.Name).ConfigureAwait(false)) using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options)) { //Verify that all blocks are in the file foreach (var b in rd.Blocks) - await bl.AddBlock(b.Key, b.Value); + await bl + .AddBlock(b.Key, b.Value) + .ConfigureAwait(false); //Select random blocks and verify their hashes match the filename and size var hashsamples = new List>(rd.Blocks); @@ -339,7 +387,15 @@ namespace Duplicati.Library.Main.Operation } } - return new KeyValuePair>>(vol.Name, changes.Union(await bl.Compare().ToListAsync())); + return new KeyValuePair>>( + vol.Name, + changes.Union( + await bl + .Compare() + .ToListAsync() + .ConfigureAwait(false) + ) + ); } } @@ -349,7 +405,9 @@ namespace Duplicati.Library.Main.Operation private async Task ReplaceFaultyIndexFilesAsync(List faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, CancellationToken cancellationToken) { - using var repairdb = await LocalRepairDatabase.CreateAsync(db); + using var repairdb = + await LocalRepairDatabase.CreateAsync(db) + .ConfigureAwait(false); foreach (var vol in faultyIndexFiles) { @@ -372,7 +430,9 @@ namespace Duplicati.Library.Main.Operation { await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_results.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(repairdb, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - await repairdb.Transaction.CommitAsync("ReplaceFaultyIndexFileCommit"); + await repairdb.Transaction + .CommitAsync("ReplaceFaultyIndexFileCommit") + .ConfigureAwait(false); } } catch (Exception ex) diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index e531e5cd6..d5846b943 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -37,9 +37,11 @@ namespace Duplicati.Library.Main.Operation public virtual async Task RunAsync() { - using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache); + using var db = + await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache) + .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); - await db.Vacuum(); + await db.Vacuum().ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; } } diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 0073acf96..24c817ee6 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -263,16 +263,18 @@ namespace Duplicati.Library.Main public async Task FlushLog(LocalDatabase db) { if (m_parent != null) - await m_parent.FlushLog(db); + await m_parent.FlushLog(db).ConfigureAwait(false); else { - await m_lock.WaitAsync(); + await m_lock.WaitAsync().ConfigureAwait(false); try { while (m_dbqueue.Count > 0) { var el = m_dbqueue.Dequeue(); - await db.LogMessage(el.Type, el.Message, el.Exception); + await db + .LogMessage(el.Type, el.Message, el.Exception) + .ConfigureAwait(false); } } finally diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index c069c65f9..bb59e0b00 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -107,7 +107,7 @@ namespace Duplicati.Library.Main internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options) { string n = null; - var opts = await db.GetDbOptions(); + var opts = await db.GetDbOptions().ConfigureAwait(false); if (opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n))) options.RawOptions["blocksize"] = opts["blocksize"] + "b"; @@ -124,8 +124,8 @@ namespace Duplicati.Library.Main /// true if the database contains options that need to be verified; false otherwise internal static async Task ContainsOptionsForVerification(LocalDatabase db) { - var opts = await db.GetDbOptions(); - await db.Transaction.CommitAsync(); + var opts = await db.GetDbOptions().ConfigureAwait(false); + await db.Transaction.CommitAsync().ConfigureAwait(false); return new[] { "blocksize", "blockhash", @@ -149,7 +149,7 @@ namespace Duplicati.Library.Main { "blockhash", options.BlockHashAlgorithm }, { "filehash", options.FileHashAlgorithm } }; - var opts = await db.GetDbOptions(); + var opts = await db.GetDbOptions().ConfigureAwait(false); if (options.NoEncryption) { @@ -199,7 +199,7 @@ namespace Duplicati.Library.Main } //Extra sanity check - if (await db.GetBlocksLargerThan(options.Blocksize) > 0) + if (await db.GetBlocksLargerThan(options.Blocksize).ConfigureAwait(false) > 0) throw new Duplicati.Library.Interface.UserInformationException("You have attempted to change the block-size on an existing backup, which is not supported. Please configure a new clean backup if you want to change the block-size.", "BlockSizeChangeNotSupported"); if (needsUpdate) @@ -209,7 +209,7 @@ namespace Duplicati.Library.Main if (!newDict.ContainsKey(k.Key)) newDict[k.Key] = k.Value; - await db.SetDbOptions(newDict); + await db.SetDbOptions(newDict).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs b/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs index e399f23f2..4e0841891 100644 --- a/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs @@ -49,7 +49,7 @@ namespace Duplicati.Library.Main.Volumes //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, using (var s = m_compression.CreateFile(Library.Utility.Utility.Base64PlainToBase64Url(hash), hint, DateTime.UtcNow)) - await s.WriteAsync(data.AsMemory(offset, size)); + await s.WriteAsync(data.AsMemory(offset, size)).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 7480b2f6b..be3b743da 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -84,12 +84,14 @@ namespace Duplicati.Library.Main.Volumes public async Task AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) { - await AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); + await AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes) + .ConfigureAwait(false); } public async Task AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) { - await AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); + await AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes) + .ConfigureAwait(false); } private async Task AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) @@ -114,12 +116,12 @@ namespace Duplicati.Library.Main.Volumes //Slightly awkward, but we avoid writing if there are no entries await using (var en = blocklisthashes.GetAsyncEnumerator()) { - if (await en.MoveNextAsync() && !string.IsNullOrEmpty(en.Current)) + if (await en.MoveNextAsync().ConfigureAwait(false) && !string.IsNullOrEmpty(en.Current)) { m_writer.WritePropertyName("blocklists"); m_writer.WriteStartArray(); m_writer.WriteValue(en.Current); - while (await en.MoveNextAsync()) + while (await en.MoveNextAsync().ConfigureAwait(false)) m_writer.WriteValue(en.Current); m_writer.WriteEndArray(); } @@ -251,7 +253,8 @@ namespace Duplicati.Library.Main.Volumes /// The maximum number of tries to probe /// The first unused filename internal static async Task ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) - => await ProbeUnusedFilenameName(database.GetRemoteVolumeID, options, start, increment, maxTries); + => await ProbeUnusedFilenameName(database.GetRemoteVolumeID, options, start, increment, maxTries) + .ConfigureAwait(false); /// /// Probes for an unused filename, using the current time as a starting point diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index c9b1fe6be..7de8de5ad 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -68,12 +68,13 @@ namespace Duplicati.Library.SQLiteHelper try { //Attempt to open in preferred state - await OpenSQLiteFileAsync(con, databasePath); - await TestSQLiteFileAsync(con); + await OpenSQLiteFileAsync(con, databasePath) + .ConfigureAwait(false); + await TestSQLiteFileAsync(con).ConfigureAwait(false); } catch { - try { await con.DisposeAsync(); } + try { await con.DisposeAsync().ConfigureAwait(false); } catch { } throw; @@ -104,7 +105,7 @@ namespace Duplicati.Library.SQLiteHelper catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, "Failed to load connection."); - await DisposeConnectionAsync(con); + await DisposeConnectionAsync(con).ConfigureAwait(false); throw; } @@ -144,7 +145,7 @@ namespace Duplicati.Library.SQLiteHelper try { cmd.CommandText = string.Format(CultureInfo.InvariantCulture, "PRAGMA {0}", opt); - await cmd.ExecuteNonQueryAsync(); + await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } catch (Exception ex) { @@ -171,22 +172,23 @@ namespace Duplicati.Library.SQLiteHelper if (string.IsNullOrWhiteSpace(targetpath)) throw new ArgumentNullException(nameof(targetpath)); - var con = await LoadConnectionAsync(); + var con = await LoadConnectionAsync().ConfigureAwait(false); try { - await OpenSQLiteFileAsync(con, targetpath); + await OpenSQLiteFileAsync(con, targetpath).ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, @"Failed to load connection with path '{0}'.", targetpath); - await DisposeConnectionAsync(con); + await DisposeConnectionAsync(con).ConfigureAwait(false); throw; } // set custom Sqlite options - return await ApplyCustomPragmasAsync(con, pagecachesize); + return await ApplyCustomPragmasAsync(con, pagecachesize) + .ConfigureAwait(false); } /// @@ -241,7 +243,7 @@ namespace Duplicati.Library.SQLiteHelper private static async Task DisposeConnectionAsync(Microsoft.Data.Sqlite.SqliteConnection? con) { if (con != null) - try { await con.DisposeAsync(); } + try { await con.DisposeAsync().ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "ConnectionDisposeError", ex, "Failed to dispose connection"); } } @@ -253,7 +255,7 @@ namespace Duplicati.Library.SQLiteHelper private static async Task OpenSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con, string path) { con.ConnectionString = $"Data Source={path};Pooling=false"; - await con.OpenAsync(); + await con.OpenAsync().ConfigureAwait(false); // Make the file only accessible by the current user, unless opting out if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile))) @@ -270,7 +272,7 @@ namespace Duplicati.Library.SQLiteHelper // Do a dummy query to make sure we have a working db using var cmd = con.CreateCommand(); cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER"; - await cmd.ExecuteScalarAsync(); + await cmd.ExecuteScalarAsync().ConfigureAwait(false); } } } diff --git a/Duplicati/UnitTest/DirectListHandlerTests.cs b/Duplicati/UnitTest/DirectListHandlerTests.cs index 125bb4340..3a82641f9 100644 --- a/Duplicati/UnitTest/DirectListHandlerTests.cs +++ b/Duplicati/UnitTest/DirectListHandlerTests.cs @@ -326,7 +326,8 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, 1) + .ConfigureAwait(false); SeedTestData(db, [ "/folder1/", // 1 "/folder1/sub1/", @@ -336,7 +337,10 @@ namespace Duplicati.UnitTest ]); // Act - var result = await db.GetRootPrefixes(1).ToListAsync(); + var result = await db + .GetRootPrefixes(1) + .ToListAsync() + .ConfigureAwait(false); // Assert Assert.That(result, Does.Contain(1L)); @@ -352,7 +356,8 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, 1) + .ConfigureAwait(false); SeedTestData(db, [ "C:\\folder1\\", "C:\\folder1\\sub1\\", @@ -361,7 +366,10 @@ namespace Duplicati.UnitTest ]); // Act - var result = await db.GetRootPrefixes(1).ToListAsync(); + var result = await db + .GetRootPrefixes(1) + .ToListAsync() + .ConfigureAwait(false); // Assert Assert.That(result, Does.Contain(1L)); @@ -376,7 +384,8 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, 1) + .ConfigureAwait(false); SeedTestData(db, [ "\\\\server\\share\\folder\\", "\\\\server\\share\\folder\\subfolder\\", @@ -384,7 +393,10 @@ namespace Duplicati.UnitTest ]); // Act - var result = await db.GetRootPrefixes(1).ToListAsync(); + var result = await db + .GetRootPrefixes(1) + .ToListAsync() + .ConfigureAwait(false); // Assert Assert.That(result, Does.Contain(1L)); @@ -398,7 +410,8 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, 1) + .ConfigureAwait(false); SeedTestData(db, [ "C:\\data\\", // 1 "C:\\data\\sub1\\", @@ -410,7 +423,10 @@ namespace Duplicati.UnitTest ]); // Act - var result = await db.GetRootPrefixes(1).ToListAsync(); + var result = await db + .GetRootPrefixes(1) + .ToListAsync() + .ConfigureAwait(false); // Assert Assert.That(result, Does.Contain(1L)); diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index f0b0a61c9..605656b0c 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -133,8 +133,8 @@ namespace Duplicati.UnitTest if (File.Exists(dbpath)) File.Delete(dbpath); - var con = SQLiteLoader.LoadConnection(); - await SQLiteLoader.OpenDatabaseAsync(con, dbpath, null); + var con = await SQLiteLoader.LoadConnectionAsync().ConfigureAwait(false); + await SQLiteLoader.OpenDatabaseAsync(con, dbpath, null).ConfigureAwait(false); DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); using (var connection = new Connection(con, true, null, this.serverDatafolder, null)) diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 554cc4349..f4ccbb995 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -426,18 +426,30 @@ namespace Duplicati.UnitTest } // Check database block link - using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, 0)) + using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, 0).ConfigureAwait(false)) { - var indexVolumeId = await db.GetRemoteVolumeID(Path.GetFileName(origFile)); - var duplicateVolumeId = await db.GetRemoteVolumeID(Path.GetFileName(dupFile)); + var indexVolumeId = await db + .GetRemoteVolumeID(Path.GetFileName(origFile)) + .ConfigureAwait(false); + var duplicateVolumeId = await db + .GetRemoteVolumeID(Path.GetFileName(dupFile)) + .ConfigureAwait(false); Assert.AreNotEqual(-1, indexVolumeId); Assert.AreNotEqual(-1, duplicateVolumeId); using (var cmd = db.Connection.CreateCommand()) { var sql = @"SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" = @VolumeId"; - var linkedIndexId = await cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", indexVolumeId).ExecuteScalarInt64Async(-1); - var linkedDuplicateId = await cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", duplicateVolumeId).ExecuteScalarInt64Async(-1); + var linkedIndexId = await cmd + .SetCommandAndParameters(sql) + .SetParameterValue("@VolumeId", indexVolumeId) + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); + var linkedDuplicateId = await cmd + .SetCommandAndParameters(sql) + .SetParameterValue("@VolumeId", duplicateVolumeId) + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); Assert.AreEqual(linkedIndexId, linkedDuplicateId); } } diff --git a/Duplicati/UnitTest/SQLiteTests.cs b/Duplicati/UnitTest/SQLiteTests.cs index 7eb5d5ae3..92f881c64 100755 --- a/Duplicati/UnitTest/SQLiteTests.cs +++ b/Duplicati/UnitTest/SQLiteTests.cs @@ -39,10 +39,10 @@ namespace Duplicati.UnitTest using (var command = connection.CreateCommand()) { command.CommandText = "CREATE TABLE TestTable (ID INTEGER PRIMARY KEY, Name TEXT)"; - await command.ExecuteNonQueryAsync(); + await command.ExecuteNonQueryAsync().ConfigureAwait(false); command.CommandText = "INSERT INTO TestTable (ID, Name) VALUES (1, 'Test1'), (2, 'Test2'), (3, 'Test3')"; - await command.ExecuteNonQueryAsync(); + await command.ExecuteNonQueryAsync().ConfigureAwait(false); } return connection; @@ -54,13 +54,14 @@ namespace Duplicati.UnitTest public async Task TestEmptyTransaction() { using var tf = new TempFile(); - using var connection = await CreateDummyDatabase(tf); + using var connection = await CreateDummyDatabase(tf) + .ConfigureAwait(false); using var t1 = connection.BeginTransaction(); - await t1.CommitAsync(); // No exception should be thrown + await t1.CommitAsync().ConfigureAwait(false); // No exception should be thrown using var t2 = connection.BeginTransaction(); - await t2.RollbackAsync(); // No exception should be thrown + await t2.RollbackAsync().ConfigureAwait(false); // No exception should be thrown } [Test] @@ -68,31 +69,31 @@ namespace Duplicati.UnitTest public async Task TestListExpansion() { using var tf = new TempFile(); - using var connection = await CreateDummyDatabase(tf); + using var connection = await CreateDummyDatabase(tf).ConfigureAwait(false); using var rtr = new ReusableTransaction(connection); using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1, 2, 3 }); - Assert.AreEqual(3, await command.ExecuteScalarInt64Async()); + Assert.AreEqual(3, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1, 2 }); - Assert.AreEqual(2, await command.ExecuteScalarInt64Async()); + Assert.AreEqual(2, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1 }); - Assert.AreEqual(1, await command.ExecuteScalarInt64Async()); + Assert.AreEqual(1, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[0]); - Assert.AreEqual(0, await command.ExecuteScalarInt64Async()); + Assert.AreEqual(0, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); } var list = new List(); @@ -101,9 +102,10 @@ namespace Duplicati.UnitTest using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { - using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list); + using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list) + .ConfigureAwait(false); command.ExpandInClauseParameter("@List", list); - Assert.AreEqual(3, await command.ExecuteScalarInt64Async()); + Assert.AreEqual(3, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) @@ -111,9 +113,10 @@ namespace Duplicati.UnitTest list.Remove(1); list.Remove(2); - using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list); + using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list) + .ConfigureAwait(false); command.ExpandInClauseParameter("@List", list); - Assert.AreEqual(1, await command.ExecuteScalarInt64Async()); + Assert.AreEqual(1, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); } } } From e81100de04f8b0f94c5515ba4849ab7e2a81cd77 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 09:59:22 +0200 Subject: [PATCH 299/362] Started on adding docstrings --- .../DatabaseTool/Commands/Downgrade.cs | 1 + .../DatabaseTool/Commands/Upgrade.cs | 1 + Duplicati/CommandLine/DatabaseTool/Helper.cs | 4 +- .../BackendManager.DatabaseCollector.cs | 7 +- .../Library/Main/Backend/BackendManager.cs | 21 +- .../Library/Main/Database/ChangeStatistics.cs | 64 ++--- .../Library/Main/Database/ExtensionMethods.cs | 239 +++++++++++++++++- .../Main/Database/LocalBackupDatabase.cs | 119 +++++++-- 8 files changed, 379 insertions(+), 77 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs index cc08250fb..dcada0b8c 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs @@ -134,6 +134,7 @@ public static class Downgrade /// The target database version /// The downgrade scripts /// Flag to disable backups + /// A task that completes when the downgrade is done. private static async Task ApplyDowngrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) { if (dbversion > targetversion) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs index 12ada136f..a2c37be64 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs @@ -147,6 +147,7 @@ public static class Upgrade /// The target database version /// The upgrade scripts /// Flag to disable backups + /// A task that completes when the upgrade is done. private static async Task ApplyUpgrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) { if (targetversion > dbversion) diff --git a/Duplicati/CommandLine/DatabaseTool/Helper.cs b/Duplicati/CommandLine/DatabaseTool/Helper.cs index 2b71aa549..edb258025 100644 --- a/Duplicati/CommandLine/DatabaseTool/Helper.cs +++ b/Duplicati/CommandLine/DatabaseTool/Helper.cs @@ -41,7 +41,7 @@ public static class Helper /// The input databases /// The folder to scan /// Whether to scan for extra databases - /// The list of databases + /// A task that when awaited contains the list of databases. public static async Task FindAllDatabases(string[]? databases, string datafolder, bool scanExtra) { databases ??= []; @@ -90,7 +90,7 @@ public static class Helper /// Examines the database and returns the version and whether it is a server database /// /// The path to the database - /// A tuple containing the version and whether it is a server database + /// A task that when awaited contains a tuple with the version and whether it is a server database public static async Task<(int Version, bool isserver)> ExamineDatabase(string db) { using (var con = await SQLiteLoader.LoadConnectionAsync(db, 0).ConfigureAwait(false)) diff --git a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs index a53e68159..8a64605e6 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs @@ -112,11 +112,10 @@ partial class BackendManager } /// - /// Flushes all messages to the database + /// Flushes all messages to the database. /// - /// The database to write pending messages to - /// The transaction to use, if any - /// Whether any messages were flushed + /// The database to write pending messages to. + /// A task that when awaited contains whether any messages were flushed. public async Task FlushPendingMessages(LocalDatabase db) { List entries; diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index 560d14a37..1e96ca2ab 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -292,22 +292,21 @@ internal partial class BackendManager : IBackendManager } /// - /// Flushes the database messages to the database + /// Flushes the database messages to the database. /// - /// The database to write to - /// The transaction to use - /// The cancellation token - /// + /// The database to write to. + /// The cancellation token. + /// A task that completes when the messages are flushed. public async Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken) { await context.Database.FlushPendingMessages(database).ConfigureAwait(false); } /// - /// Waits for the backend queue to be empty + /// Waits for the backend queue to be empty. /// - /// The cancellation token - /// An awaitable task + /// The cancellation token. + /// An awaitable task. public async Task WaitForEmptyAsync(CancellationToken cancellationToken) { var op = new WaitForEmptyOperation(context, cancellationToken); @@ -330,10 +329,10 @@ internal partial class BackendManager : IBackendManager } /// - /// Stops the backend manager and flushes any pending messages to the database + /// Stops the backend manager and flushes any pending messages to the database. /// - /// The database to write pending messages to - /// The transaction to use, if any + /// The database to write pending messages to. + /// A task that completes when the runner is stopped and messages are flushed. public async Task StopRunnerAndFlushMessages(LocalDatabase database) { await requestChannel.RetireAsync().ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Database/ChangeStatistics.cs b/Duplicati/Library/Main/Database/ChangeStatistics.cs index 0332882e7..5963196b7 100644 --- a/Duplicati/Library/Main/Database/ChangeStatistics.cs +++ b/Duplicati/Library/Main/Database/ChangeStatistics.cs @@ -37,12 +37,13 @@ public static class ChangeStatistics private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ChangeStatistics)); /// - /// Calculates the change statistics for the current and previous fileset + /// Calculates the change statistics for the current and previous fileset. /// - /// The database command to use - /// The results object to update with the statistics - /// The ID of the current fileset - /// The ID of the previous fileset + /// The database command to use. + /// The results object to update with the statistics. + /// The ID of the current fileset. + /// The ID of the previous fileset. + /// A task that completes when the statistics are updated. internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId) { var tmpName = $"TmpFileState_{Guid.NewGuid():N}"; @@ -138,13 +139,13 @@ public static class ChangeStatistics } /// - /// Counts the number of added files or folders + /// Counts the number of added files or folders. /// - /// The database command to use - /// The name of the temporary table - /// The ID of the blockset to count, or null for all - /// The blocksets to exclude from the count - /// The number of added files or folders + /// The database command to use. + /// The name of the temporary table. + /// The ID of the blockset to count, or null for all. + /// The blocksets to exclude from the count. + /// A task that when awaited contains the number of added files or folders. private static async Task CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) { var conditions = $@" @@ -164,13 +165,13 @@ public static class ChangeStatistics } /// - /// Counts the number of deleted files or folders + /// Counts the number of deleted files or folders. /// - /// The database command to use - /// The name of the temporary table - /// The ID of the blockset to count, or null for all - /// The blocksets to exclude from the count - /// The number of deleted files or folders + /// The database command to use. + /// The name of the temporary table. + /// The ID of the blockset to count, or null for all. + /// The blocksets to exclude from the count. + /// A task that when awaited contains the number of deleted files or folders. private static async Task CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) { var conditions = @$" @@ -190,14 +191,13 @@ public static class ChangeStatistics } /// - /// Counts the number of modified files or folders + /// Counts the number of modified files or folders. /// - /// The database command to use - /// The name of the temporary table - /// The ID of the blockset to count, or null for all - /// The hash column to compare - /// The blocksets to exclude from the count - /// The number of modified files or folders + /// The database command to use. + /// The name of the temporary table. + /// The ID of the blockset to count, or null for all. + /// The blocksets to exclude from the count. + /// A task that when awaited contains the number of modified files or folders. private static async Task CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) { string conditions; @@ -246,15 +246,15 @@ public static class ChangeStatistics } /// - /// Counts the number of files or folders with a specific condition + /// Counts the number of files or folders with a specific condition. /// - /// The database command to use - /// The name of the temporary table - /// The alias for the temporary table - /// The base condition for the count - /// The ID of the blockset to count, or null for all - /// The blocksets to exclude from the count - /// The number of files or folders that match the condition + /// The database command to use. + /// The name of the temporary table. + /// The alias for the temporary table. + /// The base condition for the count. + /// The ID of the blockset to count, or null for all. + /// The blocksets to exclude from the count. + /// A task that when awaited contains the number of files or folders that match the condition. private static async Task CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets) { var fullCondition = baseCondition; diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index dca4b64fb..cae4e2e05 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -36,6 +36,14 @@ namespace Duplicati.Library.Main.Database; /// public static partial class ExtensionMethods { + + /// + /// Converts the value at the given index in the reader to an Int64. + /// + /// The to read from. + /// The index of the value to convert. + /// The default value to return if the value is null or cannot be converted. + /// The converted Int64 value, or the default value if the conversion fails. public static long ConvertValueToInt64(this SqliteDataReader reader, int index, long defaultvalue = -1) { try @@ -43,13 +51,17 @@ public static partial class ExtensionMethods if (!reader.IsDBNull(index)) return reader.GetInt64(index); } - catch - { - } + catch { } return defaultvalue; } + /// + /// Converts the value at the given index in the reader to a string. + /// + /// The to read from. + /// The index of the value to convert. + /// The converted string value, or null if the value is null or cannot be converted. public static string? ConvertValueToString(this SqliteDataReader reader, int index) { var v = reader.GetValue(index); @@ -59,11 +71,23 @@ public static partial class ExtensionMethods return v.ToString(); } + /// + /// Creates a new with the given command text. + /// + /// The to create the command for. + /// The command text to set for the command. + /// A new with the command text set. public static SqliteCommand CreateCommand(this SqliteConnection self, string cmdtext) { return CreateCommandAsync(self, cmdtext).Await(); } + /// + /// Creates a new with the given command text and prepares it asynchronously. + /// + /// The to create the command for. + /// The command text to set for the command. + /// A new with the command text set and prepared asynchronously. public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext) { var cmd = self.CreateCommand() @@ -73,6 +97,12 @@ public static partial class ExtensionMethods return cmd; } + /// + /// Creates a new with the given transaction. + /// + /// The to create the command for. + /// The to set for the command. + /// A new with the transaction set. public static SqliteCommand CreateCommand(this SqliteConnection self, SqliteTransaction transaction) { var cmd = self.CreateCommand(); @@ -81,23 +111,50 @@ public static partial class ExtensionMethods return cmd; } + /// + /// Creates a new with the given reusable transaction. + /// + /// The to create the command for. + /// The to set for the command. + /// A new with the transaction set. internal static SqliteCommand CreateCommand(this SqliteConnection self, ReusableTransaction rtr) { return self.CreateCommand(rtr.Transaction); } + /// + /// Executes the command asynchronously and returns the number of rows affected. + /// + /// The instance to execute on. + /// The command text to execute. + /// A task that when awaited contains the number of rows affected. public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string? cmdtext) { return await ExecuteNonQueryAsync(self, true, cmdtext, null) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the number of rows affected. + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// A task that when awaited contains the number of rows affected. public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary values) { return await ExecuteNonQueryAsync(self, true, cmd, values) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the number of rows affected. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// A task that when awaited contains the number of rows affected. public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null) { if (cmd != null) @@ -110,18 +167,39 @@ public static partial class ExtensionMethods return await self.ExecuteNonQueryAsync().ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns a . + /// + /// The instance to execute on. + /// The command text to execute. + /// A task that when awaited contains the . public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext) { return await ExecuteReaderAsync(self, true, cmdtext, null) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns a . + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// A task that when awaited contains the . public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary? values) { return await ExecuteReaderAsync(self, true, cmdtext, values) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns a . + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// A task that when awaited contains the . public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext = null, Dictionary? values = null) { if (cmdtext != null) @@ -134,21 +212,47 @@ public static partial class ExtensionMethods return await self.ExecuteReaderAsync().ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns an enumerable of . + /// + /// The instance to execute on. + /// An asynchronous enumerable of . public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self) { return ExecuteReaderEnumerableAsync(self, true, null, null); } + /// + /// Executes the command asynchronously and returns an enumerable of . + /// + /// The instance to execute on. + /// The command text to execute. + /// An asynchronous enumerable of . public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext) { return ExecuteReaderEnumerableAsync(self, true, cmdtext, null); } + /// + /// Executes the command asynchronously and returns an enumerable of . + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// An asynchronous enumerable of . public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, Dictionary? values) { return ExecuteReaderEnumerableAsync(self, true, cmdtext, values); } + /// + /// Executes the command asynchronously and returns an enumerable of . + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// An asynchronous enumerable of . public static async IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, bool writeLog, string? cmdtext = null, Dictionary? values = null) { if (cmdtext != null) @@ -163,24 +267,50 @@ public static partial class ExtensionMethods yield return rd; } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. public static async Task ExecuteScalarAsync(this SqliteCommand self) { return await self.ExecuteScalarAsync(true, null, null) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// The command text to execute. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext) { return await self.ExecuteScalarAsync(true, cmdtext, null) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext, Dictionary? values) { return await self.ExecuteScalarAsync(true, cmdtext, values) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null) { if (cmd != null) @@ -193,30 +323,65 @@ public static partial class ExtensionMethods return await self.ExecuteScalarAsync().ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue = -1) { return await ExecuteScalarInt64Async(self, true, null, null, defaultvalue) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The default value to return if no rows are returned or the value cannot be converted. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue) { return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The command text to execute. + /// The default value to return if no rows are returned or the value cannot be converted. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, long defaultvalue = -1) { return await ExecuteScalarInt64Async(self, true, cmd, null, defaultvalue) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// The default value to return if no rows are returned or the value cannot be converted. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary? values, long defaultvalue = -1) { return await ExecuteScalarInt64Async(self, true, cmd, values, defaultvalue) .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// The default value to return if no rows are returned or the value cannot be converted. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null, long defaultvalue = -1) { if (cmd != null) @@ -233,6 +398,14 @@ public static partial class ExtensionMethods return defaultvalue; } + /// + /// Expands an IN clause parameter in a SqliteCommand by replacing the parameter with multiple parameters for each value in the collection. + /// + /// The to expand the IN clause parameter for. + /// The name of the original parameter to replace, must start with '@'. + /// The collection of values to expand the IN clause with. + /// The type of the values in the collection. + /// The modified with the IN clause expanded. public static SqliteCommand ExpandInClauseParameterMssqlite(this SqliteCommand cmd, string originalParamName, IEnumerable values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) @@ -261,6 +434,14 @@ public static partial class ExtensionMethods return cmd; } + /// + /// Expands an IN clause parameter in a SqliteCommand by replacing the parameter with multiple parameters for each value in the collection. + /// If a temporary table is used, it replaces the parameter with the table name. + /// + /// The to expand the IN clause parameter for. + /// The name of the original parameter to replace, must start with '@'. + /// The collection of values to expand the IN clause with. + /// A task that when awaited contains the modified with the IN clause expanded. internal static async Task ExpandInClauseParameterMssqliteAsync(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) @@ -279,6 +460,13 @@ public static partial class ExtensionMethods return cmd; } + /// + /// Sets the command text and parameters for the given . + /// + /// The to set the command text and parameters for. + /// The command text to set for the command, must not contain '?' as a parameter placeholder. + /// The modified with the command text and parameters set. + /// If the command text contains '?' as a parameter placeholder. public static SqliteCommand SetCommandAndParameters(this SqliteCommand cmd, string cmdtext) { cmd.CommandText = Library.Utility.Utility.FormatInvariant(cmdtext); @@ -303,12 +491,12 @@ public static partial class ExtensionMethods } /// - /// New edition + /// Sets the value of a parameter in the command. /// - /// - /// - /// - /// + /// The to set the parameter value for. + /// The name of the parameter to set the value for. + /// The value to set for the parameter, or null to set it to DBNull.Value. + /// The with the parameter value set. public static SqliteCommand SetParameterValue(this SqliteCommand self, string name, object? value) { #if DEBUG @@ -321,6 +509,13 @@ public static partial class ExtensionMethods } // Special case for DateTime, as we need to convert it to a long + /// + /// Sets the value of a DateTime parameter in the command. + /// + /// The to set the parameter value for. + /// The name of the parameter to set the value for. + /// The DateTime value to set for the parameter. + /// The with the parameter value set. public static SqliteCommand SetParameterValue(this SqliteCommand self, string name, DateTime value) { self.Parameters[name].Value = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(value); @@ -328,6 +523,12 @@ public static partial class ExtensionMethods return self; } + /// + /// Sets the parameter values for the command, the parameters must already be added. + /// + /// The to set the parameter values for. + /// The values to set the parameters to, where the key is the parameter name. + /// The with the parameter values set. public static SqliteCommand SetParameterValues(this SqliteCommand self, Dictionary values) { foreach (var kvp in values) @@ -336,6 +537,12 @@ public static partial class ExtensionMethods return self; } + /// + /// Sets the transaction for the command. + /// + /// The to set the transaction for. + /// The to set for the command. + /// The with the transaction set. public static SqliteCommand SetTransaction(this SqliteCommand self, SqliteTransaction transaction) { self.Transaction = transaction; @@ -343,6 +550,12 @@ public static partial class ExtensionMethods return self; } + /// + /// Sets the transaction for the command using a . + /// + /// The to set the transaction for. + /// The to set for the command. + /// The with the transaction set. internal static SqliteCommand SetTransaction(this SqliteCommand self, ReusableTransaction rtr) { self.Transaction = rtr.Transaction; @@ -355,6 +568,7 @@ public static partial class ExtensionMethods /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ExtensionMethods)); + // This regex matches named parameters in the form of @paramName [GeneratedRegex(@"@\w+")] private static partial Regex MyRegex(); @@ -380,6 +594,13 @@ public static partial class ExtensionMethods // Microsoft.Data.Sqlite, the extension methods using IDb* should be // removed. Currently, they're kept for compatibility. + /// + /// Adds a named parameter to the command with the given value. + /// + /// The command to add the parameter to + /// The name of the parameter + /// The optional value of the parameter + /// The command with the parameter added public static IDbCommand AddNamedParameter(this IDbCommand self, string name, object? value = null) { var p = self.CreateParameter(); @@ -430,7 +651,7 @@ public static partial class ExtensionMethods /// The type of the command /// The command to set the transaction for /// The transaction to set for the command - /// >The command with the transaction set + /// The command with the transaction set public static T SetTransaction(this T self, IDbTransaction? transaction) where T : IDbCommand { diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index d708260ac..c460b440c 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -31,37 +31,97 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// A local backup database that stores blocks, files, and metadata for backup operations. + /// This database is used to track the state of backups and to allow for efficient retrieval of blocks and files. + /// It supports operations such as finding blocks by hash and size, inserting new blocks, files, and blocksets, + /// and managing metadata datasets. + /// internal class LocalBackupDatabase : LocalDatabase { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + /// + /// The command used to find a block by its hash and size. + /// private SqliteCommand m_findblockCommand = null!; + /// + /// The command used to find a blockset by its full hash and length. + /// private SqliteCommand m_findblocksetCommand = null!; + /// + /// The command used to find a metadataset by its block hash and size. + /// private SqliteCommand m_findfilesetCommand = null!; + /// + /// The command used to find a metadataset by its block hash and size. + /// private SqliteCommand m_findmetadatasetCommand = null!; + /// + /// The command used to insert a block into the database. + /// private SqliteCommand m_insertblockCommand = null!; + /// + /// The command used to insert a file into the database. + /// private SqliteCommand m_insertfileCommand = null!; + /// + /// The command used to insert a blockset into the database. + /// private SqliteCommand m_insertblocksetCommand = null!; + /// + /// The command used to insert a blockset entry into the database. + /// private SqliteCommand m_insertblocksetentryCommand = null!; + /// + /// The command used to insert a blocklist hash into the database. + /// private SqliteCommand m_insertblocklistHashesCommand = null!; + /// + /// The command used to insert a metadataset into the database. + /// private SqliteCommand m_insertmetadatasetCommand = null!; + /// + /// The command used to find a file in the database. + /// private SqliteCommand m_findfileCommand = null!; + /// + /// The command used to select the last modified time of a file. + /// private SqliteCommand m_selectfilelastmodifiedCommand = null!; + /// + /// The command used to select the last modified time and size of a file. + /// private SqliteCommand m_selectfilelastmodifiedWithSizeCommand = null!; + /// + /// The command used to select the hash and size of a file's metadata. + /// private SqliteCommand m_selectfileHashCommand = null!; + /// + /// The command used to insert a file operation into the database. + /// private SqliteCommand m_insertfileOperationCommand = null!; + /// + /// The command used to select the metadata hash and size of a file. + /// private SqliteCommand m_selectfilemetadatahashandsizeCommand = null!; + /// + /// The command used to find the first fileset with a block in a blockset. + /// private SqliteCommand m_getfirstfilesetwithblockinblockset = null!; + /// + /// HashSet of blocklist hashes to track whether a blocklist hash has been seen before. + /// private HashSet m_blocklistHashes = []; /// @@ -81,10 +141,24 @@ namespace Duplicati.Library.Main.Database /// private SqliteCommand? m_findindeletedCommand; + /// + /// The ID of the fileset currently being processed. + /// private long m_filesetId; + /// + /// Indicates whether the database should log queries for profiling purposes. + /// private bool m_logQueries; + /// + /// Initializes a new instance of the class. + /// This constructor is private to enforce the use of the static and asynchronous CreateAsync methods for instantiation. + /// + /// The path to the database file. + /// The Duplicati options used by the current operation. + /// An optional existing instance of to reuse. Used when derived classes need to call the base constructor. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(string path, Options options, LocalBackupDatabase? dbnew = null) { dbnew ??= new LocalBackupDatabase(); @@ -98,6 +172,14 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Creates a new instance of using an existing parent database. + /// This method is used to create a new backup database based on an existing local database. + /// + /// The parent local database from which to create the new backup database. + /// The Duplicati options used by the current operation. + /// An optional existing instance of to reuse. Used when derived classes need to call the base constructor. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(LocalDatabase dbparent, Options options, LocalBackupDatabase? dbnew = null) { dbnew ??= new LocalBackupDatabase(); @@ -646,11 +728,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Probes to see if a block already exists + /// Probes to see if a block already exists. /// - /// The block key - /// The size of the block - /// True if the block should be added to the current output + /// The block key. + /// The size of the block. + /// A task that when awaited contains true if the block should be added to the current output. public async Task FindBlockID(string key, long size) { return await m_findblockCommand @@ -662,11 +744,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Adds a block to the local database, returning a value indicating if the value presents a new block + /// Adds a block to the local database, returning a value indicating if the value presents a new block. /// - /// The block key - /// The size of the block - /// True if the block should be added to the current output + /// The block key. + /// The size of the block. + /// A taskt that when awaited contains true if the block should be added to the current output. public async Task AddBlock(string key, long size, long volumeid) { var r = await FindBlockID(key, size).ConfigureAwait(false); @@ -740,13 +822,14 @@ namespace Duplicati.Library.Main.Database /// - /// Adds a blockset to the database, returns a value indicating if the blockset is new + /// Adds a blockset to the database, returns a value indicating if the blockset is new. /// - /// The hash of the blockset - /// The size of the blockset - /// The list of hashes - /// The id of the blockset, new or old - /// True if the blockset was created, false otherwise + /// The hash of the blockset. + /// The size of the blockset. + /// The size of the blocks in the blockset. + /// The list of hashes. + /// The id of the blockset, new or old. + /// A task that when awaited contains a tuple with the first value indicating whether the blockset was created, and the second value being the blockset ID. public async Task<(bool, long)> AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes) { long blocksetid = await m_findblocksetCommand @@ -845,13 +928,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Gets the metadataset ID from the filehash + /// Gets the metadataset ID from the filehash. /// - /// true, if metadataset found, false if does not exist. /// The metadata hash. /// The size of the metadata. - /// The ID of the metadataset. - /// An optional transaction. + /// A task that when awaited contains a tuple with the first value indicating if the metadataset was found, and the second value being the metadataset ID. public async Task<(bool, long)> GetMetadatasetID(string filehash, long size) { long metadataid; From 1ece8398b61e430cc25ef2b11a27068730ee2ed1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 14:02:18 +0200 Subject: [PATCH 300/362] Reordered tuple to match name --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 6 +++--- Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index c460b440c..68049adb4 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -1155,7 +1155,7 @@ namespace Duplicati.Library.Main.Database } } - public async Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFile(long fileid) + public async Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFile(long fileid) { m_selectfilemetadatahashandsizeCommand .SetTransaction(m_rtr) @@ -1164,8 +1164,8 @@ namespace Duplicati.Library.Main.Database using (var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync().ConfigureAwait(false)) if (await rd.ReadAsync().ConfigureAwait(false)) return ( - rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null") + rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null"), + rd.ConvertValueToInt64(0) ); return null; diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index f21a60bb1..74d355c87 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -117,7 +117,7 @@ namespace Duplicati.Library.Main.Operation.Backup ); } - public Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFileAsync(long fileid) + public Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFileAsync(long fileid) { return RunOnMain(async () => await m_database From 1f97832546154177c25b542834ae776b1a087c5e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 15:11:25 +0200 Subject: [PATCH 301/362] Classes implementing DisposeAsync should also implement the interface --- .../Library/Main/Database/LocalBackupDatabase.cs | 4 ++-- Duplicati/Library/Main/Database/LocalDatabase.cs | 12 ++++++------ .../Library/Main/Database/LocalRestoreDatabase.cs | 4 ++-- .../Library/Main/Database/ReusableTransaction.cs | 6 +++--- .../Library/Main/Database/TemporaryDbValueList.cs | 6 +++--- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 68049adb4..596195dfa 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -1188,10 +1188,10 @@ namespace Duplicati.Library.Main.Database public override void Dispose() { - this.DisposeAsync().Await(); + this.DisposeAsync().AsTask().Await(); } - public override async Task DisposeAsync() + public override async ValueTask DisposeAsync() { if (!string.IsNullOrWhiteSpace(m_tempDeletedBlockTable)) try diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 9f7828edb..b78522f23 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1728,7 +1728,7 @@ namespace Duplicati.Library.Main.Database private class BlocklistHashEnumerable : IAsyncEnumerable { - private class BlocklistHashEnumerator : IAsyncEnumerator + private class BlocklistHashEnumerator : IDisposable, IAsyncEnumerator { private readonly SqliteDataReader m_reader; private readonly BlocklistHashEnumerable m_parent; @@ -2075,7 +2075,7 @@ namespace Duplicati.Library.Main.Database /// /// Keeps a list of filenames in a temporary table with a single column Path /// - public class FilteredFilenameTable : IDisposable + public class FilteredFilenameTable : IDisposable, IAsyncDisposable { public string Tablename { get; private set; } private readonly LocalDatabase m_db; @@ -2209,10 +2209,10 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (Tablename != null) try @@ -2621,10 +2621,10 @@ namespace Duplicati.Library.Main.Database public virtual void Dispose() { - this.DisposeAsync().Await(); + this.DisposeAsync().AsTask().Await(); } - public virtual async Task DisposeAsync() + public virtual async ValueTask DisposeAsync() { if (IsDisposed) return; diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 7fc94c784..f371ad80a 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -2093,10 +2093,10 @@ namespace Duplicati.Library.Main.Database public override void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public override async Task DisposeAsync() + public override async ValueTask DisposeAsync() { await DisposePoolAsync().ConfigureAwait(false); await DropRestoreTable().ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 3f2ead105..013708b22 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.Database; /// /// The database to use. /// The transaction to use. If null, a new transaction is created. -internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? transaction = null) : IDisposable +internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? transaction = null) : IDisposable, IAsyncDisposable { /// /// The tag used for logging. @@ -92,14 +92,14 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran /// public void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } /// /// Async version of Dispose: /// /// An awaitable task. - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (!m_disposed) { diff --git a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs index 191f7d7db..275421b53 100644 --- a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs +++ b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs @@ -33,7 +33,7 @@ namespace Duplicati.Library.Main.Database; /// /// A list of values that can be used in a query /// -internal class TemporaryDbValueList : IDisposable +internal class TemporaryDbValueList : IDisposable, IAsyncDisposable { /// /// The tag used for logging @@ -185,10 +185,10 @@ internal class TemporaryDbValueList : IDisposable /// public void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (_disposed) return; From 8814fdebccf546bb894eb60ca00ca232cb2dbd71 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 15:13:33 +0200 Subject: [PATCH 302/362] Added more docstrings --- .../Main/Database/LocalBackupDatabase.cs | 193 ++++-- .../Main/Database/LocalBugReportDatabase.cs | 15 + .../Library/Main/Database/LocalDatabase.cs | 564 +++++++++++++++--- 3 files changed, 644 insertions(+), 128 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 596195dfa..62fd954d4 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -748,6 +748,7 @@ namespace Duplicati.Library.Main.Database /// /// The block key. /// The size of the block. + /// The ID of the volume to which the block belongs. /// A taskt that when awaited contains true if the block should be added to the current output. public async Task AddBlock(string key, long size, long volumeid) { @@ -828,7 +829,7 @@ namespace Duplicati.Library.Main.Database /// The size of the blockset. /// The size of the blocks in the blockset. /// The list of hashes. - /// The id of the blockset, new or old. + /// The list of hashes for the blocklist, or null if no blocklist is used. /// A task that when awaited contains a tuple with the first value indicating whether the blockset was created, and the second value being the blockset ID. public async Task<(bool, long)> AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes) { @@ -954,14 +955,12 @@ namespace Duplicati.Library.Main.Database } /// - /// Adds a metadata set to the database, and returns a value indicating if the record was new + /// Adds a metadata set to the database, and returns a tuple indicating if the record was new and the ID of the metadata set. /// - /// The metadata hash - /// The size of the metadata - /// The transaction to execute under - /// The id of the blockset to add - /// The id of the metadata set - /// True if the set was added to the database, false otherwise + /// The metadata hash. + /// The size of the metadata. + /// The id of the blockset to add. + /// A task that when awaited contains a tuple with the first value indicating if the metadata set was added, and the second value being the metadata ID. public async Task<(bool, long)> AddMetadataset(string filehash, long size, long blocksetid) { var (metadatafound, metadataid) = await GetMetadatasetID(filehash, size) @@ -981,14 +980,14 @@ namespace Duplicati.Library.Main.Database } /// - /// Adds a file record to the database + /// Adds a file record to the database. /// - /// The path prefix ID - /// The path to the file - /// The time the file was modified - /// The ID of the hashkey for the file - /// The ID for the metadata - /// The transaction to use for insertion, or null for no transaction + /// The path prefix ID. + /// The path to the file. + /// The time the file was modified. + /// The ID of the hashkey for the file. + /// The ID for the metadata. + /// A task that completes when the file is added. public async Task AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID) { var fileidobj = await m_findfilesetCommand @@ -1017,13 +1016,13 @@ namespace Duplicati.Library.Main.Database } /// - /// Adds a file record to the database + /// Adds a file record to the database. /// - /// The path to the file - /// The time the file was modified - /// The ID of the hashkey for the file - /// The ID for the metadata - /// The transaction to use for insertion, or null for no transaction + /// The path to the file. + /// The time the file was modified. + /// The ID of the hashkey for the file. + /// The ID for the metadata. + /// A task that completes when the file is added. public async Task AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID) { var split = SplitIntoPrefixAndName(filename); @@ -1039,11 +1038,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Adds a known file to the fileset + /// Adds a known file to the fileset. /// - /// Id of the file - /// The time the file was modified - /// The transaction to use for insertion, or null for no transaction + /// Id of the file. + /// The time the file was modified. + /// A task that completes when the file is added. public async Task AddKnownFile(long fileid, DateTime lastmodified) { await m_insertfileOperationCommand @@ -1055,18 +1054,40 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Adds a directory entry to the fileset. + /// + /// The path to the directory. + /// The ID for the metadata. + /// The time the directory was modified. + /// A task that completes when the directory entry is added. public async Task AddDirectoryEntry(string path, long metadataID, DateTime lastmodified) { await AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID) .ConfigureAwait(false); } + /// + /// Adds a symlink entry to the fileset. + /// + /// The path to the symlink. + /// The ID for the metadata. + /// The time the symlink was modified. + /// A task that completes when the symlink entry is added. public async Task AddSymlinkEntry(string path, long metadataID, DateTime lastmodified) { await AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID) .ConfigureAwait(false); } + /// + /// Gets the ID, last modified time and size of a file in the fileset. + /// + /// The ID of the path prefix. + /// The path to the file. + /// The ID of the fileset. + /// Whether to include the file length in the result. + /// A task that when awaited contains a tuple with the file ID, last modified time, and file length. public async Task<(long, DateTime, long)> GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength) { DateTime oldModified; @@ -1155,6 +1176,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets the metadata hash and size for a file. + /// + /// The ID of the file. + /// A task that when awaited contains a tuple with the metadata hash and size, or null if the file does not exist. public async Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFile(long fileid) { m_selectfilemetadatahashandsizeCommand @@ -1171,7 +1197,11 @@ namespace Duplicati.Library.Main.Database return null; } - + /// + /// Gets the hash of a file. + /// + /// The ID of the file. + /// A task that when awaited contains the hash of the file, or null if the file does not exist. public async Task GetFileHash(long fileid) { var r = await m_selectfileHashCommand @@ -1209,6 +1239,10 @@ namespace Duplicati.Library.Main.Database await base.DisposeAsync().ConfigureAwait(false); } + /// + /// Gets the size of the last written DBlock volume. + /// + /// A task that when awaited contains the size of the last written DBlock volume, or -1 if no such volume exists. public async Task GetLastWrittenDBlockVolumeSize() { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -1227,12 +1261,24 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Gets the ID of the previous fileset based on the operation timestamp and current fileset ID. + /// + /// The command to use for the query. + /// A task that when awaited contains the ID of the previous fileset, or -1 if no such fileset exists. private async Task GetPreviousFilesetID(SqliteCommand cmd) { return await GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId) .ConfigureAwait(false); } + /// + /// Gets the ID of the previous fileset based on the operation timestamp and current fileset ID. + /// + /// The command to use for the query. + /// The timestamp to use for the query. + /// The current fileset ID. + /// A task that when awaited contains the ID of the previous fileset, or -1 if no such fileset exists. private async Task GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid) { return await cmd @@ -1251,6 +1297,10 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Gets the count and size of files in the last backup fileset. + /// + /// A task that when awaited contains a tuple with the count of files and the total size of files in the last backup fileset. internal async Task> GetLastBackupFileCountAndSize() { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -1306,6 +1356,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Updates the change statistics for the current fileset based on the results of a backup operation. + /// + /// The results of the backup operation. + /// A task that completes when the change statistics are updated. internal async Task UpdateChangeStatistics(BackupResults results) { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -1321,8 +1376,8 @@ namespace Duplicati.Library.Main.Database /// Populates FilesetEntry table with files from previous fileset, which aren't /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. /// - /// Transaction - /// List of deleted paths, or null + /// List of deleted paths, or null. + /// A task that completes when the files are appended. public async Task AppendFilesFromPreviousSet(IEnumerable? deleted = null) { await AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp) @@ -1333,11 +1388,11 @@ namespace Duplicati.Library.Main.Database /// Populates FilesetEntry table with files from previous fileset, which aren't /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. /// - /// Transaction - /// List of deleted paths, or null - /// Current file-set ID - /// Source file-set ID - /// If filesetid == -1, used to locate previous file-set + /// List of deleted paths, or null. + /// Current file-set ID. + /// Source file-set ID. + /// If filesetid == -1, used to locate previous file-set. + /// A task that completes when the files are appended. public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) { using (var cmd = m_connection.CreateCommand()) @@ -1415,8 +1470,8 @@ namespace Duplicati.Library.Main.Database /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// - /// Transaction - /// Optional exclusion predicate (true = exclude file) + /// Optional exclusion predicate (true = exclude file). + /// A task that completes when the files are appended. public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate) { await AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp) @@ -1428,11 +1483,11 @@ namespace Duplicati.Library.Main.Database /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// - /// Transaction /// Optional exclusion predicate (true = exclude file) /// Current fileset ID /// Source fileset ID /// If prevFileSetId == -1, used to locate previous fileset + /// A task that completes when the files are appended. public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) { if (exclusionPredicate == null) @@ -1542,15 +1597,20 @@ namespace Duplicati.Library.Main.Database /// /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. /// - /// The ID of the fileset volume to update - /// The timestamp of the operation to create - /// An optional external transaction + /// The ID of the fileset volume to update. + /// The timestamp of the operation to create. + /// A task that when awaited contains the ID of the created fileset. public override async Task CreateFileset(long volumeid, DateTime timestamp) { return m_filesetId = await base.CreateFileset(volumeid, timestamp) .ConfigureAwait(false); } + /// + /// Retrieves the names of temporary fileset volumes that are incomplete. + /// + /// If true, only the latest incomplete fileset volume will be returned. + /// A task that when awaited contains a list of volume names. public async Task> GetTemporaryFilelistVolumeNames(bool latestOnly) { var incompleteFilesetIDs = GetIncompleteFilesets().OrderBy(x => x.Value).Select(x => x.Key); @@ -1574,6 +1634,10 @@ namespace Duplicati.Library.Main.Database return volumeNames; } + /// + /// Retrieves the names of remote volumes that are missing index files. + /// + /// An asynchronous enumerable of volume names that are missing index files. public async IAsyncEnumerable GetMissingIndexFiles() { using var cmd = m_connection.CreateCommand(m_rtr) @@ -1599,6 +1663,14 @@ namespace Duplicati.Library.Main.Database yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } + /// + /// Moves a block from one volume to another. + /// + /// The hash of the block to move. + /// The size of the block to move. + /// The ID of the source volume. + /// The ID of the target volume. + /// A task that completes when the block is moved. public async Task MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid) { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -1623,6 +1695,13 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Safely deletes a remote volume by checking if it has any associated blocks. + /// If it does, an exception is thrown; otherwise, the volume is removed. + /// + /// The name of the remote volume to delete. + /// A task that completes when the remote volume is safely deleted. + /// Thrown if the volume has associated blocks. public async Task SafeDeleteRemoteVolume(string name) { var volumeid = await GetRemoteVolumeID(name).ConfigureAwait(false); @@ -1645,6 +1724,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves the hashes of blocks that are on the blocklist for a given volume. + /// + /// The name of the volume to check. + /// A task that when awaited contains an array of blocklist hashes. public async Task GetBlocklistHashes(string name) { var volumeid = GetRemoteVolumeID(name); @@ -1670,6 +1754,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves the first path in the database, ordered by length in descending order. + /// + /// A task that when awaited contains the first path, or null if no paths exist. public async Task GetFirstPath() { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -1690,9 +1778,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Retrieves change journal data for file set + /// Retrieves the change journal data for file set. /// - /// Fileset-ID + /// The Fileset-ID. + /// An asynchronous enumerable of USN journal data entries. public async IAsyncEnumerable GetChangeJournalData(long fileSetId) { var data = new List(); @@ -1725,10 +1814,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Adds NTFS change journal data for file set and volume + /// Adds NTFS change journal data for file set and volume. /// - /// Data to add - /// An optional external transaction + /// Data to add. + /// A task that completes when the data is added. + /// Thrown if unable to add change journal entry. public async Task CreateChangeJournalData(IEnumerable data) { foreach (var entry in data) @@ -1768,11 +1858,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Adds NTFS change journal data for file set and volume + /// Adds NTFS change journal data for file set and volume. /// - /// Data to add - /// Existing file set to update - /// An optional external transaction + /// Data to add. + /// Existing file set to update. + /// A task that completes when the data is added. public async Task UpdateChangeJournalData(IEnumerable data, long fileSetId) { foreach (var entry in data) @@ -1801,11 +1891,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Checks if a blocklist hash is known + /// Checks if a blocklist hash is known. /// - /// The hash to check - /// An optional external transaction - /// True if the hash is known, false otherwise + /// The hash to check. + /// A task that when awaited returns true if the hash is known, false otherwise. public async Task IsBlocklistHashKnown(string hash) { var res = await m_getfirstfilesetwithblockinblockset diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 6470afa09..3b6674488 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -27,6 +27,10 @@ using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Database { + + /// + /// A local database for bug reports, which obfuscates sensitive data before generating a bug report. + /// internal class LocalBugReportDatabase : LocalDatabase { /// @@ -34,6 +38,13 @@ namespace Duplicati.Library.Main.Database /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalBugReportDatabase)); + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// The size of the page cache. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(string path, long pagecachesize, LocalBugReportDatabase? dbnew = null) { dbnew ??= new LocalBugReportDatabase(); @@ -46,6 +57,10 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Obfuscates sensitive data in the database, readying it for a bug report. + /// + /// A task that completes when the obfuscation is finished. public async Task Fix() { using (var cmd = m_connection.CreateCommand(m_rtr)) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index b78522f23..4f0cc920b 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -40,7 +40,12 @@ using System.Threading; namespace Duplicati.Library.Main.Database { - internal class LocalDatabase : IDisposable + /// + /// Represents a local database for Duplicati operations. + /// This class provides methods to interact with the local SQLite database, including + /// managing remote volumes, logging operations, and handling transactions. + /// + internal class LocalDatabase : IDisposable, IAsyncDisposable { /// /// The tag used for logging @@ -53,39 +58,114 @@ namespace Duplicati.Library.Main.Database /// SQLite has a limit of 999 parameters in a single statement public const int CHUNK_SIZE = 128; - // All of the required fields have been set to null! to ignore the compiler warning, as they will be initialized in the Create* factory functions. + // All of the required fields have been set to null! to ignore the compiler warning, as they will be initialized in the Create* factory methods. + /// + /// The SQLite connection to the local database. + /// protected SqliteConnection m_connection = null!; + /// + /// The operation ID for the current operation. + /// protected long m_operationid = -1; + /// + /// The size of the SQLite page cache. + /// protected long m_pagecachesize; + /// + /// Indicates whether the database has executed a vacuum operation. + /// private bool m_hasExecutedVacuum; + /// + /// The reusable transaction for the current operation. + /// protected ReusableTransaction m_rtr = null!; + /// + /// A read-only property that provides access to the current transaction. + /// public ReusableTransaction Transaction { get { return m_rtr; } } + /// + /// The command used to update a remote volume in the database. + /// private SqliteCommand m_updateremotevolumeCommand = null!; + /// + /// The command used to select remote volumes from the database. + /// private SqliteCommand m_selectremotevolumesCommand = null!; + /// + /// The command used to select a specific remote volume by name. + /// private SqliteCommand m_selectremotevolumeCommand = null!; + /// + /// The command used to remove a remote volume from the database. + /// private SqliteCommand m_removeremotevolumeCommand = null!; + /// + /// The command used to remove deleted remote volumes from the database. + /// private SqliteCommand m_removedeletedremotevolumeCommand = null!; + /// + /// The command used to select the ID of a remote volume by name. + /// private SqliteCommand m_selectremotevolumeIdCommand = null!; + /// + /// The command used to create a new remote volume in the database. + /// private SqliteCommand m_createremotevolumeCommand = null!; + /// + /// The command used to select duplicate remote volumes from the database. + /// private SqliteCommand m_selectduplicateRemoteVolumesCommand = null!; + /// + /// The command used to insert log data into the database. + /// private SqliteCommand m_insertlogCommand = null!; + /// + /// The command used to insert remote operation logs into the database. + /// private SqliteCommand m_insertremotelogCommand = null!; + /// + /// The command used to insert index block links into the database. + /// private SqliteCommand m_insertIndexBlockLink = null!; + /// + /// The command used to find a path prefix in the database. + /// private SqliteCommand m_findpathprefixCommand = null!; + /// + /// The command used to insert a new path prefix into the database. + /// private SqliteCommand m_insertpathprefixCommand = null!; + /// + /// A constant representing the blockset ID for a folder. + /// public const long FOLDER_BLOCKSET_ID = -100; + /// + /// A constant representing the blockset ID for a symlink. + /// public const long SYMLINK_BLOCKSET_ID = -200; + /// + /// The timestamp of the operation being performed. + /// public DateTime OperationTimestamp { get; private set; } + /// + /// A read-only property that provides access to the internal SQLite connection. + /// internal SqliteConnection Connection { get { return m_connection; } } + /// + /// Indicates whether the database has been disposed. + /// public bool IsDisposed { get; private set; } + /// + /// Indicates whether the connection should be closed when the database is disposed. + /// public bool ShouldCloseConnection { get; set; } // Constructor is private to force use of CreateLocalDatabaseAsync @@ -98,6 +178,13 @@ namespace Duplicati.Library.Main.Database throw new NotImplementedException("Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead"); } + /// + /// Creates a new SQLite connection to the specified database path with the given page cache size. + /// This method ensures that the directory for the database exists and upgrades the database schema if necessary. + /// + /// The path to the SQLite database file. + /// The size of the SQLite page cache in bytes. + /// A task that, when awaited, returns a new to the specified database. protected static async Task CreateConnectionAsync(string path, long pagecachesize) { path = Path.GetFullPath(path); @@ -121,6 +208,16 @@ namespace Duplicati.Library.Main.Database return c; } + /// + /// Creates a new instance of with the specified parameters. + /// This method initializes the database connection, sets the operation timestamp, and prepares the necessary commands for database operations. + /// + /// The path to the SQLite database file. + /// The description of the operation being performed. + /// Indicates whether the connection should be closed when the database is disposed. + /// The size of the SQLite page cache in bytes. + /// An optional existing instance to use. If not provided, a new instance will be created. + /// A task that, when awaited, returns a new instance of . public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize, LocalDatabase? db = null) { db ??= new LocalDatabase(); @@ -136,6 +233,12 @@ namespace Duplicati.Library.Main.Database return db; } + /// + /// Creates a new instance of based on an existing database instance. + /// This method copies the connection and transaction from the parent database and initializes the new database with the same operation timestamp and ID. + /// + /// The parent instance to copy from. + /// An optional existing instance to use. If not provided, a new instance will be created. public static async Task CreateLocalDatabaseAsync(LocalDatabase dbparent, LocalDatabase? dbnew = null) { dbnew ??= new LocalDatabase(); @@ -152,6 +255,14 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Creates a new instance of with the specified SQLite connection and operation description. + /// This method initializes the database connection, sets the operation timestamp, and prepares the necessary commands for database operations. + /// + /// The SQLite connection to use for the database operations. + /// The description of the operation being performed. + /// An optional existing instance to use. If not provided, a new instance will be created. + /// A task that, when awaited, returns a new instance of . public static async Task CreateLocalDatabaseAsync(SqliteConnection connection, string operation, LocalDatabase? dbnew = null) { dbnew ??= new LocalDatabase(); @@ -206,6 +317,13 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Creates a new instance of with the specified SQLite connection. + /// This method initializes the database connection, sets up the reusable transaction, and prepares the necessary commands for database operations. + /// + /// The SQLite connection to use for the database operations. + /// An optional existing instance to use. If not provided, a new instance will be created. + /// A task that, when awaited, returns a new instance of . private static async Task CreateLocalDatabaseAsync(SqliteConnection connection, LocalDatabase? dbnew = null) { dbnew ??= new LocalDatabase(); @@ -382,25 +500,55 @@ namespace Duplicati.Library.Main.Database } /// - /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value + /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value. /// + /// The number of seconds since the EPOCH (January 1, 1970). + /// A DateTime instance representing the specified number of seconds since the EPOCH. public static DateTime ParseFromEpochSeconds(long seconds) { return Library.Utility.Utility.EPOCH.AddSeconds(seconds); } + /// + /// Updates the state of a remote volume in the database. + /// + /// The name of the remote volume to update. + /// The new state of the remote volume. + /// The size of the remote volume in bytes. + /// The hash of the remote volume, or null if not applicable. + /// A task that completes when the remote volume has been updated. public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash) { await UpdateRemoteVolume(name, state, size, hash, false) .ConfigureAwait(false); } + /// + /// Updates the state of a remote volume in the database. + /// + /// The name of the remote volume to update. + /// The new state of the remote volume. + /// The size of the remote volume in bytes. + /// The hash of the remote volume, or null if not applicable. + /// If true, suppresses cleanup of the remote volume after updating. + /// A task that completes when the remote volume has been updated. public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup) { await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null) .ConfigureAwait(false); } + /// + /// Updates the state of a remote volume in the database. + /// + /// The name of the remote volume to update. + /// The new state of the remote volume. + /// The size of the remote volume in bytes. + /// The hash of the remote volume, or null if not applicable. + /// If true, suppresses cleanup of the remote volume after updating. + /// The time after which the remote volume can be deleted. + /// If true, sets the remote volume as archived. + /// A task that completes when the remote volume has been updated. public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived) { var c = await m_updateremotevolumeCommand.SetTransaction(m_rtr) @@ -461,6 +609,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets the ID and timestamp of all filesets in the database, ordered by timestamp in descending order. + /// + /// An asynchronous enumerable of key-value pairs, where each pair contains the fileset ID and its timestamp. public async IAsyncEnumerable> FilesetTimes() { using var cmd = await m_connection.CreateCommandAsync(@" @@ -480,6 +632,15 @@ namespace Duplicati.Library.Main.Database ); } + /// + /// Generates a SQL WHERE clause for filtering file lists based on a specified time and versions. + /// + /// The time to filter files by. If Ticks is 0, it will not be used in the query. + /// An array of versions to filter files by. If null or empty, it will not be used in the query. + /// An optional list of filesets to use for filtering. If null, it will fetch all filesets from the database. + /// If true, matches files with the exact timestamp; otherwise, matches files with timestamps less than or equal to the specified time. + /// A task that, when awaited, returns a tuple containing the SQL WHERE clause and a dictionary of parameter values. + /// Thrown if the provided time is unspecified. public async Task<(string Query, Dictionary Values)> GetFilelistWhereClause(DateTime time, long[]? versions, IEnumerable>? filesetslist = null, bool singleTimeMatch = false) { KeyValuePair[] filesets; @@ -540,6 +701,11 @@ namespace Duplicati.Library.Main.Database return (query.ToString(), args); } + /// + /// Gets the ID of a remote volume by its name. + /// + /// The name of the remote volume. + /// A task that, when awaited, returns the ID of the remote volume. If the volume does not exist, it returns -1. public async Task GetRemoteVolumeID(string file) { return await m_selectremotevolumeIdCommand @@ -549,6 +715,11 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Gets the IDs of remote volumes for a list of files. + /// + /// An enumerable collection of file names. + /// An asynchronous enumerable of key-value pairs, where each pair contains the file name and its corresponding remote volume ID. public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -571,6 +742,11 @@ namespace Duplicati.Library.Main.Database yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } + /// + /// Gets a remote volume entry by its file name. + /// + /// The name of the remote volume file. + /// A task that, when awaited, returns a representing the remote volume. If the volume does not exist, it returns an empty entry. public async Task GetRemoteVolume(string file) { m_selectremotevolumeCommand @@ -593,6 +769,10 @@ namespace Duplicati.Library.Main.Database return RemoteVolumeEntry.Empty; } + /// + /// Gets remote volumes that are duplicates, meaning they have the same name but different states. + /// + /// An asynchronous enumerable of key-value pairs, where each pair contains the name of the remote volume and its state. public async IAsyncEnumerable> DuplicateRemoteVolumes() { m_selectduplicateRemoteVolumesCommand.SetTransaction(m_rtr); @@ -608,6 +788,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets all remote volumes from the database. + /// + /// An asynchronous enumerable of representing all remote volumes. public async IAsyncEnumerable GetRemoteVolumes() { m_selectremotevolumesCommand.SetTransaction(m_rtr); @@ -631,11 +815,12 @@ namespace Duplicati.Library.Main.Database } /// - /// Log an operation performed on the remote backend + /// Log an operation performed on the remote backend. /// - /// The operation performed - /// The path involved - /// Any data relating to the operation + /// The operation performed. + /// The path involved. + /// Any data relating to the operation. + /// A task that completes when the operation log has been recorded. public async Task LogRemoteOperation(string operation, string path, string? data) { await m_insertremotelogCommand @@ -650,11 +835,12 @@ namespace Duplicati.Library.Main.Database } /// - /// Log a debug message + /// Log a debug message. /// - /// The message type - /// The message - /// An optional exception + /// The message type. + /// The message. + /// An optional exception. + /// A task that completes when the log message has been recorded. public async Task LogMessage(string type, string message, Exception? exception) { await m_insertlogCommand @@ -668,6 +854,14 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Unlinks a remote volume from the database. + /// This operation removes the remote volume entry from the database without deleting the actual volume. + /// + /// The name of the remote volume to unlink. + /// The state of the remote volume to unlink. + /// A task that completes when the remote volume has been unlinked. + /// Thrown if the number of unlinked remote volumes is not equal to 1. public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -688,11 +882,21 @@ namespace Duplicati.Library.Main.Database throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); } + /// + /// Removes a remote volume from the database. + /// + /// The name of the remote volume to remove. + /// A task that completes when the remote volume has been removed. public async Task RemoveRemoteVolume(string name) { await RemoveRemoteVolumes([name]).ConfigureAwait(false); } + /// + /// Removes multiple remote volumes from the database. + /// + /// An enumerable collection of names of remote volumes to remove. + /// A task that completes when the remote volumes have been removed. public async Task RemoveRemoteVolumes(IEnumerable names) { if (names == null || !names.Any()) return; @@ -1031,6 +1235,12 @@ namespace Duplicati.Library.Main.Database throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); } + /// + /// Performs a VACUUM operation on the database to reclaim unused space. + /// This operation can help optimize the database performance by defragmenting it. + /// Note: This operation can take a significant amount of time depending on the size of the database. + /// + /// A task that completes when the VACUUM operation has finished. public async Task Vacuum() { m_hasExecutedVacuum = true; @@ -1039,24 +1249,56 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync("VACUUM").ConfigureAwait(false); } + /// + /// Registers a new remote volume in the database. + /// + /// The name of the remote volume. + /// The type of the remote volume. + /// The size of the remote volume in bytes. Use -1 for unknown size. + /// The state of the remote volume. + /// A task that, when awaited, returns the ID of the newly registered remote volume. public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state) { return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0)) .ConfigureAwait(false); } + /// + /// Registers a new remote volume in the database. + /// + /// The name of the remote volume. + /// The type of the remote volume. + /// The state of the remote volume. + /// A task that, when awaited, returns the ID of the newly registered remote volume. public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state) { return await RegisterRemoteVolume(name, type, state, new TimeSpan(0)) .ConfigureAwait(false); } + /// + /// Registers a new remote volume in the database. + /// + /// The name of the remote volume. + /// The type of the remote volume. + /// The state of the remote volume. + /// The time after which the remote volume can be deleted. + /// A task that, when awaited, returns the ID of the newly registered remote volume. public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime) { return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime) .ConfigureAwait(false); } + /// + /// Registers a new remote volume in the database. + /// + /// The name of the remote volume. + /// The type of the remote volume. + /// The state of the remote volume. + /// The size of the remote volume in bytes. Use -1 for unknown size. + /// The time after which the remote volume can be deleted. + /// A task that, when awaited, returns the ID of the newly registered remote volume. public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime) { var r = await m_createremotevolumeCommand @@ -1075,6 +1317,16 @@ namespace Duplicati.Library.Main.Database return r; } + /// + /// Retrieves the IDs of filesets that match a specific restore time and optional versions. + /// If no filesets match the criteria, it returns the newest fileset ID. + /// + /// The time to restore from. + /// Optional array of versions to match against the filesets. + /// If true, only match filesets that exactly match the restore time. + /// An asynchronous enumerable of fileset IDs that match the criteria. + /// Thrown if the provided DateTime is unspecified. + /// Thrown if no backups are found at the specified date. public async IAsyncEnumerable GetFilesetIDs(DateTime restoretime, long[]? versions, bool singleTimeMatch = false) { if (restoretime.Kind == DateTimeKind.Unspecified) @@ -1118,6 +1370,12 @@ namespace Duplicati.Library.Main.Database yield return el; } + /// + /// Finds filesets that match a specific restore time and optional versions. + /// + /// The time to restore from. + /// Optional array of versions to match against the filesets. + /// An asynchronous task that returns a collection of fileset IDs that match the criteria. public async Task> FindMatchingFilesets(DateTime restoretime, long[]? versions) { if (restoretime.Kind == DateTimeKind.Unspecified) @@ -1143,6 +1401,12 @@ namespace Duplicati.Library.Main.Database return res; } + /// + /// Checks if a fileset is a full backup. + /// A fileset is considered a full backup if its "IsFullBackup" field is set to FULL_BACKUP. + /// + /// The timestamp of the fileset to check. + /// A task that, when awaited, returns true if the fileset is a full backup, otherwise false. public async Task IsFilesetFullBackup(DateTime filesetTime) { using var cmd = m_connection.CreateCommand(); @@ -1161,6 +1425,10 @@ namespace Duplicati.Library.Main.Database return isFullBackup == BackupType.FULL_BACKUP; } + /// + /// Retrieves a list of database options from the "Configuration" table. + /// + /// An asynchronous enumerable of key-value pairs representing the database options. private async IAsyncEnumerable> GetDbOptionList() { using var cmd = m_connection.CreateCommand(@" @@ -1179,6 +1447,10 @@ namespace Duplicati.Library.Main.Database ); } + /// + /// Retrieves all database options as a dictionary. + /// + /// A task that, when awaited, returns a dictionary containing all database options. public async Task> GetDbOptions() { var res = await GetDbOptionList() @@ -1189,10 +1461,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Updates a database option + /// Updates a database option. /// - /// The key to update - /// The value to set + /// The key to update. + /// The value to set. private async Task UpdateDbOption(string key, bool value) { var opts = await GetDbOptions().ConfigureAwait(false); @@ -1207,8 +1479,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Flag indicating if a repair is in progress + /// Flag indicating if a repair is in progress. /// + /// Optional value to set the flag to. If null, the current value in the database is returned. + /// A task that, when awaited, returns true if a repair is in progress, otherwise false. public async Task RepairInProgress(bool? value = null) { if (value is bool v) @@ -1224,8 +1498,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Flag indicating if a repair is in progress + /// Flag indicating if the database has been partially recreated. /// + /// Optional value to set the flag to. If null, the current value in the database is returned. + /// A task that, when awaited, returns true if the database has been partially recreated, otherwise false. public async Task PartiallyRecreated(bool? value = null) { if (value is bool v) @@ -1241,8 +1517,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Flag indicating if the database can contain partial uploads + /// Flag indicating if the database has been terminated with active uploads. /// + /// Optional value to set the flag to. If null, the current value in the database is returned. + /// A task that, when awaited, returns true if the database has been terminated with active uploads, otherwise false. public async Task TerminatedWithActiveUploads(bool? value = null) { if (value is bool v) @@ -1258,10 +1536,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Sets the database options + /// Sets the database options. /// - /// The options to set - /// An optional transaction + /// The options to set. + /// A task that completes when the options have been set. public async Task SetDbOptions(IDictionary options) { using var cmd = m_connection.CreateCommand(); @@ -1289,6 +1567,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Counts the number of blocks larger than a specified size. + /// + /// The size in bytes to compare against. + /// A task that, when awaited, returns the count of blocks larger than the specified size. public async Task GetBlocksLargerThan(long fhblocksize) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -1305,35 +1588,35 @@ namespace Duplicati.Library.Main.Database } /// - /// Verifies the consistency of the database + /// Verifies the consistency of the database. /// - /// The block size in bytes - /// The hash size in byts - /// Also verify filelists (can be slow) - /// The transaction to run in + /// The block size in bytes. + /// The hash size in bytes. + /// Also verify filelists (can be slow). + /// A task that completes when the consistency check is finished. public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists) => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false) .ConfigureAwait(false); /// - /// Verifies the consistency of the database prior to repair + /// Verifies the consistency of the database prior to repair. /// - /// The block size in bytes - /// The hash size in byts - /// Also verify filelists (can be slow) - /// The transaction to run in + /// The block size in bytes. + /// The hash size in bytes. + /// Also verify filelists (can be slow). + /// A task that completes when the consistency check is finished. public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists) => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true) .ConfigureAwait(false); /// - /// Verifies the consistency of the database + /// Verifies the consistency of the database. /// - /// The block size in bytes - /// The hash size in byts - /// Also verify filelists (can be slow) - /// Disable verify for errors that will be fixed by repair - /// The transaction to run in + /// The block size in bytes. + /// The hash size in bytes. + /// Also verify filelists (can be slow). + /// Disable verify for errors that will be fixed by repair. + /// A task that completes when the consistency check is finished. private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair) { using var cmd = m_connection.CreateCommand() @@ -1692,18 +1975,37 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Represents a block in the database. + /// public interface IBlock { + /// + /// Gets the hash of the block. + /// string Hash { get; } + /// + /// Gets the size of the block in bytes. + /// long Size { get; } } + /// + /// Represents a block in the database with its hash and size. + /// + /// The hash of the block. + /// The size of the block in bytes. internal class Block(string hash, long size) : IBlock { public string Hash { get; private set; } = hash; public long Size { get; private set; } = size; } + /// + /// Retrieves all blocks associated with a specific volume ID. + /// + /// The ID of the volume to retrieve blocks for. + /// An asynchronous enumerable of blocks associated with the specified volume ID. public async IAsyncEnumerable GetBlocks(long volumeid) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -1726,25 +2028,55 @@ namespace Duplicati.Library.Main.Database ); } + /// + /// An asynchronous enumerable that retrieves blocklist hashes for files in a specific fileset. + /// private class BlocklistHashEnumerable : IAsyncEnumerable { + /// + /// An asynchronous enumerator for the blocklist hashes. + /// private class BlocklistHashEnumerator : IDisposable, IAsyncEnumerator { + /// + /// The data reader used to read blocklist hashes. + /// private readonly SqliteDataReader m_reader; + /// + /// The parent enumerable that this enumerator belongs to. + /// private readonly BlocklistHashEnumerable m_parent; + /// + /// The path of the current file being processed. + /// private string? m_path = null; + /// + /// Indicates if this is the first entry being processed. + /// private bool m_first = true; + /// + /// The current blocklist hash being processed. + /// private string? m_current = null; + /// + /// Initializes a new instance of the class. + /// + /// The parent enumerable that this enumerator belongs to. + /// The data reader used to read blocklist hashes. public BlocklistHashEnumerator(BlocklistHashEnumerable parent, SqliteDataReader reader) { m_reader = reader; m_parent = parent; } + /// + /// Gets the current blocklist hash. + /// public string Current { get { return m_current!; } } public void Dispose() { } + // The warning is suppressed because the interface requires the method, but there's nothing to dispose. #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously public async ValueTask DisposeAsync() { } #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously @@ -1783,6 +2115,9 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Resets the enumerator to its initial state. + /// public void Reset() { if (!m_first) @@ -1792,16 +2127,28 @@ namespace Duplicati.Library.Main.Database } } + /// + /// The data reader used to read blocklist hashes. + /// private readonly SqliteDataReader m_reader; + /// + /// Initializes a new instance of the class. + /// public BlocklistHashEnumerable(SqliteDataReader reader) { m_reader = reader; MoreData = true; } + /// + /// Indicates if there is more data to read. + /// public bool MoreData { get; protected set; } + /// + /// Gets an enumerator that iterates through the blocklist hashes. + /// public IAsyncEnumerator GetEnumerator() { return new BlocklistHashEnumerator(this, m_reader); @@ -1813,6 +2160,9 @@ namespace Duplicati.Library.Main.Database } } + /// + /// SQL query for listing all filesets in the database, including their metadata and blocklist hashes. + /// public const string LIST_FILESETS = @" SELECT ""L"".""Path"", @@ -1894,6 +2244,9 @@ namespace Duplicati.Library.Main.Database ON ""M"".""BlocksetID"" = ""L"".""MetablocksetID"" "; + /// + /// SQL query for listing folders and symlinks in a specific fileset, including their metadata and blocklist hashes. + /// public const string LIST_FOLDERS_AND_SYMLINKS = @" SELECT ""G"".""BlocksetID"", @@ -1941,6 +2294,12 @@ namespace Duplicati.Library.Main.Database ""H"".""Index"" "; + /// + /// Writes the contents of a fileset to a specified volume writer. + /// + /// The volume writer to which the fileset will be written. + /// The ID of the fileset to write. + /// A task that completes when the fileset has been written. public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId) { using var cmd = m_connection.CreateCommand() @@ -2027,6 +2386,12 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Links a fileset to a specific volume by updating the VolumeID in the Fileset table. + /// + /// The ID of the fileset to link. + /// The ID of the volume to link the fileset to. + /// A task that completes when the link operation is finished. public async Task LinkFilesetToVolume(long filesetid, long volumeid) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -2046,6 +2411,11 @@ namespace Duplicati.Library.Main.Database throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); } + /// + /// Pushes timestamp changes from the latest version of a fileset to the previous version. + /// + /// The ID of the fileset whose timestamp changes will be pushed. + /// A task that completes when the timestamp changes have been pushed. public async Task PushTimestampChangesToPreviousVersion(long filesetId) { var query = @" @@ -2073,13 +2443,23 @@ namespace Duplicati.Library.Main.Database } /// - /// Keeps a list of filenames in a temporary table with a single column Path + /// Keeps a list of filenames in a temporary table with a single column Path. /// public class FilteredFilenameTable : IDisposable, IAsyncDisposable { + /// + /// The name of the temporary table that holds the filtered filenames. + /// public string Tablename { get; private set; } + /// + /// The database used to create and manage the temporary table. + /// private readonly LocalDatabase m_db; + /// + /// Initializes a new instance of the class. + /// + /// The database to use for creating the temporary table. private FilteredFilenameTable(LocalDatabase db) { Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -2092,6 +2472,12 @@ namespace Duplicati.Library.Main.Database throw new NotImplementedException("Use the Create method instead."); } + /// + /// Creates a new instance of the class asynchronously. + /// + /// The database to use for creating the temporary table. + /// The filter to apply to the filenames. + /// A task that represents the asynchronous operation, with a as the result. public static async Task CreateFilteredFilenameTableAsync(LocalDatabase db, IFilter filter) { var ftt = new FilteredFilenameTable(db); @@ -2228,6 +2614,13 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Renames a remote file in the database, preserving its ID links. + /// + /// The current name of the remote file. + /// The new name for the remote file. + /// A task that completes when the renaming operation is finished. + /// Thrown if the renaming operation does not affect exactly one row. public async Task RenameRemoteFile(string oldname, string newname) { //Rename the old entry, to preserve ID links @@ -2280,9 +2673,9 @@ namespace Duplicati.Library.Main.Database /// /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. /// - /// The ID of the fileset volume to update - /// The timestamp of the operation to create - /// An optional external transaction + /// The ID of the fileset volume to update. + /// The timestamp of the operation to create. + /// A task that when awaited contains the ID of the newly created fileset. public virtual async Task CreateFileset(long volumeid, DateTime timestamp) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -2319,7 +2712,8 @@ namespace Duplicati.Library.Main.Database /// /// The ID of the index volume. /// The ID of the block volume. - /// An optional transaction. + /// A task that completes when the link has been added. + /// Thrown if either volume ID is less than or equal to 0. public async Task AddIndexBlockLink(long indexVolumeID, long blockVolumeID) { if (indexVolumeID <= 0) @@ -2336,12 +2730,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Returns all unique blocklists for a given volume + /// Returns all unique blocklists for a given volume. /// - /// The volume ID to get blocklists for - /// The blocksize - /// The size of the hash - /// An optional external transaction + /// The volume ID to get blocklists for. + /// The blocksize. + /// The size of the hash. /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data public async IAsyncEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize) { @@ -2435,11 +2828,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Update fileset with full backup state + /// Update fileset with full backup state. /// - /// Existing file set to update - /// Full backup state - /// An optional external transaction + /// Existing file set to update. + /// Full backup state. + /// A task that completes when the update is finished. public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -2457,10 +2850,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Removes all entries in the fileset entry table for a given fileset ID + /// Removes all entries in the fileset entry table for a given fileset ID. /// - /// The fileset ID to clear - /// The transaction to use + /// The fileset ID to clear. + /// A task that completes when the entries have been cleared. public async Task ClearFilesetEntries(long filesetId) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -2476,10 +2869,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Gets the last previous fileset that was incomplete + /// Gets the last previous fileset that was incomplete. /// - /// The transaction to use - /// The last incomplete fileset or default + /// The transaction to use. + /// A task that when awaited returns the last incomplete fileset or default public async Task GetLastIncompleteFilesetVolume() { var candidates = GetIncompleteFilesets() @@ -2497,10 +2890,9 @@ namespace Duplicati.Library.Main.Database } /// - /// Gets a list of incomplete filesets + /// Gets a list of incomplete filesets. /// - /// An optional transaction - /// A list of fileset IDs and timestamps + /// An asynchronous enumerable of key-value pairs where the key is the fileset ID and the value is the timestamp of the fileset. public async IAsyncEnumerable> GetIncompleteFilesets() { using var cmd = await m_connection.CreateCommandAsync(@$" @@ -2538,11 +2930,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Gets the remote volume entry from the fileset ID + /// Gets the remote volume entry from the fileset ID. /// - /// The fileset ID - /// An optional transaction - /// The remote volume entry or default + /// The fileset ID. + /// A task that when awaited returns the remote volume entry associated with the fileset ID, or default if not found. public async Task GetRemoteVolumeFromFilesetID(long filesetID) { using var cmd = await m_connection.CreateCommandAsync(@" @@ -2584,6 +2975,11 @@ namespace Duplicati.Library.Main.Database return default(RemoteVolumeEntry); } + /// + /// Purges log data and remote operations older than the specified threshold. + /// + /// The threshold date and time; all log data and remote operations older than this will be purged. + /// A task that completes when the purge operation is finished. public async Task PurgeLogData(DateTime threshold) { using var cmd = m_connection.CreateCommand(m_rtr); @@ -2608,6 +3004,11 @@ namespace Duplicati.Library.Main.Database await m_rtr.CommitAsync().ConfigureAwait(false); } + /// + /// Purges deleted remote volumes that have not been modified since the specified threshold. + /// + /// The threshold date and time; all deleted remote volumes older than this will be purged. + /// A task that completes when the purge operation is finished. public async Task PurgeDeletedVolumes(DateTime threshold) { await m_removedeletedremotevolumeCommand @@ -2665,11 +3066,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Disposes all fields of a certain type, in the instance and its bases + /// Disposes all fields of a certain type, in the instance and its bases. /// - /// The type of fields to find - /// The item to dispose - /// True if an aggregate exception should be thrown, or false if exceptions are silently captured + /// The type of fields to find. + /// The item to dispose. + /// True if an aggregate exception should be thrown, or false if exceptions are silently captured. public static void DisposeAllFields(object item, bool throwExceptions) where T : IDisposable { @@ -2711,6 +3112,11 @@ namespace Duplicati.Library.Main.Database throw new AggregateException(exceptions); } + /// + /// Writes the results of a basic operation to the log. + /// + /// The results to write. + /// A task that completes when the results have been written. public async Task WriteResults(IBasicResults result) { if (IsDisposed) @@ -2735,20 +3141,19 @@ namespace Duplicati.Library.Main.Database } /// - /// The current index into the path prefix buffer + /// The current index into the path prefix buffer. /// private int m_pathPrefixIndex = 0; /// - /// The path prefix lookup list + /// The path prefix lookup list. /// private readonly KeyValuePair[] m_pathPrefixLookup = new KeyValuePair[5]; /// - /// Gets the path prefix ID, optionally creating it in the process. + /// Gets the path prefix ID, optionally creating it in the process.. /// - /// The path prefix ID. /// The path to get the prefix for. - /// The transaction to use for insertion, or null for no transaction + /// A task that when awaited returns the path prefix ID. public async Task GetOrCreatePathPrefix(string prefix) { // Ring-buffer style lookup @@ -2781,7 +3186,7 @@ namespace Duplicati.Library.Main.Database } /// - /// The path separators on this system + /// The path separators on this system. /// private static readonly char[] _pathseparators = [ Path.DirectorySeparatorChar, @@ -2789,10 +3194,10 @@ namespace Duplicati.Library.Main.Database ]; /// - /// Helper method that splits a path on the last path separator + /// Helper method that splits a path on the last path separator. /// - /// The prefix and name. /// The path to split. + /// The prefix and name. public static KeyValuePair SplitIntoPrefixAndName(string path) { if (string.IsNullOrEmpty(path)) @@ -2807,11 +3212,18 @@ namespace Duplicati.Library.Main.Database } /// - /// Defines the backups types + /// Defines the backups types. /// public static class BackupType { + /// + /// Indicates a partial backup. + /// public const int PARTIAL_BACKUP = 0; + /// + /// Indicates a full backup. + /// public const int FULL_BACKUP = 1; } + } From 0960a60462cee25d152d4dd15fa06ad2ac0527c7 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 17:45:20 +0200 Subject: [PATCH 303/362] IStorageHelper now implements IAsyncDisposable --- Duplicati/Library/Main/Database/LocalListChangesDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index b1c0e240b..097faa9f9 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -44,7 +44,7 @@ namespace Duplicati.Library.Main.Database return dbnew; } - public interface IStorageHelper : IDisposable + public interface IStorageHelper : IDisposable, IAsyncDisposable { Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew); @@ -644,10 +644,10 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (m_insertPreviousElementCommand != null) { From a32a5acaed62440f32495b88a5dcb549d3ae3dbc Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 18:01:12 +0200 Subject: [PATCH 304/362] Added more docstrings --- .../Main/Database/LocalDeleteDatabase.cs | 215 +++++++++++++++--- .../Database/LocalListAffectedDatabase.cs | 76 +++++++ .../Database/LocalListBrokenFilesDatabase.cs | 82 +++++-- .../Main/Database/LocalListChangesDatabase.cs | 158 ++++++++++++- 4 files changed, 483 insertions(+), 48 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index c4d20dae2..48b9e4c37 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -30,22 +30,39 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// A local database for deleting filesets and blocks. + /// internal class LocalDeleteDatabase : LocalDatabase { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); /// - /// Flag for toggling temporary tables; set to empty string if debugging + /// Flag for toggling temporary tables; set to empty string if debugging. /// private const string TEMPORARY = "TEMPORARY"; + /// + /// SQL command to register a duplicate block. + /// private const string REGISTER_COMMAND = @"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""ID"", @VolumeId FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "; + /// + /// The command to register a duplicate block. + /// private SqliteCommand m_registerDuplicateBlockCommand = null!; + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// The operation name. + /// The size of the page cache. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(string path, string operation, long pagecachesize, LocalDeleteDatabase? dbnew = null) { dbnew ??= new LocalDeleteDatabase(); @@ -61,6 +78,12 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Creates a new instance of the class with a parent database. + /// + /// The parent database to use. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(LocalDatabase dbparent, LocalDeleteDatabase? dbnew = null) { dbnew ??= new LocalDeleteDatabase(); @@ -79,9 +102,8 @@ namespace Duplicati.Library.Main.Database /// /// Drops all entries related to operations listed in the table. /// - /// The fileset entries to delete - /// The transaction to execute the commands in - /// A list of filesets to delete + /// The fileset entries to delete. + /// An async enumerable of key-value pairs, where the key is the fileset name and the value is the size of the fileset. public async IAsyncEnumerable> DropFilesetsFromTable(DateTime[] toDelete) { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -94,8 +116,8 @@ namespace Duplicati.Library.Main.Database DELETE FROM ""Fileset"" WHERE ""Timestamp"" IN (@Timestamps) ") - .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable) - .ConfigureAwait(false) + .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable) + .ConfigureAwait(false) ) .ExecuteNonQueryAsync() .ConfigureAwait(false); @@ -262,6 +284,7 @@ namespace Duplicati.Library.Main.Database /// exposed to the user. This is in contrast to other cases where the Version is the ID in the /// Fileset table. /// + /// An async enumerable of IListResultFileset. internal async IAsyncEnumerable FilesetsWithBackupVersion() { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -295,13 +318,35 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Represents the usage of a volume in the database. + /// private struct VolumeUsage { + /// + /// The name of the volume. + /// public readonly string Name; + /// + /// The size of the data stored in the volume. + /// public readonly long DataSize; + /// + /// The size of the data that is no longer needed in the volume. + /// public readonly long WastedSize; + /// + /// The size of the data that is compressed in the volume. + /// public readonly long CompressedSize; + /// + /// Initializes a new instance of the struct. + /// + /// The name of the volume. + /// The size of the data stored in the volume. + /// The size of the data that is no longer needed in the volume. + /// The size of the data that is compressed in the volume. public VolumeUsage(string name, long datasize, long wastedsize, long compressedsize) { Name = name; @@ -316,7 +361,11 @@ namespace Duplicati.Library.Main.Database /// and the number of bytes no longer needed in each volume. /// The sizes are the uncompressed values. /// - /// A list of tuples with name, datasize, wastedbytes. + /// + /// An asynchronous enumerable of , holding the name of the volume, + /// the size of the data stored in the volume, the size of the data that is no longer needed in the volume, + /// and the size of the data that is compressed in the volume. + /// private async IAsyncEnumerable GetWastedSpaceReport() { var tmptablename = "UsageReport-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -491,32 +540,97 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Generates a report on the volumes that can be reclaimed or compacted. + /// public interface ICompactReport { + /// + /// Gets the volumes that can be deleted. + /// IEnumerable DeleteableVolumes { get; } + /// + /// Gets the volumes that can be compacted. + /// IEnumerable CompactableVolumes { get; } + /// + /// Gets the report on whether reclamation or compaction should be performed. + /// bool ShouldReclaim { get; } + /// + /// Gets the report on whether compaction should be performed. + /// bool ShouldCompact { get; } + /// + /// Emits the report data to the log. + /// void ReportCompactData(); } + /// + /// A report on the volumes that can be reclaimed or compacted. + /// private class CompactReport : ICompactReport { + /// + /// The report to emit to the log. + /// private readonly IEnumerable m_report; + /// + /// The volumes that can be fully deleted. + /// private readonly IEnumerable m_cleandelete; + /// + /// The volumes that have wasted space. + /// private readonly IEnumerable m_wastevolumes; + /// + /// The volumes that are smaller than the specified size. + /// private readonly IEnumerable m_smallvolumes; + /// + /// The count of volumes that can be fully deleted. + /// private readonly long m_deletablevolumes; + /// + /// The total size of wasted space across all volumes. + /// private readonly long m_wastedspace; + /// + /// The total size of small volumes. + /// private readonly long m_smallspace; + /// + /// The total size of all data across all volumes. + /// private readonly long m_fullsize; + /// + /// The count of small volumes. + /// private readonly long m_smallvolumecount; + /// + /// The threshold for wasted space percentage to trigger compaction. + /// private readonly long m_wastethreshold; + /// + /// The size of the volume to trigger compaction. + /// private readonly long m_volsize; + /// + /// The maximum number of small files to trigger compaction. + /// private readonly long m_maxsmallfilecount; + /// + /// Initializes a new instance of the class. + /// + /// The size of the volume to trigger compaction. + /// The threshold for wasted space percentage to trigger compaction. + /// The size of small files to trigger compaction. + /// The maximum number of small files to trigger compaction. + /// The report data containing volume usage information. public CompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, IEnumerable report) { m_report = report; @@ -537,6 +651,9 @@ namespace Duplicati.Library.Main.Database m_smallvolumecount = m_smallvolumes.Count(); } + /// + /// Emits the report data to the log. + /// public void ReportCompactData() { var wastepercentage = ((m_wastedspace / (float)m_fullsize) * 100); @@ -556,6 +673,9 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteInformationMessage(LOGTAG, "CompactReason", "Compacting not required"); } + /// + /// Gets a value indicating whether reclamation should be performed. + /// public bool ShouldReclaim { get @@ -564,6 +684,9 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets a value indicating whether compaction should be performed. + /// public bool ShouldCompact { get @@ -572,11 +695,17 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets the volumes that can be deleted. + /// public IEnumerable DeleteableVolumes { get { return from n in m_cleandelete select n.Name; } } + /// + /// Gets the volumes that can be compacted. + /// public IEnumerable CompactableVolumes { get @@ -589,6 +718,14 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets a compact report on the volumes that can be reclaimed or compacted. + /// + /// The size of the volume to trigger compaction. + /// The threshold for wasted space percentage to trigger compaction. + /// The size of small files to trigger compaction. + /// The maximum number of small files to trigger compaction. + /// A task that when awaited contains an instance of . public async Task GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount) { return new CompactReport( @@ -602,24 +739,36 @@ namespace Duplicati.Library.Main.Database ); } - + /// + /// Registers a block as duplicated to a new volume. + /// This is used when a block is moved to a new volume, and we want to keep track of it. + /// public interface IBlockQuery : IDisposable { /// - /// Checks if a block is in use. If volumeId is not -1, check specific volume + /// Checks if a block is in use. If volumeId is not -1, check specific volume. /// - /// The hash of the block - /// The size of the block - /// The volume ID to check, or -1 to check all volumes - /// The transaction to execute the command in - /// True if the block is in use + /// The hash of the block. + /// The size of the block. + /// The volume ID to check, or -1 to check all volumes. + /// The transaction to execute the command in. + /// A task that when awaited returns true if the block is in use, false otherwise. Task UseBlock(string hash, long size, long volumeId); } + /// + /// A helper class to query blocks in the database. + /// private class BlockQuery : IBlockQuery { + /// + /// The database instance to use for the queries. + /// private LocalDatabase m_db = null!; + /// + /// The command used to query blocks in the database. + /// private SqliteCommand m_command = null!; [Obsolete("Calling this constructor will throw an exception. Use CreateAsync() instead.")] @@ -628,11 +777,17 @@ namespace Duplicati.Library.Main.Database throw new NotImplementedException("Use CreateAsync() instead"); } - private BlockQuery() - { - // Private constructor to prevent instantiation without CreateAsync - } + /// + /// Constructs a new instance of the class. + /// It is private to prevent instantiation without CreateAsync. + /// + private BlockQuery() { } + /// + /// Creates a new instance of the class. + /// + /// The local database to use for the queries. + /// A task that when awaited returns a new instance of . public static async Task CreateAsync(LocalDatabase db) { return new BlockQuery @@ -683,7 +838,7 @@ namespace Duplicati.Library.Main.Database } /// - /// Builds a lookup table to enable faster response to block queries + /// Builds a lookup table to enable faster response to block queries. /// public async Task CreateBlockQueryHelper() { @@ -691,11 +846,12 @@ namespace Duplicati.Library.Main.Database } /// - /// Registers a block as moved to a new volume + /// Registers a block as moved to a new volume. /// - /// The hash of the block - /// The size of the block - /// The new volume ID + /// The hash of the block. + /// The size of the block. + /// The new volume ID. + /// A task that when completed indicates the block has been registered. public async Task RegisterDuplicatedBlock(string hash, long size, long volumeID) { // Using INSERT OR IGNORE to avoid duplicate entries, result may be 1 or 0 @@ -709,11 +865,11 @@ namespace Duplicati.Library.Main.Database } /// - /// After new volumes are uploaded, this method will update the blocks from the old volumes to point to the new volumes + /// After new volumes are uploaded, this method will update the blocks from the old volumes to point to the new volumes. /// - /// The file to remove - /// The volume IDs that will be removed - /// The transaction to execute the command in + /// The file to remove. + /// The volume IDs that will be removed. + /// A task that when completed indicates the operation has finished. public async Task PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved) { var deletedVolume = await GetRemoteVolume(filename) @@ -844,8 +1000,7 @@ namespace Duplicati.Library.Main.Database /// Calculates the sequence in which files should be deleted based on their relations. /// /// Block volumes slated for deletion. - /// The transaction to execute the command in - /// The deletable volumes. + /// An asynchronous enumerable of that represents the order in which volumes should be deleted. public async IAsyncEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes) { using (var cmd = m_connection.CreateCommand(m_rtr)) diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 423cd1436..9e40f97f8 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -28,8 +28,19 @@ using System.Threading.Tasks; namespace Duplicati.Library.Main.Database { + /// + /// LocalListAffectedDatabase is a specialized database class for listing + /// affected filesets, files, remote logs, and remote volumes. + /// internal class LocalListAffectedDatabase : LocalDatabase { + /// + /// Creates a new instance of . + /// + /// The path to the database file. + /// The size of the page cache in bytes. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited returns a new instance of . public static async Task CreateAsync(string path, long pagecachesize, LocalListAffectedDatabase? dbnew = null) { dbnew ??= new LocalListAffectedDatabase(); @@ -42,32 +53,80 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Represents a fileset result for listing affected filesets. + /// private class ListResultFileset : Interface.IListResultFileset { + /// + /// Gets or sets the version of the fileset. + /// public long Version { get; set; } + /// + /// Gets or sets whether the backup related to this fileset is a full backup. + /// public int IsFullBackup { get; set; } + /// + /// Gets or sets the time when the fileset was created. + /// public DateTime Time { get; set; } + /// + /// Gets or sets the number of files in the fileset. + /// public long FileCount { get; set; } + /// + /// Gets or sets the total size of files in the fileset. + /// public long FileSizes { get; set; } } + /// + /// Represents a file result for listing affected files. + /// private class ListResultFile : Interface.IListResultFile { + /// + /// Gets or sets the path of the file reperesented by this result entry. + /// public required string Path { get; set; } + /// + /// Gets or sets the sizes of the file blocks, if available. + /// This can be null if the sizes are not available or not applicable. + /// public required IEnumerable? Sizes { get; set; } } + /// + /// Represents a remote log entry for listing affected remote logs. + /// private class ListResultRemoteLog : Interface.IListResultRemoteLog { + /// + /// Gets or sets the timestamp of the log entry. + /// public required DateTime Timestamp { get; set; } + /// + /// Gets or sets the message of the log entry. + /// public required string Message { get; set; } } + /// + /// Represents a remote volume entry for listing affected remote volumes. + /// private class ListResultRemoteVolume : Interface.IListResultRemoteVolume { + /// + /// Gets or sets the name of the remote volume. + /// public required string Name { get; set; } } + /// + /// Retrieves the fileset times from the database. + /// + /// The items to filter the filesets by. + /// An asynchronous enumerable of fileset times. public async IAsyncEnumerable GetFilesets(IEnumerable items) { var filesets = await FilesetTimes() @@ -125,6 +184,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves the list of files from the database. + /// + /// The items to filter the files by. + /// An asynchronous enumerable of file results. public async IAsyncEnumerable GetFiles(IEnumerable items) { var sql = $@" @@ -196,6 +260,12 @@ namespace Duplicati.Library.Main.Database }; } + /// + /// Retrieves the log lines from the database that match the specified items. + /// The log lines are retrieved from both the LogData and RemoteOperation tables. + /// + /// The items to filter the log lines by. + /// An asynchronous enumerable of log line results. public async IAsyncEnumerable GetLogLines(IEnumerable items) { using var cmd = m_connection.CreateCommand(); @@ -239,6 +309,12 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves the remote volumes that match the specified items. + /// This method queries the database for remote volumes associated with filesets and metadata datasets. + /// + /// The names of the remote volumes to filter by. + /// An asynchronous enumerable of remote volume results. public async IAsyncEnumerable GetVolumes(IEnumerable items) { var sql = $@" diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index bae48a759..80422b44f 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -28,8 +28,14 @@ using System.Threading.Tasks; namespace Duplicati.Library.Main.Database { + /// + /// A database for listing broken files, i.e., files that reference blocksets or blocks that are not available. + /// internal class LocalListBrokenFilesDatabase : LocalDatabase { + /// + /// SQL query to get the IDs of all block volumes. + /// private static readonly string BLOCK_VOLUME_IDS = $@" SELECT ""ID"" FROM ""RemoteVolume"" @@ -42,6 +48,11 @@ namespace Duplicati.Library.Main.Database // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. + + /// + /// SQL query to get the IDs of broken files. + /// A broken file is one that has a BlocksetID that is not in the list of valid blocksets, or that has a MetadataID that is not in the list of valid metadata blocksets. + /// private static readonly string BROKEN_FILE_IDS = $@" SELECT DISTINCT ""ID"" FROM ( @@ -97,6 +108,9 @@ namespace Duplicati.Library.Main.Database ) "; + /// + /// SQL query to get the broken filesets, i.e., filesets that contain broken files. + /// private static readonly string BROKEN_FILE_SETS = $@" SELECT DISTINCT ""B"".""Timestamp"", @@ -110,6 +124,9 @@ namespace Duplicati.Library.Main.Database AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS}) "; + /// + /// SQL query to get the names and lengths of broken files. + /// private static readonly string BROKEN_FILE_NAMES = $@" SELECT ""A"".""Path"", @@ -126,6 +143,10 @@ namespace Duplicati.Library.Main.Database ) "; + /// + /// SQL query to insert broken file IDs into a specified table. + /// The table must have a single column with the same name as the ID field name. + /// private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => $@" INSERT INTO ""{tablename}"" ( ""{IDfieldname}"" @@ -138,6 +159,13 @@ namespace Duplicati.Library.Main.Database ) "; + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// The size of the page cache. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(string path, long pagecachesize, LocalListBrokenFilesDatabase? dbnew = null) { dbnew ??= new LocalListBrokenFilesDatabase(); @@ -150,6 +178,12 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Creates a new instance of the class. + /// + /// The parent database to use for the new database. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(LocalDatabase dbparent, LocalListBrokenFilesDatabase? dbnew = null) { dbnew ??= new LocalListBrokenFilesDatabase(); @@ -162,6 +196,12 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Returns all broken file IDs, i.e., files that reference blocksets or blocks that are not available. + /// + /// The time to filter filesets by. + /// Optional array of versions to filter filesets by. If null, all versions are considered. + /// An asynchronous enumerable of broken file IDs. public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[]? versions) { var query = BROKEN_FILE_SETS; @@ -192,6 +232,11 @@ namespace Duplicati.Library.Main.Database ); } + /// + /// Returns all broken file IDs, i.e., files that reference blocksets or blocks that are not available. + /// + /// The fileset ID to filter by. + /// An asynchronous enumerable of broken file IDs. public async IAsyncEnumerable> GetBrokenFilenames(long filesetid) { using var cmd = Connection.CreateCommand(m_rtr) @@ -209,8 +254,7 @@ namespace Duplicati.Library.Main.Database /// /// Returns all index files that are orphaned, i.e., not referenced by any block files. /// - /// Transaction to use for the query. - /// >All index files that are orphaned. + /// An asynchronous enumerable of representing the orphaned index files. public async IAsyncEnumerable GetOrphanedIndexFiles() { using var cmd = Connection.CreateCommand($@" @@ -239,10 +283,10 @@ namespace Duplicati.Library.Main.Database /// /// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name. /// - /// The filset id for the current operation - /// The name of the table to insert into - /// The name of the ID field in the table - /// The transaction to use for the query + /// The filset id for the current operation. + /// The name of the table to insert into. + /// The name of the ID field in the table. + /// A task that completes when the insertion is finished. public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname) { using var cmd = Connection.CreateCommand(m_rtr) @@ -257,11 +301,10 @@ namespace Duplicati.Library.Main.Database /// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs. /// If no such blockset is found, it returns -1. /// - /// The volume ids to ignore when searching for a suitable metadata block - /// The hash of the empty blockset - /// The size of the empty blockset - /// The transaction to use for the query - /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found + /// The volume ids to ignore when searching for a suitable metadata block. + /// The hash of the empty blockset. + /// The size of the empty blockset. + /// A task that when awaited contains the ID of the empty metadata blockset, or -1 if no suitable blockset is found public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize) { using var cmd = Connection.CreateCommand(@" @@ -338,10 +381,9 @@ namespace Duplicati.Library.Main.Database /// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume. /// This is used to clean up the metadata blocksets that are now missing. /// - /// The filesetId to target - /// The empty blockset ID to replace with - /// The transaction to use for the query - /// The number of rows affected + /// The filesetId to target. + /// The empty blockset ID to replace with. + /// A task that when awaited contains the number of rows affected public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId) { using var cmd = m_connection.CreateCommand(@" @@ -372,6 +414,11 @@ namespace Duplicati.Library.Main.Database return await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } + /// + /// Removes all blocks, blocksets, and index links that are missing from the specified volumes. + /// + /// The names of the volumes to check for missing blocks. + /// A task that completes when the removal is finished. public async Task RemoveMissingBlocks(IEnumerable names) { if (names == null || !names.Any()) return; @@ -453,6 +500,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets the count of files in a specific fileset. + /// + /// The ID of the fileset to count files in. + /// A task that when awaited contains the count of files in the specified fileset. public async Task GetFilesetFileCount(long filesetid) { using var cmd = m_connection.CreateCommand(@" diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 097faa9f9..7ced47e41 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -29,9 +29,19 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// A local database for tracking changes in file lists, such as added, deleted, or modified files. + /// internal class LocalListChangesDatabase : LocalDatabase { + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// The size of the page cache. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(string path, long pagecachesize, LocalListChangesDatabase? dbnew = null) { dbnew ??= new LocalListChangesDatabase(); @@ -44,41 +54,123 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Interface for storage helper that manages temporary storage of file changes. + /// public interface IStorageHelper : IDisposable, IAsyncDisposable { + /// + /// Adds an element to the temporary storage. + /// + /// The path of the file or folder. + /// The file hash, if applicable. + /// The metadata hash. + /// The size of the file or folder. + /// The type of the element (file, folder, symlink). + /// If true, adds to the current table; otherwise, adds to the previous table. + /// A task that completes when the element is added. Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew); + /// + /// Adds elements from the database to the temporary storage. + /// + /// The ID of the fileset to add. + /// If true, adds to the current table; otherwise, adds to the previous table. + /// An optional filter to apply when adding elements. + /// A task that completes when the elements are added. Task AddFromDb(long filesetId, bool asNew, IFilter filter); + /// + /// Creates a report containing the count of added, deleted, and modified elements. + /// + /// A task that, when awaited, returns an with the change counts. Task CreateChangeCountReport(); + + /// + /// Creates a report containing the size information for added, deleted, previous, and current elements. + /// + /// A task that, when awaited, returns an with the size details. Task CreateChangeSizeReport(); + + /// + /// Asynchronously generates a report of changed files, yielding tuples that describe the change type, element type, and file path. + /// + /// An asynchronous enumerable of tuples containing the change type, element type, and file path. IAsyncEnumerable> CreateChangedFileReport(); } + /// + /// Interface for reporting changes in file counts and sizes. + /// public interface IChangeCountReport { + /// + /// Gets the count of added folders. + /// long AddedFolders { get; } + /// + /// Gets the count of added symlinks. + /// long AddedSymlinks { get; } + /// + /// Gets the count of added files. + /// long AddedFiles { get; } + /// + /// Gets the count of deleted folders. + /// long DeletedFolders { get; } + /// + /// Gets the count of deleted symlinks. + /// long DeletedSymlinks { get; } + /// + /// Gets the count of deleted files. + /// long DeletedFiles { get; } + /// + /// Gets the count of modified folders. + /// long ModifiedFolders { get; } + /// + /// Gets the count of modified symlinks. + /// long ModifiedSymlinks { get; } + /// + /// Gets the count of modified files. + /// long ModifiedFiles { get; } } + /// + /// Interface for a report describing changes in file sizes. + /// public interface IChangeSizeReport { + /// + /// Gets the total size of added files. + /// long AddedSize { get; } + /// + /// Gets the total size of deleted files. + /// long DeletedSize { get; } + /// + /// Gets the size of files in the previous state. + /// long PreviousSize { get; } + /// + /// Gets the size of files in the current state. + /// long CurrentSize { get; } } + /// + /// Internal class that implements the interface to report changes in file counts. + /// internal class ChangeCountReport : IChangeCountReport { public long AddedFolders { get; internal set; } @@ -94,6 +186,9 @@ namespace Duplicati.Library.Main.Database public long ModifiedFiles { get; internal set; } } + /// + /// Internal class that implements the interface to report changes in file sizes. + /// internal class ChangeSizeReport : IChangeSizeReport { public long AddedSize { get; internal set; } @@ -103,23 +198,54 @@ namespace Duplicati.Library.Main.Database public long CurrentSize { get; internal set; } } + /// + /// Helper class for managing temporary storage of file changes. + /// Implements the interface. + /// private class StorageHelper : IStorageHelper { + /// + /// The database instance used for storage operations. + /// private LocalDatabase m_db = null!; + /// + /// Command for inserting elements into the previous table. + /// private SqliteCommand m_insertPreviousElementCommand = null!; + /// + /// Command for inserting elements into the current table. + /// private SqliteCommand m_insertCurrentElementCommand = null!; + /// + /// The name of the temporary table for previous elements. + /// private string m_previousTable = null!; + /// + /// The name of the temporary table for current elements. + /// private string m_currentTable = null!; + /// + /// Private constructor to prevent direct instantiation. + /// This constructor is obsolete and will throw an exception if called. + /// Use the method to create an instance instead. + /// [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] - public StorageHelper(SqliteConnection con) - { - } + public StorageHelper(SqliteConnection con) { } + /// + /// Private constructor to prevent direct instantiation. + /// This class should be created using the CreateAsync method. + /// private StorageHelper() { } + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database instance to use. + /// A task that, when awaited, returns a new instance of . public static async Task CreateAsync(LocalDatabase db) { var sh = new StorageHelper @@ -437,6 +563,11 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Converts a SqliteDataReader to an asynchronous enumerable of strings. + /// + /// The SqliteDataReader to read from. + /// An asynchronous enumerable of strings, where each string is a value from the first column of the reader. private static async IAsyncEnumerable ReaderToStringList(SqliteDataReader rd) { using (rd) @@ -450,6 +581,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves SQL queries for added, deleted, and modified files based on the current and previous tables. + /// + /// If true, retrieves all types of changes; otherwise, filters by type. + /// A tuple containing SQL queries for added, deleted, and modified files. private (string Added, string Deleted, string Modified) GetSqls(bool allTypes) { return ( @@ -490,6 +626,10 @@ namespace Duplicati.Library.Main.Database ); } + /// + /// Creates a report of changes in file sizes, including added, deleted, previous, and current sizes. + /// + /// A task that, when awaited, returns an with the size details. public async Task CreateChangeSizeReport() { var (Added, Deleted, Modified) = GetSqls(true); @@ -529,6 +669,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Creates a report containing the count of added, deleted, and modified elements. + /// + /// A task that, when awaited, returns an with the change counts. public async Task CreateChangeCountReport() { var (Added, Deleted, Modified) = GetSqls(false); @@ -602,6 +746,10 @@ namespace Duplicati.Library.Main.Database return result; } + /// + /// Asynchronously creates a report of changed files, yielding tuples that describe the change type, element type, and file path. + /// + /// An asynchronous enumerable of tuples containing the change type, element type, and file path. public async IAsyncEnumerable> CreateChangedFileReport() { var (Added, Deleted, Modified) = GetSqls(false); @@ -688,6 +836,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Creates a new instance of the for managing temporary storage of file changes. + /// + /// A task that, when awaited, returns an instance of . public async Task CreateStorageHelper() { return await StorageHelper.CreateAsync(this).ConfigureAwait(false); From 3c36e227ef69ed7bad360f45a8ea997b72e99fa4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 21:03:41 +0200 Subject: [PATCH 305/362] LocalListDatabase.IFileSets now implements IAsyncDisposable --- Duplicati/Library/Main/Database/LocalListDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 3d8859b01..bdd38165e 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -61,7 +61,7 @@ namespace Duplicati.Library.Main.Database long FileSizes { get; } } - public interface IFileSets : IDisposable + public interface IFileSets : IDisposable, IAsyncDisposable { IAsyncEnumerable Sets(); IAsyncEnumerable QuickSets(); @@ -687,10 +687,10 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (m_tablename != null) { From f167254efef56bfe03afbf7506f9026352d1d341 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 21:04:41 +0200 Subject: [PATCH 306/362] LocalPurgeDatabase.ITemporaryFileset now implements IAsyncDisposable --- Duplicati/Library/Main/Database/LocalPurgeDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index fd9049484..9a857ada1 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -100,7 +100,7 @@ namespace Duplicati.Library.Main.Database return 0; } - public interface ITemporaryFileset : IDisposable + public interface ITemporaryFileset : IDisposable, IAsyncDisposable { long ParentID { get; } long RemovedFileCount { get; } @@ -360,10 +360,10 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { try { From b4e5f2739ee1ae9ba16458a7c7c2f22db6182ff1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 21:05:34 +0200 Subject: [PATCH 307/362] Found incorrect type signature for DisposeAsync() methods --- Duplicati/Library/Main/Database/LocalRecreateDatabase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 9814ae6af..c74ce42ab 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -1278,11 +1278,11 @@ namespace Duplicati.Library.Main.Database public override void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public async override Task DisposeAsync() + public async override ValueTask DisposeAsync() { using (var cmd = m_connection.CreateCommand(m_rtr)) { From 273b0f093e8b16127e7cec6d9237d8cebaa04d17 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 21:30:11 +0200 Subject: [PATCH 308/362] Added more docstrings --- .../Main/Database/LocalListDatabase.cs | 181 ++++++++++++++-- .../Main/Database/LocalPurgeDatabase.cs | 102 +++++++++ .../Main/Database/LocalRecreateDatabase.cs | 201 +++++++++++++++++- 3 files changed, 458 insertions(+), 26 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index bdd38165e..d8771a859 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -29,11 +29,22 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Utility; using Duplicati.Library.Interface; using Microsoft.Data.Sqlite; +using Google.Protobuf.WellKnownTypes; namespace Duplicati.Library.Main.Database { + /// + /// A local database that stores file lists and filesets. + /// internal class LocalListDatabase : LocalDatabase { + /// + /// Creates a new instance of the LocalListDatabase. + /// + /// The path to the database file. + /// The size of the page cache. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of LocalListDatabase. public static async Task CreateAsync(string path, long pagecachesize, LocalListDatabase? dbnew = null) { dbnew ??= new LocalListDatabase(); @@ -46,45 +57,115 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Interface for file versions. + /// public interface IFileversion { + /// + /// Gets the path of the file, or null if not available. + /// string? Path { get; } + /// + /// Asynchronously retrieves the sizes of the file versions. + /// IAsyncEnumerable Sizes(); } + /// + /// Interface for filesets. + /// public interface IFileset { + /// + /// Gets the version of the fileset. + /// long Version { get; } + /// + /// Gets or sets a value indicating whether the fileset is a full backup. + /// int IsFullBackup { get; set; } + /// + /// Gets the timestamp of the fileset. + /// DateTime Time { get; } + /// + /// Gets the count of files in the fileset. + /// long FileCount { get; } + /// + /// Gets the total sizes of files in the fileset. + /// long FileSizes { get; } } + /// + /// Interface for managing file sets in the local database. + /// public interface IFileSets : IDisposable, IAsyncDisposable { + /// + /// Gets all of the filesets in the database as an asynchronous enumerable, including file count and total size. + /// IAsyncEnumerable Sets(); + /// + /// Gets all of the filesets in the database as an asynchronous enumerable, omitting file count and total size. + /// IAsyncEnumerable QuickSets(); + /// + /// Gets the files in the filesets that match the provided filter as an asynchronous enumerable. + /// IAsyncEnumerable SelectFiles(IFilter filter); + /// + /// Gets the largest prefix of files in the filesets that match the provided filter as an asynchronous enumerable. + /// IAsyncEnumerable GetLargestPrefix(IFilter filter); + /// + /// Selects the contents of a folder in the filesets that match the provided filter as an asynchronous enumerable. + /// IAsyncEnumerable SelectFolderContents(IFilter filter); Task TakeFirst(); } + /// + /// Creates a new instance of the class implementing the interface. + /// private class FileSets : IFileSets { + /// + /// The database instance to execute commands against. + /// private LocalDatabase m_db = null!; + /// + /// The name of the temporary table used to store filesets. + /// private string m_tablename = null!; + /// + /// An array of key-value pairs representing filesets, where the key is the fileset ID and the value is the timestamp. + /// private KeyValuePair[] m_filesets = []; + /// + /// Private constructor to prevent instantiation without using CreateAsync. + /// [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public FileSets(LocalListDatabase owner, DateTime time, long[] versions) { throw new NotSupportedException("Use CreateAsync instead."); } + /// + /// Private constructor to prevent instantiation without using CreateAsync. + /// private FileSets() { } + /// + /// Asynchronously creates a new instance of the class. + /// + /// The owner database instance. + /// The time to filter filesets by. + /// An array of versions to filter filesets by. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(LocalListDatabase owner, DateTime time, long[] versions) { var fs = new FileSets @@ -126,14 +207,40 @@ namespace Duplicati.Library.Main.Database return fs; } + /// + /// Internal class representing a fileset. + /// private class Fileset : IFileset { + /// + /// The backup version this fileset is part of. + /// public long Version { get; private set; } + /// + /// Indicates whether this fileset is a full backup (1) or a partial backup (0). + /// public int IsFullBackup { get; set; } + /// + /// The timestamp of the fileset, indicating when it was created. + /// public DateTime Time { get; private set; } + /// + /// The count of files in this fileset. + /// public long FileCount { get; private set; } + /// + /// The total sizes of files in this fileset. + /// public long FileSizes { get; private set; } + /// + /// Initializes a new instance of the class. + /// + /// The backup version of the fileset. + /// Indicates whether this fileset is a full backup (1) or a partial backup (0). + /// The timestamp of the fileset, indicating when it was created. + /// The count of files in this fileset. + /// The total sizes of files in this fileset. public Fileset(long version, int isFullBackup, DateTime time, long filecount, long filesizes) { Version = version; @@ -144,12 +251,28 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Implementation of the interface. + /// private class Fileversion : IFileversion { + /// + /// The SqliteDataReader used to read file version data. + /// private readonly SqliteDataReader m_reader; + /// + /// The path of the file, or null if not available. + /// public string? Path { get; private set; } + /// + /// Indicates whether there are more file versions to read. + /// public bool More { get; private set; } + /// + /// Initializes a new instance of the class. + /// + /// The SqliteDataReader used to read file version data. public Fileversion(SqliteDataReader reader) { m_reader = reader; @@ -167,6 +290,9 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Internal class representing a fixed file version with no sizes. + /// private class FileversionFixed : IFileversion { public string? Path { get; internal set; } @@ -178,6 +304,12 @@ namespace Duplicati.Library.Main.Database return GetLargestPrefix(filter, null); } + /// + /// Recursively finds the largest prefix of files in the filesets that match the provided filter. + /// + /// The filter to apply to the files. + /// The prefix rule to apply, or null if no prefix rule is specified. + /// An asynchronous enumerable of file versions representing the largest prefix. private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) { using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false)) @@ -302,6 +434,13 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Asynchronously retrieves all folder entries from the specified table that match the given prefix. + /// + /// The SqliteCommand to execute. + /// The prefix to filter folder entries by. + /// The name of the table to query. + /// An asynchronous enumerable of folder entries. private async IAsyncEnumerable SelectFolderEntries(SqliteCommand cmd, string prefix, string table) { if (!string.IsNullOrEmpty(prefix)) @@ -584,6 +723,9 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Deletes all filesets except the most recent one. + /// public async Task TakeFirst() { using var cmd = m_db.Connection.CreateCommand(); @@ -706,6 +848,12 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Creates a new instance of with the specified time and versions. + /// + /// The time to filter filesets by. + /// An array of versions to filter filesets by. + /// A task that when awaited returns a new instance of . public async Task SelectFileSets(DateTime time, long[] versions) { return await FileSets.CreateAsync(this, time, versions) @@ -715,17 +863,17 @@ namespace Duplicati.Library.Main.Database /// /// Represents a fileset entry in the list fileset results. /// - /// The fileset version - /// Flag indicating if the backup is a full backup or synthetic backup - /// The timestamp of the fileset - /// The number of files in the fileset - /// >The total size of files in the fileset + /// The fileset version. + /// Flag indicating if the backup is a full backup or synthetic backup. + /// The timestamp of the fileset. + /// The number of files in the fileset. + /// >The total size of files in the fileset. private sealed record FilesetEntry(long Version, bool? IsFullBackup, DateTime Time, long? FileCount, long? FileSizes) : IListFilesetResultFileset; /// - /// Lists all filesets with summary data + /// Lists all filesets with summary data. /// - /// A collection of fileset entries + /// An asynchronous enumerable of fileset entries. public async IAsyncEnumerable ListFilesetsExtended() { const string query = @" @@ -782,7 +930,7 @@ namespace Duplicati.Library.Main.Database /// The fileset ID to filter the folder entries. /// The offset for pagination. /// >The limit for pagination. - /// A paginated result set of folder entries. + /// A task that when awaited returns a paginated result set of folder entries. public async Task> ListFolder(IEnumerable prefixIds, long filesetid, long offset, long limit) { if (offset != 0 && limit <= 0) @@ -890,9 +1038,10 @@ namespace Duplicati.Library.Main.Database /// Lists the folder entries for a given fileset and prefix IDs. /// /// The prefix IDs to return entries for. + /// The fileset ID to filter the folder entries. /// The offset for pagination. - /// >The limit for pagination. - /// A paginated result set of folder entries. + /// The limit for pagination. + /// A task that when awaited returns a paginated result set of folder entries. public async Task> ListFilesetEntries(IEnumerable prefixIds, long filesetid, long offset, long limit) { if (offset != 0 && limit <= 0) @@ -992,7 +1141,7 @@ namespace Duplicati.Library.Main.Database /// Gets the prefix IDs for a given set of folder-prefixes. /// /// The prefixes to get the IDs for. - /// >A list of prefix IDs. + /// An asynchronous enumerable of prefix IDs. public IAsyncEnumerable GetPrefixIds(IEnumerable prefixes) { using var cmd = m_connection.CreateCommand(); @@ -1012,8 +1161,8 @@ namespace Duplicati.Library.Main.Database /// /// Gathers root prefixes from the fileset entries. /// - /// The filesetId to get the prefixes for - /// A list of root prefixes + /// The filesetId to get the prefixes for. + /// An asynchronous enumerable of root prefixes. public IAsyncEnumerable GetRootPrefixes(long filesetid) { using var cmd = m_connection.CreateCommand(); @@ -1053,7 +1202,7 @@ namespace Duplicati.Library.Main.Database /// Gathers all prefixes and their parent prefix IDs from the fileset entries. /// /// The filesetId to get the prefixes for - /// A list of (PrefixID, ParentPrefixID) pairs + /// A task that when awaited returns an asynchronous enumerable of (PrefixID, ParentPrefixID) pairs. public async Task> GetRootPrefixesWithParents(long filesetid) { using var cmd = m_connection.CreateCommand(); @@ -1120,7 +1269,7 @@ namespace Duplicati.Library.Main.Database /// Optional fileset IDs to restrict the search to. /// Pagination offset. /// Pagination limit. - /// All matching file versions ordered by path and then version time. + /// A task that when awaited returns all matching file versions ordered by path and then version time. public async Task> ListFileVersions(IEnumerable paths, long[]? filesetIds, long offset, long limit) { if (paths == null || !paths.Any()) @@ -1265,7 +1414,7 @@ namespace Duplicati.Library.Main.Database /// Optional fileset IDs to restrict the search to. /// The offset for pagination. /// The limit for pagination. - /// All matching file versions ordered by path and then version time. + /// A task that when awaited returns all matching file versions ordered by path and then version time. public async Task> SearchEntries(IEnumerable? pathprefixes, IFilter filter, long[]? filesetIds, long offset, long limit) { if (offset != 0 && limit <= 0) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 9a857ada1..2e1dd466d 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -29,8 +29,18 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// A local database for purging files from a fileset. + /// internal class LocalPurgeDatabase : LocalDeleteDatabase { + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// The size of the page cache. + /// The optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(string path, long pagecachesize, LocalPurgeDatabase? dbnew = null) { dbnew ??= new LocalPurgeDatabase(); @@ -44,6 +54,12 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Creates a new instance of the class. + /// + /// The parent database to use. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(LocalDatabase dbparent, LocalPurgeDatabase? dbnew = null) { dbnew ??= new LocalPurgeDatabase(); @@ -55,12 +71,22 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Creates a new instance of the class. + /// + /// The ID of the parent fileset. + /// A task that when awaited contains a new instance of . public async Task CreateTemporaryFileset(long parentid) { return await TemporaryFileset.CreateAsync(parentid, this) .ConfigureAwait(false); } + /// + /// Gets the remote volume name for a fileset with the specified ID. + /// + /// The ID of the fileset. + /// A task that when awaited contains the name of the remote volume. public async Task GetRemoteVolumeNameForFileset(long id) { using var cmd = m_connection.CreateCommand(@" @@ -82,6 +108,10 @@ namespace Duplicati.Library.Main.Database return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); } + /// + /// Counts the number of orphan files in the database. + /// + /// A task that when awaited contains the count of orphan files. internal async Task CountOrphanFiles() { using var cmd = m_connection.CreateCommand(m_rtr); @@ -100,22 +130,67 @@ namespace Duplicati.Library.Main.Database return 0; } + /// + /// Interface for a temporary fileset used for purging files from a fileset. + /// public interface ITemporaryFileset : IDisposable, IAsyncDisposable { + /// + /// Gets the ID of the parent fileset. + /// long ParentID { get; } + /// + /// Gets the count of removed files. + /// long RemovedFileCount { get; } + /// + /// Gets the total size of removed files. + /// long RemovedFileSize { get; } + /// + /// Gets the count of updated files. + /// long UpdatedFileCount { get; } + /// + /// Applies a filter to the temporary fileset. + /// + /// The filter to apply. + /// A task that completes when the filter has been applied. Task ApplyFilter(Library.Utility.IFilter filter); + /// + /// Applies a filter using a custom command. + /// + /// The command to execute for filtering. + /// A task that completes when the filter has been applied. Task ApplyFilter(Func> filtercommand); + /// + /// Converts the temporary fileset to a permanent fileset. + /// + /// The name of the new fileset. + /// The timestamp for the new fileset. + /// Indicates if this is a full backup. + /// A task that when awaited contains a tuple with the remote volume ID and the new fileset ID. Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); + /// + /// Lists all deleted files in the temporary fileset. + /// + /// An asynchronous enumerable of key-value pairs where the key is the file path and the value is the file size. IAsyncEnumerable> ListAllDeletedFiles(); } + /// + /// A temporary fileset implementation for purging files from a fileset. + /// private class TemporaryFileset : ITemporaryFileset { + /// + /// The name of the temporary table used for storing deleted files. + /// private string m_tablename = null!; + /// + /// The database instance used for queries. + /// private LocalDatabase m_db = null!; public long ParentID { get; private set; } @@ -123,14 +198,26 @@ namespace Duplicati.Library.Main.Database public long RemovedFileSize { get; private set; } public long UpdatedFileCount { get; private set; } + /// + /// Calling this constructor will throw an exception. Use CreateAsync instead. + /// [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, SqliteConnection connection, SqliteTransaction transaction) { throw new NotImplementedException("Use CreateAsync instead"); } + /// + /// Private constructor to prevent direct instantiation. + /// private TemporaryFileset() { } + /// + /// Creates a new instance of the class asynchronously. + /// + /// The ID of the parent fileset. + /// The database instance to use. + /// A task that when awaited contains a new instance of . public static async Task CreateAsync(long parentid, LocalDatabase db) { var tempf = new TemporaryFileset() @@ -151,6 +238,11 @@ namespace Duplicati.Library.Main.Database return tempf; } + /// + /// Applies a filter to the temporary fileset using a custom command. + /// + /// The command to execute for filtering. + /// A task that completes when the filter has been applied. public async Task ApplyFilter(Func> filtercommand) { int updated; @@ -161,6 +253,11 @@ namespace Duplicati.Library.Main.Database await PostFilterChecks(updated).ConfigureAwait(false); } + /// + /// Applies a filter to the temporary fileset. + /// + /// The filter to apply. + /// A task that completes when the filter has been applied. public async Task ApplyFilter(Library.Utility.IFilter filter) { if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) @@ -257,6 +354,11 @@ namespace Duplicati.Library.Main.Database await PostFilterChecks(0).ConfigureAwait(false); } + /// + /// Applies the filter checks after filtering. + /// + /// The number of files updated by the filter. + /// A task that completes when the checks have been applied. private async Task PostFilterChecks(int updated) { using (var cmd = m_db.Connection.CreateCommand()) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index c74ce42ab..ef46ae871 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -29,15 +29,25 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// This class is used to recreate a local database from a backup. + /// internal class LocalRecreateDatabase : LocalRestoreDatabase { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRecreateDatabase)); + /// + /// Keeps track of the path entries in a sorted manner. + /// This is used to ensure that the entries are unique and sorted by their keys. + /// private class PathEntryKeeper { + /// + /// Comparer for KeyValuePair to sort by key and value. + /// private struct KeyValueComparer : IComparer> { public readonly int Compare(KeyValuePair x, KeyValuePair y) @@ -51,36 +61,89 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Inserts a new file entry into the FileLookup table and returns the new row ID. + /// private SqliteCommand m_insertFileCommand = null!; + /// + /// Inserts a new entry into the FilesetEntry table linking a fileset to a file with a last-modified timestamp. + /// private SqliteCommand m_insertFilesetEntryCommand = null!; + /// + /// Inserts a new row into the Metadataset table for a given blockset and returns the new row ID. + /// private SqliteCommand m_insertMetadatasetCommand = null!; + /// + /// Inserts a new blockset with its length and full hash into the Blockset table and returns the new row ID. + /// private SqliteCommand m_insertBlocksetCommand = null!; + /// + /// Inserts a new blocklist hash entry into the BlocklistHash table for a given blockset and index. + /// private SqliteCommand m_insertBlocklistHashCommand = null!; + /// + /// Updates the VolumeID of a block in the Block table matching a specific hash and size. + /// private SqliteCommand m_updateBlockVolumeCommand = null!; + /// + /// Inserts a blocklist hash, block hash, and index into the temporary blocklist table. + /// private SqliteCommand m_insertTempBlockListHash = null!; + /// + /// Inserts or ignores a file hash, block hash, and block size into the temporary small blockset table. + /// private SqliteCommand m_insertSmallBlockset = null!; + /// + /// Selects the ID of a blockset from the Blockset table matching a given length and full hash. + /// private SqliteCommand m_findBlocksetCommand = null!; + /// + /// Selects the ID of a Metadataset whose blockset matches a given full hash and length. + /// private SqliteCommand m_findMetadatasetCommand = null!; + /// + /// Selects the ID of a file from the FileLookup table matching a given prefix, path, blockset, and metadata. + /// private SqliteCommand m_findFilesetCommand = null!; + /// + /// Selects distinct blocklist hashes from the temporary blocklist table matching a given hash. + /// private SqliteCommand m_findTempBlockListHashCommand = null!; + /// + /// Selects the VolumeID of a block in the Block table matching a specific hash and size. + /// private SqliteCommand m_findHashBlockCommand = null!; + /// + /// Inserts a new block with hash, size, and volume ID into the Block table. + /// private SqliteCommand m_insertBlockCommand = null!; + /// + /// Inserts or ignores a duplicate block entry linking a block and a volume into the DuplicateBlock table. + /// private SqliteCommand m_insertDuplicateBlockCommand = null!; + /// + /// Temporary table name for blocklist hashes and their indices. + /// private string m_tempblocklist = null!; + /// + /// Temporary table name for small blocksets, containing file hashes, block hashes, and block sizes. + /// private string m_tempsmalllist = null!; /// - /// A lookup table that prevents multiple downloads of the same volume + /// A lookup table that prevents multiple downloads of the same volume. /// private readonly Dictionary m_proccessedVolumes = new Dictionary(); - // SQL that finds index and block size for all blocklist hashes, based on the temporary hash list - // with vars Used: - // {0} --> Blocksize - // {1} --> BlockHash-Size - // {2} --> Temp-Table - // {3} --> FullBlocklist-BlockCount [equals ({0} / {1}), if SQLite pays respect to ints] + /// + /// SQL that finds index and block size for all blocklist hashes, based on the temporary hash list. + /// + /// The size of the blocks in the blocklist. + /// The size of the block hashes. + /// The name of the temporary table containing blocklist hashes and indices. + /// The number of blocks in the full block list, used to calculate the full index. + /// A SQL query string that selects blocklist entries with their full index and block size. private static string SELECT_BLOCKLIST_ENTRIES(long blocksize, long blockhashsize, string temptable, long fullBlockListBlockCount) => $@" SELECT ""E"".""BlocksetID"", @@ -120,6 +183,13 @@ namespace Duplicati.Library.Main.Database ""FullIndex"" "; + /// + /// Asynchronously creates a new instance of the class. + /// + /// The parent database from which to restore. + /// The options to use for the database creation. + /// An optional existing instance of to use; if null, a new instance will be created. + /// A task that represents the asynchronous operation, containing the newly created instance. public static async Task CreateAsync(LocalDatabase parentdb, Options options, LocalRecreateDatabase? dbnew = null) { dbnew ??= new LocalRecreateDatabase(); @@ -362,6 +432,12 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Identifies and inserts any missing blocks and blockset entries into the database by comparing temporary tables of blocklist and small blockset data with the main tables, ensuring that all required block and blockset relationships are present for database consistency. + /// + /// The size of the hashes in the blocklist. + /// The size of the blocks in the blocklist. + /// A task that completes when the operation is finished. public async Task FindMissingBlocklistHashes(long hashsize, long blocksize) { using var cmd = m_connection.CreateCommand(m_rtr); @@ -557,6 +633,10 @@ namespace Duplicati.Library.Main.Database /// temp small list structure: filehash, blockhash, blocksize: as the small files are defined /// by the fact that they are contained in a single block, blockhash is the same as the filehash, /// and blocksize can vary from 0 to the configured block size for the backup + /// The size of the hashes in the blocklist. + /// The size of the blocks in the blocklist. + /// If true, only hash entries are processed, ignoring small blocks. + /// A task that completes when the operation is finished. public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, bool hashOnly = false) { using var cmd = m_connection.CreateCommand(m_rtr); @@ -688,24 +768,64 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Adds a directory entry to the FilesetEntry table, linking it to a fileset and a path prefix. + /// + /// The ID of the fileset to which the entry belongs. + /// The ID of the path prefix for the entry. + /// The path of the directory entry. + /// The last modified time of the directory entry. + /// The ID of the metadata associated with the directory entry. + /// A task that when completed, indicates that the directory entry has been added. public async Task AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid) { await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid) .ConfigureAwait(false); } + /// + /// Adds a symlink entry to the FilesetEntry table, linking it to a fileset and a path prefix. + /// + /// The ID of the fileset to which the entry belongs. + /// The ID of the path prefix for the entry. + /// The path of the symlink entry. + /// The last modified time of the symlink entry. + /// The ID of the metadata associated with the symlink entry. + /// A task that when completed, indicates that the symlink entry has been added. public async Task AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid) { await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid) .ConfigureAwait(false); } + /// + /// Adds a file entry to the FilesetEntry table, linking it to a fileset and a path prefix. + /// + /// The ID of the fileset to which the entry belongs. + /// The ID of the path prefix for the entry. + /// The path of the file entry. + /// The last modified time of the file entry. + /// The ID of the blockset associated with the file entry. + /// The ID of the metadata associated with the file entry. + /// A task that when completed, indicates that the file entry has been added. public async Task AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid) { await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid) .ConfigureAwait(false); } + /// + /// Adds a file and fileset entry to the FilesetEntry table, linking it to a fileset and a path prefix. + /// If the file already exists, it retrieves its ID; otherwise, it inserts a new file entry. + /// Then, it inserts the fileset entry linking the file to the fileset. + /// + /// The ID of the fileset to which the entry belongs. + /// The ID of the path prefix for the entry. + /// The path of the file entry. + /// The last modified time of the file entry. + /// The ID of the blockset associated with the file entry. + /// The ID of the metadata associated with the file entry. + /// A task that when completed, indicates that the file and fileset entry has been added. private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid) { var fileid = await m_findFilesetCommand @@ -738,6 +858,15 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Adds a new metadataset to the database, creating a blockset if it does not already exist. + /// If the metadataset already exists, it returns its ID. + /// + /// The full hash of the metadataset. + /// The size of the metadataset in bytes. + /// A collection of blocklist hashes associated with the metadataset. + /// The expected number of blocklist hashes for the metadataset. + /// A task that returns the ID of the added or existing metadataset. public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes) { var metadataid = -1L; @@ -767,6 +896,15 @@ namespace Duplicati.Library.Main.Database return metadataid; } + /// + /// Adds a new blockset to the database, creating it if it does not already exist. + /// If the blockset already exists, it returns its ID. + /// + /// The full hash of the blockset. + /// The size of the blockset in bytes. + /// A collection of blocklist hashes associated with the blockset. + /// The expected number of blocklist hashes for the blockset. + /// A task that returns the ID of the added or existing blockset. public async Task AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes) { var blocksetid = await m_findBlocksetCommand @@ -817,6 +955,14 @@ namespace Duplicati.Library.Main.Database return blocksetid; } + /// + /// Updates the block information in the database, inserting a new block if it does not exist, + /// updating the volume ID if it does, or inserting a duplicate block entry if the block already exists with a different volume ID. + /// + /// The hash of the block to update. + /// The size of the block in bytes. + /// The ID of the volume to which the block belongs. + /// A task that returns a tuple indicating whether any changes were made and whether the block was newly inserted. public async Task<(bool, bool)> UpdateBlock(string hash, long size, long volumeID) { var anyChange = false; @@ -875,6 +1021,14 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Adds a link between a small blockset and a block in the database. + /// This method is used to track small blocks that are part of a larger blockset, allowing for efficient storage and retrieval of small files. + /// + /// The hash of the file that contains the small block. + /// The hash of the small block. + /// The size of the small block in bytes. + /// A task that completes when the small blockset link has been added. public async Task AddSmallBlocksetLink(string filehash, string blockhash, long blocksize) { await m_insertSmallBlockset @@ -886,6 +1040,13 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Adds a temporary blocklist hash to the database, along with its associated block hashes. + /// This method is used to track temporary blocklists that are being processed, allowing for efficient management of blocklist data. + /// + /// The hash of the temporary blocklist. + /// A collection of block hashes associated with the temporary blocklist. + /// A task that returns a boolean indicating whether the temporary blocklist hash was successfully added. public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes) { var r = await m_findTempBlockListHashCommand @@ -915,7 +1076,12 @@ namespace Duplicati.Library.Main.Database return true; } - + /// + /// Retrieves a list of block hashes associated with a specific volume ID from the database. + /// This method is used to obtain the block hashes for a given volume, allowing for efficient retrieval of block data. + /// + /// The ID of the volume for which to retrieve block hashes. + /// An asynchronous enumerable collection of block hashes associated with the specified volume ID. public async IAsyncEnumerable GetBlockLists(long volumeid) { using var cmd = m_connection.CreateCommand(@" @@ -935,6 +1101,15 @@ namespace Duplicati.Library.Main.Database yield return rd.ConvertValueToString(0) ?? ""; } + /// + /// Retrieves a list of remote volumes that are missing blocks based on the specified parameters. + /// This method is used to identify volumes that need to be processed for missing blocks, allowing for efficient management of remote storage. + /// + /// The current pass number, used to determine the state of processing. + /// The size of the blocks in the blocklist. + /// The size of the hashes in the blocklist. + /// A boolean indicating whether to force the use of blocks, even if they are not missing. + /// An asynchronous enumerable collection of remote volumes that are missing blocks. public async IAsyncEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -1061,6 +1236,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Cleans up missing volumes by replacing blocks for non-present remote files with existing ones, + /// removing references to non-present remote files, and marking index files for later removal. + /// + /// A task that completes when the cleanup operation is finished. public async Task CleanupMissingVolumes() { var tablename = "SwapBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -1213,8 +1393,9 @@ namespace Duplicati.Library.Main.Database /// /// Move blocks that are not referenced by any files to DeletedBlock table. - /// /// Needs to be called after the last FindMissingBlocklistHashes, otherwise the tables are not up to date. + /// + /// A task that completes when the cleanup operation is finished. public async Task CleanupDeletedBlocks() { // Find out which blocks are deleted and move them into DeletedBlock, so that compact notices these blocks are empty From 2296d040a2457dac2d9f095703f0348eebd53014 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 17 Jun 2025 22:15:57 +0200 Subject: [PATCH 309/362] LocalRestoreDatabase.IFilesAndMetadata now implements IAsyncDisposable --- Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index f371ad80a..97b2e59eb 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1290,7 +1290,7 @@ namespace Duplicati.Library.Main.Database } } - public interface IFilesAndMetadata : IDisposable + public interface IFilesAndMetadata : IDisposable, IAsyncDisposable { IAsyncEnumerable FilesWithMissingBlocks(); IAsyncEnumerable MetadataWithMissingBlocks(); @@ -1360,10 +1360,10 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (m_tmptable != null) { From 86b75b395ae6db01188d90c776ac14b355499a8d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 06:44:18 +0200 Subject: [PATCH 310/362] LocalRestoreDatabase.IBlockMarker now implements IAsyncDisposable --- .../Main/Database/LocalRestoreDatabase.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 97b2e59eb..af6ff4a1b 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1871,7 +1871,7 @@ namespace Duplicati.Library.Main.Database await m_rtr.CommitAsync().ConfigureAwait(false); } - public interface IBlockMarker : IDisposable + public interface IBlockMarker : IDisposable, IAsyncDisposable { Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata); Task SetAllBlocksMissing(long targetfileid); @@ -2067,11 +2067,16 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - m_insertblockCommand?.Dispose(); - m_resetfileCommand?.Dispose(); - m_updateAsRestoredCommand?.Dispose(); - m_updateFileAsDataVerifiedCommand?.Dispose(); - m_statUpdateCommand?.Dispose(); + DisposeAsync().AsTask().Await(); + } + + public async ValueTask DisposeAsync() + { + await m_insertblockCommand.DisposeAsync().ConfigureAwait(false); + await m_resetfileCommand.DisposeAsync().ConfigureAwait(false); + await m_updateAsRestoredCommand.DisposeAsync().ConfigureAwait(false); + await m_updateFileAsDataVerifiedCommand.DisposeAsync().ConfigureAwait(false); + await m_statUpdateCommand.DisposeAsync().ConfigureAwait(false); } } From eb4f76383188daa19204b6735a36d5a285b986da Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 06:59:57 +0200 Subject: [PATCH 311/362] LocalTestDatabase.Basiclist now implements IAsyncDisposable --- Duplicati/Library/Main/Database/LocalTestDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index ea909caa6..e427b3fe2 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -294,7 +294,7 @@ namespace Duplicati.Library.Main.Database } } - private abstract class Basiclist : IDisposable + private abstract class Basiclist : IDisposable, IAsyncDisposable { protected LocalDatabase m_db = null!; protected string m_volumename = null!; @@ -337,10 +337,10 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public virtual async Task DisposeAsync() + public virtual async ValueTask DisposeAsync() { if (m_tablename != null) try From 20716e3fa6015c0bcee1099a50478b4dfc86a782 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 07:25:24 +0200 Subject: [PATCH 312/362] Made some more of the internal classes in LocalTestDatabase implement IAsyncDisposable --- Duplicati/Library/Main/Database/LocalTestDatabase.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index e427b3fe2..4b4049e9b 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -356,7 +356,7 @@ namespace Duplicati.Library.Main.Database } } - public interface IFilelist : IDisposable + public interface IFilelist : IDisposable, IAsyncDisposable { Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time); IAsyncEnumerable> Compare(); @@ -538,7 +538,7 @@ namespace Duplicati.Library.Main.Database } } - public interface IIndexlist : IDisposable + public interface IIndexlist : IDisposable, IAsyncDisposable { Task AddBlockLink(string filename, string hash, long length); IAsyncEnumerable> Compare(); @@ -689,13 +689,13 @@ namespace Duplicati.Library.Main.Database } } - public interface IBlocklist : IDisposable + public interface IBlocklist : IDisposable, IAsyncDisposable { Task AddBlock(string key, long value); IAsyncEnumerable> Compare(); } - public interface IBlocklistHashList : IDisposable + public interface IBlocklistHashList : IDisposable, IAsyncDisposable { Task AddBlockHash(string hash, long size); IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize); From 224e883e8d5d3892d964e0da01cfe0e8d34c4eac Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 07:30:39 +0200 Subject: [PATCH 313/362] Formatted SQL query strings --- .../Main/Database/LocalTestDatabase.cs | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 4b4049e9b..bf3c7c2ee 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -704,9 +704,18 @@ namespace Duplicati.Library.Main.Database private class Blocklist : Basiclist, IBlocklist { private const string TABLE_PREFIX = "Blocklist"; - private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; - private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; - + private const string TABLE_FORMAT = @"( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + )"; + private const string INSERT_COMMAND = @"( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size + )"; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Blocklist(SqliteConnection connection, string volumename, ReusableTransaction rtr) { @@ -867,9 +876,19 @@ namespace Duplicati.Library.Main.Database private class BlocklistHashList : Basiclist, IBlocklistHashList { private const string TABLE_PREFIX = "BlocklistHashList"; - private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; - private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; + private const string TABLE_FORMAT = @"( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + )"; + private const string INSERT_COMMAND = @"( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size + )"; [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public BlocklistHashList(SqliteConnection connection, string volumename, ReusableTransaction rtr) { From ae633fe0e8707b1cf5b56725b55717de987c420c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 07:42:57 +0200 Subject: [PATCH 314/362] Added more docstrings --- .../Library/Main/Database/IRemoteVolume.cs | 42 +- .../Main/Database/LocalRepairDatabase.cs | 263 +++++--- .../Main/Database/LocalRestoreDatabase.cs | 594 +++++++++++++++++- .../Main/Database/LocalTestDatabase.cs | 273 ++++++++ .../Library/Main/Database/RemoteVolume.cs | 47 +- .../Main/Database/RemoteVolumeEntry.cs | 65 +- .../Main/Database/ReusableTransaction.cs | 22 +- .../Main/Database/TemporaryDbValueList.cs | 77 ++- 8 files changed, 1207 insertions(+), 176 deletions(-) diff --git a/Duplicati/Library/Main/Database/IRemoteVolume.cs b/Duplicati/Library/Main/Database/IRemoteVolume.cs index b5047e2dc..9cb24f49c 100644 --- a/Duplicati/Library/Main/Database/IRemoteVolume.cs +++ b/Duplicati/Library/Main/Database/IRemoteVolume.cs @@ -1,29 +1,41 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. namespace Duplicati.Library.Main.Database { + /// + /// Interface for a remote volume, containing its name, hash, and size. + /// public interface IRemoteVolume { + /// + /// Gets the name of the remote volume. + /// string Name { get; } + /// + /// Gets the hash of the remote volume. + /// string Hash { get; } + /// + /// Gets the size of the remote volume in bytes. + /// long Size { get; } } } diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 331420983..5f9019c2b 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -29,18 +29,30 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// Represents a specialized local database used for repair operations in Duplicati. + /// Provides methods for creating and managing repair databases, handling missing or duplicate blocks, + /// repairing metadata and file entries, and verifying consistency of backup data. + /// + /// + /// This class extends and provides additional functionality + /// for repairing and maintaining the integrity of backup databases, including block and fileset management, + /// duplicate detection and correction, and blocklist hash repairs. + /// internal class LocalRepairDatabase : LocalDatabase { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRepairDatabase)); /// - /// Creates a new local repair database + /// Creates a new local repair database. /// - /// The path to the database - /// The page cache size + /// The path to the database. + /// The page cache size. + /// An optional existing database instance to use. + /// A task that when awaited returns a new instance of . public static async Task CreateRepairDatabase(string path, long pagecachesize, LocalRepairDatabase? dbnew = null) { dbnew ??= new LocalRepairDatabase(); @@ -53,9 +65,11 @@ namespace Duplicati.Library.Main.Database } /// - /// Creates a new local repair database from an existing local database + /// Creates a new local repair database from an existing local database. /// - /// The existing local database + /// The parent local database to use. + /// An optional existing database instance to use. + /// A task that when awaited returns a new instance of . public static async Task CreateAsync(LocalDatabase dbparent, LocalRepairDatabase? dbnew = null) { dbnew ??= new LocalRepairDatabase(); @@ -63,15 +77,14 @@ namespace Duplicati.Library.Main.Database return (LocalRepairDatabase) await CreateLocalDatabaseAsync(dbparent, dbnew) .ConfigureAwait(false); - } /// - /// Gets the fileset ID from the remote name + /// Gets the fileset ID from the remote name. /// - /// The remote name of the fileset - /// An optional transaction - /// >The fileset ID + /// The remote name of the fileset. + /// A task that when awaited returns a tuple containing the fileset ID, timestamp, and whether it is a full backup. + /// Thrown if the remote file does not exist. public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist) { using var cmd = m_connection.CreateCommand(@" @@ -102,11 +115,12 @@ namespace Duplicati.Library.Main.Database } /// - /// Moves entries in the FilesetEntry table from previous fileset to current fileset + /// Moves entries in the FilesetEntry table from previous fileset to current fileset. /// - /// Current fileset ID - /// Source fileset ID - /// The transaction + /// Current fileset ID. + /// Source fileset ID. + /// A task that represents the asynchronous operation. + /// Thrown if either fileset ID is less than or equal to zero. public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId) { if (filesetid <= 0) @@ -128,10 +142,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Gets the list of index files that reference a given block file + /// Gets the list of index files that reference a given block file. /// - /// The block file ID - /// >A list of index file IDs + /// The block file ID. + /// An asynchronous enumerable of index file names that reference the specified block file. public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid) { using var cmd = m_connection.CreateCommand(@" @@ -153,10 +167,9 @@ namespace Duplicati.Library.Main.Database } /// - /// Gets a list of filesets that are missing files + /// Gets a list of filesets that are missing files. /// - /// An optional transaction - /// A list of fileset IDs and timestamps + /// An asynchronous enumerable of key-value pairs where the key is the fileset ID and the value is the timestamp of the fileset. public async IAsyncEnumerable> GetFilesetsWithMissingFiles() { using var cmd = m_connection.CreateCommand(@" @@ -186,11 +199,10 @@ namespace Duplicati.Library.Main.Database } /// - /// Deletes all fileset entries for a given fileset + /// Deletes all fileset entries for a given fileset. /// - /// The fileset ID - /// An optional transaction - /// The number of deleted entries + /// The fileset ID. + /// A task that when awaited returns the number of deleted entries. public async Task DeleteFilesetEntries(long filesetid) { using var cmd = m_connection.CreateCommand(@" @@ -203,12 +215,30 @@ namespace Duplicati.Library.Main.Database return await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } + /// + /// Represents a remote volume with its name, hash, and size. + /// private class RemoteVolume : IRemoteVolume { + /// + /// The name of the remote volume. + /// public string Name { get; private set; } + /// + /// The hash of the remote volume. + /// public string Hash { get; private set; } + /// + /// The size of the remote volume in bytes. + /// public long Size { get; private set; } + /// + /// Initializes a new instance of the class. + /// + /// The name of the remote volume. + /// The hash of the remote volume. + /// The size of the remote volume in bytes. public RemoteVolume(string name, string hash, long size) { this.Name = name; @@ -217,6 +247,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets a list of block volumes that are associated with a given index name. + /// + /// The name of the index volume. + /// An asynchronous enumerable of representing the block volumes. public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName) { using var cmd = m_connection.CreateCommand(@" @@ -247,31 +282,31 @@ namespace Duplicati.Library.Main.Database } /// - /// A single block with source data for repair + /// A single block with source data for repair. /// - /// The block hash - /// The block size - /// The file that contains the block - /// The offset of the block in the file + /// The block hash. + /// The block size. + /// The file that contains the block. + /// The offset of the block in the file. public sealed record BlockWithSourceData(string Hash, long Size, string File, long Offset); /// - /// A single block with metadata source data for repair + /// A single block with metadata source data for repair. /// - /// The block hash - /// The block size - /// The path of the file or directory that contains the block + /// The block hash. + /// The block size. + /// The path of the file or directory that contains the block. public sealed record BlockWithMetadataSourceData(string Hash, long Size, string Path); /// - /// A single blocklist hash entry + /// A single blocklist hash entry. /// - /// The blockset id - /// The hash of the blocklist entry (when done) - /// The total length of the blockset - /// The index of the blocklist hash - /// The index of the block in the blockset - /// The hash of the entry + /// The blockset id. + /// The hash of the blocklist entry (when done). + /// The total length of the blockset. + /// The index of the blocklist hash. + /// The index of the block in the blockset. + /// The hash of the entry. public sealed record BlocklistHashesEntry( long BlocksetId, string BlocklistHash, @@ -281,54 +316,54 @@ namespace Duplicati.Library.Main.Database string Hash); /// - /// Helper interface for the missing block list + /// Helper interface for the missing block list. /// - public interface IMissingBlockList : IDisposable + public interface IMissingBlockList : IDisposable, IAsyncDisposable { /// - /// Registers a block as restored + /// Registers a block as restored. /// - /// The block hash - /// The block size - /// The volume ID of the new target volume - /// The restored blocks + /// The block hash. + /// The block size. + /// The volume ID of the new target volume. + /// A task that when awaited returns true if the block was successfully marked as restored, false otherwise. Task SetBlockRestored(string hash, long size, long volumeId); /// - /// Gets the list of files that contains missing blocks + /// Gets the list of files that contains missing blocks. /// - /// The blocksize setting - /// A list of files with missing blocks + /// The blocksize setting. + /// An asynchronous enumerable of representing the files with missing blocks. IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize); /// - /// Gets a list for filesystem entries that contain missing blocks in metadata + /// Gets a list for filesystem entries that contain missing blocks in metadata. /// - /// A list of filesystem entries with missing blocks + /// An asynchronous enumerable of representing the metadata blocks. IAsyncEnumerable GetSourceItemsWithMetadataBlocks(); /// - /// Gets missing blocklist hashes + /// Gets missing blocklist hashes. /// - /// The number of hashes for each block - /// >A list of blocklist hashes + /// The number of hashes for each block. + /// An asynchronous enumerable of representing the blocklist hashes. IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock); /// - /// Gets the number of missing blocks + /// Gets the number of missing blocks. /// - /// >The number of missing blocks + /// A task that when awaited returns the count of missing blocks. Task GetMissingBlockCount(); /// - /// Gets all the filesets that are affected by missing blocks + /// Gets all the filesets that are affected by missing blocks. /// - /// >A list of filesets + /// An asynchronous enumerable of representing the filesets that contain missing blocks. IAsyncEnumerable GetFilesetsUsingMissingBlocks(); /// - /// Gets a list of remote files that may contain missing blocks + /// Gets a list of remote files that may contain missing blocks. /// - /// >A list of remote files + /// An asynchronous enumerable of representing the remote volumes that may contain missing blocks. IAsyncEnumerable GetMissingBlockSources(); } /// - /// Implementation of the missing block list + /// Implementation of the missing block list. /// private class MissingBlockList : IMissingBlockList { @@ -341,39 +376,46 @@ namespace Duplicati.Library.Main.Database /// private readonly ReusableTransaction m_rtr; /// - /// The insert command to use for restoring blocks + /// Updates the "Restored" status of a block in the temporary missing blocks table for a given hash and size. /// private SqliteCommand m_insertCommand = null!; /// - /// The command to copy blocks into the duplicate block table + /// Inserts or ignores a block and its volume assignment into the "DuplicateBlock" table for a given hash and size. /// private SqliteCommand m_copyIntoDuplicatedBlocks = null!; /// - /// The command to assign blocks to a new volume + /// Updates the "VolumeID" of a block in the "Block" table to assign it to a new volume for a given hash and size. /// private SqliteCommand m_assignBlocksToNewVolume = null!; - private SqliteCommand m_missingBlocksCommand = null!; - private SqliteCommand m_missingBlocksCountCommand = null!; /// - /// The name of the temporary table + /// Selects the hash and size of all missing blocks (where "Restored" is 0) from the temporary missing blocks table. + /// + private SqliteCommand m_missingBlocksCommand = null!; + /// + /// Counts the number of missing blocks (where "Restored" is 0) in the temporary missing blocks table. + /// + private SqliteCommand m_missingBlocksCountCommand = null!; + + /// + /// The name of the temporary table. /// private readonly string m_tablename; /// - /// The name of the volume where blocks are missing + /// The name of the volume where blocks are missing. /// private readonly string m_volumename; /// - /// Whether the object has been disposed + /// Whether the object has been disposed. /// private bool m_isDisposed = false; /// - /// Creates a new missing block list + /// Creates a new missing block list. /// - /// The name of the volume with missing blocks - /// The connection to the database - /// The transaction to use + /// The name of the volume with missing blocks. + /// The connection to the database. + /// The transaction to use. private MissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction rtr) { m_connection = connection; @@ -383,6 +425,13 @@ namespace Duplicati.Library.Main.Database m_tablename = tablename; } + /// + /// Creates a new instance of the missing block list. + /// + /// The name of the volume with missing blocks. + /// The connection to the database. + /// The transaction to use. + /// A task that when awaited returns a new instance of . public static async Task CreateMissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) { var blocklist = new MissingBlockList(volumename, connection, transaction); @@ -703,9 +752,9 @@ namespace Duplicati.Library.Main.Database } /// - /// Gets the list of missing blocks + /// Gets the list of missing blocks. /// - /// >A list of missing blocks + /// An asynchronous enumerable of tuples containing the block hash and size. public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks() { m_missingBlocksCommand @@ -777,6 +826,7 @@ namespace Duplicati.Library.Main.Database return updated; } + /// public async IAsyncEnumerable GetFilesetsUsingMissingBlocks() { var blocks = $@" @@ -842,6 +892,7 @@ namespace Duplicati.Library.Main.Database ); } + /// public async IAsyncEnumerable GetMissingBlockSources() { using var cmd = m_connection.CreateCommand($@" @@ -877,10 +928,10 @@ namespace Duplicati.Library.Main.Database if (m_isDisposed) return; - DisposeAsync().Await(); + DisposeAsync().AsTask().Await(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { if (m_isDisposed) return; @@ -906,12 +957,21 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Creates a new missing block list for the specified volume. + /// + /// The name of the volume with missing blocks. + /// A task that when awaited returns an instance of . public async Task CreateBlockList(string volumename) { return await MissingBlockList.CreateMissingBlockList(volumename, m_connection, m_rtr) .ConfigureAwait(false); } + /// + /// Fixes duplicate metadata hashes in the database. + /// + /// A task that when completed indicates that the repair has been attempted. public async Task FixDuplicateMetahash() { using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -1093,6 +1153,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Fixes duplicate file entries in the database. + /// + /// A task that when completed indicates that the repair has been attempted. public async Task FixDuplicateFileentries() { using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -1201,6 +1265,12 @@ namespace Duplicati.Library.Main.Database } + /// + /// Fixes missing blocklist hashes in the database. + /// + /// The hash algorithm used for the blocklist hashes. + /// The size of each block in bytes. + /// A task that when completed indicates that the repair has been attempted. public async Task FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize) { var blocklistbuffer = new byte[blocksize]; @@ -1458,6 +1528,12 @@ namespace Duplicati.Library.Main.Database } + /// + /// Fixes duplicate blocklist hashes in the database. + /// + /// The size of each block in bytes. + /// The size of each hash in bytes. + /// A task that when completed indicates that the repair has been attempted. public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize) { using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -1560,6 +1636,13 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Checks if all blocks in the specified volume are present in the database. + /// + /// The name of the volume to check. + /// A collection of blocks to check, represented as key-value pairs where the key is the block hash and the value is the block size. + /// Thrown if not all blocks are found in the specified volume. + /// A task that when awaited indicates the completion of the check. public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks) { using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -1629,6 +1712,16 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Checks if the provided blocklist matches the expected entries in the database for a given block hash and size. + /// + /// The hash of the blocklist to check. + /// The size of the blocklist in bytes. + /// The expected blocklist entries to compare against. + /// The size of each block in bytes. + /// The length of each block hash in bytes. + /// A task that when awaited indicates the completion of the check. + /// Thrown if the blocklist does not match the expected entries. public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength) { using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -1682,6 +1775,10 @@ namespace Duplicati.Library.Main.Database throw new Exception($"Too many source blocklist entries in {hash}"); } + /// + /// Checks if there are any missing local filesets in the database. + /// + /// An asynchronous enumerable of missing local fileset names. public async IAsyncEnumerable MissingLocalFilesets() { using var cmd = m_connection.CreateCommand(@" @@ -1706,6 +1803,10 @@ namespace Duplicati.Library.Main.Database yield return rd.ConvertValueToString(0) ?? ""; } + /// + /// Checks if there are any missing remote filesets in the database. + /// + /// An asynchronous enumerable of tuples containing the fileset ID, timestamp, and whether it is a full backup. public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets() { using var cmd = m_connection.CreateCommand(@" @@ -1737,6 +1838,10 @@ namespace Duplicati.Library.Main.Database ); } + /// + /// Checks if there are any empty index files in the database. + /// + /// An asynchronous enumerable of remote volumes that are empty index files. public async IAsyncEnumerable EmptyIndexFiles() { using var cmd = m_connection.CreateCommand(@" diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index af6ff4a1b..5420c6044 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -36,29 +36,70 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// Represents a local database used during restore. + /// Provides methods for preparing and tracking restore file lists, managing temporary tables for files and blocks, + /// tracking restore progress, and efficiently querying and updating restore-related metadata. + /// This class supports advanced restore scenarios, including filtering, progress tracking via triggers, + /// cross-platform path mapping, and efficient block and file restoration workflows. + /// internal class LocalRestoreDatabase : LocalDatabase { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRestoreDatabase)); + /// + /// A unique identifier for the temporary table set, used to ensure that temporary tables do not conflict with others. + /// protected readonly string m_temptabsetguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); /// /// The name of the temporary table in the database, which is used to store the list of files to restore. /// protected string? m_tempfiletable; + /// + /// The name of the temporary table in the database, which is used to store the blocks associated with the files to restore. + /// protected string? m_tempblocktable; + /// + /// A pool of connections and transactions that are used for concurrent access to the database. + /// protected ConcurrentBag<(SqliteConnection, ReusableTransaction)> m_connection_pool = []; + /// + /// The latest block table used to track the most recent blocks processed during a restore operation. + /// protected string? m_latestblocktable; + /// + /// The name of the temporary table used to track progress of file restoration. + /// protected string? m_fileprogtable; + /// + /// The name of the temporary table used to track overall restoration progress across all files. + /// protected string? m_totalprogtable; + /// + /// The name of the temporary table used to track files that have been fully restored (all blocks restored). + /// protected string? m_filesnewlydonetable; + /// + /// The time at which the restore operation begins. + /// protected DateTime m_restoreTime; + /// + /// Gets the time that the restore operation was initiated. + /// public DateTime RestoreTime { get { return m_restoreTime; } } + /// + /// Initializes a new instance of the class. + /// + /// The path to the database file. + /// The size of the page cache for the database. + /// An optional existing instance of to use, or null to create a new one. Used to mimic constructor chaining. + /// A task that when awaited on, returns a new instance of . public static async Task CreateAsync(string path, long pagecachesize, LocalRestoreDatabase? dbnew = null) { dbnew ??= new LocalRestoreDatabase(); @@ -71,6 +112,12 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Initializes a new instance of the class using an existing parent database. + /// + /// The parent database from which to create the restore database. + /// An optional existing instance of to use, or null to create a new one. Used to mimic constructor chaining. + /// A task that when awaited on, returns a new instance of . public static async Task CreateAsync(LocalDatabase dbparent, LocalRestoreDatabase? dbnew = null) { dbnew ??= new LocalRestoreDatabase(); @@ -85,8 +132,8 @@ namespace Duplicati.Library.Main.Database /// This replaces continuous requerying of block progress by iterating over blocks table. /// SQLite is much faster keeping information up to date with internal triggers. /// - /// This allows to create another table that keeps track - /// of all files that are done (all data blocks restored). + /// This allows to create another table that keeps track + /// of all files that are done (all data blocks restored). /// /// The method is prepared to create a table that keeps track of all files being done completely. /// That means, it fires for files where the number of restored blocks equals the number of all blocks. @@ -95,6 +142,7 @@ namespace Duplicati.Library.Main.Database /// Note: If a file is done once and then set back to a none restored state, the file is not automatically removed. /// But if it reaches a restored state later, it will be re-added (trigger will fire) /// + /// A task that completes when the progress tracker is created. public async Task CreateProgressTracker(bool createFilesNewlyDoneTracker) { m_fileprogtable = "FileProgress-" + m_temptabsetguid; @@ -344,6 +392,13 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Prepares the restore file list by querying the database for filesets and their associated files. + /// + /// The time at which the restore operation is being performed. + /// An array of version identifiers to filter the filesets. + /// An optional filter to apply to the files being restored. + /// A task that, when awaited, returns a tuple containing the count of files to restore and the total size of those files. public async Task> PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter) { m_tempfiletable = "Fileset-" + m_temptabsetguid; @@ -651,6 +706,10 @@ namespace Duplicati.Library.Main.Database return new Tuple(0, 0); } + /// + /// Retrieves the first path from the temporary file table. + /// + /// A task that, when awaited, returns the first path found in the temporary file table, or null if no paths are found. public async Task GetFirstPath() { using var cmd = m_connection.CreateCommand($@" @@ -668,6 +727,10 @@ namespace Duplicati.Library.Main.Database return v0.ToString(); } + /// + /// Retrieves the largest prefix path from the temporary file table. + /// + /// A task that, when awaited, returns the largest prefix path found in the temporary file table. public async Task GetLargestPrefix() { using var cmd = m_connection.CreateCommand($@" @@ -723,6 +786,13 @@ namespace Duplicati.Library.Main.Database return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); } + /// + /// Sets the target paths for files in the temporary file table. + /// This method adjusts the target paths based on the largest prefix and destination provided. + /// + /// The largest prefix path to use for adjusting target paths. + /// The destination path to prepend to the target paths. + /// A task that completes when the target paths have been set. public async Task SetTargetPaths(string largest_prefix, string destination) { var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? @@ -865,6 +935,11 @@ namespace Duplicati.Library.Main.Database await m_rtr.CommitAsync().ConfigureAwait(false); } + /// + /// Finds missing blocks in the temporary block table. + /// + /// If true, skips metadata blocks when searching for missing blocks. + /// A task that completes when the missing blocks have been found and inserted into the temporary block table. public async Task FindMissingBlocks(bool skipMetadata) { using var cmd = m_connection.CreateCommand() @@ -958,6 +1033,12 @@ namespace Duplicati.Library.Main.Database await m_rtr.CommitAsync().ConfigureAwait(false); } + /// + /// Updates the target path of a file in the temporary file table. + /// + /// The ID of the file in the temporary file table. + /// The new target path to set for the file. + /// A task that completes when the target path has been updated. public async Task UpdateTargetPath(long ID, string newname) { using var cmd = m_connection.CreateCommand($@" @@ -975,43 +1056,118 @@ namespace Duplicati.Library.Main.Database await m_rtr.CommitAsync().ConfigureAwait(false); } + /// + /// Interface for an object describing a block in an existing file. + /// public interface IExistingFileBlock { + /// + /// Gets the hash of the block. + /// string Hash { get; } + /// + /// Gets the index of the block within the file. + /// long Index { get; } + /// + /// Gets the size of the block in bytes. + /// long Size { get; } } + /// + /// Interface for an existing file with its blocks. + /// public interface IExistingFile { + /// + /// Gets the target path of the file. + /// string TargetPath { get; } + /// + /// Gets the hash of the file. + /// string TargetHash { get; } + /// + /// Gets the ID of the file in the database. + /// long TargetFileID { get; } + /// + /// Gets the length of the file in bytes. + /// long Length { get; } + /// + /// An asynchronous enumerable of the blocks in the file. + /// IAsyncEnumerable Blocks(); } + /// + /// Interface for a block source, which can be a file or a metadata source. + /// public interface IBlockSource { + /// + /// Gets the path of the block source. + /// string Path { get; } + /// + /// Gets the offset of the block within the source. + /// long Offset { get; } + /// + /// Indicates whether the block source is metadata. + /// bool IsMetadata { get; } } + /// + /// Interface for a block descriptor, which describes a block in a file. + /// public interface IBlockDescriptor { + /// + /// Gets the hash of the block. + /// string Hash { get; } + /// + /// Gets the size of the block in bytes. + /// long Size { get; } + /// + /// Gets the offset of the block within the file. + /// long Offset { get; } + /// + /// Gets the index of the block within the file. + /// long Index { get; } + /// + /// Indicates whether the block is metadata. + /// bool IsMetadata { get; } + /// + /// An asynchronous enumerable of block sources for this block descriptor. + /// IAsyncEnumerable BlockSources(); } + /// + /// Interface for a local block source, which provides information about a file and its blocks. + /// public interface ILocalBlockSource { + /// + /// Gets the target path of the file. + /// string TargetPath { get; } + /// + /// Gets the ID of the target file in the database. + /// long TargetFileID { get; } + /// + /// An asynchronous enumerable of block descriptors for the file. + /// IAsyncEnumerable Blocks(); } @@ -1020,49 +1176,115 @@ namespace Duplicati.Library.Main.Database /// public interface IFileToRestore { + /// + /// Gets the target path of the file. + /// string Path { get; } + /// + /// Gets the hash of the file. + /// string Hash { get; } + /// + /// Gets the ID of the file in the database. + /// long Length { get; } } + /// + /// Interface for a remote volume, which contains blocks to be restored. + /// public interface IPatchBlock { + /// + /// Gets the hash of the block. + /// long Offset { get; } + /// + /// Gets the size of the block in bytes. + /// long Size { get; } + /// + /// Gets the path of the block source. + /// string Key { get; } } + /// + /// Interface for a remote volume that contains blocks to be restored. + /// public interface IVolumePatch { + /// + /// Gets the name of the remote volume. + /// string Path { get; } + /// + /// Gets the hash of the remote volume. + /// long FileID { get; } + /// + /// Gets the size of the remote volume in bytes. + /// IAsyncEnumerable Blocks(); } + /// + /// Implementation of the IExistingFile interface that retrieves existing files and their blocks from the database. + /// private class ExistingFile : IExistingFile { + /// + /// The data reader used to read existing files from the database. + /// private readonly SqliteDataReader m_reader; - public ExistingFile(SqliteDataReader rd) { m_reader = rd; HasMore = true; } + /// + /// Initializes a new instance of the ExistingFile class. + /// + /// The SqliteDataReader used to read existing files. + public ExistingFile(SqliteDataReader rd) + { + m_reader = rd; + HasMore = true; + } public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } public string TargetHash { get { return m_reader.ConvertValueToString(1) ?? ""; } } public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } } public long Length { get { return m_reader.ConvertValueToInt64(3); } } + /// + /// Indicates whether there are more blocks to read for the current file. + /// public bool HasMore { get; private set; } + /// + /// Represents a block in an existing file. + /// private class ExistingFileBlock : IExistingFileBlock { + /// + /// The data reader used to read block information for existing files. + /// private readonly SqliteDataReader m_reader; - public ExistingFileBlock(SqliteDataReader rd) { m_reader = rd; } + /// + /// Initializes a new instance of the ExistingFileBlock class. + /// + public ExistingFileBlock(SqliteDataReader rd) + { + m_reader = rd; + } public string Hash { get { return m_reader.ConvertValueToString(4) ?? ""; } } public long Index { get { return m_reader.ConvertValueToInt64(5); } } public long Size { get { return m_reader.ConvertValueToInt64(6); } } } + /// + /// Gets the blocks in the existing file. + /// + /// An asynchronous enumerable of IExistingFileBlock objects representing the blocks in the file. public async IAsyncEnumerable Blocks() { string p = TargetPath; @@ -1073,6 +1295,12 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves existing files with their blocks from the database. + /// + /// The LocalDatabase instance to query. + /// The name of the table containing existing files. + /// An asynchronous enumerable of IExistingFile objects representing the existing files with their blocks. public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename) { using var cmd = db.Connection.CreateCommand($@" @@ -1116,6 +1344,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves existing files with their blocks from the temporary file table. + /// + /// An asynchronous enumerable of IExistingFile objects representing the existing files with their blocks. public IAsyncEnumerable GetExistingFilesWithBlocks() { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) @@ -1123,23 +1355,58 @@ namespace Duplicati.Library.Main.Database return ExistingFile.GetExistingFilesWithBlocks(this, m_tempfiletable); } + /// + /// Implements the ILocalBlockSource interface to provide access to local block sources. + /// private class LocalBlockSource : ILocalBlockSource { + /// + /// Implements the IBlockDescriptor interface to provide access to block descriptors. + /// private class BlockDescriptor : IBlockDescriptor { + /// + /// Implements the IBlockSource interface to provide access to block sources. + /// private class BlockSource : IBlockSource { + /// + /// The data reader used to read block source information. + /// private readonly SqliteDataReader m_reader; - public BlockSource(SqliteDataReader rd) { m_reader = rd; } + + /// + /// Initializes a new instance of the class. + /// + /// The SqliteDataReader used to read block source information. + public BlockSource(SqliteDataReader rd) + { + m_reader = rd; + } public string Path { get { return m_reader.ConvertValueToString(6) ?? ""; } } public long Offset { get { return m_reader.ConvertValueToInt64(7); } } public bool IsMetadata { get { return false; } } } + /// + /// The data reader used to read block descriptor information. + /// private readonly SqliteDataReader m_reader; - public BlockDescriptor(SqliteDataReader rd) { m_reader = rd; HasMore = true; } + /// + /// Initializes a new instance of the class. + /// + /// The SqliteDataReader used to read block descriptor information. + public BlockDescriptor(SqliteDataReader rd) + { + m_reader = rd; + HasMore = true; + } + + /// + /// Gets the target path of the block source. + /// private string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } public string Hash { get { return m_reader.ConvertValueToString(2) ?? ""; } } @@ -1148,6 +1415,9 @@ namespace Duplicati.Library.Main.Database public long Size { get { return m_reader.ConvertValueToInt64(5); } } public bool IsMetadata { get { return !(m_reader.ConvertValueToInt64(9) == 0); } } + /// + /// Indicates whether there are more block sources to read for the current block descriptor. + /// public bool HasMore { get; private set; } public async IAsyncEnumerable BlockSources() @@ -1165,12 +1435,27 @@ namespace Duplicati.Library.Main.Database } + /// + /// The data reader used to read local block source information. + /// private readonly SqliteDataReader m_reader; - public LocalBlockSource(SqliteDataReader rd) { m_reader = rd; HasMore = true; } + + /// + /// Initializes a new instance of the class. + /// + /// The SqliteDataReader used to read local block source information. + public LocalBlockSource(SqliteDataReader rd) + { + m_reader = rd; + HasMore = true; + } public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } public long TargetFileID { get { return m_reader.ConvertValueToInt64(1); } } + /// + /// Indicates whether there are more blocks to read for the current file. + /// public bool HasMore { get; private set; } public async IAsyncEnumerable Blocks() @@ -1190,6 +1475,15 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves files and their source blocks from the database. + /// + /// The LocalDatabase instance to query. + /// The name of the file table to query. + /// The name of the block table to query. + /// The size of each block in bytes. + /// If true, skips metadata blocks when retrieving files and source blocks. + /// An asynchronous enumerable of ILocalBlockSource objects representing the files and their source blocks. public static async IAsyncEnumerable GetFilesAndSourceBlocks(LocalDatabase db, string filetablename, string blocktablename, long blocksize, bool skipMetadata) { // TODO: Skip metadata as required @@ -1240,6 +1534,12 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves files and their source blocks from the temporary file and block tables. + /// + /// If true, skips metadata blocks when retrieving files and source blocks. + /// The size of each block in bytes. + /// An asynchronous enumerable of ILocalBlockSource objects representing the files and their source blocks. public IAsyncEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize) { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) @@ -1247,6 +1547,10 @@ namespace Duplicati.Library.Main.Database return LocalBlockSource.GetFilesAndSourceBlocks(this, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata); } + /// + /// Retrieves the missing remote volumes that need to be restored. + /// + /// An asynchronous enumerable of IRemoteVolume objects representing the missing remote volumes. public async IAsyncEnumerable GetMissingVolumes() { using var cmd = m_connection.CreateCommand($@" @@ -1290,8 +1594,13 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Interface for retrieving files and metadata with missing blocks. + /// public interface IFilesAndMetadata : IDisposable, IAsyncDisposable { + /// + /// Retrieves files with missing blocks. IAsyncEnumerable FilesWithMissingBlocks(); IAsyncEnumerable MetadataWithMissingBlocks(); } @@ -1374,20 +1683,45 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Implements the IVolumePatch interface to provide access to files with missing blocks. + /// private class VolumePatch : IVolumePatch { + /// + /// Implements the IPatchBlock interface to provide access to blocks in a volume. + /// private class PatchBlock : IPatchBlock { + /// + /// The data reader used to read patch block information. + /// private readonly SqliteDataReader m_reader; - public PatchBlock(SqliteDataReader rd) { m_reader = rd; } + /// + /// Initializes a new instance of the class. + /// + public PatchBlock(SqliteDataReader rd) + { + m_reader = rd; + } public long Offset { get { return m_reader.ConvertValueToInt64(2); } } public long Size { get { return m_reader.ConvertValueToInt64(3); } } public string Key { get { return m_reader.ConvertValueToString(4) ?? ""; } } } + /// + /// The data reader used to read volume patch information. + /// private readonly SqliteDataReader m_reader; - public VolumePatch(SqliteDataReader rd) { m_reader = rd; HasMore = true; } + /// + /// Initializes a new instance of the class. + /// + public VolumePatch(SqliteDataReader rd) + { + m_reader = rd; + HasMore = true; + } public string Path { get { return m_reader.ConvertValueToString(0) ?? ""; } } public long FileID { get { return m_reader.ConvertValueToInt64(1); } } @@ -1404,6 +1738,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves the files with missing blocks. + /// + /// An asynchronous enumerable of IVolumePatch objects representing the files with missing blocks. public async IAsyncEnumerable FilesWithMissingBlocks() { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! @@ -1453,6 +1791,10 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves the metadata with missing blocks. + /// + /// An asynchronous enumerable of IVolumePatch objects representing the metadata with missing blocks. public async IAsyncEnumerable MetadataWithMissingBlocks() { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! @@ -1503,6 +1845,12 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves files and metadata with missing blocks for the given volume. + /// + /// The size of each block in bytes. + /// The current volume reader containing the blocks to restore. + /// A task that when awaited returns an IFilesAndMetadata instance containing the files and metadata with missing blocks. public async Task GetMissingBlockData(BlockVolumeReader curvolume, long blocksize) { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) @@ -1512,9 +1860,9 @@ namespace Duplicati.Library.Main.Database } /// - /// Returns a connection from the connection pool. + /// Returns a connection and transaction from the connection pool. /// - /// A connection from the connection pool. + /// A task that when awaited returns a tuple containing the SqliteConnection and ReusableTransaction. public async Task<(SqliteConnection, ReusableTransaction)> GetConnectionFromPool() { if (!m_connection_pool.TryTake(out var entry)) @@ -1535,12 +1883,22 @@ namespace Duplicati.Library.Main.Database return entry; } + /// + /// Represents a file to restore, containing its path, hash, and length. + /// private class FileToRestore : IFileToRestore { public string Path { get; private set; } public string Hash { get; private set; } public long Length { get; private set; } + /// + /// Initializes a new instance of the class. + /// + /// The ID of the file to restore. + /// The path of the file to restore. + /// The hash of the file to restore. + /// The length of the file to restore in bytes. public FileToRestore(long id, string path, string hash, long length) { Path = path; @@ -1549,6 +1907,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Returns a list of files to restore, including their target paths, hashes, and lengths. + /// + /// If true, only returns files that have not been verified. + /// An asynchronous enumerable of IFileToRestore objects representing the files to restore. public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified) { using var cmd = m_connection.CreateCommand($@" @@ -1576,7 +1939,7 @@ namespace Duplicati.Library.Main.Database /// /// Returns a list of files and symlinks to restore. /// - /// A list of files and symlinks to restore. + /// An asynchronous enumerable of FileRequest objects representing the files and symlinks to restore. public async IAsyncEnumerable GetFilesAndSymlinksToRestore() { // Order by length descending, so that larger files are restored first. @@ -1610,7 +1973,7 @@ namespace Duplicati.Library.Main.Database /// /// Returns a list of folders to restore. Used to restore folder metadata. /// - /// A list of folders to restore. + /// An asynchronous enumerable of FileRequest objects representing the folders to restore. public async IAsyncEnumerable GetFolderMetadataToRestore() { using var cmd = m_connection.CreateCommand(); @@ -1646,7 +2009,7 @@ namespace Duplicati.Library.Main.Database /// Returns a list of blocks and their volume IDs. Used by the to keep track of blocks and volumes to automatically evict them from the respective caches. /// /// Flag indicating whether the returned blocks should exclude the metadata blocks. - /// A list of tuples containing the block ID and the volume ID of the block. + /// An asynchronous enumerable of tuples containing the block ID and volume ID. public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) { var metadata_query = skipMetadata ? "" : $@" @@ -1688,7 +2051,7 @@ namespace Duplicati.Library.Main.Database /// Returns a list of for the given blockset ID. It is used by the to restore the blocks of a file. /// /// The BlocksetID of the file. - /// A list of needed to restore the given file. + /// An asynchronous enumerable of representing the blocks of the file. public async IAsyncEnumerable GetBlocksFromFile(long blocksetID) { var (connection, transaction) = await GetConnectionFromPool() @@ -1735,7 +2098,7 @@ namespace Duplicati.Library.Main.Database /// Returns a list of for the metadata blocks of the given file. It is used by the to restore the metadata of a file. /// /// The ID of the file. - /// A list of needed to restore the metadata of the given file. + /// An asynchronous enumerable of representing the metadata blocks of the file. public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID) { var (connection, transaction) = await GetConnectionFromPool() @@ -1785,7 +2148,7 @@ namespace Duplicati.Library.Main.Database /// Returns the volume information for the given volume ID. It is used by the to get the volume information for the given volume ID. /// /// The ID of the volume. - /// A tuple containing the name, size, and hash of the volume. + /// An asynchronous enumerable of tuples containing the volume name, size, and hash. public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) { using var cmd = m_connection.CreateCommand(@" @@ -1809,6 +2172,10 @@ namespace Duplicati.Library.Main.Database ); } + /// + /// Drops all temporary tables used for the restore process and commits the transaction. + /// + /// A task that completes when the temporary tables are dropped and the transaction is committed. public async Task DropRestoreTable() { using var cmd = m_connection.CreateCommand() @@ -1871,13 +2238,55 @@ namespace Duplicati.Library.Main.Database await m_rtr.CommitAsync().ConfigureAwait(false); } + /// + /// Interface for marking blocks as restored, missing, or verified. + /// public interface IBlockMarker : IDisposable, IAsyncDisposable { + /// + /// Marks a specific block as restored. + /// + /// The ID of the target file. + /// The index of the block. + /// The hash of the block. + /// The size of the block. + /// Indicates whether the block is metadata. + /// A task that completes when the block is marked as restored. Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata); + + /// + /// Marks all blocks of a specific file as missing. + /// + /// The ID of the target file. + /// A task that completes when all blocks of the file are marked as missing. Task SetAllBlocksMissing(long targetfileid); + + /// + /// Marks all blocks of a specific file as restored. + /// + /// The ID of the target file. + /// Indicates whether to include metadata blocks. + /// A task that completes when all blocks of the file are marked as restored. Task SetAllBlocksRestored(long targetfileid, bool includeMetadata); + + /// + /// Marks the data of a specific file as verified. + /// + /// The ID of the target file. + /// A task that completes when the file data is marked as verified. Task SetFileDataVerified(long targetfileid); + + /// + /// Commits the changes made by the block marker. + /// + /// A task that completes when the changes are committed. Task CommitAsync(); + + /// + /// Updates the processed statistics for the blocks and files. + /// + /// An operation progress updater to report the processed statistics. + /// A task that completes when the processed statistics are updated. Task UpdateProcessed(IOperationProgressUpdater writer); } @@ -1888,25 +2297,72 @@ namespace Duplicati.Library.Main.Database /// private class DirectBlockMarker : IBlockMarker { + /// + /// Updates the "Restored" status to 1 for a specific block in the block table, matching by file ID, index, hash, size, and metadata, but only if it is currently not restored. + /// private SqliteCommand m_insertblockCommand = null!; + + /// + /// Resets the "Restored" status to 0 for all blocks belonging to a specific file in the block table. + /// private SqliteCommand m_resetfileCommand = null!; + + /// + /// Updates the "Restored" status to 1 for all blocks of a file in the block table, optionally including metadata blocks depending on the parameter. + /// private SqliteCommand m_updateAsRestoredCommand = null!; + + /// + /// Marks a file as data verified by setting the "DataVerified" field to 1 in the file table for a specific file ID. + /// private SqliteCommand m_updateFileAsDataVerifiedCommand = null!; + + /// + /// Retrieves the number of fully restored files and the total restored size from the progress/statistics table, or, if unavailable, counts restored blocks and their size directly from the block table. + /// private SqliteCommand m_statUpdateCommand = null!; + + /// + /// The database instance used for executing commands. + /// private LocalDatabase m_db = null!; + + /// + /// Indicates whether there are updates to be processed. + /// private bool m_hasUpdates = false; + /// + /// The name of the temporary block table used for marking blocks as restored. + /// private string m_blocktablename = null!; + /// + /// The name of the temporary file table used for marking files as data verified. + /// private string m_filetablename = null!; + /// + /// Calling this constructor will throw an exception. Use CreateAsync instead. + /// [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public DirectBlockMarker(SqliteConnection connection, string blocktablename, string filetablename, string statstablename) { throw new NotImplementedException("Use CreateAsync instead of the constructor"); } + /// + /// Private constructor to prevent direct instantiation. Use CreateAsync to create an instance. + /// private DirectBlockMarker() { } + /// + /// Creates a new instance of the class asynchronously. + /// + /// The local database instance to use for executing commands. + /// The name of the temporary block table. + /// The name of the temporary file table. + /// The name of the statistics table, or null if not available. + /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalDatabase db, string blocktablename, string filetablename, string statstablename) { var dbm = new DirectBlockMarker() @@ -2080,12 +2536,10 @@ namespace Duplicati.Library.Main.Database } } - [Obsolete("Calling this constructor will throw an exception. Use CreateBlockMarkerAsync instead.")] - public IBlockMarker CreateBlockMarker() - { - throw new NotImplementedException("Use CreateBlockMarkerAsync instead of the constructor"); - } - + /// + /// Creates a new block marker for marking blocks as restored, missing, or verified. + /// + /// A task that when awaited returns an instance of . public async Task CreateBlockMarkerAsync() { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) @@ -2108,6 +2562,10 @@ namespace Duplicati.Library.Main.Database await base.DisposeAsync().ConfigureAwait(false); } + /// + /// Disposes all connections and their transactions in the connection pool. + /// + /// A task that completes when all connections and transactions are disposed. public async Task DisposePoolAsync() { foreach (var (connection, transaction) in m_connection_pool) @@ -2119,6 +2577,10 @@ namespace Duplicati.Library.Main.Database m_connection_pool.Clear(); } + /// + /// Returns a list of target folders for the folder blockset. + /// + /// An asynchronous enumerable of strings representing the target folders. public async IAsyncEnumerable GetTargetFolders() { using var cmd = m_connection.CreateCommand($@" @@ -2134,39 +2596,114 @@ namespace Duplicati.Library.Main.Database yield return rd.ConvertValueToString(0) ?? ""; } + /// + /// Returns a list of files and their source blocks for fast restoration. + /// public interface IFastSource { + /// + /// The target path of the file to restore. + /// string TargetPath { get; } + /// + /// The ID of the target file. + /// long TargetFileID { get; } + /// + /// The source path of the file to restore. + /// string SourcePath { get; } + /// + /// Gets an asynchronous enumerable of block entries for the file. + /// IAsyncEnumerable Blocks(); } + /// + /// Interface representing a block entry in the fast source. + /// public interface IBlockEntry { + /// + /// The offset of the block in the source file. + /// long Offset { get; } + /// + /// The size of the block. + /// long Size { get; } + /// + /// The index of the block in the source file. + /// long Index { get; } + /// + /// The hash of the block. + /// string Hash { get; } } + /// + /// Implementation of that provides access to file and block information for fast restoration. + /// private class FastSource : IFastSource { + /// + /// Implementation of that provides access to block information. + /// private class BlockEntry : IBlockEntry { + /// + /// The SqliteDataReader used to read block data. + /// private readonly SqliteDataReader m_rd; + /// + /// The block size used to calculate the offset. + /// private readonly long m_blocksize; - public BlockEntry(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } + + /// + /// Initializes a new instance of the class with the specified SqliteDataReader and block size. + /// + /// The SqliteDataReader used to read block data. + /// The block size used to calculate the offset. + public BlockEntry(SqliteDataReader rd, long blocksize) + { + m_rd = rd; + m_blocksize = blocksize; + } + public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } } public long Index { get { return m_rd.ConvertValueToInt64(3); } } public long Size { get { return m_rd.ConvertValueToInt64(5); } } public string Hash { get { return m_rd.ConvertValueToString(4) ?? ""; } } } + /// + /// The SqliteDataReader used to read file and block data. + /// private readonly SqliteDataReader m_rd; + /// + /// The block size used to calculate the offset of blocks. + /// private readonly long m_blocksize; - public FastSource(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } + + /// + /// Initializes a new instance of the class with the specified SqliteDataReader and block size. + /// + /// The SqliteDataReader used to read file and block data. + /// The block size used to calculate the offset of blocks. + public FastSource(SqliteDataReader rd, long blocksize) + { + m_rd = rd; + m_blocksize = blocksize; + MoreData = true; + } + + /// + /// Indicates whether there is more data to read from the SqliteDataReader. + /// public bool MoreData { get; private set; } + public string TargetPath { get { return m_rd.ConvertValueToString(0) ?? ""; } } public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } public string SourcePath { get { return m_rd.ConvertValueToString(1) ?? ""; } } @@ -2182,6 +2719,11 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Returns a list of files and their source blocks for fast restoration. + /// + /// The size of the blocks to be used for restoration. + /// An asynchronous enumerable of representing the files and their source blocks. public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize) { using (var cmdReader = m_connection.CreateCommand()) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index bf3c7c2ee..9423aff23 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -30,8 +30,20 @@ using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Database { + /// + /// Represents a local database used for testing and verification. + /// Provides methods for creating test databases, updating verification counts, and comparing file, index, and block lists + /// to support integrity checks and remote volume verification during backup testing. + /// internal class LocalTestDatabase : LocalDatabase { + /// + /// Initializes a new instance of the class. + /// + /// The path to the database file. + /// The size of the page cache in bytes. + /// An optional existing instance to use. Used to mimic constructor chaining. + /// A task that represents the asynchronous operation. The task result contains the created instance. public static async Task CreateAsync(string path, long pagecachesize, LocalTestDatabase? dbnew = null) { dbnew ??= new LocalTestDatabase(); @@ -45,6 +57,12 @@ namespace Duplicati.Library.Main.Database return dbnew; } + /// + /// Creates a new instance of the class using an existing parent database. + /// + /// The parent database to use for creating the new test database. + /// An optional existing instance to use. Used to mimic constructor chaining. + /// A task that represents the asynchronous operation. The task result contains the created instance. public static async Task CreateAsync(LocalDatabase dbparent, LocalTestDatabase? dbnew = null) { dbnew ??= new LocalTestDatabase(); @@ -54,6 +72,12 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// Updates the verification count for a remote volume with the specified name. + /// Increments the count by 1, or sets it to the maximum of 1 if it was previously 0 or negative. + /// + /// The name of the remote volume to update. + /// A task that represents the asynchronous operation. public async Task UpdateVerificationCount(string name) { using (var cmd = m_connection.CreateCommand(m_rtr)) @@ -76,14 +100,27 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); } + /// + /// A record representing a remote volume, which implements the interface. + /// private record RemoteVolume : IRemoteVolume { + /// + /// Gets the ID of the remote volume. + /// public long ID { get; init; } public string Name { get; init; } public long Size { get; init; } public string Hash { get; init; } + /// + /// Gets the verification count of the remote volume, indicating how many times it has been verified. + /// public long VerificationCount { get; init; } + /// + /// Initializes a new instance of the record from a SqliteDataReader. + /// + /// The SqliteDataReader containing the data for the remote volume. public RemoteVolume(SqliteDataReader rd) { ID = rd.ConvertValueToInt64(0); @@ -94,6 +131,14 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Filters a list of remote volumes based on their verification count. + /// The method selects volumes that have not been verified, those with a low verification count, and finally those with a high verification count, ensuring a balanced selection. + /// + /// The collection of remote volumes to filter. + /// The number of samples to select. + /// The maximum verification count to consider for filtering. + /// A list of remote volumes filtered by verification count. private static List FilterByVerificationCount(IEnumerable volumes, long samples, long maxverification) { var rnd = new Random(); @@ -146,6 +191,13 @@ namespace Duplicati.Library.Main.Database return res; } + /// + /// Asynchronously selects a set of remote volumes to be tested for integrity, based on the specified sample count and selection options. + /// This method retrieves candidate remote volumes from the database, prioritizes them according to verification count and state, and yields the selected targets for verification. + /// + /// The number of remote volumes to select for testing. + /// The options that define selection criteria, such as time, version, and verification strategy. + /// An asynchronous enumerable of representing the selected test targets. public async IAsyncEnumerable SelectTestTargets(long samples, Options options) { var tp = await GetFilelistWhereClause(options.Time, options.Version) @@ -294,21 +346,53 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Base class for basic lists used in the local test database. + /// Provides methods for creating temporary tables, inserting data, and disposing of resources. + /// private abstract class Basiclist : IDisposable, IAsyncDisposable { + /// + /// The database connection used for executing commands. + /// protected LocalDatabase m_db = null!; + /// + /// The name of the volume associated with this list. + /// protected string m_volumename = null!; + /// + /// The name of the temporary table used for this list. + /// protected string m_tablename = null!; + /// + /// Command used for inserting data into the temporary table. + /// protected SqliteCommand m_insertCommand = null!; + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] protected Basiclist(SqliteConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) { throw new NotSupportedException("Use CreateAsync method instead."); } + /// + /// Protected constructor to allow derived classes to initialize without parameters. + /// protected Basiclist() { } + /// + /// Creates a new instance of the class asynchronously. + /// + /// The instance of the to initialize. + /// The local database to use for the list. + /// The name of the volume associated with this list. + /// The prefix for the temporary table name. + /// The SQL format for creating the temporary table. + /// The SQL command for inserting data into the temporary table. + /// A task that represents the asynchronous operation. The task result contains the initialized instance. protected static async Task CreateAsync(Basiclist bl, LocalDatabase db, string volumename, string tablePrefix, string tableFormat, string insertCommand) { bl.m_db = db; @@ -356,15 +440,46 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Interface for a file list used in the local test database. + /// Provides methods for adding entries, comparing the list with remote volumes, and disposing of resources. + /// public interface IFilelist : IDisposable, IAsyncDisposable { + /// + /// Asynchronously adds a file entry to the file list. + /// + /// The path of the file. + /// The size of the file in bytes. + /// The hash of the file, or null if not applicable. + /// The size of the metadata associated with the file. + /// The hash of the metadata associated with the file. + /// A collection of blocklist hashes associated with the file. + /// The type of the file entry. + /// The timestamp of the file entry. + /// A task that completes when the file entry has been added. Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time); + + /// + /// Asynchronously compares the file list with remote volumes and yields differences. + /// + /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path. IAsyncEnumerable> Compare(); } + /// + /// Implementation of the interface that manages a list of files in a local test database. + /// private class Filelist : Basiclist, IFilelist { + /// + /// The prefix for the temporary table name used for the file list. + /// private const string TABLE_PREFIX = "Filelist"; + + /// + /// The SQL format for creating the temporary table used for the file list. + /// private const string TABLE_FORMAT = @" ( ""Path"" TEXT NOT NULL, @@ -374,6 +489,10 @@ namespace Duplicati.Library.Main.Database ""Metahash"" TEXT NOT NULL ) "; + + /// + /// The SQL command for inserting data into the temporary table used for the file list. + /// private const string INSERT_COMMAND = @" ( ""Path"", @@ -391,6 +510,9 @@ namespace Duplicati.Library.Main.Database ) "; + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Filelist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) @@ -398,8 +520,17 @@ namespace Duplicati.Library.Main.Database throw new NotSupportedException("Use CreateAsync method instead."); } + /// + /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. + /// private Filelist() { } + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database to use for the file list. + /// The name of the volume associated with this file list. + /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Filelist(); @@ -538,15 +669,41 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Interface for an index list used in the local test database. + /// Provides methods for adding block links, comparing the index list with remote volumes, and disposing of resources. + /// public interface IIndexlist : IDisposable, IAsyncDisposable { + /// + /// Asynchronously adds a block link to the index list. + /// + /// The name of the file associated with the block link. + /// The hash of the block link. + /// The length of the block link in bytes. + /// A task that completes when the block link has been added. Task AddBlockLink(string filename, string hash, long length); + + /// + /// Asynchronously compares the index list with remote volumes and yields differences. + /// + /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path. IAsyncEnumerable> Compare(); } + /// + /// Implementation of the interface that manages a list of index entries in a local test database. + /// private class Indexlist : Basiclist, IIndexlist { + /// + /// The prefix for the temporary table name used for the index list. + /// private const string TABLE_PREFIX = "Indexlist"; + + /// + /// The SQL format for creating the temporary table used for the index list. + /// private const string TABLE_FORMAT = @" ( ""Name"" TEXT NOT NULL, @@ -555,6 +712,9 @@ namespace Duplicati.Library.Main.Database ) "; + /// + /// The SQL command for inserting data into the temporary table used for the index list. + /// private const string INSERT_COMMAND = @" ( ""Name"", @@ -568,6 +728,9 @@ namespace Duplicati.Library.Main.Database ) "; + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Indexlist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) @@ -575,8 +738,17 @@ namespace Duplicati.Library.Main.Database throw new NotSupportedException("Use CreateAsync method instead."); } + /// + /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. + /// private Indexlist() { } + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database to use for the index list. + /// The name of the volume associated with this index list. + /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Indexlist(); @@ -689,25 +861,67 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Interface for a blocklist used in the local test database. + /// Provides methods for adding blocks, comparing the blocklist with remote volumes, and disposing of resources. + /// public interface IBlocklist : IDisposable, IAsyncDisposable { + /// + /// Asynchronously adds a block to the blocklist. + /// + /// The key (hash) of the block. + /// The size of the block in bytes. + /// A task that completes when the block has been added. Task AddBlock(string key, long value); + + /// + /// Asynchronously compares the blocklist with remote volumes and yields differences. + /// + /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash. IAsyncEnumerable> Compare(); } public interface IBlocklistHashList : IDisposable, IAsyncDisposable { + /// + /// Asynchronously adds a block hash to the blocklist hash list. + /// + /// The hash of the block. + /// The size of the block in bytes. + /// A task that completes when the block hash has been added. Task AddBlockHash(string hash, long size); + + /// + /// Asynchronously compares the blocklist hash list with remote volumes and yields differences. + /// + /// The number of hashes per block. + /// The size of each hash in bytes. + /// The size of each block in bytes. + /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash. IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize); } + /// + /// Implementation of the interface that manages a list of blocks in a local test database. + /// Provides methods for adding blocks, comparing the blocklist with remote volumes, and disposing of resources. + /// private class Blocklist : Basiclist, IBlocklist { + /// + /// The prefix for the temporary table name used for the blocklist. + /// private const string TABLE_PREFIX = "Blocklist"; + /// + /// The SQL format for creating the temporary table used for the blocklist. + /// private const string TABLE_FORMAT = @"( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )"; + /// + /// The SQL command for inserting data into the temporary table used for the blocklist. + /// private const string INSERT_COMMAND = @"( ""Hash"", ""Size"" @@ -716,14 +930,26 @@ namespace Duplicati.Library.Main.Database @Hash, @Size )"; + + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Blocklist(SqliteConnection connection, string volumename, ReusableTransaction rtr) { throw new NotSupportedException("Use CreateAsync method instead."); } + /// + /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. + /// private Blocklist() { } + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database to use for the blocklist. + /// The name of the volume associated with this blocklist. public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Blocklist(); @@ -873,14 +1099,28 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Implementation of the interface that manages a list of block hashes in a local test database. + /// Provides methods for adding block hashes, comparing the blocklist hash list with remote volumes, and disposing of resources. + /// private class BlocklistHashList : Basiclist, IBlocklistHashList { + /// + /// The prefix for the temporary table name used for the blocklist hash list. + /// private const string TABLE_PREFIX = "BlocklistHashList"; + + /// + /// The SQL format for creating the temporary table used for the blocklist hash list. + /// private const string TABLE_FORMAT = @"( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )"; + /// + /// The SQL command for inserting data into the temporary table used for the blocklist hash list. + /// private const string INSERT_COMMAND = @"( ""Hash"", ""Size"" @@ -889,14 +1129,27 @@ namespace Duplicati.Library.Main.Database @Hash, @Size )"; + + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public BlocklistHashList(SqliteConnection connection, string volumename, ReusableTransaction rtr) { throw new NotSupportedException("Use CreateAsync method instead."); } + /// + /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. + /// private BlocklistHashList() { } + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database to use for the blocklist hash list. + /// The name of the volume associated with this blocklist hash list. + /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new BlocklistHashList(); @@ -1053,21 +1306,41 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Creates a new filelist in the local test database. + /// + /// The name of the filelist to create. + /// A task that when awaited returns a new instance of the interface. public async Task CreateFilelist(string name) { return await Filelist.CreateAsync(this, name).ConfigureAwait(false); } + /// + /// Creates a new indexlist in the local test database. + /// + /// The name of the indexlist to create. + /// A task that when awaited returns a new instance of the interface. public async Task CreateIndexlist(string name) { return await Indexlist.CreateAsync(this, name).ConfigureAwait(false); } + /// + /// Creates a new blocklist in the local test database. + /// + /// The name of the blocklist to create. + /// A task that when awaited returns a new instance of the interface. public async Task CreateBlocklist(string name) { return await Blocklist.CreateAsync(this, name).ConfigureAwait(false); } + /// + /// Creates a new blocklist hash list in the local test database. + /// + /// The name of the blocklist hash list to create. + /// A task that when awaited returns a new instance of the interface. public async Task CreateBlocklistHashList(string name) { return await BlocklistHashList.CreateAsync(this, name) diff --git a/Duplicati/Library/Main/Database/RemoteVolume.cs b/Duplicati/Library/Main/Database/RemoteVolume.cs index 92f3bc77b..4d5f5940c 100644 --- a/Duplicati/Library/Main/Database/RemoteVolume.cs +++ b/Duplicati/Library/Main/Database/RemoteVolume.cs @@ -1,28 +1,38 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. namespace Duplicati.Library.Main.Database { + /// + /// Represents a remote volume with its name, hash, size, and an optional file entry. + /// Implements the IRemoteVolume interface. + /// public class RemoteVolume : IRemoteVolume { + /// + /// Initializes a new instance of the class with a file + /// entry and an optional hash. + /// + /// The file entry associated with the remote volume. + /// The hash of the remote volume, can be null. public RemoteVolume(Interface.IFileEntry file, string hash = null) { Name = file.Name; @@ -31,6 +41,13 @@ namespace Duplicati.Library.Main.Database File = file; } + /// + /// Initializes a new instance of the class with a name, + /// hash, and size. + /// + /// The name of the remote volume. + /// The hash of the remote volume. + /// The size of the remote volume in bytes. public RemoteVolume(string name, string hash, long size) { Name = name; diff --git a/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs b/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs index e496f0698..3c264087d 100644 --- a/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs +++ b/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs @@ -1,40 +1,75 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; namespace Duplicati.Library.Main.Database { + /// + /// Represents a remote volume entry with its ID, name, hash, size, type, state, + /// delete grace period, and archive time. + /// Implements the IRemoteVolume interface. + /// public struct RemoteVolumeEntry : IRemoteVolume { + /// + /// The ID of the remote volume entry. + /// public long ID { get; private set; } public string Name { get; private set; } public string Hash { get; private set; } public long Size { get; private set; } + /// + /// The type of the remote volume, indicating whether it is a file or a directory. + /// public RemoteVolumeType Type { get; private set; } + /// + /// The state of the remote volume, indicating its current status. + /// public RemoteVolumeState State { get; private set; } + /// + /// The time period during which the remote volume can be deleted without permanent loss. + /// public DateTime DeleteGracePeriod { get; private set; } + /// + /// The time when the remote volume was archived, indicating when it was moved to a less accessible state. + /// public DateTime ArchiveTime { get; private set; } + /// + /// Represents an empty remote volume entry with default values. + /// This is useful for initializing or resetting remote volume entries. + /// public static readonly RemoteVolumeEntry Empty = new RemoteVolumeEntry(-1, null, null, -1, (RemoteVolumeType)(-1), (RemoteVolumeState)(-1), default(DateTime), default(DateTime)); + /// + /// Initializes a new instance of the RemoteVolumeEntry struct with specified values. + /// + /// The ID of the remote volume entry. + /// The name of the remote volume entry. + /// The hash of the remote volume entry. + /// The size of the remote volume entry. + /// The type of the remote volume entry (file or directory). + /// The state of the remote volume entry. + /// The delete grace period for the remote volume entry. + /// The archive time for the remote volume entry. public RemoteVolumeEntry(long id, string name, string hash, long size, RemoteVolumeType type, RemoteVolumeState state, DateTime deleteGracePeriod, DateTime archiveTime) { ID = id; diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 013708b22..30c66eb8f 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -56,6 +56,11 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran /// private bool m_disposed = false; + /// + /// Creates a new reusable transaction. + /// + /// The database this transaction relates to. + /// An optional existing transaction to use. If null, a new transaction is created. public ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) : this(db.Connection, transaction) { } /// @@ -68,7 +73,7 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran /// /// The log message to use. /// True if the transaction should be restarted. - /// An awaitable task. + /// A task that completes when the commit is done and (potentially) a new transaction has been started. /// If the transaction is already Disposed. public async Task CommitAsync(string? message = null, bool restart = true) { @@ -87,18 +92,12 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran } /// - /// - /// Calls the Async version of this method and awaits it. - /// public void Dispose() { DisposeAsync().AsTask().Await(); } - /// - /// Async version of Dispose: - /// - /// An awaitable task. + /// public async ValueTask DisposeAsync() { if (!m_disposed) @@ -121,6 +120,13 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran } } + /// + /// Rolls back the transaction and optionally restarts it. + /// + /// Message to log. + /// Whether to restart the transaction after rolling back. + /// A task that completes when the rollback is done and (potentially) a new transaction has been started. + /// If the transaction has already been disposed. public async Task RollBackAsync(string? message = null, bool restart = true) { if (m_disposed) diff --git a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs index 275421b53..644410517 100644 --- a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs +++ b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs @@ -31,49 +31,56 @@ using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database; /// -/// A list of values that can be used in a query +/// A list of values that can be used in a query. /// internal class TemporaryDbValueList : IDisposable, IAsyncDisposable { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(TemporaryDbValueList)); /// - /// The command to use + /// The command to use. /// private SqliteCommand? _cmd; + /// + /// The connection to use. + /// private SqliteConnection _con; + /// + /// The reusable transaction to use. + /// private ReusableTransaction _rtr; /// - /// The table to use + /// The table to use. /// private readonly string _tableName = $"TemporaryList-{Guid.NewGuid():N}"; /// - /// Flag indicating if the object has been disposed + /// Flag indicating if the object has been disposed. /// private bool _disposed = false; /// - /// Flag indicating if the values are in a table + /// Flag indicating if the values are in a table. /// private bool _isTable = false; /// - /// The values to use + /// The values to use. /// private readonly IEnumerable _values; /// - /// The type of the values + /// The type of the values. /// private readonly string _valuesType; /// - /// Creates a new reusable transaction + /// Creates a new TemporaryDbValueList with the given values. /// - /// The command to use - /// The connection to use - /// The values to use + /// The connection to use. + /// The reusable transaction to use. + /// The values to use. + /// The type of the values (e.g., "INTEGER", "TEXT"). private TemporaryDbValueList(SqliteConnection con, ReusableTransaction rtr, IEnumerable values, string valuesType) { _con = con; @@ -84,30 +91,62 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable ArgumentNullException.ThrowIfNull(values); } + /// + /// Asynchronously creates a new TemporaryDbValueList with the given values. + /// + /// The database to use. + /// The values to use. + /// A task that when awaited contains the created TemporaryDbValueList. internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) { return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "INTEGER")) .ConfigureAwait(false); } + /// + /// Asynchronously creates a new TemporaryDbValueList with the given values. + /// + /// The database to use. + /// The values to use. + /// A task that when awaited contains the created TemporaryDbValueList. internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) { return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "TEXT")) .ConfigureAwait(false); } + /// + /// Asynchronously creates a new TemporaryDbValueList with the given values. + /// + /// The connection to use. + /// The reusable transaction to use. + /// The values to use. + /// A task that when awaited contains the created TemporaryDbValueList. internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values) { return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "INTEGER")) .ConfigureAwait(false); } + /// + /// Asynchronously creates a new TemporaryDbValueList with the given values. + /// + /// The connection to use. + /// The reusable transaction to use. + /// The values to use. + /// A task that when awaited contains the created TemporaryDbValueList. internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values) { return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "TEXT")) .ConfigureAwait(false); } + /// + /// Performs the actual creation of the TemporaryDbValueList, ensuring that if the number of values exceeds the chunk size, + /// the values are forced into a table. + /// + /// The TemporaryDbValueList to create. + /// A task that when awaited contains the created TemporaryDbValueList. private static async Task DoCreateAsync(TemporaryDbValueList valueList) { if (valueList._values.Count() > LocalDatabase.CHUNK_SIZE) @@ -117,19 +156,19 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable } /// - /// The values in the list + /// The values in the list. /// public IEnumerable Values => _values; /// - /// Flag indicating if the table has been created + /// Flag indicating if the table has been created. /// public bool IsTableCreated => _isTable; /// - /// Get the in clause for the values, creating the table if needed + /// Get the in clause for the values, creating the table if needed. /// - /// The in clause for the values + /// A task that when awaited contains the SQL in clause string. public async Task GetInClause() { if (_disposed) @@ -144,8 +183,9 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable } /// - /// Force the values to be written to a table, if not already done + /// Force the values to be written to a table, if not already done. /// + /// A task that completes when the table has been created and the values inserted. public async Task ForceToTable() { if (_disposed) @@ -178,7 +218,7 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable } /// - /// The table name to use + /// The table name to use. /// public string TableName => _tableName; @@ -188,6 +228,7 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable DisposeAsync().AsTask().Await(); } + /// public async ValueTask DisposeAsync() { if (_disposed) From baab37004685b7ca364d5aac23b7f95075dec75c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 08:26:20 +0200 Subject: [PATCH 315/362] Added the final docstrings --- Duplicati/Library/Main/IBackendManager.cs | 2 - .../Operation/Backup/MetadataPreProcess.cs | 2 +- .../Main/Operation/FilelistProcessor.cs | 67 ++++---- .../Main/Operation/RecreateDatabaseHandler.cs | 43 ++--- .../Library/Main/Operation/RepairHandler.cs | 147 ++++++++++-------- .../Main/Operation/Restore/BlockManager.cs | 9 ++ .../Library/Main/Operation/TestHandler.cs | 11 +- Duplicati/Library/Main/Utility.cs | 22 +-- .../Main/Volumes/FilesetVolumeWriter.cs | 15 +- .../Library/SQLiteHelper/SQLiteLoader.cs | 54 +++++-- 10 files changed, 212 insertions(+), 160 deletions(-) diff --git a/Duplicati/Library/Main/IBackendManager.cs b/Duplicati/Library/Main/IBackendManager.cs index e44d69e49..8e6d8c86a 100644 --- a/Duplicati/Library/Main/IBackendManager.cs +++ b/Duplicati/Library/Main/IBackendManager.cs @@ -50,7 +50,6 @@ internal interface IBackendManager : IDisposable /// Waits for the backend queue to be empty and flushes any pending messages to the database /// /// The database to write pending messages to - /// The transaction to use, if any /// The cancellation token /// An awaitable task Task WaitForEmptyAsync(LocalDatabase database, CancellationToken cancellationToken); @@ -130,7 +129,6 @@ internal interface IBackendManager : IDisposable /// Flushes the database messages to the database /// /// The database to write to - /// The transaction to use /// The cancellation token /// Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken); diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index cf9f8534c..97135856d 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -252,11 +252,11 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// Adds metadata to output, and returns the metadataset ID /// - /// The metadataset ID. /// The path for which metadata is processed. /// The metadata entry. /// The database connection. /// The channel to write streams to. + /// A task that when awaited returns a tuple with the first value indicating if the metadata set was added, and the second value being the metadata ID. internal static async Task<(bool, long)> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) { StreamProcessResult res; diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 4b0455e61..39d0e8134 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -63,9 +63,12 @@ namespace Duplicati.Library.Main.Operation /// /// Helper method that verifies uploaded volumes and updates their state in the database. - /// Throws an error if there are issues with the remote storage + /// Throws an error if there are issues with the remote storage. /// - /// The database to compare with + /// The backend manager instance to use. + /// The database to compare with. + /// The cancellation token to use. + /// A task that completes when the verification is done. public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) { var locallist = database.GetRemoteVolumes(); @@ -104,15 +107,15 @@ namespace Duplicati.Library.Main.Operation /// /// Helper method that verifies uploaded volumes and updates their state in the database. - /// Throws an error if there are issues with the remote storage + /// Throws an error if there are issues with the remote storage. /// - /// The backend instance to use - /// The options used - /// The database to compare with - /// The backend writer to use - /// True if only the latest volumes should be verified - /// The mode to use for verification - /// An awaitable task + /// The backend instance to use. + /// The options used. + /// The database to compare with. + /// The backend writer to use. + /// True if only the latest volumes should be verified. + /// The mode to use for verification. + /// An awaitable task. public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode) { if (!options.NoBackendverification) @@ -131,17 +134,17 @@ namespace Duplicati.Library.Main.Operation /// /// Helper method that verifies uploaded volumes and updates their state in the database. - /// Throws an error if there are issues with the remote storage + /// Throws an error if there are issues with the remote storage. /// - /// The backend instance to use - /// The options used - /// The database to compare with - /// The log instance to use - /// Filenames that should be exempted from deletion - /// Filenames that should be exempted from strict verification - /// The mode to use for verification - /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless - /// An awaitable task + /// The backend instance to use. + /// The options used. + /// The database to compare with. + /// The log instance to use. + /// Filenames that should be exempted from deletion. + /// Filenames that should be exempted from strict verification. + /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless. + /// The mode to use for verification. + /// An awaitable task. public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) { var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); @@ -219,9 +222,10 @@ namespace Duplicati.Library.Main.Operation /// /// Uploads the verification file. /// - /// The backend url - /// The options to use - /// The attached database + /// The backend manager instance to use. + /// The options to use. + /// The attached database. + /// A task that completes when the upload is done. public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db) { using (var tempfile = new Library.Utility.TempFile()) @@ -244,15 +248,16 @@ namespace Duplicati.Library.Main.Operation /// /// Helper method that verifies uploaded volumes and updates their state in the database. - /// Throws an error if there are issues with the remote storage + /// Throws an error if there are issues with the remote storage. /// - /// The backend instance to use - /// The options used - /// The database to compare with - /// Filenames that should be exempted from deletion - /// Filenames that should be exempted from strict verification - /// The mode to use for verification - /// The result of the analysis + /// The backend manager instance to use. + /// The options used. + /// The database to compare with. + /// The log instance to use. + /// Filenames that should be exempted from deletion. + /// Filenames that should be exempted from strict verification. + /// The mode to use for verification. + /// A task that when awaited contains the analysis result. public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) { // If the last operation completed, no cleanup should be required diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index a230b3438..660f0c6f9 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -52,13 +52,13 @@ namespace Duplicati.Library.Main.Operation } /// - /// Run the recreate procedure + /// Run the recreate procedure. /// - /// Path to the database that will be created - /// The backend manager to use for downloading files - /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist - /// Filters the files in a filelist to prevent downloading unwanted data - /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists + /// Path to the database that will be created. + /// The backend manager to use for downloading files. + /// Filters the files in a filelist to prevent downloading unwanted data. + /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist. + /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists. public async Task RunAsync(string path, IBackendManager backendManager, IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) { if (File.Exists(path)) @@ -77,11 +77,12 @@ namespace Duplicati.Library.Main.Operation } /// - /// Updates a database with new path information from a remote fileset + /// Updates a database with new path information from a remote fileset. /// - /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist - /// Filters the files in a filelist to prevent downloading unwanted data - /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists + /// The backend manager to use for downloading files. + /// Filters the files in a filelist to prevent downloading unwanted data. + /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist. + /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists. public async Task RunUpdateAsync(IBackendManager backendManager, Library.Utility.IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) { if (!m_options.RepairOnlyPaths) @@ -119,13 +120,15 @@ namespace Duplicati.Library.Main.Operation } /// - /// Run the recreate procedure + /// Run the recreate procedure. /// - /// The database to restore into - /// True if this is an update call, false otherwise - /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist - /// Filters the files in a filelist to prevent downloading unwanted data - /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists + /// The backend manager to use for downloading files. + /// The database to restore into. + /// True if this is an update call, false otherwise. + /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist. + /// Filters the files in a filelist to prevent downloading unwanted data. + /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists. + /// A task that completes when the operation is done. internal async Task DoRunAsync(IBackendManager backendManager, LocalDatabase dbparent, bool updating, IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running); @@ -781,11 +784,11 @@ namespace Duplicati.Library.Main.Operation } /// - /// Look in the database for filenames similar to the current filename, but with a different compression and encryption module + /// Look in the database for filenames similar to the current filename, but with a different compression and encryption module. /// - /// The volume id of the item - /// The filename read and written - /// The database to query + /// The filename read and written. + /// The database to query. + /// The volume id of the item. public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb) { var p = VolumeBase.ParseFilename(filename); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 7353805a0..2b839856d 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -654,16 +654,15 @@ namespace Duplicati.Library.Main.Operation } /// - /// Repairs a single fileset by recreating it from the database content and uploading it + /// Repairs a single fileset by recreating it from the database content and uploading it. /// - /// The backend manager to use for uploading - /// The database to use for the repair - /// The transaction to use for the repair - /// The volume writer to use for the repair - /// The remote volume entry to repair - /// The time of the new fileset to create - /// Cancellation token to cancel the operation - /// A task representing the asynchronous operation + /// The backend manager to use for uploading. + /// The database to use for the repair. + /// The volume writer to use for the repair. + /// The remote volume entry to repair. + /// The time of the new fileset to create. + /// Cancellation token to cancel the operation. + /// A task representing the asynchronous operation. private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) { volumeWriter.VolumeID = await db @@ -722,18 +721,15 @@ namespace Duplicati.Library.Main.Operation } /// - /// Repairs a single index file by recreating it from the database content and uploading it + /// Repairs a single index file by recreating it from the database content and uploading it. /// - /// The backend manager to use for uploading - /// The database to use for the repair - /// The transaction to use for the repair - /// The volume writer to use for the repair - /// The remote volume entry to repair - /// The time of the new fileset to create - /// The options for the current operation - /// Cancellation token to cancel the operation - /// A task representing the asynchronous operation - + /// The backend manager to use for uploading. + /// The database to use for the repair. + /// The volume writer to use for the repair. + /// The remote volume entry to repair. + /// The options for the current operation. + /// Cancellation token to cancel the operation. + /// A task representing the asynchronous operation. public static async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, IndexVolumeWriter indexWriter, IRemoteVolume originalVolume, Options options, CancellationToken cancellationToken) { indexWriter.VolumeID = await db @@ -792,45 +788,44 @@ namespace Duplicati.Library.Main.Operation } /// - /// Class for holding the state of a dblock volume that is being constructed + /// Class for holding the state of a dblock volume that is being constructed. /// private sealed class InProgressDblockVolumes { /// - /// Database to use for the operation + /// Database to use for the operation. /// private readonly LocalDatabase m_database; /// - /// List of volumes to delete after the upload is complete + /// List of volumes to delete after the upload is complete. /// private readonly List m_toDelete = new List(); /// - /// Options for the current operation + /// Options for the current operation. /// private readonly Options m_options; /// - /// Maximum size of the volume before it is considered full + /// Maximum size of the volume before it is considered full. /// private readonly long m_maxVolumeSize; /// - /// The writers that are completed and ready to be uploaded + /// The writers that are completed and ready to be uploaded. /// private readonly List m_completedWriters = new List(); /// - /// Writer for the current in-progress volume + /// Writer for the current in-progress volume. /// private BlockVolumeWriter m_activeWriter; /// - /// Flag indicating if any data has been added to the volume + /// Flag indicating if any data has been added to the volume. /// private bool m_anyData = false; /// - /// Constructor for the InProgressDblockVolume class + /// Constructor for the InProgressDblockVolume class. /// - /// The options for the current operation - /// The database to use for registering a volume in progress - /// The transaction to use for the operation + /// The options for the current operation. + /// The database to use for registering a volume in progress. private InProgressDblockVolumes(Options options, LocalDatabase db) { m_database = db; @@ -839,6 +834,12 @@ namespace Duplicati.Library.Main.Operation m_activeWriter = new BlockVolumeWriter(options); } + /// + /// Asynchronously creates a new instance of and registers the initial volume. + /// + /// The options for the current operation. + /// The database to use for registering the initial volume. + /// A task that completes with the new instance of . public static async Task CreateAsync(Options options, LocalDatabase db) { var ipdv = new InProgressDblockVolumes(options, db); @@ -849,39 +850,39 @@ namespace Duplicati.Library.Main.Operation } /// - /// Checks if any data has been added to the current volume + /// Checks if any data has been added to the current volume. /// public bool AnyData => m_anyData; /// - /// Checks if the current volume is full + /// Checks if the current volume is full. /// private bool IsFull => m_activeWriter.Filesize > m_maxVolumeSize; /// - /// Gets the volume ID of the current volume + /// Gets the volume ID of the current volume. /// public long VolumeID => m_activeWriter.VolumeID; /// - /// Gets the remote filename of the current volume + /// Gets the remote filename of the current volume. /// public string RemoteFilename => m_activeWriter.RemoteFilename; /// - /// Gets the list of volumes to delete after the upload is complete + /// Gets the list of volumes to delete after the upload is complete. /// public List VolumesToDelete => m_toDelete; /// - /// Gets the list of completed writers + /// Gets the list of completed writers. /// public List CompletedWriters => m_completedWriters; /// - /// Adds a block to the current volume, starting a new volume if the current one is full + /// Adds a block to the current volume, starting a new volume if the current one is full. /// - /// The hash of the block - /// The buffer containing the block data - /// The offset in the buffer where the block data starts - /// The size of the block data - /// The compression hint for the block + /// The hash of the block. + /// The buffer containing the block data. + /// The offset in the buffer where the block data starts. + /// The size of the block data. + /// The compression hint for the block. public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) { await m_activeWriter @@ -893,8 +894,9 @@ namespace Duplicati.Library.Main.Operation } /// - /// Starts a new volume for the current operation + /// Starts a new volume for the current operation. /// + /// A task that completes when the new volume is started. public async Task StartNewVolume() { if (!m_anyData) @@ -911,15 +913,14 @@ namespace Duplicati.Library.Main.Operation } /// - /// Runs the repair process for missing dblock files + /// Runs the repair process for missing dblock files. /// - /// The backend manager to use for uploading - /// The database to use for the repair - /// The transaction to use for the repair - /// The list of missing dblock files - /// - /// Cancellation token to cancel the operation - /// >A task representing the asynchronous operation + /// The backend manager to use for uploading. + /// The database to use for the repair. + /// The list of missing dblock files. + /// + /// Cancellation token to cancel the operation. + /// >A task that completes when the repair is finished. private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) { var currentVolume = @@ -967,14 +968,13 @@ namespace Duplicati.Library.Main.Operation } /// - /// Uploads all completed block volumes to the backend and deletes the old files + /// Uploads all completed block volumes to the backend and deletes the old files. /// - /// The backend manager to use for uploading - /// The completed volume state to upload - /// The database to use for the upload - /// >The transaction to use for the upload - /// Cancellation token to cancel the operation - /// >A task representing the asynchronous operation + /// The backend manager to use for uploading. + /// The completed volume state to upload. + /// The database to use for the upload. + /// Cancellation token to cancel the operation. + /// >A task that completes when the upload is finished. async Task UploadCompletedBlockVolumes(IBackendManager backendManager, InProgressDblockVolumes activeVolume, LocalRepairDatabase db, CancellationToken cancellationToken) { foreach (var completedVolume in activeVolume.CompletedWriters) @@ -1054,15 +1054,14 @@ namespace Duplicati.Library.Main.Operation } /// - /// Repairs a single dblock file by recreating it from available data + /// Repairs a single dblock file by recreating it from available data. /// - /// The backend manager to use for uploading - /// The database to use for the repair - /// The transaction to use for the repair - /// The remote volume entry to recreate - /// The in-progress dblock volume to use for the recreate - /// Cancellation token to cancel the operation - /// >A task representing the asynchronous operation + /// The backend manager to use for uploading. + /// The database to use for the repair. + /// The remote volume entry to recreate. + /// The in-progress dblock volume to use for the recreate. + /// Cancellation token to cancel the operation. + /// >A task that completes when the repair is finished. private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, RemoteVolumeEntry originalVolume, InProgressDblockVolumes pendingVolume, CancellationToken cancellationToken) { // The dblock files are the most complex to recreate @@ -1315,6 +1314,12 @@ namespace Duplicati.Library.Main.Operation } } + /// + /// Runs the repair process for broken filesets in the database. + /// + /// The backend manager to use for accessing remote files. + /// >A task that completes when the repair is finished. + /// If the database doesn't exist. public async Task RunRepairBrokenFilesets(IBackendManager backendManager) { if (!File.Exists(m_options.Dbpath)) @@ -1362,6 +1367,12 @@ namespace Duplicati.Library.Main.Operation } } + /// + /// Runs the common repair operations on the database, such as fixing duplicate entries and missing blocklist hashes. + /// This method does not require any remote access and can be run without a backend manager. + /// + /// A task that completes when the repair is finished. + /// If the database file does not exist. public async Task RunRepairCommon() { if (!File.Exists(m_options.Dbpath)) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index be179ef37..5e77a0ef4 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -173,6 +173,15 @@ namespace Duplicati.Library.Main.Operation.Restore sw_set_wake_set = options.InternalProfiling ? new() : null; } + /// + /// Asynchronously creates a new instance of the class. + /// This method initializes the block and volume counts based on the data in the database. + /// + /// The database holding information about how many of each block this restore requires. + /// CoCoL channel for submitting block requests from a volume. + /// The restore options. + /// The number of readers accessing this dictionary. Used during shutdown / cleanup. + /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers) { var sd = new SleepableDictionary(db, volume_request, options, readers); diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index c09284738..d58e4dfe0 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -263,11 +263,14 @@ namespace Duplicati.Library.Main.Operation } /// - /// Tests the volume by examining the internal contents + /// Tests the volume by examining the internal contents. /// - /// The remote volume being examined - /// The path to the downloaded copy of the file - /// A value between 0 and 1 that indicates how many blocks are tested in a dblock file + /// The local test database to use for verification. + /// The remote volume being examined. + /// The path to the downloaded copy of the file. + /// The options to use for the test. + /// A value between 0 and 1 that indicates how many blocks are tested in a dblock file. + /// A task that when awaited returns a key-value pair where the key is the volume name and the value is a list of test results. public static async Task>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent) { var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm); diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index bb59e0b00..feaf14dbc 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -99,11 +99,11 @@ namespace Duplicati.Library.Main } /// - /// Updates the options with settings from the data, if any + /// Updates the options with settings from the data, if any. /// - /// The database to read from - /// The options to update - /// The transaction to use, if any + /// The database to read from. + /// The options to update. + /// A task that completes when the options have been updated. internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options) { string n = null; @@ -118,10 +118,10 @@ namespace Duplicati.Library.Main } /// - /// Checks if the database contains options that need to be verified, such as the blocksize + /// Checks if the database contains options that need to be verified, such as the blocksize. /// - /// The database to check - /// true if the database contains options that need to be verified; false otherwise + /// The database to check. + /// A task that when awaited returns true if the database contains options that need to be verified; false otherwise. internal static async Task ContainsOptionsForVerification(LocalDatabase db) { var opts = await db.GetDbOptions().ConfigureAwait(false); @@ -136,11 +136,11 @@ namespace Duplicati.Library.Main } /// - /// Verifies the parameters in the database, and updates the database if needed + /// Verifies the parameters in the database, and updates the database if needed. /// - /// The database to check - /// The options to verify - /// The transaction to use, if any + /// The database to check. + /// The options to verify. + /// A task that completes when the options have been verified and the database has been updated if needed. internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options) { var newDict = new Dictionary diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index be3b743da..ebaf50ab3 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -243,15 +243,14 @@ namespace Duplicati.Library.Main.Volumes => ProbeUnusedFilenameName(name => database.GetRemoteVolumeIDAsync(name), options, start, increment, maxTries); /// - /// Probes for an unused filename, using the current time as a starting point + /// Probes for an unused filename, using the current time as a starting point. /// - /// The database to check for clashes - /// The options to use for the filename - /// The transaction to use - /// The starting time to probe from - /// The time to increment by each probe - /// The maximum number of tries to probe - /// The first unused filename + /// The database to check for clashes. + /// The options to use for the filename. + /// The starting time to probe from. + /// The time to increment by each probe. + /// The maximum number of tries to probe. + /// A task that when awaited returns the first unused filename. internal static async Task ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) => await ProbeUnusedFilenameName(database.GetRemoteVolumeID, options, start, increment, maxTries) .ConfigureAwait(false); diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 7de8de5ad..af086b424 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -30,24 +30,29 @@ using Duplicati.Library.Utility; namespace Duplicati.Library.SQLiteHelper { + /// + /// Provides methods to load and manage SQLite connections, including handling encrypted databases. + /// public static class SQLiteLoader { /// - /// The minimum value for the SQLite page cache size + /// The minimum value for the SQLite page cache size. /// public const long MINIMUM_SQLITE_PAGE_CACHE_SIZE = 2048000L; /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(SQLiteLoader)); /// - /// Helper method with logic to handle opening a database in possibly encrypted format + /// Helper method with logic to handle opening a database in possibly encrypted format. /// - /// The SQLite connection object + /// The SQLite connection object. /// The location of Duplicati's database. /// The password to use for decryption. + /// A task that completes when the database is opened. + /// Thrown if the database cannot be opened or decrypted. public static async Task OpenDatabaseAsync(Microsoft.Data.Sqlite.SqliteConnection con, string databasePath, string? decryptionPassword) { if (!string.IsNullOrWhiteSpace(decryptionPassword) && SQLiteRC4Decrypter.IsDatabaseEncrypted(databasePath)) @@ -84,15 +89,22 @@ namespace Duplicati.Library.SQLiteHelper throw new UserInformationException("Failed to open database for unknown reason, check the logs to see error messages", "DatabaseOpenFailed"); } + /// + /// Loads an SQLite connection instance and opening the database. + /// + /// The SQLite connection instance. + /// + /// This method is synchronous and should be used when you need to load the connection immediately. It calls the asynchronous version and waits for it to complete. + /// public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection() { return LoadConnectionAsync().Await(); } /// - /// Loads an SQLite connection instance and opening the database + /// Loads an SQLite connection instance and opening the database. /// - /// The SQLite connection instance. + /// A task that when awaited returns the SQLite connection instance. public static async Task LoadConnectionAsync() { Microsoft.Data.Sqlite.SqliteConnection? con = null; @@ -114,11 +126,11 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Applies user-supplied custom pragmas to the SQLite connection + /// Applies user-supplied custom pragmas to the SQLite connection. /// /// The connection to apply the pragmas to. /// The page cache size to set. - /// The connection with the pragmas applied. + /// A task that when awaited returns the connection with the pragmas applied. public static async Task ApplyCustomPragmasAsync(Microsoft.Data.Sqlite.SqliteConnection con, long pagecachesize) { // TODO more default custom options @@ -156,17 +168,26 @@ namespace Duplicati.Library.SQLiteHelper return con; } + /// + /// Loads an SQLite connection instance and opening the database with a specified page cache size. + /// + /// The optional path to the database. + /// The page cache size to set. + /// The SQLite connection instance. + /// + /// This method is synchronous and should be used when you need to load the connection immediately. It calls the asynchronous version and waits for it to complete. + /// public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection(string targetpath, long pagecachesize) { return LoadConnectionAsync(targetpath, pagecachesize).Await(); } /// - /// Loads an SQLite connection instance and opening the database + /// Loads an SQLite connection instance and opening the database. /// - /// The SQLite connection instance. /// The optional path to the database. /// The page cache size to set. + /// A task that when waited returns the SQLite connection instance. public static async Task LoadConnectionAsync(string targetpath, long pagecachesize) { if (string.IsNullOrWhiteSpace(targetpath)) @@ -192,7 +213,7 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Returns the SQLiteCommand type for the current architecture + /// Returns the SQLiteCommand type for the current architecture. /// public static Type SQLiteConnectionType { @@ -203,7 +224,7 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Returns the version string from the SQLite type + /// Returns the version string from the SQLite type. /// public static string? SQLiteVersion { @@ -237,9 +258,10 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Wrapper to dispose the SQLite connection + /// Wrapper to dispose the SQLite connection. /// /// The connection to close. + /// A task that completes when the connection is disposed. private static async Task DisposeConnectionAsync(Microsoft.Data.Sqlite.SqliteConnection? con) { if (con != null) @@ -248,10 +270,11 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Opens the SQLite file in the given connection, creating the file if required + /// Opens the SQLite file in the given connection, creating the file if required. /// /// The connection to use. /// Path to the file to open, which may not exist. + /// A task that completes when the file is opened. private static async Task OpenSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con, string path) { con.ConnectionString = $"Data Source={path};Pooling=false"; @@ -264,9 +287,10 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Tests the SQLite connection, throwing an exception if the connection does not work + /// Tests the SQLite connection, throwing an exception if the connection does not work. /// /// The connection to test. + /// A task that completes when the test query is executed. private static async Task TestSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con) { // Do a dummy query to make sure we have a working db From f221098c28fd6a57708defe14b82a3372e64219e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 10:42:08 +0200 Subject: [PATCH 316/362] Formatted SQL queries - mainly adding double quotes where they were missing. --- .../DatabaseTool/Commands/Downgrade.cs | 4 +- .../DatabaseTool/Commands/Upgrade.cs | 4 +- .../Library/Main/Database/ChangeStatistics.cs | 82 ++--- .../Main/Database/LocalBackupDatabase.cs | 64 ++-- .../Library/Main/Database/LocalDatabase.cs | 158 ++++----- .../Main/Database/LocalDeleteDatabase.cs | 24 +- .../Database/LocalListBrokenFilesDatabase.cs | 16 +- .../Main/Database/LocalListChangesDatabase.cs | 6 +- .../Main/Database/LocalListDatabase.cs | 306 +++++++++--------- .../Main/Database/LocalPurgeDatabase.cs | 34 +- .../Main/Database/LocalRecreateDatabase.cs | 144 +++++---- .../Main/Database/LocalRepairDatabase.cs | 79 ++--- .../Main/Database/LocalRestoreDatabase.cs | 55 ++-- .../Main/Database/LocalTestDatabase.cs | 118 +++---- 14 files changed, 559 insertions(+), 535 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs index dcada0b8c..af5c58841 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs @@ -176,8 +176,8 @@ public static class Downgrade throw new UserInformationException($"{r}", "DowngradeScriptFailure"); await cmd.SetCommandAndParameters(@" - UPDATE Version - SET Version = @Version + UPDATE ""Version"" + SET ""Version"" = @Version ") .SetParameterValue("@Version", script.Version - 1) .ExecuteNonQueryAsync() diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs index a2c37be64..437c583bf 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs @@ -188,8 +188,8 @@ public static class Upgrade if (r != null) throw new UserInformationException($"{r}", "UpgradeScriptFailure"); await cmd.SetCommandAndParameters(@" - UPDATE Version - SET Version = @Version + UPDATE ""Version"" + SET ""Version"" = @Version ") .SetParameterValue("@Version", script.Version) .ExecuteNonQueryAsync() diff --git a/Duplicati/Library/Main/Database/ChangeStatistics.cs b/Duplicati/Library/Main/Database/ChangeStatistics.cs index 5963196b7..47f80b371 100644 --- a/Duplicati/Library/Main/Database/ChangeStatistics.cs +++ b/Duplicati/Library/Main/Database/ChangeStatistics.cs @@ -54,24 +54,24 @@ public static class ChangeStatistics await cmd.SetCommandAndParameters($@" CREATE TEMP TABLE ""{tmpName}"" AS SELECT - FL.""PrefixID"", - FL.""Path"", - FL.""BlocksetID"", - BS_Meta.""Fullhash"" AS ""Metahash"", - CASE FE.""FilesetID"" + ""FL"".""PrefixID"", + ""FL"".""Path"", + ""FL"".""BlocksetID"", + ""BS_Meta"".""Fullhash"" AS ""Metahash"", + CASE ""FE"".""FilesetID"" WHEN @LastFilesetId THEN 0 WHEN @CurrentFilesetId THEN 1 - END AS Source - FROM ""FileLookup"" FL - JOIN ""FilesetEntry"" FE - ON FL.""ID"" = FE.""FileID"" - LEFT JOIN ""Blockset"" BS_Data - ON FL.""BlocksetID"" = BS_Data.""ID"" + END AS ""Source"" + FROM ""FileLookup"" ""FL"" + JOIN ""FilesetEntry"" ""FE"" + ON ""FL"".""ID"" = ""FE"".""FileID"" + LEFT JOIN ""Blockset"" ""BS_Data"" + ON ""FL"".""BlocksetID"" = ""BS_Data"".""ID"" LEFT JOIN ""Metadataset"" M - ON FL.""MetadataID"" = M.""ID"" - LEFT JOIN ""Blockset"" BS_Meta - ON M.""BlocksetID"" = BS_Meta.""ID"" - WHERE FE.""FilesetID"" IN ( + ON ""FL"".""MetadataID"" = M.""ID"" + LEFT JOIN ""Blockset"" ""BS_Meta"" + ON M.""BlocksetID"" = ""BS_Meta"".""ID"" + WHERE ""FE"".""FilesetID"" IN ( @LastFilesetId, @CurrentFilesetId ); @@ -149,14 +149,14 @@ public static class ChangeStatistics private static async Task CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) { var conditions = $@" - Source = 1 + ""Source"" = 1 AND NOT EXISTS ( SELECT 1 - FROM ""{tmpName}"" B + FROM ""{tmpName}"" ""B"" WHERE - B.Source = 0 - AND B.PrefixID = A.PrefixID - AND B.Path = A.Path + ""B"".""Source"" = 0 + AND ""B"".""PrefixID"" = ""A"".""PrefixID"" + AND ""B"".""Path"" = ""A"".""Path"" ) "; @@ -175,14 +175,14 @@ public static class ChangeStatistics private static async Task CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) { var conditions = @$" - Source = 0 + ""Source"" = 0 AND NOT EXISTS ( SELECT 1 - FROM ""{tmpName}"" B + FROM ""{tmpName}"" ""B"" WHERE - B.Source = 1 - AND B.PrefixID = A.PrefixID - AND B.Path = A.Path + ""B"".""Source"" = 1 + AND ""B"".""PrefixID"" = ""A"".""PrefixID"" + AND ""B"".""Path"" = ""A"".""Path"" ) "; @@ -205,23 +205,23 @@ public static class ChangeStatistics if (blocksetId == LocalDatabase.FOLDER_BLOCKSET_ID || blocksetId == LocalDatabase.SYMLINK_BLOCKSET_ID) { conditions = @" - A.Source = 0 - AND B.Source = 1 - AND A.PrefixID = B.PrefixID - AND A.Path = B.Path - AND A.Metahash IS NOT B.Metahash + ""A"".""Source"" = 0 + AND ""B"".""Source"" = 1 + AND ""A"".""PrefixID"" = ""B"".""PrefixID"" + AND ""A"".""Path"" = ""B"".""Path"" + AND ""A"".""Metahash"" IS NOT ""B"".""Metahash"" "; } else { conditions = @" - A.Source = 0 - AND B.Source = 1 - AND A.PrefixID = B.PrefixID - AND A.Path = B.Path + ""A"".""Source"" = 0 + AND ""B"".""Source"" = 1 + AND ""A"".""PrefixID"" = ""B"".""PrefixID"" + AND ""A"".""Path"" = ""B"".""Path"" AND ( - A.BlocksetID IS NOT B.BlocksetID - OR A.Metahash IS NOT B.Metahash + ""A"".""BlocksetID"" IS NOT ""B"".""BlocksetID"" + OR ""A"".""Metahash"" IS NOT ""B"".""Metahash"" ) "; } @@ -232,10 +232,10 @@ public static class ChangeStatistics string sql = $@" SELECT COUNT(*) - FROM ""{tmpName}"" A - JOIN ""{tmpName}"" B - ON A.PrefixID = B.PrefixID - AND A.Path = B.Path + FROM ""{tmpName}"" ""A"" + JOIN ""{tmpName}"" ""B"" + ON ""A"".""PrefixID"" = ""B"".""PrefixID"" + AND ""A"".""Path"" = ""B"".""Path"" WHERE {conditions} "; @@ -264,7 +264,7 @@ public static class ChangeStatistics var sql = $@" SELECT COUNT(*) - FROM ""{tmpName}"" {alias} + FROM ""{tmpName}"" ""{alias}"" WHERE {fullCondition} "; diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 62fd954d4..5590124db 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -210,9 +210,9 @@ namespace Duplicati.Library.Main.Database dbnew.m_findmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" SELECT ""A"".""ID"" FROM - ""Metadataset"" A, - ""BlocksetEntry"" B, - ""Block"" C + ""Metadataset"" ""A"", + ""BlocksetEntry"" ""B"", + ""Block"" ""C"" WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" @@ -297,8 +297,8 @@ namespace Duplicati.Library.Main.Database ""BlockID"" ) SELECT - @BlocksetId AS A, - @Index AS B, + @BlocksetId AS ""A"", + @Index AS ""B"", ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash @@ -641,25 +641,25 @@ namespace Duplicati.Library.Main.Database case 3: findQuery = @" SELECT - FileLookup.ID as FileID, - FilesetEntry.Lastmodified, - FileBlockset.Length, - MetaBlockset.FullHash AS Metahash, - MetaBlockset.Length as Metasize - FROM FilesetEntry - INNER JOIN Fileset - ON (FileSet.ID = FilesetEntry.FilesetID) - INNER JOIN FileLookup - ON (FileLookup.ID = FilesetEntry.FileID) - INNER JOIN Metadataset - ON (Metadataset.ID = FileLookup.MetadataID) - INNER JOIN Blockset AS MetaBlockset - ON (MetaBlockset.ID = Metadataset.BlocksetID) - LEFT JOIN Blockset AS FileBlockset - ON (FileBlockset.ID = FileLookup.BlocksetID) + ""FileLookup"".""ID"" as ""FileID"", + ""FilesetEntry"".""Lastmodified"", + ""FileBlockset"".""Length"", + ""MetaBlockset"".""FullHash"" AS ""Metahash"", + ""MetaBlockset"".""Length"" as ""Metasize"" + FROM ""FilesetEntry"" + INNER JOIN ""Fileset"" + ON (""FileSet"".""ID"" = ""FilesetEntry"".FilesetID) + INNER JOIN ""FileLookup"" + ON (""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"") + INNER JOIN ""Metadataset"" + ON (""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"") + INNER JOIN Blockset AS ""MetaBlockset"" + ON (""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"") + LEFT JOIN Blockset AS ""FileBlockset"" + ON (""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"") WHERE - FileLookup.PrefixID = @PrefixId - AND FileLookup.Path = @Path + ""FileLookup"".""PrefixID"" = @PrefixId + AND ""FileLookup"".""Path"" = @Path AND FilesetID = @FilesetId LIMIT 1 "; @@ -1546,20 +1546,20 @@ namespace Duplicati.Library.Main.Database // enumerate files from new temporary file set, and remove any entries handled by UNC cmd.SetCommandAndParameters($@" SELECT - f.""Path"", - fs.""FileID"", - fs.""Lastmodified"", - COALESCE(bs.""Length"", -1) + ""f"".""Path"", + ""fs"".""FileID"", + ""fs"".""Lastmodified"", + COALESCE(""bs"".""Length"", -1) FROM ( SELECT DISTINCT ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"" - ) AS fs - LEFT JOIN ""File"" AS f - ON fs.""FileID"" = f.""ID"" - LEFT JOIN ""Blockset"" AS bs - ON f.""BlocksetID"" = bs.""ID""; + ) AS ""fs"" + LEFT JOIN ""File"" AS ""f"" + ON ""fs"".""FileID"" = ""f"".""ID"" + LEFT JOIN ""Blockset"" AS ""bs"" + ON ""f"".""BlocksetID"" = ""bs"".""ID""; "); await foreach (var row in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 4f0cc920b..bef009aba 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -972,34 +972,34 @@ namespace Duplicati.Library.Main.Database // Create a temp table to associate metadata that is being deleted to a fileset var metadataFilesetQuery = $@" SELECT - Metadataset.ID, - FilesetEntry.FilesetID - FROM Metadataset - INNER JOIN FileLookup - ON FileLookup.MetadataID = Metadataset.ID - INNER JOIN FilesetEntry - ON FilesetEntry.FileID = FileLookup.ID - WHERE Metadataset.BlocksetID IN ({bsIdsSubQuery}) - OR Metadataset.ID IN ( - SELECT MetadataID - FROM FileLookup - WHERE BlocksetID IN ({bsIdsSubQuery}) + ""Metadataset"".""ID"", + ""FilesetEntry"".""FilesetID"" + FROM ""Metadataset"" + INNER JOIN ""FileLookup"" + ON ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""FilesetEntry"" + ON ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + WHERE ""Metadataset"".""BlocksetID"" IN ({bsIdsSubQuery}) + OR ""Metadataset"".""ID"" IN ( + SELECT ""MetadataID"" + FROM ""FileLookup"" + WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) ) "; var metadataFilesetTable = $"DelMetadataFilesetIds-{temptransguid}"; await deletecmd.ExecuteNonQueryAsync($@" CREATE TEMP TABLE ""{metadataFilesetTable}"" ( - MetadataID INTEGER PRIMARY KEY, - FilesetID INTEGER + ""MetadataID"" INTEGER PRIMARY KEY, + ""FilesetID"" INTEGER ) ") .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" INSERT OR IGNORE INTO ""{metadataFilesetTable}"" ( - MetadataID, - FilesetID + ""MetadataID"", + ""FilesetID"" ) {metadataFilesetQuery} ") @@ -1007,19 +1007,19 @@ namespace Duplicati.Library.Main.Database // Delete FilesetEntry rows that had their metadata deleted await deletecmd.ExecuteNonQueryAsync($@" - DELETE FROM FilesetEntry + DELETE FROM ""FilesetEntry"" WHERE - FilesetEntry.FilesetID IN ( - SELECT DISTINCT FilesetID + ""FilesetEntry"".""FilesetID"" IN ( + SELECT DISTINCT ""FilesetID"" FROM ""{metadataFilesetTable}"" ) - AND FilesetEntry.FileID IN ( - SELECT FilesetEntry.FileID - FROM FilesetEntry - INNER JOIN FileLookup - ON FileLookup.ID = FilesetEntry.FileID - WHERE FileLookup.MetadataID IN ( - SELECT MetadataID + AND ""FilesetEntry"".""FileID"" IN ( + SELECT ""FilesetEntry"".""FileID"" + FROM ""FilesetEntry"" + INNER JOIN ""FileLookup"" + ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + WHERE ""FileLookup"".""MetadataID"" IN ( + SELECT ""MetadataID"" FROM ""{metadataFilesetTable}"" ) ) @@ -1028,19 +1028,19 @@ namespace Duplicati.Library.Main.Database // Delete FilesetEntry rows that had their blocks deleted await deletecmd.ExecuteNonQueryAsync($@" - DELETE FROM FilesetEntry - WHERE FilesetEntry.FileID IN ( - SELECT ID - FROM FileLookup - WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}) + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetEntry"".""FileID"" IN ( + SELECT ""ID"" + FROM ""FileLookup"" + WHERE ""FileLookup"".""BlocksetID"" IN ({bsIdsSubQuery}) ) ") .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" - DELETE FROM FileLookup - WHERE FileLookup.MetadataID IN ( - SELECT MetadataID + DELETE FROM ""FileLookup"" + WHERE ""FileLookup"".""MetadataID"" IN ( + SELECT ""MetadataID"" FROM ""{metadataFilesetTable}"" ) ") @@ -1107,11 +1107,11 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" - DELETE FROM FilesetEntry - WHERE FilesetID IN ( - SELECT ID - FROM Fileset - WHERE VolumeID IN ({volIdsSubQuery}) + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) ) ") .ConfigureAwait(false); @@ -1143,7 +1143,7 @@ namespace Duplicati.Library.Main.Database ) AND ""Fileset"".""ID"" NOT IN ( SELECT DISTINCT ""FilesetID"" - FROM FilesetEntry + FROM ""FilesetEntry"" ) ") .ConfigureAwait(false); @@ -1627,7 +1627,7 @@ namespace Duplicati.Library.Main.Database ""A"".""ID"" AS ""BlocksetID"", IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"", ""A"".""Length"" - FROM ""Blockset"" A + FROM ""Blockset"" ""A"" LEFT OUTER JOIN ( SELECT ""BlocksetEntry"".""BlocksetID"", @@ -1636,7 +1636,7 @@ namespace Duplicati.Library.Main.Database LEFT OUTER JOIN ""Block"" ON ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" GROUP BY ""BlocksetEntry"".""BlocksetID"" - ) B + ) ""B"" ON ""A"".""ID"" = ""B"".""BlocksetID"" "; @@ -1647,7 +1647,7 @@ namespace Duplicati.Library.Main.Database ""Length"", ""A"".""BlocksetID"", ""File"".""Path"" FROM - ({combinedLengths}) A, + ({combinedLengths}) ""A"", ""File"" WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" @@ -1676,13 +1676,13 @@ namespace Duplicati.Library.Main.Database } var real_count = await cmd.ExecuteScalarInt64Async(@" - SELECT Count(*) + SELECT COUNT(*) FROM ""BlocklistHash"" ", 0) .ConfigureAwait(false); var unique_count = await cmd.ExecuteScalarInt64Async(@" - SELECT Count(*) + SELECT COUNT(*) FROM ( SELECT DISTINCT ""BlocksetID"", @@ -2205,22 +2205,22 @@ namespace Duplicati.Library.Main.Database ""H"".""Hash"" AS ""FirstMetaBlockHash"", ""H"".""Size"" AS ""FirstMetaBlockSize"", ""C"".""BlocksetID"" AS ""MetablocksetID"" - FROM ""File"" A - LEFT JOIN ""Blockset"" B + FROM ""File"" ""A"" + LEFT JOIN ""Blockset"" ""B"" ON ""A"".""BlocksetID"" = ""B"".""ID"" - LEFT JOIN ""Metadataset"" C + LEFT JOIN ""Metadataset"" ""C"" ON ""A"".""MetadataID"" = ""C"".""ID"" - LEFT JOIN ""FilesetEntry"" D + LEFT JOIN ""FilesetEntry"" ""D"" ON ""A"".""ID"" = ""D"".""FileID"" - LEFT JOIN ""Blockset"" E + LEFT JOIN ""Blockset"" ""E"" ON ""E"".""ID"" = ""C"".""BlocksetID"" - LEFT JOIN ""BlocksetEntry"" G + LEFT JOIN ""BlocksetEntry"" ""G"" ON ""B"".""ID"" = ""G"".""BlocksetID"" - LEFT JOIN ""Block"" F + LEFT JOIN ""Block"" ""F"" ON ""G"".""BlockID"" = ""F"".""ID"" - LEFT JOIN ""BlocksetEntry"" I + LEFT JOIN ""BlocksetEntry"" ""I"" ON ""E"".""ID"" = ""I"".""BlocksetID"" - LEFT JOIN ""Block"" H + LEFT JOIN ""Block"" ""H"" ON ""I"".""BlockID"" = ""H"".""ID"" WHERE ""A"".""BlocksetId"" >= 0 @@ -2233,14 +2233,14 @@ namespace Duplicati.Library.Main.Database ""G"".""Index"" = 0 OR ""G"".""Index"" IS NULL ) - ) J - LEFT OUTER JOIN ""BlocklistHash"" K + ) ""J"" + LEFT OUTER JOIN ""BlocklistHash"" ""K"" ON ""K"".""BlocksetID"" = ""J"".""BlocksetID"" ORDER BY ""J"".""Path"", ""K"".""Index"" - ) L - LEFT OUTER JOIN ""BlocklistHash"" M + ) ""L"" + LEFT OUTER JOIN ""BlocklistHash"" ""M"" ON ""M"".""BlocksetID"" = ""L"".""MetablocksetID"" "; @@ -2268,12 +2268,12 @@ namespace Duplicati.Library.Main.Database ""F"".""Hash"" AS ""FirstMetaBlockHash"", ""C"".""BlocksetID"" AS ""MetaBlocksetID"" FROM - ""FilesetEntry"" A, - ""File"" B, - ""Metadataset"" C, - ""Blockset"" D, - ""BlocksetEntry"" E, - ""Block"" F + ""FilesetEntry"" ""A"", + ""File"" ""B"", + ""Metadataset"" ""C"", + ""Blockset"" ""D"", + ""BlocksetEntry"" ""E"", + ""Block"" ""F"" WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" @@ -2286,8 +2286,8 @@ namespace Duplicati.Library.Main.Database OR ""B"".""BlocksetID"" = @SymlinkBlocksetId ) AND ""A"".""FilesetID"" = @FilesetId - ) G - LEFT OUTER JOIN ""BlocklistHash"" H + ) ""G"" + LEFT OUTER JOIN ""BlocklistHash"" ""H"" ON ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID"" ORDER BY ""G"".""Path"", @@ -2419,17 +2419,17 @@ namespace Duplicati.Library.Main.Database public async Task PushTimestampChangesToPreviousVersion(long filesetId) { var query = @" - UPDATE FilesetEntry AS oldVersion - SET Lastmodified = tempVersion.Lastmodified - FROM FilesetEntry AS tempVersion + UPDATE ""FilesetEntry"" AS ""oldVersion"" + SET ""Lastmodified"" = ""tempVersion"".""Lastmodified"" + FROM ""FilesetEntry"" AS ""tempVersion"" WHERE - oldVersion.FileID = tempVersion.FileID - AND tempVersion.FilesetID = @FilesetId - AND oldVersion.FilesetID = ( - SELECT ID - FROM Fileset - WHERE ID != @FilesetId - ORDER BY Timestamp DESC + ""oldVersion"".""FileID"" = ""tempVersion"".""FileID"" + AND ""tempVersion"".""FilesetID"" = @FilesetId + AND ""oldVersion"".""FilesetID"" = ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""ID"" != @FilesetId + ORDER BY ""Timestamp"" DESC LIMIT 1 ) "; @@ -2784,9 +2784,9 @@ namespace Duplicati.Library.Main.Database GROUP BY ""AllBlocksInVolume"".""Hash"", ""AllBlocksInVolume"".""Size"" - ) A, - ""BlocksetEntry"" B, - ""Block"" C + ) ""A"", + ""BlocksetEntry"" ""B"", + ""Block"" ""C"" WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 48b9e4c37..2347629c4 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -48,7 +48,19 @@ namespace Duplicati.Library.Main.Database /// /// SQL command to register a duplicate block. /// - private const string REGISTER_COMMAND = @"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""ID"", @VolumeId FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "; + private const string REGISTER_COMMAND = @" + INSERT OR IGNORE INTO ""DuplicateBlock"" ( + ""BlockID"", + ""VolumeID"" + ) + SELECT + ""ID"", + @VolumeId + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + "; /// /// The command to register a duplicate block. @@ -447,8 +459,8 @@ namespace Duplicati.Library.Main.Database THEN 0 ELSE ""B"".""Sorttime"" END AS ""Sorttime"" - FROM ({usedBlocks}) A - LEFT OUTER JOIN ({scantime}) B + FROM ({usedBlocks}) ""A"" + LEFT OUTER JOIN ({scantime}) ""B"" ON ""B"".""VolumeID"" = ""A"".""VolumeID"" "; @@ -512,8 +524,8 @@ namespace Duplicati.Library.Main.Database ""B"".""InactiveSize"", ""A"".""Size"" FROM - ""Remotevolume"" A, - ""{tmptablename}"" B + ""Remotevolume"" ""A"", + ""{tmptablename}"" ""B"" WHERE ""A"".""ID"" = ""B"".""VolumeID"" ORDER BY ""B"".""Sorttime"" ASC "); @@ -957,7 +969,7 @@ namespace Duplicati.Library.Main.Database ""RB"".""BlockID"" || ':' || ""RB"".""VolumeID"" - FROM ""{replacementBlocks}"" RB + FROM ""{replacementBlocks}"" ""RB"" ) ") .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 80422b44f..c7bcafcf1 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -67,8 +67,8 @@ namespace Duplicati.Library.Main.Database SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" - FROM ""FileLookup"" A - LEFT JOIN ""Metadataset"" B + FROM ""FileLookup"" ""A"" + LEFT JOIN ""Metadataset"" ""B"" ON ""A"".""MetadataID"" = ""B"".""ID"" ) WHERE @@ -93,8 +93,8 @@ namespace Duplicati.Library.Main.Database ) UNION SELECT ""A"".""ID"" AS ""BlocksetID"" - FROM ""Blockset"" A - LEFT JOIN ""BlocksetEntry"" B + FROM ""Blockset"" ""A"" + LEFT JOIN ""BlocksetEntry"" ""B"" ON ""A"".""ID"" = ""B"".""BlocksetID"" WHERE ""A"".""Length"" > 0 @@ -117,8 +117,8 @@ namespace Duplicati.Library.Main.Database ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM - ""FilesetEntry"" A, - ""Fileset"" B + ""FilesetEntry"" ""A"", + ""Fileset"" ""B"" WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS}) @@ -131,8 +131,8 @@ namespace Duplicati.Library.Main.Database SELECT ""A"".""Path"", ""B"".""Length"" - FROM ""File"" A - LEFT JOIN ""Blockset"" B + FROM ""File"" ""A"" + LEFT JOIN ""Blockset"" ""B"" ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 7ced47e41..3504981e2 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -407,7 +407,7 @@ namespace Duplicati.Library.Main.Database ""MetaHash"", ""Size"", ""Type"" - FROM {combined} A + FROM {combined} ""A"" WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) @@ -478,7 +478,7 @@ namespace Duplicati.Library.Main.Database ""MetaHash"", ""Size"", ""Type"" - FROM {combined} A + FROM {combined} ""A"" WHERE {whereClause} ") .SetParameterValue("@FilesetId", filesetId) @@ -500,7 +500,7 @@ namespace Duplicati.Library.Main.Database ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" - FROM {combined} A + FROM {combined} ""A"" WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index d8771a859..40c9be61e 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -399,7 +399,7 @@ namespace Duplicati.Library.Main.Database if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) { var paths = cmd.ExecuteReaderEnumerableAsync($@" - SELECT Path + SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ") .Select(x => x.ConvertValueToString(0) ?? ""); @@ -560,8 +560,8 @@ namespace Duplicati.Library.Main.Database ""A"".""Path"", ""B"".""FilesetID"" FROM - ""{tbname}"" A, - ({filesets}) B + ""{tbname}"" ""A"", + ({filesets}) ""B"" ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC @@ -590,8 +590,8 @@ namespace Duplicati.Library.Main.Database ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" - FROM ({cartesianPathFileset}) C - LEFT OUTER JOIN ({filesWithSizes}) D + FROM ({cartesianPathFileset}) ""C"" + LEFT OUTER JOIN ({filesWithSizes}) ""D"" ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"" "; @@ -672,8 +672,8 @@ namespace Duplicati.Library.Main.Database ""A"".""Path"", ""B"".""FilesetID"" FROM - ""{tmpnames.Tablename}"" A, - ({filesets}) B + ""{tmpnames.Tablename}"" ""A"", + ({filesets}) ""B"" ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC @@ -702,8 +702,8 @@ namespace Duplicati.Library.Main.Database ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" - FROM ({cartesianPathFileset}) C - LEFT OUTER JOIN ({filesWithSizes}) D + FROM ({cartesianPathFileset}) ""C"" + LEFT OUTER JOIN ({filesWithSizes}) ""D"" ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"" "; @@ -783,9 +783,9 @@ namespace Duplicati.Library.Main.Database COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM - ""FilesetEntry"" A, - ""File"" B, - ""Blockset"" C + ""FilesetEntry"" ""A"", + ""File"" ""B"", + ""Blockset"" ""C"" WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" @@ -802,8 +802,8 @@ namespace Duplicati.Library.Main.Database ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" - FROM ""{m_tablename}"" A - LEFT OUTER JOIN ( {summation} ) B + FROM ""{m_tablename}"" ""A"" + LEFT OUTER JOIN ( {summation} ) ""B"" ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ") @@ -878,20 +878,20 @@ namespace Duplicati.Library.Main.Database { const string query = @" SELECT - f.""ID"" AS ""FilesetID"", - f.""Timestamp"", - f.""IsFullBackup"", - COUNT(fe.""FileID"") AS ""NumberOfFiles"", - COALESCE(SUM(b.""Length""), 0) AS ""TotalFileSize"" - FROM ""Fileset"" f - LEFT JOIN ""FilesetEntry"" fe - ON f.""ID"" = fe.""FilesetID"" - LEFT JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" - LEFT JOIN ""Blockset"" b - ON fl.""BlocksetID"" = b.""ID"" - GROUP BY f.""ID"" - ORDER BY f.""Timestamp"" DESC; + ""f"".""ID"" AS ""FilesetID"", + ""f"".""Timestamp"", + ""f"".""IsFullBackup"", + COUNT(""fe"".""FileID"") AS ""NumberOfFiles"", + COALESCE(SUM(""b"".""Length""), 0) AS ""TotalFileSize"" + FROM ""Fileset"" ""f"" + LEFT JOIN ""FilesetEntry"" ""fe"" + ON ""f"".""ID"" = ""fe"".""FilesetID"" + LEFT JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + LEFT JOIN ""Blockset"" ""b"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" + GROUP BY ""f"".""ID"" + ORDER BY ""f"".""Timestamp"" DESC; "; using (var cmd = m_connection.CreateCommand(query)) @@ -946,32 +946,32 @@ namespace Duplicati.Library.Main.Database // Then query the matching files cmd.SetCommandAndParameters($@" SELECT - pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - b.""Length"", + ""pp"".""Prefix"" || ""fl"".""Path"" AS ""FullPath"", + ""b"".""Length"", (CASE - WHEN fl.""BlocksetID"" = @FolderBlocksetId + WHEN ""fl"".""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END) AS ""IsDirectory"", (CASE - WHEN fl.""BlocksetID"" = @SymlinkBlocksetId + WHEN ""fl"".""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END) AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp - ON fl.""PrefixID"" = pp.""ID"" - LEFT JOIN ""Blockset"" b - ON fl.""BlocksetID"" = b.""ID"" + ""fe"".""Lastmodified"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" + LEFT JOIN ""Blockset"" ""b"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" WHERE - fe.""FilesetID"" = @filesetid - AND fl.""PrefixID"" IN (@PrefixIds) + ""fe"".""FilesetID"" = @filesetid + AND ""fl"".""PrefixID"" IN (@PrefixIds) ORDER BY - pp.""Prefix"" ASC, - fl.""Path"" ASC + ""pp"".""Prefix"" ASC, + ""fl"".""Path"" ASC LIMIT @limit OFFSET @offset ") @@ -1016,12 +1016,12 @@ namespace Duplicati.Library.Main.Database // Get the total count of files for the given fileset and prefix IDs totalCount = await cmd.SetCommandAndParameters($@" SELECT COUNT(*) - FROM FilesetEntry fe - INNER JOIN FileLookup fl - ON fe.FileID = fl.ID + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" WHERE - fe.FilesetID = @filesetid - AND fl.PrefixID IN (@PrefixIds) + ""fe"".""FilesetID"" = @filesetid + AND ""fl"".""PrefixID"" IN (@PrefixIds) ") .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) .SetParameterValue("@filesetid", filesetid) @@ -1057,32 +1057,32 @@ namespace Duplicati.Library.Main.Database // Fetch entries cmd.SetCommandAndParameters(@" SELECT - pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - b.""Length"", + ""pp"".""Prefix"" || ""fl"".""Path"" AS ""FullPath"", + ""b"".""Length"", CASE - WHEN fl.""BlocksetID"" = @FolderBlocksetId + WHEN ""fl"".""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", CASE - WHEN fl.""BlocksetID"" = @SymlinkBlocksetId + WHEN ""fl"".""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp - ON fl.""PrefixID"" = pp.""ID"" + ""fe"".""Lastmodified"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" LEFT JOIN ""Blockset"" b - ON fl.""BlocksetID"" = b.""ID"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" WHERE - fe.""FilesetId"" = @FilesetId - AND fl.""PrefixId"" IN (@PrefixIds) + ""fe"".""FilesetId"" = @FilesetId + AND ""fl"".""PrefixId"" IN (@PrefixIds) ORDER BY - pp.""Prefix"" ASC, - fl.""Path"" ASC + ""pp"".""Prefix"" ASC, + ""fl"".""Path"" ASC LIMIT @limit OFFSET @offset ") @@ -1120,12 +1120,12 @@ namespace Duplicati.Library.Main.Database { totalCount = await cmd.SetCommandAndParameters(@" SELECT COUNT(*) - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" WHERE - fe.""FilesetId"" = @FilesetId - AND fl.""PrefixId"" IN (@PrefixIds) + ""fe"".""FilesetId"" = @FilesetId + AND ""fl"".""PrefixId"" IN (@PrefixIds) ") .SetParameterValue("@FilesetId", filesetid) .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) @@ -1170,26 +1170,26 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(@" WITH Prefixes AS ( SELECT DISTINCT - fl.""PrefixID"", - pp.""Prefix"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp - ON fl.""PrefixID"" = pp.""ID"" - WHERE fe.""FilesetID"" = @filesetid + ""fl"".""PrefixID"", + ""pp"".""Prefix"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" + WHERE ""fe"".""FilesetID"" = @filesetid ) - SELECT p1.""PrefixID"" - FROM Prefixes p1 + SELECT ""p1"".""PrefixID"" + FROM Prefixes ""p1"" WHERE NOT EXISTS ( SELECT 1 - FROM Prefixes p2 + FROM Prefixes ""p2"" WHERE - p2.""PrefixID"" != p1.""PrefixID"" - AND p1.""Prefix"" LIKE p2.""Prefix"" || '%' - AND LENGTH(p1.""Prefix"") > LENGTH(p2.""Prefix"") + ""p2"".""PrefixID"" != ""p1"".""PrefixID"" + AND ""p1"".""Prefix"" LIKE ""p2"".""Prefix"" || '%' + AND LENGTH(""p1"".""Prefix"") > LENGTH(""p2"".""Prefix"") ) - ORDER BY p1.""Prefix"" ASC + ORDER BY ""p1"".""Prefix"" ASC ") .SetParameterValue("@filesetid", filesetid); @@ -1218,39 +1218,39 @@ namespace Duplicati.Library.Main.Database )?.ToString()); cmd.SetCommandAndParameters(@" - WITH Prefixes AS ( + WITH ""Prefixes"" AS ( SELECT DISTINCT - fl.""PrefixID"", - pp.""Prefix"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp - ON fl.""PrefixID"" = pp.""ID"" - WHERE fe.""FilesetID"" = @filesetid + ""fl"".""PrefixID"", + ""pp"".""Prefix"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" + WHERE ""fe"".""FilesetID"" = @filesetid ), - PrefixParents AS ( + ""PrefixParents"" AS ( SELECT - child.""PrefixID"" AS ChildPrefixID, - parent.""PrefixID"" AS ParentPrefixID, - LENGTH(parent.""Prefix"") AS ParentLength - FROM Prefixes child - JOIN Prefixes parent - ON LENGTH(child.""Prefix"") > LENGTH(parent.""Prefix"") - AND SUBSTR(child.""Prefix"", 1, LENGTH(parent.""Prefix"")) = parent.""Prefix"" - AND (SUBSTR(child.""Prefix"", LENGTH(parent.""Prefix"") + 1, 1) = @dirSeparator) + ""child"".""PrefixID"" AS ""ChildPrefixID"", + ""parent"".""PrefixID"" AS ""ParentPrefixID"", + LENGTH(""parent"".""Prefix"") AS ""ParentLength"" + FROM ""Prefixes"" ""child"" + JOIN ""Prefixes"" ""parent"" + ON LENGTH(""child"".""Prefix"") > LENGTH(""parent"".""Prefix"") + AND SUBSTR(""child"".""Prefix"", 1, LENGTH(""parent"".""Prefix"")) = ""parent"".""Prefix"" + AND (SUBSTR(""child"".""Prefix"", LENGTH(""parent"".""Prefix"") + 1, 1) = @dirSeparator) ) SELECT - child.""PrefixID"", + ""child"".""PrefixID"", ( - SELECT ParentPrefixID - FROM PrefixParents pp - WHERE pp.ChildPrefixID = child.""PrefixID"" - ORDER BY pp.ParentLength DESC + SELECT ""ParentPrefixID"" + FROM PrefixParents ""pp"" + WHERE ""pp"".""ChildPrefixID"" = ""child"".""PrefixID"" + ORDER BY ""pp"".""ParentLength"" DESC LIMIT 1 - ) AS ParentPrefixID - FROM Prefixes child - ORDER BY child.""Prefix"" ASC + ) AS ""ParentPrefixID"" + FROM ""Prefixes"" ""child"" + ORDER BY ""child"".""Prefix"" ASC ") .SetParameterValue("@dirSeparator", dirSeparator) .SetParameterValue("@filesetid", filesetid); @@ -1301,32 +1301,32 @@ namespace Duplicati.Library.Main.Database // Then fetch the actual data var fetchSql = @$" SELECT - fe.""FilesetID"", - f.""Timestamp"", - fl.""Path"", - COALESCE(b.""Length"", 0) AS ""Size"", + ""fe"".""FilesetID"", + ""f"".""Timestamp"", + ""fl"".""Path"", + COALESCE(""b"".""Length"", 0) AS ""Size"", CASE - WHEN fl.""BlocksetID"" = @FolderBlocksetId + WHEN ""fl"".""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", CASE - WHEN fl.""BlocksetID"" = @SymlinkBlocksetId + WHEN ""fl"".""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""Fileset"" f - ON fe.""FilesetID"" = f.""ID"" - LEFT JOIN ""Blockset"" b - ON fl.""BlocksetID"" = b.""ID"" + ""fe"".""Lastmodified"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""Fileset"" ""f"" + ON ""fe"".""FilesetID"" = ""f"".""ID"" + LEFT JOIN ""Blockset"" ""b"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" {countWhere} ORDER BY - fl.""Path"" ASC, - f.""Timestamp"" ASC + ""fl"".""Path"" ASC, + ""f"".""Timestamp"" ASC LIMIT @limit OFFSET @offset "; @@ -1379,9 +1379,9 @@ namespace Duplicati.Library.Main.Database // Get total count var countSql = @" SELECT COUNT(*) - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" "; cmd.SetCommandAndParameters(countSql + "\n" + countWhere) @@ -1557,35 +1557,35 @@ namespace Duplicati.Library.Main.Database // Fetch results var fetchSql = $@" SELECT - fe.""FilesetID"", - f.""Timestamp"", - pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - COALESCE(b.""Length"", 0) AS ""Size"", + ""fe"".""FilesetID"", + ""f"".""Timestamp"", + ""pp"".""Prefix"" || ""fl"".""Path"" AS ""FullPath"", + COALESCE(""b"".""Length"", 0) AS ""Size"", CASE - WHEN fl.""BlocksetID"" = @FolderBlocksetId + WHEN ""fl"".""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", CASE - WHEN fl.""BlocksetID"" = @SymlinkBlocksetId + WHEN ""fl"".""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""Fileset"" f - ON fe.""FilesetID"" = f.""ID"" - LEFT JOIN ""Blockset"" b - ON fl.""BlocksetID"" = b.""ID"" - INNER JOIN ""PathPrefix"" pp - ON fl.""PrefixID"" = pp.""ID"" + ""fe"".""Lastmodified"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""Fileset"" ""f"" + ON ""fe"".""FilesetID"" = ""f"".""ID"" + LEFT JOIN ""Blockset"" ""b"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" WHERE {whereClause} ORDER BY - pp.""Prefix"" ASC, - fl.""Path"" ASC, - f.""Timestamp"" ASC + ""pp"".""Prefix"" ASC, + ""fl"".""Path"" ASC, + ""f"".""Timestamp"" ASC LIMIT @limit OFFSET @offset "; @@ -1636,11 +1636,11 @@ namespace Duplicati.Library.Main.Database // Calculate the total var countSql = $@" SELECT COUNT(*) - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl - ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp - ON fl.""PrefixID"" = pp.""ID"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" WHERE {whereClause} "; diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 2e1dd466d..9ee6af83a 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -93,8 +93,8 @@ namespace Duplicati.Library.Main.Database SELECT ""B"".""Name"" FROM - ""Fileset"" A, - ""RemoteVolume"" B + ""Fileset"" ""A"", + ""RemoteVolume"" ""B"" WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = @FilesetId @@ -292,8 +292,8 @@ namespace Duplicati.Library.Main.Database INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM - ""FilesetEntry"" A, - ""File"" B + ""FilesetEntry"" ""A"", + ""File"" ""B"" WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" @@ -327,8 +327,8 @@ namespace Duplicati.Library.Main.Database ""B"".""Path"", ""A"".""FileID"" FROM - ""FilesetEntry"" A, - ""File"" B + ""FilesetEntry"" ""A"", + ""File"" ""B"" WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" @@ -374,10 +374,10 @@ namespace Duplicati.Library.Main.Database RemovedFileSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""C"".""Length"") FROM - ""{m_tablename}"" A, - ""FileLookup"" B, - ""Blockset"" C, - ""Metadataset"" D + ""{m_tablename}"" ""A"", + ""FileLookup"" ""B"", + ""Blockset"" ""C"", + ""Metadataset"" ""D"" WHERE ""A"".""FileID"" = ""B"".""ID"" AND ( @@ -390,11 +390,13 @@ namespace Duplicati.Library.Main.Database ", 0) .ConfigureAwait(false); - var filesetcount = await cmd.ExecuteScalarInt64Async($@" + var filesetcount = await cmd.SetCommandAndParameters($@" SELECT COUNT(*) FROM ""FilesetEntry"" - WHERE ""FilesetID"" = {ParentID} - ", 0) + WHERE ""FilesetID"" = @ParentID + ") + .SetParameterValue("@ParentID", ParentID) + .ExecuteScalarInt64Async(0) .ConfigureAwait(false); if (filesetcount == RemovedFileCount) @@ -446,9 +448,9 @@ namespace Duplicati.Library.Main.Database ""B"".""Path"", ""C"".""Length"" FROM - ""{m_tablename}"" A, - ""File"" B, - ""Blockset"" C + ""{m_tablename}"" ""A"", + ""File"" ""B"", + ""Blockset"" ""C"" WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index ef46ae871..f6c621de1 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -165,17 +165,17 @@ namespace Duplicati.Library.Main.Database ""A"".""Hash"" AS ""BlocklistHash"", ""B"".""Length"" FROM - ""BlocklistHash"" A, - ""Blockset"" B + ""BlocklistHash"" ""A"", + ""Blockset"" ""B"" WHERE ""B"".""ID"" = ""A"".""BlocksetID"" - ) C, - ""Block"" D + ) ""C"", + ""Block"" ""D"" WHERE ""C"".""BlocklistHash"" = ""D"".""Hash"" AND ""C"".""BlocklistSize"" = ""D"".""Size"" - ) E, - ""{temptable}"" F + ) ""E"", + ""{temptable}"" ""F"" WHERE ""F"".""BlocklistHash"" = ""E"".""Hash"" ORDER BY @@ -488,8 +488,8 @@ namespace Duplicati.Library.Main.Database THEN -1 ELSE ""B"".""Size"" END AS ""Size"" - FROM ({selectAllBlocks}) A - LEFT OUTER JOIN ""Block"" B + FROM ({selectAllBlocks}) ""A"" + LEFT OUTER JOIN ""Block"" ""B"" ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" ) @@ -520,10 +520,10 @@ namespace Duplicati.Library.Main.Database ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM - ({SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) D, - ""BlocklistHash"" E, - ""Block"" F, - ""Block"" G + ({SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) ""D"", + ""BlocklistHash"" ""E"", + ""Block"" ""F"", + ""Block"" ""G"" WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" @@ -541,7 +541,7 @@ namespace Duplicati.Library.Main.Database FROM ""Blockset"", ""Block"", - ""{m_tempsmalllist}"" S + ""{m_tempsmalllist}"" ""S"" WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" @@ -639,48 +639,49 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the operation is finished. public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, bool hashOnly = false) { + // TODO should values be parameters, rather than hardcoded into the SQL queries? using var cmd = m_connection.CreateCommand(m_rtr); var extra = hashOnly ? "" : $@" UNION SELECT - TS.BlockHash, - TS.BlockSize - FROM {m_tempsmalllist} TS + ""TS"".""BlockHash"", + ""TS"".""BlockSize"" + FROM ""{m_tempsmalllist}"" ""TS"" WHERE NOT EXISTS ( SELECT ""X"" - FROM Block AS B + FROM Block AS ""B"" WHERE - B.Hash = TS.BlockHash - AND B.Size = TS.BlockSize + ""B"".""Hash"" = ""TS"".""BlockHash"" + AND ""B"".""Size"" = ""TS"".""BlockSize"" ) "; var insertBlocksCommand = $@" - INSERT INTO BLOCK ( - Hash, - Size, - VolumeID + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" ) SELECT DISTINCT - BlockHash AS Hash, - BlockSize AS Size, - -1 AS VolumeID + ""BlockHash"" AS ""Hash"", + ""BlockSize"" AS ""Size"", + -1 AS ""VolumeID"" FROM ( SELECT - NB.BlockHash, - MIN({blocksize}, BS.Length - ((NB.""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS BlockSize + ""NB"".""BlockHash"", + MIN({blocksize}, ""BS"".""Length"" - ((""NB"".""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS ""BlockSize"" FROM ( SELECT - TBL.BlockListHash, - TBL.BlockHash, - TBL.""Index"" FROM {m_tempblocklist} TBL - LEFT OUTER JOIN Block B - ON (B.Hash = TBL.BlockHash) - WHERE B.Hash IS NULL - ) NB - JOIN BlocklistHash BH - ON (BH.Hash = NB.BlocklistHash) - JOIN Blockset BS - ON (BS.ID = BH.Blocksetid) + ""TBL"".""BlockListHash"", + ""TBL"".""BlockHash"", + ""TBL"".""Index"" FROM ""{m_tempblocklist}"" ""TBL"" + LEFT OUTER JOIN ""Block"" ""B"" + ON (""B"".""Hash"" = ""TBL"".""BlockHash"") + WHERE ""B"".""Hash"" IS NULL + ) ""NB"" + JOIN ""BlocklistHash"" ""BH"" + ON (""BH"".""Hash"" = ""NB"".""BlocklistHash"") + JOIN ""Blockset"" ""BS"" + ON (""BS"".""ID"" = ""BH"".""Blocksetid"") {extra} ) "; @@ -688,50 +689,51 @@ namespace Duplicati.Library.Main.Database extra = hashOnly ? "" : $@" UNION SELECT - BS.ID AS BlocksetID, + ""BS"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", - BL.ID AS BlockID - FROM {m_tempsmalllist} TS - JOIN Blockset BS + ""BL"".""ID"" AS ""BlockID"" + FROM ""{m_tempsmalllist}"" ""TS"" + JOIN ""Blockset"" ""BS"" ON ( - BS.FullHash = TS.FileHash - AND BS.Length = TS.BlockSize - AND BS.Length <= {blocksize} + ""BS"".""FullHash"" = ""TS"".""FileHash"" + AND ""BS"".""Length"" = ""TS"".""BlockSize"" + AND ""BS"".""Length"" <= {blocksize} ) - JOIN Block BL + JOIN ""Block"" ""BL"" ON ( - BL.Hash = TS.BlockHash - AND BL.Size = TS.BlockSize + ""BL"".""Hash"" = ""TS"".""BlockHash"" + AND ""BL"".""Size"" = ""TS"".""BlockSize"" ) - LEFT OUTER JOIN BlocksetEntry BE + LEFT OUTER JOIN ""BlocksetEntry"" ""BE"" ON ( - BE.BlocksetID = BS.ID - AND BE.""Index"" = 0 + ""BE"".""BlocksetID"" = ""BS"".""ID"" + AND ""BE"".""Index"" = 0 ) - WHERE BE.BlocksetID IS NULL + WHERE ""BE"".""BlocksetID"" IS NULL "; var insertBlocksetEntriesCommand = $@" - INSERT INTO BlocksetEntry ( - BlocksetID, + INSERT INTO ""BlocksetEntry"" ( + ""BlocksetID"", ""Index"", - BlockID + ""BlockID"" ) SELECT DISTINCT - BH.blocksetid, - (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" as FullIndex, - BK.ID AS BlockID - FROM {m_tempblocklist} TBL - JOIN blocklisthash BH - ON (BH.hash = TBL.blocklisthash) - JOIN block BK - ON (BK.Hash = TBL.BlockHash) - LEFT OUTER JOIN BlocksetEntry BE + ""BH"".""blocksetid"", + (""BH"".""Index"" * {blocksize / hashsize}) + ""TBL"".""Index"" as ""FullIndex"", + ""BK"".""ID"" AS ""BlockID"" + FROM ""{m_tempblocklist}"" ""TBL"" + JOIN ""blocklisthash"" ""BH"" + ON (""BH"".""Hash"" = ""TBL"".""blocklisthash"") + JOIN ""Block"" ""BK"" + ON (""BK"".""Hash"" = ""TBL"".""BlockHash"") + LEFT OUTER JOIN ""BlocksetEntry"" ""BE"" ON ( - BE.BlockSetID = BH.BlocksetID - AND BE.""Index"" = (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" + BE.""BlockSetID"" = ""BH"".""BlocksetID"" + AND BE.""Index"" = (""BH"".""Index"" * {blocksize / hashsize})+""TBL"".""Index"" ) - WHERE BE.BlockSetID IS NULL - {extra}"; + WHERE ""BE"".""BlockSetID"" IS NULL + {extra} + "; try { @@ -1137,14 +1139,14 @@ namespace Duplicati.Library.Main.Database FROM ""Block"" WHERE ""VolumeID"" < 0 - AND SIZE > 0 + AND ""Size"" > 0 "; var missingBlocklistVolumes = $@" SELECT ""VolumeID"" FROM ""Block"", - ({missingBlocklistEntries}) A + ({missingBlocklistEntries}) ""A"" WHERE ""A"".""Hash"" = ""Block"".""Hash"" "; diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 5f9019c2b..2e1a252c7 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -174,11 +174,11 @@ namespace Duplicati.Library.Main.Database { using var cmd = m_connection.CreateCommand(@" SELECT - ID, - Timestamp - FROM Fileset - WHERE ID IN ( - SELECT FilesetID + ""ID"", + ""Timestamp"" + FROM ""Fileset"" + WHERE ""ID"" IN ( + SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN ( SELECT ""ID"" @@ -661,32 +661,32 @@ namespace Duplicati.Library.Main.Database await cmd.SetCommandAndParameters($@" CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS SELECT - b.""Hash"" AS ""BlockHash"", - bs.""Id"" AS ""BlocksetId"", - bs.""Length"" AS ""BlocksetLength"", - bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", + ""b"".""Hash"" AS ""BlockHash"", + ""bs"".""Id"" AS ""BlocksetId"", + ""bs"".""Length"" AS ""BlocksetLength"", + ""bse"".""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", ( - SELECT blh.""Hash"" - FROM ""BlocklistHash"" blh + SELECT ""blh"".""Hash"" + FROM ""BlocklistHash"" ""blh"" WHERE - blh.""BlocksetID"" = bs.""ID"" - AND blh.""Index"" == bse.""Index"" / @HashesPerBlock + ""blh"".""BlocksetID"" = ""bs"".""ID"" + AND ""blh"".""Index"" == ""bse"".""Index"" / @HashesPerBlock LIMIT 1 ) AS ""BlocklistHashHash"", - bse.""Index"" AS ""BlocksetEntryIndex"" - FROM ""BlocksetEntry"" bse - JOIN ""Block"" b - ON b.""ID"" = bse.""BlockID"" - JOIN ""Blockset"" bs - ON bs.""ID"" = bse.""BlocksetID"" + ""bse"".""Index"" AS ""BlocksetEntryIndex"" + FROM ""BlocksetEntry"" ""bse"" + JOIN ""Block"" ""b"" + ON ""b"".""ID"" = ""bse"".""BlockID"" + JOIN ""Blockset"" ""bs"" + ON ""bs"".""ID"" = ""bse"".""BlocksetID"" WHERE EXISTS ( SELECT 1 - FROM ""BlocklistHash"" blh - JOIN ""{m_tablename}"" mt - ON mt.""Hash"" = blh.""Hash"" + FROM ""BlocklistHash"" ""blh"" + JOIN ""{m_tablename}"" ""mt"" + ON ""mt"".""Hash"" = ""blh"".""Hash"" WHERE - blh.""BlocksetID"" = bs.""ID"" - AND mt.""Restored"" = @Restored + ""blh"".""BlocksetID"" = ""bs"".""ID"" + AND ""mt"".""Restored"" = @Restored ) ") .SetParameterValue("@HashesPerBlock", hashesPerBlock) @@ -781,12 +781,12 @@ namespace Duplicati.Library.Main.Database ""VolumeID"" ) SELECT - b.""ID"", - b.""VolumeID"" - FROM ""Block"" b + ""b"".""ID"", + ""b"".""VolumeID"" + FROM ""Block"" ""b"" WHERE - b.""VolumeID"" = @SourceVolumeId - AND (b.""Hash"", b.""Size"") IN ( + ""b"".""VolumeID"" = @SourceVolumeId + AND (""b"".""Hash"", ""b"".""Size"") IN ( SELECT ""Hash"", ""Size"" @@ -979,7 +979,7 @@ namespace Duplicati.Library.Main.Database var sql_count = @" SELECT COUNT(*) FROM ( - SELECT DISTINCT c1 + SELECT DISTINCT ""C1"" FROM ( SELECT COUNT(*) AS ""C1"" FROM ( @@ -1003,7 +1003,8 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tablename}"" - AS SELECT * FROM ""File"" + AS SELECT * + FROM ""File"" ") .ConfigureAwait(false); @@ -1132,7 +1133,7 @@ namespace Duplicati.Library.Main.Database UPDATE ""FileLookup"" SET ""MetadataID"" = ( SELECT ""MetadataID"" - FROM ""{tablename}"" A + FROM ""{tablename}"" ""A"" WHERE ""A"".""ID"" = ""FileLookup"".""ID"" ) ") @@ -1691,8 +1692,8 @@ namespace Duplicati.Library.Main.Database SELECT COUNT(*) FROM ( SELECT ""A"".""VolumeID"" - FROM ""{tablename}"" B - LEFT OUTER JOIN ""Block"" A + FROM ""{tablename}"" ""B"" + LEFT OUTER JOIN ""Block"" ""A"" ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"" ) @@ -1731,7 +1732,7 @@ namespace Duplicati.Library.Main.Database ""C"".""Hash"", ""C"".""Size"" FROM - ""BlocksetEntry"" A, + ""BlocksetEntry"" ""A"", ( SELECT ""Y"".""BlocksetID"", @@ -1740,15 +1741,15 @@ namespace Duplicati.Library.Main.Database ""Z"".""Size"" AS ""BlocklistSize"", ""Z"".""ID"" AS ""BlocklistHashBlockID"" FROM - ""BlocklistHash"" Y, - ""Block"" Z + ""BlocklistHash"" ""Y"", + ""Block"" ""Z"" WHERE ""Y"".""Hash"" = ""Z"".""Hash"" AND ""Y"".""Hash"" = @Hash AND ""Z"".""Size"" = @Size LIMIT 1 - ) B, - ""Block"" C + ) ""B"", + ""Block"" ""C"" WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""A"".""BlockID"" = ""C"".""ID"" diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 5420c6044..63bab97c6 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1505,7 +1505,7 @@ namespace Duplicati.Library.Main.Database ""{blocktablename}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", - ""Block"" E + ""Block"" ""E"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" @@ -1650,7 +1650,8 @@ namespace Duplicati.Library.Main.Database ) VALUES ( @Hash, - @Size) + @Size + ) "); foreach (var s in curvolume.Blocks) { @@ -1945,16 +1946,16 @@ namespace Duplicati.Library.Main.Database // Order by length descending, so that larger files are restored first. using var cmd = m_connection.CreateCommand($@" SELECT - F.ID, - F.Path, - F.TargetPath, - IFNULL(B.FullHash, ''), - IFNULL(B.Length, 0) AS ""Length"", - F.BlocksetID - FROM ""{m_tempfiletable}"" F - LEFT JOIN Blockset B - ON F.BlocksetID = B.ID - WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID} + ""F"".""ID"", + ""F"".""Path"", + ""F"".""TargetPath"", + IFNULL(""B"".""FullHash"", ''), + IFNULL(""B"".""Length"", 0) AS ""Length"", + ""F"".""BlocksetID"" + FROM ""{m_tempfiletable}"" ""F"" + LEFT JOIN ""Blockset"" ""B"" + ON ""F"".""BlocksetID"" = ""B"".""ID"" + WHERE ""F"".""BlocksetID"" != {FOLDER_BLOCKSET_ID} ORDER BY ""Length"" DESC ") .SetTransaction(m_rtr); @@ -1980,17 +1981,17 @@ namespace Duplicati.Library.Main.Database cmd.SetTransaction(m_rtr); using var rd = await cmd.ExecuteReaderAsync($@" SELECT - F.ID, + ""F"".""ID"", '', - F.TargetPath, + ""F"".""TargetPath"", '', 0, {FOLDER_BLOCKSET_ID} - FROM ""{m_tempfiletable}"" F + FROM ""{m_tempfiletable}"" ""F"" WHERE - F.BlocksetID = {FOLDER_BLOCKSET_ID} - AND F.MetadataID IS NOT NULL - AND F.MetadataID >= 0 + ""F"".""BlocksetID"" = {FOLDER_BLOCKSET_ID} + AND ""F"".""MetadataID"" IS NOT NULL + AND ""F"".""MetadataID"" >= 0 ") .ConfigureAwait(false); @@ -2153,11 +2154,11 @@ namespace Duplicati.Library.Main.Database { using var cmd = m_connection.CreateCommand(@" SELECT - Name, - Size, - Hash - FROM RemoteVolume - WHERE ID = @VolumeID + ""Name"", + ""Size"", + ""Hash"" + FROM ""RemoteVolume"" + WHERE ""ID"" = @VolumeID ") .SetTransaction(m_rtr) .SetParameterValue("@VolumeID", VolumeID); @@ -2734,7 +2735,7 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters($@" UPDATE ""{m_tempfiletable}"" SET ""LocalSourceExists"" = 1 - WHERE Path = @Path + WHERE ""Path"" = @Path "); cmdReader.SetCommandAndParameters($@" SELECT DISTINCT ""{m_tempfiletable}"".""Path"" @@ -2819,10 +2820,10 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" - SET LatestBlocksetId = ( - SELECT BlocksetId + SET ""LatestBlocksetId"" = ( + SELECT ""BlocksetId"" FROM ""{m_latestblocktable}"" - WHERE Path = ""{m_tempfiletable}"".Path + WHERE ""Path"" = ""{m_tempfiletable}"".""Path"" ) ") .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 9423aff23..e4fbfd300 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -266,7 +266,7 @@ namespace Duplicati.Library.Main.Database @State1, @State2 ) - ) A, + ) ""A"", ""Fileset"" {whereClause} ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" @@ -584,10 +584,10 @@ namespace Duplicati.Library.Main.Database AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" - ) A - LEFT OUTER JOIN ""Blockset"" B + ) ""A"" + LEFT OUTER JOIN ""Blockset"" ""B"" ON ""B"".""ID"" = ""A"".""FileBlocksetID"" - LEFT OUTER JOIN ""Blockset"" C + LEFT OUTER JOIN ""Blockset"" ""C"" ON ""C"".""ID""=""A"".""MetadataBlocksetID"" "; @@ -617,8 +617,8 @@ namespace Duplicati.Library.Main.Database @TypeModified AS ""Type"", ""E"".""Path"" AS ""Path"" FROM - ""{m_tablename}"" E, - ""{cmpName}"" D + ""{m_tablename}"" ""E"", + ""{cmpName}"" ""D"" WHERE ""D"".""Path"" = ""E"".""Path"" AND ( @@ -778,8 +778,8 @@ namespace Duplicati.Library.Main.Database ""A"".""Hash"", ""A"".""Size"" FROM - ""Remotevolume"" A, - ""Remotevolume"" B, + ""Remotevolume"" ""A"", + ""Remotevolume"" ""B"", ""IndexBlockLink"" WHERE ""B"".""Name"" = @Name @@ -814,8 +814,8 @@ namespace Duplicati.Library.Main.Database @TypeModified AS ""Type"", ""E"".""Name"" AS ""Name"" FROM - ""{m_tablename}"" E, - ""{cmpName}"" D + ""{m_tablename}"" ""E"", + ""{cmpName}"" ""D"" WHERE ""D"".""Name"" = ""E"".""Name"" AND ( @@ -1183,87 +1183,93 @@ namespace Duplicati.Library.Main.Database ""Size"" ) SELECT - b.""Hash"", - b.""Size"" - FROM Block b + ""b"".""Hash"", + ""b"".""Size"" + FROM ""Block"" ""b"" JOIN ( SELECT - blh.""Hash"", + ""blh"".""Hash"", CASE - WHEN blh.""Index"" = (((bs.""Length"" + {blockSize} - 1) / {blockSize} - 1) / {hashesPerBlock}) - AND ((bs.""Length"" + {blockSize} - 1) / {blockSize}) % {hashesPerBlock} != 0 - THEN {hashSize} * ((bs.""Length"" + {blockSize} - 1) / {blockSize} % {hashesPerBlock}) + WHEN ""blh"".""Index"" = (((""bs"".""Length"" + {blockSize} - 1) / {blockSize} - 1) / {hashesPerBlock}) + AND ((""bs"".""Length"" + {blockSize} - 1) / {blockSize}) % {hashesPerBlock} != 0 + THEN {hashSize} * ((""bs"".""Length"" + {blockSize} - 1) / {blockSize} % {hashesPerBlock}) ELSE {hashSize} * {hashesPerBlock} END AS ""Size"" - FROM BlocklistHash blh - JOIN Blockset bs - ON bs.""ID"" = blh.""BlocksetID"" - ) expected + FROM ""BlocklistHash"" ""blh"" + JOIN ""Blockset"" ""bs"" + ON ""bs"".""ID"" = ""blh"".""BlocksetID"" + ) ""Expected"" ON - b.""Hash"" = expected.""Hash"" - AND b.""Size"" = expected.""Size"" - WHERE b.""VolumeID"" IN ( - SELECT ibl.""BlockVolumeID"" - FROM Remotevolume idx - JOIN IndexBlockLink ibl - ON ibl.""IndexVolumeID"" = idx.""ID"" - WHERE idx.""Name"" = @Name + ""b"".""Hash"" = ""Expected"".""Hash"" + AND ""b"".""Size"" = ""Expected"".""Size"" + WHERE ""b"".""VolumeID"" IN ( + SELECT ""ibl"".""BlockVolumeID"" + FROM ""Remotevolume"" ""idx"" + JOIN IndexBlockLink ""ibl"" + ON ""ibl"".""IndexVolumeID"" = ""idx"".""ID"" + WHERE ""idx"".""Name"" = @Name ); "; var compare = $@" WITH - Expected AS ( + ""Expected"" AS ( SELECT ""Hash"", ""Size"" FROM ""{cmpName}"" ), - Actual AS ( + ""Actual"" AS ( SELECT ""Hash"", ""Size"" FROM ""{m_tablename}"" ), - Extra AS ( - SELECT @TypeExtra AS Type, a.""Hash"" - FROM Actual a - LEFT JOIN Expected e + ""Extra"" AS ( + SELECT + @TypeExtra AS ""Type"", + ""a"".""Hash"" + FROM ""Actual"" ""a"" + LEFT JOIN ""Expected"" ""e"" ON - a.""Hash"" = e.""Hash"" - AND a.""Size"" = e.""Size"" - WHERE e.""Hash"" IS NULL + ""a"".""Hash"" = ""e"".""Hash"" + AND ""a"".""Size"" = ""e"".""Size"" + WHERE ""e"".""Hash"" IS NULL ), - Missing AS ( - SELECT @TypeMissing AS Type, e.""Hash"" - FROM Expected e - LEFT JOIN Actual a + ""Missing"" AS ( + SELECT + @TypeMissing AS ""Type"", + ""e"".""Hash"" + FROM ""Expected"" ""e"" + LEFT JOIN ""Actual"" ""a"" ON - a.""Hash"" = e.""Hash"" - AND a.""Size"" = e.""Size"" - WHERE a.""Hash"" IS NULL + ""a"".""Hash"" = ""e"".""Hash"" + AND ""a"".""Size"" = ""e"".""Size"" + WHERE ""a"".""Hash"" IS NULL ), - Modified AS ( - SELECT @TypeModified AS Type, a.""Hash"" - FROM Actual a - JOIN Expected e - ON a.""Hash"" = e.""Hash"" + ""Modified"" AS ( + SELECT + @TypeModified AS ""Type"", + ""a"".""Hash"" + FROM ""Actual"" ""a"" + JOIN ""Expected"" ""e"" + ON ""a"".""Hash"" = ""e"".""Hash"" WHERE - a.""Size"" != e.""Size"" + ""a"".""Size"" != ""e"".""Size"" AND NOT EXISTS ( SELECT 1 - FROM Extra x - WHERE x.""Hash"" = a.""Hash"" + FROM ""Extra"" ""x"" + WHERE ""x"".""Hash"" = ""a"".""Hash"" ) ) SELECT * - FROM Extra + FROM ""Extra"" UNION SELECT * - FROM Missing + FROM ""Missing"" UNION SELECT * - FROM Modified; + FROM ""Modified""; "; var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"""; From 859b65054d539579e3e23b3672044c652fc4afba Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 11:18:25 +0200 Subject: [PATCH 317/362] Simplified using statements that uses databases or accesses their methods --- Duplicati/Library/Main/Controller.cs | 60 +- .../Main/Database/LocalBackupDatabase.cs | 537 +++++----- .../Main/Database/LocalBugReportDatabase.cs | 100 +- .../Library/Main/Database/LocalDatabase.cs | 58 +- .../Main/Database/LocalDeleteDatabase.cs | 390 ++++--- .../Database/LocalListAffectedDatabase.cs | 68 +- .../Database/LocalListBrokenFilesDatabase.cs | 74 +- .../Main/Database/LocalListChangesDatabase.cs | 170 ++-- .../Main/Database/LocalListDatabase.cs | 382 ++++--- .../Main/Database/LocalPurgeDatabase.cs | 136 ++- .../Main/Database/LocalRecreateDatabase.cs | 144 ++- .../Main/Database/LocalTestDatabase.cs | 358 ++++--- .../Library/Main/Operation/BackupHandler.cs | 24 +- .../Library/Main/Operation/DeleteHandler.cs | 26 +- .../Library/Main/Operation/ListAffected.cs | 74 +- .../Main/Operation/ListBrokenFilesHandler.cs | 16 +- .../Main/Operation/ListChangesHandler.cs | 216 ++-- .../Main/Operation/ListControlFilesHandler.cs | 78 +- .../Main/Operation/ListFilesHandler.cs | 130 ++- .../Main/Operation/PurgeBrokenFilesHandler.cs | 283 +++--- .../Main/Operation/PurgeFilesHandler.cs | 24 +- .../Library/Main/Operation/RepairHandler.cs | 36 +- .../Operation/RestoreControlFilesHandler.cs | 102 +- .../Library/Main/Operation/RestoreHandler.cs | 952 +++++++++--------- .../Library/Main/Operation/TestHandler.cs | 28 +- 25 files changed, 2175 insertions(+), 2291 deletions(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 3d0a25912..b16d9b920 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -277,17 +277,15 @@ namespace Duplicati.Library.Main // and uses the same prefix (see issues #2678, #3845, and #4244). if (File.Exists(m_options.Dbpath)) { - using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false)) - { - var dbRemoteVolumes = db.GetRemoteVolumes(); - var dbRemoteFiles = await dbRemoteVolumes - .Select(x => x.Name) - .ToHashSetAsync() - .ConfigureAwait(false); - list = list.Where(x => - dbRemoteFiles.Contains(x.File.Name) - ).ToList(); - } + using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false); + var dbRemoteVolumes = db.GetRemoteVolumes(); + var dbRemoteFiles = await dbRemoteVolumes + .Select(x => x.Name) + .ToHashSetAsync() + .ConfigureAwait(false); + list = list.Where(x => + dbRemoteFiles.Contains(x.File.Name) + ).ToList(); } result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Deleting); @@ -524,8 +522,8 @@ namespace Duplicati.Library.Main // instead of relying on the operations to correctly toggle the flag if (File.Exists(m_options.Dbpath) && !m_options.NoLocalDb) { - using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache).Await()) - backend.StopRunnerAndFlushMessages(db).Await(); + using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache).Await(); + backend.StopRunnerAndFlushMessages(db).Await(); } else { @@ -540,27 +538,25 @@ namespace Duplicati.Library.Main if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) { - using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache).Await()) - { - db.WriteResults(result).Await(); - db.PurgeLogData(m_options.LogRetention).Await(); - db.PurgeDeletedVolumes(DateTime.UtcNow).Await(); + using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache).Await(); + db.WriteResults(result).Await(); + db.PurgeLogData(m_options.LogRetention).Await(); + db.PurgeDeletedVolumes(DateTime.UtcNow).Await(); - // Vacuum is done AFTER the results are written to the database - // This means that the information about the vacuum is not stored in the database, - // but will be reported in the log output and messages sent with any of the reporting modules - if (m_options.AutoVacuum && result is IResultsWithVacuum vacuumResults && result is BasicResults basicResults) + // Vacuum is done AFTER the results are written to the database + // This means that the information about the vacuum is not stored in the database, + // but will be reported in the log output and messages sent with any of the reporting modules + if (m_options.AutoVacuum && result is IResultsWithVacuum vacuumResults && result is BasicResults basicResults) + { + try { - try - { - vacuumResults.VacuumResults = new VacuumResults(basicResults); - new Operation.VacuumHandler(m_options, (VacuumResults)vacuumResults.VacuumResults) - .RunAsync().Await(); - } - catch (Exception ex) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToVacuum", ex, "Failed to vacuum database"); - } + vacuumResults.VacuumResults = new VacuumResults(basicResults); + new Operation.VacuumHandler(m_options, (VacuumResults)vacuumResults.VacuumResults) + .RunAsync().Await(); + } + catch (Exception ex) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToVacuum", ex, "Failed to vacuum database"); } } } diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 5590124db..281bdcec7 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -401,10 +401,9 @@ namespace Duplicati.Library.Main.Database if (deletedBlockCacheSizeLong >= 0) { - using (var cmd = dbnew.Connection.CreateCommand()) - { - dbnew.m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - await cmd.SetCommandAndParameters($@" + using var cmd = dbnew.Connection.CreateCommand(); + dbnew.m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await cmd.SetCommandAndParameters($@" CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS SELECT MAX(""ID"") AS ""ID"", @@ -420,78 +419,78 @@ namespace Duplicati.Library.Main.Database ""Hash"", ""Size"" ") - .ExpandInClauseParameterMssqlite("@States", [ - RemoteVolumeState.Deleted, + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Deleted, RemoteVolumeState.Deleting - ]) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + ]) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$" + var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$" SELECT COUNT(*) FROM ""{dbnew.m_tempDeletedBlockTable}"" ", 0) + .ConfigureAwait(false); + + // There are no deleted blocks, so we can drop the table + if (deletedBlocks == 0) + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""") .ConfigureAwait(false); + dbnew.m_tempDeletedBlockTable = null; - // There are no deleted blocks, so we can drop the table - if (deletedBlocks == 0) - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""") - .ConfigureAwait(false); - dbnew.m_tempDeletedBlockTable = null; - - } - // The deleted blocks are small enough to fit in memory - else if (deletedBlocks <= deletedBlockCacheSizeLong) - { - dbnew.m_deletedBlockLookup = new Dictionary>(); - cmd.SetCommandAndParameters(@$" + } + // The deleted blocks are small enough to fit in memory + else if (deletedBlocks <= deletedBlockCacheSizeLong) + { + dbnew.m_deletedBlockLookup = new Dictionary>(); + cmd.SetCommandAndParameters(@$" SELECT ""ID"", ""Hash"", ""Size"" FROM ""{dbnew.m_tempDeletedBlockTable}"" ") - .ConfigureAwait(false); + .ConfigureAwait(false); - using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await reader.ReadAsync().ConfigureAwait(false)) - { - var id = reader.ConvertValueToInt64(0); - var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null"); - var size = reader.ConvertValueToInt64(2); + using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await reader.ReadAsync().ConfigureAwait(false)) + { + var id = reader.ConvertValueToInt64(0); + var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null"); + var size = reader.ConvertValueToInt64(2); - if (!dbnew.m_deletedBlockLookup.TryGetValue(hash, out var sizes)) - dbnew.m_deletedBlockLookup[hash] = sizes = new Dictionary(); - sizes[size] = id; - } + if (!dbnew.m_deletedBlockLookup.TryGetValue(hash, out var sizes)) + dbnew.m_deletedBlockLookup[hash] = sizes = new Dictionary(); + sizes[size] = id; + } - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""") - .ConfigureAwait(false); - dbnew.m_tempDeletedBlockTable = null; - } - // The deleted blocks are too large to fit in memory, so we use a temporary table - else - { - await cmd.ExecuteNonQueryAsync($@" + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""") + .ConfigureAwait(false); + dbnew.m_tempDeletedBlockTable = null; + } + // The deleted blocks are too large to fit in memory, so we use a temporary table + else + { + await cmd.ExecuteNonQueryAsync($@" CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" ON ""{dbnew.m_tempDeletedBlockTable}"" ( ""Hash"", ""Size"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - dbnew.m_findindeletedCommand = await dbnew.Connection.CreateCommandAsync($@" + dbnew.m_findindeletedCommand = await dbnew.Connection.CreateCommandAsync($@" SELECT ""ID"" FROM ""{dbnew.m_tempDeletedBlockTable}"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size ") - .ConfigureAwait(false); + .ConfigureAwait(false); - dbnew.m_moveblockfromdeletedCommand = await dbnew.Connection.CreateCommandAsync(@$" + dbnew.m_moveblockfromdeletedCommand = await dbnew.Connection.CreateCommandAsync(@$" INSERT INTO ""Block"" ( ""Hash"", ""Size"", @@ -512,13 +511,13 @@ namespace Duplicati.Library.Main.Database SELECT last_insert_rowid() ") - .ConfigureAwait(false); + .ConfigureAwait(false); - } + } - if (deletedBlocks > 0) - { - dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(@$" + if (deletedBlocks > 0) + { + dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(@$" INSERT INTO ""Block"" ( ""Hash"", ""Size"", @@ -536,9 +535,7 @@ namespace Duplicati.Library.Main.Database SELECT last_insert_rowid() ") - .ConfigureAwait(false); - } - + .ConfigureAwait(false); } } @@ -1100,13 +1097,13 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null).ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) - { - oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - length = rd.ConvertValueToInt64(2); - return (rd.ConvertValueToInt64(0), oldModified, length); - } + using var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null).ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) + { + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + length = rd.ConvertValueToInt64(2); + return (rd.ConvertValueToInt64(0), oldModified, length); + } } else { @@ -1115,13 +1112,13 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries).ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) - { - length = -1; - oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - return (rd.ConvertValueToInt64(0), oldModified, length); - } + using var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries).ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) + { + length = -1; + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + return (rd.ConvertValueToInt64(0), oldModified, length); + } } oldModified = new DateTime(0, DateTimeKind.Utc); @@ -1143,37 +1140,37 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = await m_findfileCommand.ExecuteReaderAsync().ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) - { - oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - lastFileSize = rd.ConvertValueToInt64(2); - oldMetahash = rd.ConvertValueToString(3); - oldMetasize = rd.ConvertValueToInt64(4); + using var rd = await m_findfileCommand.ExecuteReaderAsync().ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) + { + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + lastFileSize = rd.ConvertValueToInt64(2); + oldMetahash = rd.ConvertValueToString(3); + oldMetasize = rd.ConvertValueToInt64(4); - return ( - rd.ConvertValueToInt64(0), - oldModified, - lastFileSize, - oldMetahash, - oldMetasize - ); - } - else - { - oldModified = new DateTime(0, DateTimeKind.Utc); - lastFileSize = -1; - oldMetahash = null; - oldMetasize = -1; + return ( + rd.ConvertValueToInt64(0), + oldModified, + lastFileSize, + oldMetahash, + oldMetasize + ); + } + else + { + oldModified = new DateTime(0, DateTimeKind.Utc); + lastFileSize = -1; + oldMetahash = null; + oldMetasize = -1; - return ( - -1, - oldModified, - lastFileSize, - oldMetahash, - oldMetasize - ); - } + return ( + -1, + oldModified, + lastFileSize, + oldMetahash, + oldMetasize + ); + } } /// @@ -1187,12 +1184,12 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid); - using (var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync().ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) - return ( - rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null"), - rd.ConvertValueToInt64(0) - ); + using var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync().ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) + return ( + rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null"), + rd.ConvertValueToInt64(0) + ); return null; } @@ -1245,8 +1242,8 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the size of the last written DBlock volume, or -1 if no such volume exists. public async Task GetLastWrittenDBlockVolumeSize() { - using (var cmd = m_connection.CreateCommand(m_rtr)) - return await cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(m_rtr); + return await cmd.SetCommandAndParameters(@" SELECT ""Size"" FROM ""RemoteVolume"" WHERE @@ -1255,10 +1252,10 @@ namespace Duplicati.Library.Main.Database ORDER BY ""ID"" DESC LIMIT 1 ") - .SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExecuteScalarInt64Async(-1) - .ConfigureAwait(false); + .SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); } /// @@ -1303,17 +1300,16 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains a tuple with the count of files and the total size of files in the last backup fileset. internal async Task> GetLastBackupFileCountAndSize() { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var lastFilesetId = await cmd.ExecuteScalarInt64Async(@" + using var cmd = m_connection.CreateCommand(m_rtr); + var lastFilesetId = await cmd.ExecuteScalarInt64Async(@" SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1 ") - .ConfigureAwait(false); + .ConfigureAwait(false); - var count = await cmd.SetCommandAndParameters(@" + var count = await cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" @@ -1325,13 +1321,13 @@ namespace Duplicati.Library.Main.Database @SymlinkBlocksetId ) ") - .SetParameterValue("@FilesetId", lastFilesetId) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64Async(-1) - .ConfigureAwait(false); + .SetParameterValue("@FilesetId", lastFilesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); - var size = await cmd.SetCommandAndParameters(@" + var size = await cmd.SetCommandAndParameters(@" SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", @@ -1346,14 +1342,13 @@ namespace Duplicati.Library.Main.Database @SymlinkBlocksetId ) ") - .SetParameterValue("@FilesetId", lastFilesetId) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64Async(-1) - .ConfigureAwait(false); + .SetParameterValue("@FilesetId", lastFilesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); - return new Tuple(count, size); - } + return new Tuple(count, size); } /// @@ -1363,13 +1358,11 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the change statistics are updated. internal async Task UpdateChangeStatistics(BackupResults results) { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var prevFileSetId = await GetPreviousFilesetID(cmd) - .ConfigureAwait(false); - await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId) - .ConfigureAwait(false); - } + using var cmd = m_connection.CreateCommand(m_rtr); + var prevFileSetId = await GetPreviousFilesetID(cmd) + .ConfigureAwait(false); + await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId) + .ConfigureAwait(false); } /// @@ -1395,17 +1388,16 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the files are appended. public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) { - using (var cmd = m_connection.CreateCommand()) - using (var cmdDelete = m_connection.CreateCommand()) - { - long lastFilesetId = prevId < 0 ? - await GetPreviousFilesetID(cmd, timestamp, filesetid) - .ConfigureAwait(false) - : - prevId; + using var cmd = m_connection.CreateCommand(); + using var cmdDelete = m_connection.CreateCommand(); + long lastFilesetId = prevId < 0 ? + await GetPreviousFilesetID(cmd, timestamp, filesetid) + .ConfigureAwait(false) + : + prevId; - await cmd.SetTransaction(m_rtr) - .SetCommandAndParameters(@" + await cmd.SetTransaction(m_rtr) + .SetCommandAndParameters(@" INSERT INTO ""FilesetEntry"" ( ""FilesetID"", ""FileID"", @@ -1430,20 +1422,20 @@ namespace Duplicati.Library.Main.Database ) ) ") - .SetParameterValue("@CurrentFilesetId", filesetid) - .SetParameterValue("@PreviousFilesetId", lastFilesetId) - .ExecuteNonQueryAsync(m_logQueries) + .SetParameterValue("@CurrentFilesetId", filesetid) + .SetParameterValue("@PreviousFilesetId", lastFilesetId) + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); + + if (deleted != null) + { + using var tmplist = await TemporaryDbValueList + .CreateAsync(this, deleted) .ConfigureAwait(false); - if (deleted != null) - { - using var tmplist = await TemporaryDbValueList - .CreateAsync(this, deleted) - .ConfigureAwait(false); - - await ( - await cmdDelete.SetTransaction(m_rtr) - .SetCommandAndParameters(@" + await ( + await cmdDelete.SetTransaction(m_rtr) + .SetCommandAndParameters(@" DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId @@ -1453,16 +1445,15 @@ namespace Duplicati.Library.Main.Database WHERE ""Path"" IN (@Paths) ) ") - .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameterMssqliteAsync("@Paths", tmplist) - .ConfigureAwait(false) - ) - .ExecuteNonQueryAsync(m_logQueries) - .ConfigureAwait(false); - } - - await m_rtr.CommitAsync().ConfigureAwait(false); + .SetParameterValue("@FilesetId", filesetid) + .ExpandInClauseParameterMssqliteAsync("@Paths", tmplist) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync(m_logQueries) + .ConfigureAwait(false); } + + await m_rtr.CommitAsync().ConfigureAwait(false); } /// @@ -1497,20 +1488,19 @@ namespace Duplicati.Library.Main.Database return; } - using (var cmd = m_connection.CreateCommand()) - using (var cmdDelete = m_connection.CreateCommand()) - { - long lastFilesetId = prevFileSetId < 0 ? - await GetPreviousFilesetID(cmd, timestamp, fileSetId) - .ConfigureAwait(false) - : - prevFileSetId; + using var cmd = m_connection.CreateCommand(); + using var cmdDelete = m_connection.CreateCommand(); + long lastFilesetId = prevFileSetId < 0 ? + await GetPreviousFilesetID(cmd, timestamp, fileSetId) + .ConfigureAwait(false) + : + prevFileSetId; - // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup - var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - await cmd - .SetTransaction(m_rtr) - .SetCommandAndParameters($@" + // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup + var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await cmd + .SetTransaction(m_rtr) + .SetCommandAndParameters($@" CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", @@ -1530,21 +1520,21 @@ namespace Duplicati.Library.Main.Database ) ) ") - .SetParameterValue("@PreviousFilesetId", lastFilesetId) - .SetParameterValue("@CurrentFilesetId", fileSetId) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@PreviousFilesetId", lastFilesetId) + .SetParameterValue("@CurrentFilesetId", fileSetId) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - // now we need to remove, from the above, any entries that were enumerated by the - // UNC-driven backup - cmdDelete.SetTransaction(m_rtr) - .SetCommandAndParameters($@" + // now we need to remove, from the above, any entries that were enumerated by the + // UNC-driven backup + cmdDelete.SetTransaction(m_rtr) + .SetCommandAndParameters($@" DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = @FileId "); - // enumerate files from new temporary file set, and remove any entries handled by UNC - cmd.SetCommandAndParameters($@" + // enumerate files from new temporary file set, and remove any entries handled by UNC + cmd.SetCommandAndParameters($@" SELECT ""f"".""Path"", ""fs"".""FileID"", @@ -1562,19 +1552,19 @@ namespace Duplicati.Library.Main.Database ON ""f"".""BlocksetID"" = ""bs"".""ID""; "); - await foreach (var row in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) - { - var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path"); - var size = row.ConvertValueToInt64(3); + await foreach (var row in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + { + var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path"); + var size = row.ConvertValueToInt64(3); - if (exclusionPredicate(path, size)) - await cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); - } + if (exclusionPredicate(path, size)) + await cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + } - // now copy the temporary table into the FileSetEntry table - await cmd.SetCommandAndParameters($@" + // now copy the temporary table into the FileSetEntry table + await cmd.SetCommandAndParameters($@" INSERT INTO ""FilesetEntry"" ( ""FilesetID"", ""FileID"", @@ -1586,12 +1576,11 @@ namespace Duplicati.Library.Main.Database ""Lastmodified"" FROM ""{tempFileSetTable}"" ") - .SetParameterValue("@FilesetId", fileSetId) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@FilesetId", fileSetId) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); - } + await m_rtr.CommitAsync().ConfigureAwait(false); } /// @@ -1658,9 +1647,9 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Verified.ToString() ]); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } /// @@ -1673,9 +1662,8 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the block is moved. public async Task MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid) { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var c = await cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(m_rtr); + var c = await cmd.SetCommandAndParameters(@" UPDATE ""Block"" SET ""VolumeID"" = @NewVolumeId WHERE @@ -1683,16 +1671,15 @@ namespace Duplicati.Library.Main.Database AND ""Size"" = @Size AND ""VolumeID"" = @PreviousVolumeId ") - .SetParameterValue("@NewVolumeId", targetvolumeid) - .SetParameterValue("@Hash", blockkey) - .SetParameterValue("@Size", size) - .SetParameterValue("@PreviousVolumeId", sourcevolumeid) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@NewVolumeId", targetvolumeid) + .SetParameterValue("@Hash", blockkey) + .SetParameterValue("@Size", size) + .SetParameterValue("@PreviousVolumeId", sourcevolumeid) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - if (c != 1) - throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}"); - } + if (c != 1) + throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}"); } /// @@ -1706,22 +1693,20 @@ namespace Duplicati.Library.Main.Database { var volumeid = await GetRemoteVolumeID(name).ConfigureAwait(false); - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var c = await cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(m_rtr); + var c = await cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = @VolumeId ") - .SetParameterValue("@VolumeId", volumeid) - .ExecuteScalarInt64Async(-1) - .ConfigureAwait(false); + .SetParameterValue("@VolumeId", volumeid) + .ExecuteScalarInt64Async(-1) + .ConfigureAwait(false); - if (c != 0) - throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); + if (c != 0) + throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); - await RemoveRemoteVolume(name).ConfigureAwait(false); - } + await RemoveRemoteVolume(name).ConfigureAwait(false); } /// @@ -1732,10 +1717,9 @@ namespace Duplicati.Library.Main.Database public async Task GetBlocklistHashes(string name) { var volumeid = GetRemoteVolumeID(name); - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - // Grab the strings and return as array to avoid concurrent access to the IEnumerable - cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(m_rtr); + // Grab the strings and return as array to avoid concurrent access to the IEnumerable + cmd.SetCommandAndParameters(@" SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE @@ -1745,13 +1729,12 @@ namespace Duplicati.Library.Main.Database FROM ""BlocklistHash"" ) ") - .SetParameterValue("@VolumeId", volumeid); + .SetParameterValue("@VolumeId", volumeid); - return await cmd.ExecuteReaderEnumerableAsync() - .Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash")) - .ToArrayAsync() - .ConfigureAwait(false); - } + return await cmd.ExecuteReaderEnumerableAsync() + .Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash")) + .ToArrayAsync() + .ConfigureAwait(false); } /// @@ -1760,21 +1743,19 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the first path, or null if no paths exist. public async Task GetFirstPath() { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var v0 = await cmd.ExecuteScalarAsync(@" + using var cmd = m_connection.CreateCommand(m_rtr); + var v0 = await cmd.ExecuteScalarAsync(@" SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1 ") - .ConfigureAwait(false); + .ConfigureAwait(false); - if (v0 == null || v0 == DBNull.Value) - return null; + if (v0 == null || v0 == DBNull.Value) + return null; - return v0.ToString(); - } + return v0.ToString(); } /// @@ -1798,18 +1779,16 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) { - while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new Interface.USNJournalDataEntry { - yield return new Interface.USNJournalDataEntry - { - Volume = rd.ConvertValueToString(0), - JournalId = rd.ConvertValueToInt64(1), - NextUsn = rd.ConvertValueToInt64(2), - ConfigHash = rd.ConvertValueToString(3) - }; - } + Volume = rd.ConvertValueToString(0), + JournalId = rd.ConvertValueToInt64(1), + NextUsn = rd.ConvertValueToInt64(2), + ConfigHash = rd.ConvertValueToString(3) + }; } } @@ -1823,9 +1802,8 @@ namespace Duplicati.Library.Main.Database { foreach (var entry in data) { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var c = await cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(m_rtr); + var c = await cmd.SetCommandAndParameters(@" INSERT INTO ""ChangeJournalData"" ( ""FilesetID"", ""VolumeName"", @@ -1841,17 +1819,16 @@ namespace Duplicati.Library.Main.Database @ConfigHash ); ") - .SetParameterValue("@FilesetId", m_filesetId) - .SetParameterValue("@VolumeName", entry.Volume) - .SetParameterValue("@JournalId", entry.JournalId) - .SetParameterValue("@NextUsn", entry.NextUsn) - .SetParameterValue("@ConfigHash", entry.ConfigHash) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@FilesetId", m_filesetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@ConfigHash", entry.ConfigHash) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - if (c != 1) - throw new Exception("Unable to add change journal entry"); - } + if (c != 1) + throw new Exception("Unable to add change journal entry"); } await m_rtr.CommitAsync().ConfigureAwait(false); @@ -1867,9 +1844,8 @@ namespace Duplicati.Library.Main.Database { foreach (var entry in data) { - using (var cmd = m_connection.CreateCommand()) - { - await cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(); + await cmd.SetCommandAndParameters(@" UPDATE ""ChangeJournalData"" SET ""NextUSN"" = @NextUsn WHERE @@ -1877,14 +1853,13 @@ namespace Duplicati.Library.Main.Database AND ""VolumeName"" = @VolumeName AND ""JournalID"" = @JournalId; ") - .SetTransaction(m_rtr) - .SetParameterValue("@NextUsn", entry.NextUsn) - .SetParameterValue("@FilesetId", fileSetId) - .SetParameterValue("@VolumeName", entry.Volume) - .SetParameterValue("@JournalId", entry.JournalId) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); - } + .SetTransaction(m_rtr) + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@FilesetId", fileSetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } await m_rtr.CommitAsync().ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 3b6674488..ff73c0147 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -63,32 +63,31 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the obfuscation is finished. public async Task Fix() { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + using var cmd = m_connection.CreateCommand(m_rtr); + var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - // TODO: Rewrite this to use PathPrefix - // TODO: Needs to be much faster - using (var upcmd = m_connection.CreateCommand(m_rtr)) - { - await upcmd.ExecuteNonQueryAsync($@" + // TODO: Rewrite this to use PathPrefix + // TODO: Needs to be much faster + using (var upcmd = m_connection.CreateCommand(m_rtr)) + { + await upcmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tablename}"" ( ""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await upcmd.ExecuteNonQueryAsync($@" + await upcmd.ExecuteNonQueryAsync($@" INSERT INTO ""{tablename}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await upcmd.SetCommandAndParameters($@" + await upcmd.SetCommandAndParameters($@" UPDATE ""{tablename}"" SET ""Obfuscated"" = @StartPath @@ -103,51 +102,51 @@ namespace Duplicati.Library.Main.Database END ) ") - .SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\") - .SetParameterValue("@DirSep", Util.DirectorySeparatorString) - .SetParameterValue("@FileExt", ".bin") - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\") + .SetParameterValue("@DirSep", Util.DirectorySeparatorString) + .SetParameterValue("@FileExt", ".bin") + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - /*long id = 1; - using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename))) - while(rd.Read()) - { - upcmd.SetCommandAndParameters(@"UPDATE ""LogData"" SET ""Message"" = replace(""Message"", @RealPath, @Obfuscated), ""Exception"" = replace(""Exception"", @RealPath, @Obfuscated)") - .SetParameterValue("@RealPath", rd.GetValue(0)) - .SetParameterValue("@Obfuscated", rd.GetValue(1)) - .ExecuteNonQuery(); - id++; - } - */ - } + /*long id = 1; + using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename))) + while(rd.Read()) + { + upcmd.SetCommandAndParameters(@"UPDATE ""LogData"" SET ""Message"" = replace(""Message"", @RealPath, @Obfuscated), ""Exception"" = replace(""Exception"", @RealPath, @Obfuscated)") + .SetParameterValue("@RealPath", rd.GetValue(0)) + .SetParameterValue("@Obfuscated", rd.GetValue(1)) + .ExecuteNonQuery(); + id++; + } + */ + } - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" UPDATE ""LogData"" SET ""Message"" = 'ERASED!' WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" UPDATE ""LogData"" SET ""Exception"" = 'ERASED!' WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@" + await cmd.ExecuteNonQueryAsync($@" CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", @@ -159,24 +158,23 @@ namespace Duplicati.Library.Main.Database ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" ") - .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" ") - .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" ") - .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ") - .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" ") + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ") + .ConfigureAwait(false); - using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) - await m_rtr.CommitAsync().ConfigureAwait(false); - - cmd.SetTransaction(m_rtr); - await cmd.ExecuteNonQueryAsync("VACUUM").ConfigureAwait(false); + using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) await m_rtr.CommitAsync().ConfigureAwait(false); - } + + cmd.SetTransaction(m_rtr); + await cmd.ExecuteNonQueryAsync("VACUUM").ConfigureAwait(false); + await m_rtr.CommitAsync().ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index bef009aba..1afb6f8f8 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2738,12 +2738,11 @@ namespace Duplicati.Library.Main.Database /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data public async IAsyncEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize) { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - // Group subquery by hash to ensure that each blocklist hash appears only once in the result - // The AllBlocks CTE is used to map both active and duplicate blocks from the volume, - // because the new volume is initially registered with only duplicate blocks. - var sql = @" + using var cmd = m_connection.CreateCommand(m_rtr); + // Group subquery by hash to ensure that each blocklist hash appears only once in the result + // The AllBlocks CTE is used to map both active and duplicate blocks from the volume, + // because the new volume is initially registered with only duplicate blocks. + var sql = @" WITH ""AllBlocksInVolume"" AS ( SELECT DISTINCT ""ID"", @@ -2796,35 +2795,34 @@ namespace Duplicati.Library.Main.Database ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" "; - string? curHash = null; - var count = 0; - var buffer = new byte[blocksize]; + string? curHash = null; + var count = 0; + var buffer = new byte[blocksize]; - cmd.SetCommandAndParameters(sql) - .SetParameterValue("@VolumeId", volumeid) - .SetParameterValue("@HashesPerBlock", blocksize / hashsize) - .ConfigureAwait(false); + cmd.SetCommandAndParameters(sql) + .SetParameterValue("@VolumeId", volumeid) + .SetParameterValue("@HashesPerBlock", blocksize / hashsize) + .ConfigureAwait(false); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) + { + var blockhash = rd.ConvertValueToString(0); + if (curHash != null && (blockhash != curHash || count + hashsize > buffer.Length)) { - var blockhash = rd.ConvertValueToString(0); - if (curHash != null && (blockhash != curHash || count + hashsize > buffer.Length)) - { - yield return (curHash, buffer, count); - buffer = new byte[blocksize]; - count = 0; - } - - var hash = Convert.FromBase64String(rd.ConvertValueToString(1) ?? throw new Exception("Hash is null")); - Array.Copy(hash, 0, buffer, count, hashsize); - curHash = blockhash; - count += hashsize; + yield return (curHash, buffer, count); + buffer = new byte[blocksize]; + count = 0; } - if (curHash != null) - yield return (curHash, buffer, count); - } + var hash = Convert.FromBase64String(rd.ConvertValueToString(1) ?? throw new Exception("Hash is null")); + Array.Copy(hash, 0, buffer, count, hashsize); + curHash = blockhash; + count += hashsize; + } + + if (curHash != null) + yield return (curHash, buffer, count); } /// diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 2347629c4..a381491f7 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -118,63 +118,62 @@ namespace Duplicati.Library.Main.Database /// An async enumerable of key-value pairs, where the key is the fileset name and the value is the size of the fileset. public async IAsyncEnumerable> DropFilesetsFromTable(DateTime[] toDelete) { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var deleted = 0; + using var cmd = m_connection.CreateCommand(m_rtr); + var deleted = 0; - using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds)).ConfigureAwait(false)) - deleted += await ( - await cmd.SetCommandAndParameters(@" + using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds)).ConfigureAwait(false)) + deleted += await ( + await cmd.SetCommandAndParameters(@" DELETE FROM ""Fileset"" WHERE ""Timestamp"" IN (@Timestamps) ") - .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable) - .ConfigureAwait(false) - ) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - if (deleted != toDelete.Length) - throw new Exception($"Unexpected number of deleted filesets {deleted} vs {toDelete.Length}"); + if (deleted != toDelete.Length) + throw new Exception($"Unexpected number of deleted filesets {deleted} vs {toDelete.Length}"); - //Then we delete anything that is no longer being referenced - await cmd.ExecuteNonQueryAsync(@" + //Then we delete anything that is no longer being referenced + await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" NOT IN ( SELECT DISTINCT ""ID"" FROM ""Fileset"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" NOT IN ( SELECT DISTINCT ""ID"" FROM ""Fileset"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN ( SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN ( SELECT DISTINCT ""MetadataID"" FROM ""FileLookup"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN ( SELECT DISTINCT ""BlocksetID"" @@ -184,28 +183,28 @@ namespace Duplicati.Library.Main.Database FROM ""Metadataset"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" NOT IN ( SELECT DISTINCT ""ID"" FROM ""Blockset"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" NOT IN ( SELECT DISTINCT ""ID"" FROM ""Blockset"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - //We save the block info for the remote files, before we delete it - await cmd.ExecuteNonQueryAsync(@" + //We save the block info for the remote files, before we delete it + await cmd.ExecuteNonQueryAsync(@" INSERT INTO ""DeletedBlock"" ( ""Hash"", ""Size"", @@ -227,9 +226,9 @@ namespace Duplicati.Library.Main.Database WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@" + await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Block"" WHERE ""ID"" NOT IN ( SELECT DISTINCT ""BlockID"" @@ -242,10 +241,10 @@ namespace Duplicati.Library.Main.Database WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - //Find all remote filesets that are no longer required, and mark them as deleting - var updated = await cmd.SetCommandAndParameters(@" + //Find all remote filesets that are no longer required, and mark them as deleting + var updated = await cmd.SetCommandAndParameters(@" UPDATE ""RemoteVolume"" SET ""State"" = @NewState WHERE @@ -256,21 +255,21 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" ) ") - .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) - .SetParameterValue("@CurrentType", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameterMssqlite("@AllowedStates", [ - RemoteVolumeState.Uploaded.ToString(), + .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) + .SetParameterValue("@CurrentType", RemoteVolumeType.Files.ToString()) + .ExpandInClauseParameterMssqlite("@AllowedStates", [ + RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Deleting.ToString() - ]) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + ]) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - if (deleted != updated) - throw new Exception($"Unexpected number of remote volumes marked as deleted. Found {deleted} filesets, but {updated} volumes"); + if (deleted != updated) + throw new Exception($"Unexpected number of remote volumes marked as deleted. Found {deleted} filesets, but {updated} volumes"); - cmd.SetCommandAndParameters(@" + cmd.SetCommandAndParameters(@" SELECT ""Name"", ""Size"" @@ -279,16 +278,15 @@ namespace Duplicati.Library.Main.Database ""Type"" = @Type AND ""State"" = @State ") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleting.ToString()); + .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) + .SetParameterValue("@State", RemoteVolumeState.Deleting.ToString()); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new KeyValuePair( - rd.ConvertValueToString(0) ?? "", - rd.ConvertValueToInt64(1) - ); - } + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1) + ); } /// @@ -299,13 +297,12 @@ namespace Duplicati.Library.Main.Database /// An async enumerable of IListResultFileset. internal async IAsyncEnumerable FilesetsWithBackupVersion() { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - // TODO check if this is still the case? (shouldn't be with new sqlite driver): - // We can also use the ROW_NUMBER() window function to generate the backup versions, - // but this requires at least SQLite 3.25, which is not available in some common - // distributions (e.g., Debian) currently. - cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(m_rtr); + // TODO check if this is still the case? (shouldn't be with new sqlite driver): + // We can also use the ROW_NUMBER() window function to generate the backup versions, + // but this requires at least SQLite 3.25, which is not available in some common + // distributions (e.g., Debian) currently. + cmd.SetCommandAndParameters(@" SELECT ""IsFullBackup"", ""Timestamp"" @@ -313,20 +310,17 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC "); - using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - { - int version = 0; - while (await reader.ReadAsync().ConfigureAwait(false)) - { - yield return new ListResultFileset( - version++, - reader.GetInt32(0), - ParseFromEpochSeconds(reader.ConvertValueToInt64(1)).ToLocalTime(), - -1L, - -1L - ); - } - } + using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + int version = 0; + while (await reader.ReadAsync().ConfigureAwait(false)) + { + yield return new ListResultFileset( + version++, + reader.GetInt32(0), + ParseFromEpochSeconds(reader.ConvertValueToInt64(1)).ToLocalTime(), + -1L, + -1L + ); } } @@ -503,21 +497,20 @@ namespace Duplicati.Library.Main.Database var createtable = $"{@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS "}{collected}"; - using (var cmd = m_connection.CreateCommand(m_rtr)) + using var cmd = m_connection.CreateCommand(m_rtr); + try { - try - { - await cmd - .SetCommandAndParameters(createtable) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameterMssqlite("@AllowedStates", [ - RemoteVolumeState.Uploaded.ToString(), + await cmd + .SetCommandAndParameters(createtable) + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .ExpandInClauseParameterMssqlite("@AllowedStates", [ + RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() - ]) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + ]) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - cmd.SetCommandAndParameters($@" + cmd.SetCommandAndParameters($@" SELECT ""A"".""Name"", ""B"".""ActiveSize"", @@ -530,25 +523,24 @@ namespace Duplicati.Library.Main.Database ORDER BY ""B"".""Sorttime"" ASC "); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new VolumeUsage( - rd.ConvertValueToString(0) ?? "", - rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), - rd.ConvertValueToInt64(2, 0), - rd.ConvertValueToInt64(3, 0) - ); - } - finally + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new VolumeUsage( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), + rd.ConvertValueToInt64(2, 0), + rd.ConvertValueToInt64(3, 0) + ); + } + finally + { + try { - try - { - await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"" ") - .ConfigureAwait(false); - } - catch { } + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"" ") + .ConfigureAwait(false); } + catch { } } } @@ -889,25 +881,24 @@ namespace Duplicati.Library.Main.Database if (deletedVolume.Type != RemoteVolumeType.Blocks) return; - using (var cmd = m_connection.CreateCommand(m_rtr)) + using var cmd = m_connection.CreateCommand(m_rtr); + var updatedBlocks = "BlocksToUpdate-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var replacementBlocks = "ReplacementBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + try { - var updatedBlocks = "BlocksToUpdate-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var replacementBlocks = "ReplacementBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - try - { - await cmd.SetCommandAndParameters($@" + await cmd.SetCommandAndParameters($@" CREATE {TEMPORARY} TABLE ""{updatedBlocks}"" AS SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId ") - .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@VolumeId", deletedVolume.ID) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved).ConfigureAwait(false)) - await ( - await cmd.SetCommandAndParameters($@" + using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved).ConfigureAwait(false)) + await ( + await cmd.SetCommandAndParameters($@" CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS SELECT ""BlockID"", @@ -921,28 +912,28 @@ namespace Duplicati.Library.Main.Database ) GROUP BY ""BlockID"" ") - .ExpandInClauseParameterMssqliteAsync("@VolumeIds", tempTable) - .ConfigureAwait(false) - ) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .ExpandInClauseParameterMssqliteAsync("@VolumeIds", tempTable) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - var targetCount = await cmd.ExecuteScalarInt64Async($@" + var targetCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{updatedBlocks}"" ") - .ConfigureAwait(false); + .ConfigureAwait(false); - if (targetCount == 0) - return; + if (targetCount == 0) + return; - var replacementCount = await cmd.ExecuteScalarInt64Async($@" + var replacementCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{replacementBlocks}"" ") - .ConfigureAwait(false); + .ConfigureAwait(false); - var updateCount = await cmd.SetCommandAndParameters(@$" + var updateCount = await cmd.SetCommandAndParameters(@$" UPDATE ""Block"" SET ""VolumeID"" = ( SELECT ""VolumeID"" @@ -953,11 +944,11 @@ namespace Duplicati.Library.Main.Database ) WHERE ""Block"".""VolumeID"" = @VolumeId ") - .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@VolumeId", deletedVolume.ID) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - var deleteCount = await cmd.ExecuteNonQueryAsync(@$" + var deleteCount = await cmd.ExecuteNonQueryAsync(@$" DELETE FROM ""DuplicateBlock"" WHERE ( @@ -972,39 +963,38 @@ namespace Duplicati.Library.Main.Database FROM ""{replacementBlocks}"" ""RB"" ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - if (targetCount != updateCount - || replacementCount != deleteCount - || updateCount != deleteCount) - { - throw new Exception($"Unexpected number of rows updated. Expected {targetCount} but got updated {updateCount}, deleted {deleteCount}, and replaced {replacementCount}"); - } + if (targetCount != updateCount + || replacementCount != deleteCount + || updateCount != deleteCount) + { + throw new Exception($"Unexpected number of rows updated. Expected {targetCount} but got updated {updateCount}, deleted {deleteCount}, and replaced {replacementCount}"); + } - // Remove knowledge of any old blocks - await cmd.SetCommandAndParameters(@$" + // Remove knowledge of any old blocks + await cmd.SetCommandAndParameters(@$" DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" = @VolumeId ") - .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync() + .SetParameterValue("@VolumeId", deletedVolume.ID) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + } + finally + { + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ") .ConfigureAwait(false); } - finally + catch { } + try { - try - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ") - .ConfigureAwait(false); - } - catch { } - try - { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ") - .ConfigureAwait(false); - } - catch { } + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ") + .ConfigureAwait(false); } + catch { } } } @@ -1015,15 +1005,14 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of that represents the order in which volumes should be deleted. public async IAsyncEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes) { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - // Although the generated index volumes are always in pairs, - // this code handles many-to-many relations between - // index files and block volumes, should this be added later - var lookupBlock = new Dictionary>(); - var lookupIndexfiles = new Dictionary>(); + using var cmd = m_connection.CreateCommand(m_rtr); + // Although the generated index volumes are always in pairs, + // this code handles many-to-many relations between + // index files and block volumes, should this be added later + var lookupBlock = new Dictionary>(); + var lookupIndexfiles = new Dictionary>(); - cmd.SetCommandAndParameters(@" + cmd.SetCommandAndParameters(@" SELECT ""C"".""Name"", ""B"".""Name"", @@ -1040,47 +1029,46 @@ namespace Duplicati.Library.Main.Database AND ""B"".""Size"" IS NOT NULL "); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) + { + var name = rd.ConvertValueToString(0) ?? ""; + if (!lookupBlock.TryGetValue(name, out var indexfileList)) { - var name = rd.ConvertValueToString(0) ?? ""; - if (!lookupBlock.TryGetValue(name, out var indexfileList)) - { - indexfileList = new List(); - lookupBlock.Add(name, indexfileList); - } - - var v = new RemoteVolume( - rd.ConvertValueToString(1), - rd.ConvertValueToString(2), - rd.ConvertValueToInt64(3) - ); - indexfileList.Add(v); - - if (!lookupIndexfiles.TryGetValue(v.Name, out var blockList)) - { - blockList = new List(); - lookupIndexfiles.Add(v.Name, blockList); - } - blockList.Add(name); + indexfileList = new List(); + lookupBlock.Add(name, indexfileList); } - foreach (var r in deleteableVolumes.Distinct()) - { - // Return the input - yield return r; - if (lookupBlock.TryGetValue(r.Name, out var indexfileList)) - foreach (var sh in indexfileList) - { - if (lookupIndexfiles.TryGetValue(sh.Name, out var backref)) - { - //If this is the last reference, - // remove the index file as well - if (backref.Remove(r.Name) && backref.Count == 0) - yield return sh; - } - } + var v = new RemoteVolume( + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToInt64(3) + ); + indexfileList.Add(v); + + if (!lookupIndexfiles.TryGetValue(v.Name, out var blockList)) + { + blockList = new List(); + lookupIndexfiles.Add(v.Name, blockList); + } + blockList.Add(name); } + + foreach (var r in deleteableVolumes.Distinct()) + { + // Return the input + yield return r; + if (lookupBlock.TryGetValue(r.Name, out var indexfileList)) + foreach (var sh in indexfileList) + { + if (lookupIndexfiles.TryGetValue(sh.Name, out var backref)) + { + //If this is the last reference, + // remove the index file as well + if (backref.Remove(r.Name) && backref.Count == 0) + yield return sh; + } + } } } diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 9e40f97f8..62f5676ec 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -170,18 +170,18 @@ namespace Duplicati.Library.Main.Database ) "; - using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false)) - using (var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false)) - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + using var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false); + using var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + { + var v = dict[rd.ConvertValueToInt64(0)]; + yield return new ListResultFileset() { - var v = dict[rd.ConvertValueToInt64(0)]; - yield return new ListResultFileset() - { - Version = v, - Time = filesets[v].Value - }; - } + Version = v, + Time = filesets[v].Value + }; + } } /// @@ -249,15 +249,15 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Path"" "; - using (var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false)) - using (var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false)) - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new ListResultFile() - { - Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"), - Sizes = null - }; + using var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false); + using var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new ListResultFile() + { + Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"), + Sizes = null + }; } /// @@ -299,13 +299,13 @@ namespace Duplicati.Library.Main.Database foreach ((var x, var i) in items.Select((x, i) => (x, i))) cmd.SetParameterValue($"@Message{i}", "%" + x + "%"); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new ListResultRemoteLog() - { - Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)), - Message = rd.ConvertValueToString(1) ?? "" - }; + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new ListResultRemoteLog() + { + Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)), + Message = rd.ConvertValueToString(1) ?? "" + }; } } @@ -382,13 +382,13 @@ namespace Duplicati.Library.Main.Database ) "; - using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterMssqlite("@Names", items.ToArray())) - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new ListResultRemoteVolume() - { - Name = rd.ConvertValueToString(0) ?? "" - }; + using var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterMssqlite("@Names", items.ToArray()); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new ListResultRemoteVolume() + { + Name = rd.ConvertValueToString(0) ?? "" + }; } } } diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index c7bcafcf1..474abac6f 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -423,81 +423,79 @@ namespace Duplicati.Library.Main.Database { if (names == null || !names.Any()) return; - using (var deletecmd = m_connection.CreateCommand(m_rtr)) - { - var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var volidstable = "DelVolSetIds-" + temptransguid; + using var deletecmd = m_connection.CreateCommand(m_rtr); + var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var volidstable = "DelVolSetIds-" + temptransguid; - // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - await deletecmd.ExecuteNonQueryAsync($@" + // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. + await deletecmd.ExecuteNonQueryAsync($@" CREATE TEMP TABLE ""{volidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names).ConfigureAwait(false)) - await ( - await deletecmd.SetCommandAndParameters($@" + using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names).ConfigureAwait(false)) + await ( + await deletecmd.SetCommandAndParameters($@" INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names) ") - .ExpandInClauseParameterMssqliteAsync("@Names", tmptable) - .ConfigureAwait(false) - ) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - var volIdsSubQuery = $@" + var volIdsSubQuery = $@" SELECT ""ID"" FROM ""{volidstable}"" "; - deletecmd.Parameters.Clear(); + deletecmd.Parameters.Clear(); - await deletecmd.ExecuteNonQueryAsync($@" + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery}) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@" + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@" + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@" + await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - // Clean up temp tables for subqueries. We truncate content and then try to delete. - // Drop in try-block, as it fails in nested transactions (SQLite problem) - // SQLite.SQLiteException (0x80004005): database table is locked + // Clean up temp tables for subqueries. We truncate content and then try to delete. + // Drop in try-block, as it fails in nested transactions (SQLite problem) + // SQLite.SQLiteException (0x80004005): database table is locked + await deletecmd + .ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ") + .ConfigureAwait(false); + + try + { + deletecmd.CommandTimeout = 2; await deletecmd - .ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ") + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ") .ConfigureAwait(false); - - try - { - deletecmd.CommandTimeout = 2; - await deletecmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ") - .ConfigureAwait(false); - } - catch { /* Ignore, will be deleted on close anyway. */ } } + catch { /* Ignore, will be deleted on close anyway. */ } } /// diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 3504981e2..14ff354db 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -388,12 +388,11 @@ namespace Duplicati.Library.Main.Database var combined = $"({folders} UNION {symlinks} UNION {files})"; - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + if (filter == null || filter.Empty) { - if (filter == null || filter.Empty) - { - // Simple case, select everything - await cmd.SetCommandAndParameters($@" + // Simple case, select everything + await cmd.SetCommandAndParameters($@" INSERT INTO ""{tablename}"" ( ""Path"", ""FileHash"", @@ -410,61 +409,61 @@ namespace Duplicati.Library.Main.Database FROM {combined} ""A"" WHERE ""A"".""FilesetID"" = @FilesetId ") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); - } - else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) - { - // File list based - // unfortunately we cannot do this if the filesystem is case sensitive as - // SQLite only supports ASCII compares - var p = expression.GetSimpleList(); - var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - await cmd.ExecuteNonQueryAsync($@" + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + } + else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) + { + // File list based + // unfortunately we cannot do this if the filesystem is case sensitive as + // SQLite only supports ASCII compares + var p = expression.GetSimpleList(); + var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{filenamestable}"" ( ""Path"" TEXT NOT NULL ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.SetCommandAndParameters($@" + await cmd.SetCommandAndParameters($@" INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path) ") - .PrepareAsync() + .PrepareAsync() + .ConfigureAwait(false); + + foreach (var s in p) + await cmd + .SetParameterValue("@Path", s) + .ExecuteNonQueryAsync() .ConfigureAwait(false); - foreach (var s in p) - await cmd - .SetParameterValue("@Path", s) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); - - string whereClause; - if (expression.Result) - { - // Include filter - whereClause = $@" + string whereClause; + if (expression.Result) + { + // Include filter + whereClause = $@" ""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" IN ( SELECT DISTINCT ""Path"" FROM ""{filenamestable}"" ) "; - } - else - { - // Exclude filter - whereClause = $@" + } + else + { + // Exclude filter + whereClause = $@" ""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" NOT IN ( SELECT DISTINCT ""Path"" FROM ""{filenamestable}"" ) "; - } - await cmd.SetCommandAndParameters($@" + } + await cmd.SetCommandAndParameters($@" INSERT INTO ""{tablename}"" ( ""Path"", ""FileHash"", @@ -481,19 +480,19 @@ namespace Duplicati.Library.Main.Database FROM {combined} ""A"" WHERE {whereClause} ") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ") - .ConfigureAwait(false); - } - else - { - // Do row-wise iteration - var values = new object[5]; - await cmd.SetCommandAndParameters($@" + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ") + .ConfigureAwait(false); + } + else + { + // Do row-wise iteration + var values = new object[5]; + await cmd.SetCommandAndParameters($@" SELECT ""A"".""Path"", ""A"".""FileHash"", @@ -503,12 +502,12 @@ namespace Duplicati.Library.Main.Database FROM {combined} ""A"" WHERE ""A"".""FilesetID"" = @FilesetId ") - .SetParameterValue("@FilesetId", filesetId) - .PrepareAsync() - .ConfigureAwait(false); + .SetParameterValue("@FilesetId", filesetId) + .PrepareAsync() + .ConfigureAwait(false); - using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) - .SetCommandAndParameters($@" + using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) + .SetCommandAndParameters($@" INSERT INTO ""{tablename}"" ( ""Path"", ""FileHash"", @@ -524,27 +523,26 @@ namespace Duplicati.Library.Main.Database @Type ) "); - await cmd2.PrepareAsync().ConfigureAwait(false); + await cmd2.PrepareAsync().ConfigureAwait(false); - using var rd = await cmd - .ExecuteReaderAsync() - .ConfigureAwait(false); + using var rd = await cmd + .ExecuteReaderAsync() + .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) + { + rd.GetValues(values); + var path = values[0] as string; + if (path != null && FilterExpression.Matches(filter, path.ToString())) { - rd.GetValues(values); - var path = values[0] as string; - if (path != null && FilterExpression.Matches(filter, path.ToString())) - { - await cmd2 - .SetParameterValue("@Path", values[0]) - .SetParameterValue("@FileHash", values[1]) - .SetParameterValue("@MetaHash", values[2]) - .SetParameterValue("@Size", values[3]) - .SetParameterValue("@Type", values[4]) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); - } + await cmd2 + .SetParameterValue("@Path", values[0]) + .SetParameterValue("@FileHash", values[1]) + .SetParameterValue("@MetaHash", values[2]) + .SetParameterValue("@Size", values[3]) + .SetParameterValue("@Type", values[4]) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } } } @@ -634,39 +632,37 @@ namespace Duplicati.Library.Main.Database { var (Added, Deleted, Modified) = GetSqls(true); - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + var result = new ChangeSizeReport { - var result = new ChangeSizeReport - { - PreviousSize = await cmd.ExecuteScalarInt64Async($@" + PreviousSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_previousTable}"" ", 0) - .ConfigureAwait(false), + .ConfigureAwait(false), - CurrentSize = await cmd.ExecuteScalarInt64Async($@" + CurrentSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_currentTable}"" ", 0) - .ConfigureAwait(false), + .ConfigureAwait(false), - AddedSize = await cmd.ExecuteScalarInt64Async($@" + AddedSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({Added}) ", 0) - .ConfigureAwait(false), + .ConfigureAwait(false), - DeletedSize = await cmd.ExecuteScalarInt64Async($@" + DeletedSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({Deleted}) ", 0) - .ConfigureAwait(false) - }; + .ConfigureAwait(false) + }; - return result; - } + return result; } /// diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 40c9be61e..2e6da5087 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -312,11 +312,10 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of file versions representing the largest prefix. private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) { - using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false)) - using (var cmd = m_db.Connection.CreateCommand()) - { - //First we trim the filelist to exclude filenames not found in any of the filesets - await cmd.ExecuteNonQueryAsync($@" + using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false); + using var cmd = m_db.Connection.CreateCommand(); + //First we trim the filelist to exclude filenames not found in any of the filesets + await cmd.ExecuteNonQueryAsync($@" DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN ( SELECT DISTINCT ""Path"" @@ -331,107 +330,106 @@ namespace Duplicati.Library.Main.Database ) ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - //If we have a prefix rule, apply it - if (!string.IsNullOrWhiteSpace(prefixrule)) - await cmd.SetCommandAndParameters($@" + //If we have a prefix rule, apply it + if (!string.IsNullOrWhiteSpace(prefixrule)) + await cmd.SetCommandAndParameters($@" DELETE FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule ") - .SetParameterValue("@Rule", prefixrule) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@Rule", prefixrule) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - // Then we recursively find the largest prefix - var v0 = await cmd.ExecuteScalarAsync($@" + // Then we recursively find the largest prefix + var v0 = await cmd.ExecuteScalarAsync($@" SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1 ") - .ConfigureAwait(false); + .ConfigureAwait(false); - var maxpath = ""; - if (v0 != null) - maxpath = v0.ToString() ?? ""; + var maxpath = ""; + if (v0 != null) + maxpath = v0.ToString() ?? ""; - var dirsep = Util.GuessDirSeparator(maxpath); + var dirsep = Util.GuessDirSeparator(maxpath); - var filecount = await cmd.ExecuteScalarInt64Async($@" + var filecount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" ", 0) - .ConfigureAwait(false); + .ConfigureAwait(false); - var foundfiles = -1L; + var foundfiles = -1L; - //TODO: Handle FS case-sensitive? - await cmd.SetCommandAndParameters($@" + //TODO: Handle FS case-sensitive? + await cmd.SetCommandAndParameters($@" SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix ") - .PrepareAsync() + .PrepareAsync() + .ConfigureAwait(false); + + while (filecount != foundfiles && maxpath.Length > 0) + { + var mp = Util.AppendDirSeparator(maxpath, dirsep); + foundfiles = await cmd + .SetParameterValue("@PrefixLength", mp.Length) + .SetParameterValue("@Prefix", mp) + .ExecuteScalarInt64Async(0) .ConfigureAwait(false); - while (filecount != foundfiles && maxpath.Length > 0) + if (filecount != foundfiles) { - var mp = Util.AppendDirSeparator(maxpath, dirsep); - foundfiles = await cmd - .SetParameterValue("@PrefixLength", mp.Length) - .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64Async(0) - .ConfigureAwait(false); + var oldlen = maxpath.Length; + var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); - if (filecount != foundfiles) - { - var oldlen = maxpath.Length; - var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); - - maxpath = maxpath.Substring(0, lix + 1); - if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen || maxpath == "\\\\") - maxpath = ""; - } + maxpath = maxpath.Substring(0, lix + 1); + if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen || maxpath == "\\\\") + maxpath = ""; } + } - // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root - if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) - { - var paths = cmd.ExecuteReaderEnumerableAsync($@" + // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root + if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) + { + var paths = cmd.ExecuteReaderEnumerableAsync($@" SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ") - .Select(x => x.ConvertValueToString(0) ?? ""); + .Select(x => x.ConvertValueToString(0) ?? ""); - var roots = paths - .Select(x => x.Substring(0, 1)) - .Distinct() - .Where(x => x != "\\"); + var roots = paths + .Select(x => x.Substring(0, 1)) + .Distinct() + .Where(x => x != "\\"); - //unc path like \\server.domain\ - var regexUNCPrefix = new System.Text.RegularExpressions.Regex(@"^\\\\.*?\\"); - var rootsUNC = paths - .Select(x => regexUNCPrefix.Match(x)) - .Where(x => x.Success) - .Select(x => x.Value) - .Distinct(); + //unc path like \\server.domain\ + var regexUNCPrefix = new System.Text.RegularExpressions.Regex(@"^\\\\.*?\\"); + var rootsUNC = paths + .Select(x => regexUNCPrefix.Match(x)) + .Where(x => x.Success) + .Select(x => x.Value) + .Distinct(); - var result = roots - .Concat(rootsUNC) - .Select(x => GetLargestPrefix(filter, x) - .FirstAsync()) - .Distinct(); + var result = roots + .Concat(rootsUNC) + .Select(x => GetLargestPrefix(filter, x) + .FirstAsync()) + .Distinct(); - await foreach (var el in result.ConfigureAwait(false)) - yield return await el; - } - - yield return new FileversionFixed - { - Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) - }; + await foreach (var el in result.ConfigureAwait(false)) + yield return await el; } + + yield return new FileversionFixed + { + Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) + }; } /// @@ -451,22 +449,22 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""Path"" FROM ""{table}"" "); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - { - var s = rd.ConvertValueToString(0) ?? ""; - if (!s.StartsWith(prefix, StringComparison.Ordinal)) - continue; + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + { + var s = rd.ConvertValueToString(0) ?? ""; + if (!s.StartsWith(prefix, StringComparison.Ordinal)) + continue; - var dirsep = Util.GuessDirSeparator(s); + var dirsep = Util.GuessDirSeparator(s); - s = s.Substring(ppl); - var ix = s.IndexOf(dirsep, StringComparison.Ordinal); - if (ix > 0 && ix != s.Length - 1) - s = s.Substring(0, ix + 1); + s = s.Substring(ppl); + var ix = s.IndexOf(dirsep, StringComparison.Ordinal); + if (ix > 0 && ix != s.Length - 1) + s = s.Substring(0, ix + 1); - yield return prefix + s; - } + yield return prefix + s; + } } public async IAsyncEnumerable SelectFolderContents(IFilter filter) @@ -489,11 +487,10 @@ namespace Duplicati.Library.Main.Database if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); - using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true)).ConfigureAwait(false)) - using (var cmd = m_db.Connection.CreateCommand()) - { - //First we trim the filelist to exclude filenames not found in any of the filesets - await cmd.ExecuteNonQueryAsync($@" + using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true)).ConfigureAwait(false); + using var cmd = m_db.Connection.CreateCommand(); + //First we trim the filelist to exclude filenames not found in any of the filesets + await cmd.ExecuteNonQueryAsync($@" DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN ( SELECT DISTINCT ""Path"" @@ -508,46 +505,46 @@ namespace Duplicati.Library.Main.Database ) ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - // If we had instr support this would work: - /*var distinctPaths = @"SELECT DISTINCT :1 || " + - @"CASE(INSTR(SUBSTR(""Path"", :2), '/')) " + - @"WHEN 0 THEN SUBSTR(""Path"", :2) " + - @"ELSE SUBSTR(""Path"", :2, INSTR(SUBSTR(path, :2), '/')) " + - @"END AS ""Path"", ""FilesetID"" " + - @" FROM (" + cartesianPathFileset + @")";*/ + // If we had instr support this would work: + /*var distinctPaths = @"SELECT DISTINCT :1 || " + + @"CASE(INSTR(SUBSTR(""Path"", :2), '/')) " + + @"WHEN 0 THEN SUBSTR(""Path"", :2) " + + @"ELSE SUBSTR(""Path"", :2, INSTR(SUBSTR(path, :2), '/')) " + + @"END AS ""Path"", ""FilesetID"" " + + @" FROM (" + cartesianPathFileset + @")";*/ - // Instead we manually iterate the paths - await cmd.ExecuteNonQueryAsync($@" + // Instead we manually iterate the paths + await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tbname}"" ( ""Path"" TEXT NOT NULL ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - using (var c2 = m_db.Connection.CreateCommand()) - { - c2.SetCommandAndParameters($@" + using (var c2 = m_db.Connection.CreateCommand()) + { + c2.SetCommandAndParameters($@" INSERT INTO ""{tbname}"" (""Path"") VALUES (@Path) "); - await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct().ConfigureAwait(false)) - await c2 - .SetParameterValue("@Path", n) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct().ConfigureAwait(false)) + await c2 + .SetParameterValue("@Path", n) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await c2.ExecuteNonQueryAsync($@" + await c2.ExecuteNonQueryAsync($@" CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"") ") - .ConfigureAwait(false); - } + .ConfigureAwait(false); + } - //Then we select the matching results - var filesets = $@" + //Then we select the matching results + var filesets = $@" SELECT ""FilesetID"", ""Timestamp"" @@ -555,7 +552,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC "; - var cartesianPathFileset = $@" + var cartesianPathFileset = $@" SELECT ""A"".""Path"", ""B"".""FilesetID"" @@ -567,7 +564,7 @@ namespace Duplicati.Library.Main.Database ""B"".""Timestamp"" DESC "; - var filesWithSizes = $@" + var filesWithSizes = $@" SELECT ""Length"", ""FilesetEntry"".""FilesetID"", @@ -585,7 +582,7 @@ namespace Duplicati.Library.Main.Database ) "; - var query = @$" + var query = @$" SELECT ""C"".""Path"", ""D"".""Length"", @@ -596,38 +593,37 @@ namespace Duplicati.Library.Main.Database AND ""C"".""Path"" = ""D"".""Path"" "; - using var rd = await cmd - .ExecuteReaderAsync(query) - .ConfigureAwait(false); + using var rd = await cmd + .ExecuteReaderAsync(query) + .ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + if (await rd.ReadAsync().ConfigureAwait(false)) + { + bool more; + do { - bool more; - do + var f = new Fileversion(rd); + if (!(string.IsNullOrWhiteSpace(f.Path) || f.Path == pathprefix)) { - var f = new Fileversion(rd); - if (!(string.IsNullOrWhiteSpace(f.Path) || f.Path == pathprefix)) - { - yield return f; - more = f.More; - } - else - { - more = await rd.ReadAsync() - .ConfigureAwait(false); - } + yield return f; + more = f.More; + } + else + { + more = await rd.ReadAsync() + .ConfigureAwait(false); + } - } while (more); - } + } while (more); } } finally { try { - using (var c = m_db.Connection.CreateCommand()) - await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}""") - .ConfigureAwait(false); + using var c = m_db.Connection.CreateCommand(); + await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}""") + .ConfigureAwait(false); } catch { @@ -637,11 +633,10 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable SelectFiles(IFilter filter) { - using (var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false)) - using (var cmd = m_db.Connection.CreateCommand()) - { - //First we trim the filelist to exclude filenames not found in any of the filesets - await cmd.ExecuteNonQueryAsync($@" + using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false); + using var cmd = m_db.Connection.CreateCommand(); + //First we trim the filelist to exclude filenames not found in any of the filesets + await cmd.ExecuteNonQueryAsync($@" DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN ( SELECT DISTINCT ""Path"" @@ -656,10 +651,10 @@ namespace Duplicati.Library.Main.Database ) ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - //Then we select the matching results - var filesets = $@" + //Then we select the matching results + var filesets = $@" SELECT ""FilesetID"", ""Timestamp"" @@ -667,7 +662,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC "; - var cartesianPathFileset = $@" + var cartesianPathFileset = $@" SELECT ""A"".""Path"", ""B"".""FilesetID"" @@ -679,7 +674,7 @@ namespace Duplicati.Library.Main.Database ""B"".""Timestamp"" DESC "; - var filesWithSizes = $@" + var filesWithSizes = $@" SELECT ""Length"", ""FilesetEntry"".""FilesetID"", @@ -697,7 +692,7 @@ namespace Duplicati.Library.Main.Database ) "; - var query = @$" + var query = @$" SELECT ""C"".""Path"", ""D"".""Length"", @@ -708,18 +703,17 @@ namespace Duplicati.Library.Main.Database AND ""C"".""Path"" = ""D"".""Path"" "; - using var rd = await cmd.ExecuteReaderAsync(query) - .ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + using var rd = await cmd.ExecuteReaderAsync(query) + .ConfigureAwait(false); + if (await rd.ReadAsync().ConfigureAwait(false)) + { + bool more; + do { - bool more; - do - { - var f = new Fileversion(rd); - yield return f; - more = f.More; - } while (more); - } + var f = new Fileversion(rd); + yield return f; + more = f.More; + } while (more); } } @@ -747,28 +741,28 @@ namespace Duplicati.Library.Main.Database for (var i = 0; i < m_filesets.Length; i++) dict[m_filesets[i].Key] = i; - using (var cmd = m_db.Connection.CreateCommand()) - using (var rd = await cmd.ExecuteReaderAsync(@" + using var cmd = m_db.Connection.CreateCommand(); + using var rd = await cmd.ExecuteReaderAsync(@" SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC - ").ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - { - var id = rd.ConvertValueToInt64(0); - var backupType = rd.GetInt32(1); - var e = dict[id]; + ").ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + { + var id = rd.ConvertValueToInt64(0); + var backupType = rd.GetInt32(1); + var e = dict[id]; - yield return new Fileset( - e, - backupType, - m_filesets[e].Value, - -1L, - -1L - ); - } + yield return new Fileset( + e, + backupType, + m_filesets[e].Value, + -1L, + -1L + ); + } } public async IAsyncEnumerable Sets() @@ -838,9 +832,9 @@ namespace Duplicati.Library.Main.Database { try { - using (var cmd = m_db.Connection.CreateCommand()) - await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ") - .ConfigureAwait(false); + using var cmd = m_db.Connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ") + .ConfigureAwait(false); } catch { } finally { m_tablename = null!; } @@ -894,23 +888,23 @@ namespace Duplicati.Library.Main.Database ORDER BY ""f"".""Timestamp"" DESC; "; - using (var cmd = m_connection.CreateCommand(query)) - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) - { - var id = rd.ConvertValueToInt64(0); - var time = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); - var isFullBackup = rd.GetInt32(2) == BackupType.FULL_BACKUP; - var filecount = rd.ConvertValueToInt64(3, -1L); - var filesizes = rd.ConvertValueToInt64(4, -1L); + using var cmd = m_connection.CreateCommand(query); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + { + var id = rd.ConvertValueToInt64(0); + var time = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); + var isFullBackup = rd.GetInt32(2) == BackupType.FULL_BACKUP; + var filecount = rd.ConvertValueToInt64(3, -1L); + var filesizes = rd.ConvertValueToInt64(4, -1L); - yield return new FilesetEntry( - id, - isFullBackup, - time, - filecount, - filesizes - ); - } + yield return new FilesetEntry( + id, + isFullBackup, + time, + filecount, + filesizes + ); + } } /// diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 9ee6af83a..bbaad66fd 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -101,11 +101,11 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", id); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - if (!await rd.ReadAsync().ConfigureAwait(false)) - throw new Exception($"No remote volume found for fileset with id {id}"); - else - return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + if (!await rd.ReadAsync().ConfigureAwait(false)) + throw new Exception($"No remote volume found for fileset with id {id}"); + else + return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); } /// @@ -267,28 +267,27 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_db.Connection.CreateCommand()) - { - await cmd.ExecuteNonQueryAsync($@" + using var cmd = m_db.Connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{filenamestable}"" ( ""Path"" TEXT NOT NULL ) ") - .ConfigureAwait(false); + .ConfigureAwait(false); - await cmd.SetCommandAndParameters($@" + await cmd.SetCommandAndParameters($@" INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path) ") - .PrepareAsync() + .PrepareAsync() + .ConfigureAwait(false); + + foreach (var s in p) + await cmd.SetParameterValue("@Path", s) + .ExecuteNonQueryAsync() .ConfigureAwait(false); - foreach (var s in p) - await cmd.SetParameterValue("@Path", s) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); - - await cmd.SetCommandAndParameters($@" + await cmd.SetCommandAndParameters($@" INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM @@ -299,30 +298,28 @@ namespace Duplicati.Library.Main.Database AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}"" ") - .SetParameterValue("@FilesetId", ParentID) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@FilesetId", ParentID) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ") - .ConfigureAwait(false); - } + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ") + .ConfigureAwait(false); } else { // Do row-wise iteration var values = new object[2]; - using (var cmd = m_db.Connection.CreateCommand()) - using (var cmd2 = m_db.Connection.CreateCommand()) - { - await cmd2.SetCommandAndParameters($@" + using var cmd = m_db.Connection.CreateCommand(); + using var cmd2 = m_db.Connection.CreateCommand(); + await cmd2.SetCommandAndParameters($@" INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId) ") - .PrepareAsync() - .ConfigureAwait(false); + .PrepareAsync() + .ConfigureAwait(false); - cmd.SetCommandAndParameters(@" + cmd.SetCommandAndParameters(@" SELECT ""B"".""Path"", ""A"".""FileID"" @@ -333,21 +330,20 @@ namespace Duplicati.Library.Main.Database ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" ") - .SetParameterValue("@FilesetId", ParentID); + .SetParameterValue("@FilesetId", ParentID); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - { - rd.GetValues(values); - var path = values[0] as string; - if (path != null && Library.Utility.FilterExpression.Matches(filter, path.ToString())) - { - await cmd2 - .SetParameterValue("@FileId", values[1]) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); - } - } + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + { + rd.GetValues(values); + var path = values[0] as string; + if (path != null && Library.Utility.FilterExpression.Matches(filter, path.ToString())) + { + await cmd2 + .SetParameterValue("@FileId", values[1]) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + } } } @@ -361,17 +357,16 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the checks have been applied. private async Task PostFilterChecks(int updated) { - using (var cmd = m_db.Connection.CreateCommand()) - { - UpdatedFileCount = updated; + using var cmd = m_db.Connection.CreateCommand(); + UpdatedFileCount = updated; - RemovedFileCount = await cmd.ExecuteScalarInt64Async($@" + RemovedFileCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{m_tablename}"" ", 0) - .ConfigureAwait(false); + .ConfigureAwait(false); - RemovedFileSize = await cmd.ExecuteScalarInt64Async($@" + RemovedFileSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""C"".""Length"") FROM ""{m_tablename}"" ""A"", @@ -388,20 +383,19 @@ namespace Duplicati.Library.Main.Database ) ) ", 0) - .ConfigureAwait(false); + .ConfigureAwait(false); - var filesetcount = await cmd.SetCommandAndParameters($@" + var filesetcount = await cmd.SetCommandAndParameters($@" SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @ParentID ") - .SetParameterValue("@ParentID", ParentID) - .ExecuteScalarInt64Async(0) - .ConfigureAwait(false); + .SetParameterValue("@ParentID", ParentID) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); - if (filesetcount == RemovedFileCount) - throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); - } + if (filesetcount == RemovedFileCount) + throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } public async Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup) @@ -442,8 +436,8 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable> ListAllDeletedFiles() { - using (var cmd = m_db.Connection.CreateCommand()) - using (var rd = await cmd.ExecuteReaderAsync($@" + using var cmd = m_db.Connection.CreateCommand(); + using var rd = await cmd.ExecuteReaderAsync($@" SELECT ""B"".""Path"", ""C"".""Length"" @@ -454,12 +448,12 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" - ").ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new KeyValuePair( - rd.ConvertValueToString(0) ?? "", - rd.ConvertValueToInt64(1) - ); + ").ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1) + ); } public void Dispose() @@ -471,10 +465,10 @@ namespace Duplicati.Library.Main.Database { try { - using (var cmd = m_db.Connection.CreateCommand()) - await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") - .ConfigureAwait(false); + using var cmd = m_db.Connection.CreateCommand(); + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + .ConfigureAwait(false); } catch { } } diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index f6c621de1..836ffe44a 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -1114,9 +1114,8 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable collection of remote volumes that are missing blocks. public async IAsyncEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) { - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var selectCommand = @" + using var cmd = m_connection.CreateCommand(m_rtr); + var selectCommand = @" SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", @@ -1125,7 +1124,7 @@ namespace Duplicati.Library.Main.Database FROM ""RemoteVolume"" "; - var missingBlocklistEntries = $@" + var missingBlocklistEntries = $@" SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" @@ -1134,7 +1133,7 @@ namespace Duplicati.Library.Main.Database WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL "; - var missingBlockInfo = @" + var missingBlockInfo = @" SELECT ""VolumeID"" FROM ""Block"" WHERE @@ -1142,7 +1141,7 @@ namespace Duplicati.Library.Main.Database AND ""Size"" > 0 "; - var missingBlocklistVolumes = $@" + var missingBlocklistVolumes = $@" SELECT ""VolumeID"" FROM ""Block"", @@ -1150,7 +1149,7 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""Hash"" = ""Block"".""Hash"" "; - var countMissingInformation = $@" + var countMissingInformation = $@" SELECT COUNT(*) FROM ( SELECT DISTINCT ""VolumeID"" @@ -1161,79 +1160,78 @@ namespace Duplicati.Library.Main.Database ) "; - if (passNo == 0) - { - // On the first pass, we select all the volumes we know we need, - // which may be an empty list - cmd.SetCommandAndParameters($@" + if (passNo == 0) + { + // On the first pass, we select all the volumes we know we need, + // which may be an empty list + cmd.SetCommandAndParameters($@" {selectCommand} WHERE ""ID"" IN ({missingBlocklistVolumes}) "); - // Reset the list - m_proccessedVolumes.Clear(); - } - else + // Reset the list + m_proccessedVolumes.Clear(); + } + else + { + //On anything but the first pass, we check if we are done + var r = await cmd + .SetCommandAndParameters(countMissingInformation) + .ExecuteScalarInt64Async(0) + .ConfigureAwait(false); + + if (r == 0 && !forceBlockUse) + yield break; + + if (passNo == 1) { - //On anything but the first pass, we check if we are done - var r = await cmd - .SetCommandAndParameters(countMissingInformation) - .ExecuteScalarInt64Async(0) - .ConfigureAwait(false); + // On the second pass, we select all volumes that are not mentioned in the db - if (r == 0 && !forceBlockUse) - yield break; - - if (passNo == 1) - { - // On the second pass, we select all volumes that are not mentioned in the db - - var mentionedVolumes = @" + var mentionedVolumes = @" SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; - cmd - .SetCommandAndParameters($@" + cmd + .SetCommandAndParameters($@" {selectCommand} WHERE ""ID"" NOT IN ({mentionedVolumes}) AND ""Type"" = @Type ") - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - } - else - { - // On the final pass, we select all volumes - // the filter will ensure that we do not download anything twice - cmd - .SetCommandAndParameters($@" + } + else + { + // On the final pass, we select all volumes + // the filter will ensure that we do not download anything twice + cmd + .SetCommandAndParameters($@" {selectCommand} WHERE ""Type"" = @Type ") - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - } + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); } + } - using var rd = await cmd - .ExecuteReaderAsync() - .ConfigureAwait(false); + using var rd = await cmd + .ExecuteReaderAsync() + .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) + { + var volumeID = rd.ConvertValueToInt64(3); + + // Guard against multiple downloads of the same file + if (m_proccessedVolumes.TryAdd(volumeID, volumeID)) { - var volumeID = rd.ConvertValueToInt64(3); - - // Guard against multiple downloads of the same file - if (m_proccessedVolumes.TryAdd(volumeID, volumeID)) - { - yield return new RemoteVolume( - rd.ConvertValueToString(0), - rd.ConvertValueToString(1), - rd.ConvertValueToInt64(2, -1) - ); - } + yield return new RemoteVolume( + rd.ConvertValueToString(0), + rd.ConvertValueToString(1), + rd.ConvertValueToInt64(2, -1) + ); } } } @@ -1367,28 +1365,26 @@ namespace Duplicati.Library.Main.Database AND ""Type"" = '{RemoteVolumeType.Blocks}' "; - using (var cmd = m_connection.CreateCommand(m_rtr)) + using var cmd = m_connection.CreateCommand(m_rtr); + var cnt = await cmd.ExecuteScalarInt64Async(countsql) + .ConfigureAwait(false); + + if (cnt > 0) { - var cnt = await cmd.ExecuteScalarInt64Async(countsql) - .ConfigureAwait(false); - - if (cnt > 0) + try { - try - { - await cmd.ExecuteNonQueryAsync(sql) - .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(sql) + .ConfigureAwait(false); - var cnt2 = await cmd.ExecuteScalarInt64Async(countsql) - .ConfigureAwait(false); + var cnt2 = await cmd.ExecuteScalarInt64Async(countsql) + .ConfigureAwait(false); - Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", ex, "Found {0} missing volumes; failed while attempting to replace blocks from existing volumes", cnt); - throw; - } + Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", ex, "Found {0} missing volumes; failed while attempting to replace blocks from existing volumes", cnt); + throw; } } } diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index e4fbfd300..47f6a4b90 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -80,8 +80,8 @@ namespace Duplicati.Library.Main.Database /// A task that represents the asynchronous operation. public async Task UpdateVerificationCount(string name) { - using (var cmd = m_connection.CreateCommand(m_rtr)) - await cmd.SetCommandAndParameters(@" + using var cmd = m_connection.CreateCommand(m_rtr); + await cmd.SetCommandAndParameters(@" UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE @@ -95,9 +95,9 @@ namespace Duplicati.Library.Main.Database ) WHERE ""Name"" = @Name ") - .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } /// @@ -204,18 +204,17 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); samples = Math.Max(1, samples); - using (var cmd = m_connection.CreateCommand(m_rtr)) - { - var files = new List(); - var max = cmd.ExecuteScalarInt64(@" + using var cmd = m_connection.CreateCommand(m_rtr); + var files = new List(); + var max = cmd.ExecuteScalarInt64(@" SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"" ", 0); - if (options.FullRemoteVerification != Options.RemoteTestStrategy.IndexesOnly) - { - // Select any broken items - cmd.SetCommandAndParameters(@" + if (options.FullRemoteVerification != Options.RemoteTestStrategy.IndexesOnly) + { + // Select any broken items + cmd.SetCommandAndParameters(@" SELECT ""ID"", ""Name"", @@ -233,18 +232,18 @@ namespace Duplicati.Library.Main.Database ) AND (""ArchiveTime"" = 0) ") - .ExpandInClauseParameterMssqlite("@States", [ - RemoteVolumeState.Verified.ToString(), + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Uploaded.ToString() - ]); + ]); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new RemoteVolume(rd); + using (var rd = cmd.ExecuteReader()) + while (rd.Read()) + yield return new RemoteVolume(rd); - //First we select some filesets - var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); - using (var rd = cmd.SetCommandAndParameters(@$" + //First we select some filesets + var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); + using (var rd = cmd.SetCommandAndParameters(@$" SELECT ""A"".""VolumeID"", ""A"".""Name"", @@ -272,56 +271,27 @@ namespace Duplicati.Library.Main.Database ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" ") - .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .SetParameterValues(tp.Item2) - .ExecuteReader()) - while (rd.Read()) - files.Add(new RemoteVolume(rd)); - - if (files.Count == 0) - yield break; - - if (string.IsNullOrEmpty(tp.Item1)) - files = FilterByVerificationCount(files, samples, max).ToList(); - - foreach (var f in files) - yield return f; - - //Then we select some index files - files.Clear(); - } - - cmd.SetCommandAndParameters(@" - SELECT - ""ID"", - ""Name"", - ""Size"", - ""Hash"", - ""VerificationCount"" - FROM ""Remotevolume"" - WHERE - ""Type"" = @Type - AND ""State"" IN (@States) - AND ""ArchiveTime"" = 0 - ") - .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) - .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) + .SetParameterValues(tp.Item2) + .ExecuteReader()) + while (rd.Read()) files.Add(new RemoteVolume(rd)); - foreach (var f in FilterByVerificationCount(files, samples, max)) - yield return f; - - if (options.FullRemoteVerification == Options.RemoteTestStrategy.ListAndIndexes || options.FullRemoteVerification == Options.RemoteTestStrategy.IndexesOnly) + if (files.Count == 0) yield break; - //And finally some block files - files.Clear(); + if (string.IsNullOrEmpty(tp.Item1)) + files = FilterByVerificationCount(files, samples, max).ToList(); - cmd.SetCommandAndParameters(@" + foreach (var f in files) + yield return f; + + //Then we select some index files + files.Clear(); + } + + cmd.SetCommandAndParameters(@" SELECT ""ID"", ""Name"", @@ -334,16 +304,44 @@ namespace Duplicati.Library.Main.Database AND ""State"" IN (@States) AND ""ArchiveTime"" = 0 ") - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) + .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - files.Add(new RemoteVolume(rd)); + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) + files.Add(new RemoteVolume(rd)); - foreach (var f in FilterByVerificationCount(files, samples, max)) - yield return f; - } + foreach (var f in FilterByVerificationCount(files, samples, max)) + yield return f; + + if (options.FullRemoteVerification == Options.RemoteTestStrategy.ListAndIndexes || options.FullRemoteVerification == Options.RemoteTestStrategy.IndexesOnly) + yield break; + + //And finally some block files + files.Clear(); + + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + ""Type"" = @Type + AND ""State"" IN (@States) + AND ""ArchiveTime"" = 0 + ") + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + + using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + while (await rd.ReadAsync().ConfigureAwait(false)) + files.Add(new RemoteVolume(rd)); + + foreach (var f in FilterByVerificationCount(files, samples, max)) + yield return f; } /// @@ -429,9 +427,9 @@ namespace Duplicati.Library.Main.Database if (m_tablename != null) try { - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction)) - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") - .ConfigureAwait(false); + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + .ConfigureAwait(false); } catch { } finally { m_tablename = null!; } @@ -631,40 +629,38 @@ namespace Duplicati.Library.Main.Database var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + try + { + await cmd + .SetCommandAndParameters(create) + .SetParameterValue("@Name", m_volumename) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + + cmd + .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") + .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) + .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) + .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); + + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair( + (Interface.TestEntryStatus)rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1) ?? "" + ); + + } + finally { try { await cmd - .SetCommandAndParameters(create) - .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(drop) .ConfigureAwait(false); - - cmd - .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") - .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) - .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) - .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new KeyValuePair( - (Interface.TestEntryStatus)rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1) ?? "" - ); - - } - finally - { - try - { - await cmd - .ExecuteNonQueryAsync(drop) - .ConfigureAwait(false); - } - catch { } } + catch { } } } } @@ -826,37 +822,35 @@ namespace Duplicati.Library.Main.Database var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + try + { + await cmd + .SetCommandAndParameters(create) + .SetParameterValue("@Name", m_volumename) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + + cmd + .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") + .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) + .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) + .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); + + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); + + } + finally { try { await cmd - .SetCommandAndParameters(create) - .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(drop) .ConfigureAwait(false); - - cmd - .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") - .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) - .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) - .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); - - } - finally - { - try - { - await cmd - .ExecuteNonQueryAsync(drop) - .ConfigureAwait(false); - } - catch { } } + catch { } } } } @@ -1060,41 +1054,39 @@ namespace Duplicati.Library.Main.Database var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + try { - try - { - await cmd - .SetCommandAndParameters(create) - .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync() - .ConfigureAwait(false); + await cmd + .SetCommandAndParameters(create) + .SetParameterValue("@Name", m_volumename) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); - cmd - .SetCommandAndParameters($@" + cmd + .SetCommandAndParameters($@" {extra} UNION {missing} UNION {modified} ") - .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) - .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) - .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); + .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) + .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) + .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); - } - finally + } + finally + { + try { - try - { - await cmd - .ExecuteNonQueryAsync(drop) - .ConfigureAwait(false); - } - catch { } + await cmd + .ExecuteNonQueryAsync(drop) + .ConfigureAwait(false); } + catch { } } } } @@ -1274,40 +1266,38 @@ namespace Duplicati.Library.Main.Database var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"""; - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + try + { + // Create expected hash+size table filtered by volume + await cmd + .SetCommandAndParameters(create) + .SetParameterValue("@Name", m_volumename) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + + // Compare against actual values inserted into temp table + cmd + .SetCommandAndParameters(compare) + .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) + .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) + .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); + + using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await rd.ReadAsync().ConfigureAwait(false)) + yield return new KeyValuePair( + (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1) ?? ""); + } + finally { try { - // Create expected hash+size table filtered by volume await cmd - .SetCommandAndParameters(create) - .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(drop) .ConfigureAwait(false); - - // Compare against actual values inserted into temp table - cmd - .SetCommandAndParameters(compare) - .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) - .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) - .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) - yield return new KeyValuePair( - (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1) ?? ""); - } - finally - { - try - { - await cmd - .ExecuteNonQueryAsync(drop) - .ConfigureAwait(false); - } - catch { } } + catch { } } } } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index be52b81e0..2f7f6b9be 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -540,20 +540,20 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); - using (var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false)) - await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) - .DoRunAsync(db, true, currentIsSmall, backendManager) - .ConfigureAwait(false); + using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); + await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) + .DoRunAsync(db, true, currentIsSmall, backendManager) + .ConfigureAwait(false); } else if (currentIsSmall && !m_options.NoAutoCompact) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); - using (var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false)) - await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) - .DoCompactAsync(db, true, backendManager) - .ConfigureAwait(false); + using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); + await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) + .DoCompactAsync(db, true, backendManager) + .ConfigureAwait(false); } } @@ -582,10 +582,10 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest); m_result.TestResults = new TestResults(m_result); - using (var testdb = await LocalTestDatabase.CreateAsync(m_database).ConfigureAwait(false)) - await new TestHandler(m_options, (TestResults)m_result.TestResults) - .DoRunAsync(samplesToTest, testdb, backendManager) - .ConfigureAwait(false); + using var testdb = await LocalTestDatabase.CreateAsync(m_database).ConfigureAwait(false); + await new TestHandler(m_options, (TestResults)m_result.TestResults) + .DoRunAsync(samplesToTest, testdb, backendManager) + .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index c9392cdd7..3b34d0b66 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -51,21 +51,19 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache).ConfigureAwait(false)) - { - await Utility.UpdateOptionsFromDb(db, m_options) + using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); + + await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); + + if (!m_options.Dryrun) + await db.Transaction + .CommitAsync("ComitDelete") .ConfigureAwait(false); - - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) - .ConfigureAwait(false); - - await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); - - if (!m_options.Dryrun) - await db.Transaction - .CommitAsync("ComitDelete") - .ConfigureAwait(false); - } } public async Task DoRunAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 10920ee99..006e91f94 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -44,46 +44,44 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) + using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + if (callback == null) { - if (callback == null) - { - m_result.SetResult( - await db.GetFilesets(args) - .OrderByDescending(x => x.Time) - .ToArrayAsync() - .ConfigureAwait(false), - await db.GetFiles(args) - .ToArrayAsync() - .ConfigureAwait(false), - await db.GetLogLines(args) - .ToArrayAsync() - .ConfigureAwait(false), - await db.GetVolumes(args) - .ToArrayAsync() - .ConfigureAwait(false) - ); - } - else - { - m_result.SetResult( - await db.GetFilesets(args) - .OrderByDescending(x => x.Time) - .ToArrayAsync() - .ConfigureAwait(false), - await db.GetFiles(args) - .ToArrayAsync() - .ConfigureAwait(false), - await db.GetLogLines(args) - .ToArrayAsync() - .ConfigureAwait(false), - await db.GetVolumes(args) - .ToArrayAsync() - .ConfigureAwait(false) - ); + m_result.SetResult( + await db.GetFilesets(args) + .OrderByDescending(x => x.Time) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetFiles(args) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetLogLines(args) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetVolumes(args) + .ToArrayAsync() + .ConfigureAwait(false) + ); + } + else + { + m_result.SetResult( + await db.GetFilesets(args) + .OrderByDescending(x => x.Time) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetFiles(args) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetLogLines(args) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetVolumes(args) + .ToArrayAsync() + .ConfigureAwait(false) + ); - callback(m_result); - } + callback(m_result); } } } diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 2b7ca9224..b126115d4 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -52,15 +52,13 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) - { - await Utility.UpdateOptionsFromDb(db, m_options) - .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) - .ConfigureAwait(false); - await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); - await db.Transaction.RollBackAsync().ConfigureAwait(false); - } + using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); + await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); + await db.Transaction.RollBackAsync().ConfigureAwait(false); } public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[]?, List? Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, Options options) diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index e83d07fa2..3043fe088 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -79,120 +79,118 @@ namespace Duplicati.Library.Main.Operation DateTime baseVersionTime; DateTime compareVersionTime; - using (var tmpdb = useLocalDb ? null : new TempFile()) - using (var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache).ConfigureAwait(false)) - using (var storageKeeper = await db.CreateStorageHelper().ConfigureAwait(false)) + using var tmpdb = useLocalDb ? null : new TempFile(); + using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache).ConfigureAwait(false); + using var storageKeeper = await db.CreateStorageHelper().ConfigureAwait(false); + if (useLocalDb) { - if (useLocalDb) - { - var dbtimes = await db - .FilesetTimes() - .ToListAsync() - .ConfigureAwait(false); - - if (dbtimes.Count < 2) - throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", dbtimes.Count), "NeedTwoBackupsToStartDiff"); - - long baseVersionId; - long compareVersionId; - - var times = dbtimes.Zip(Enumerable.Range(0, dbtimes.Count), (a, b) => new Tuple(b, a.Value, a.Key)).ToList(); - var bt = SelectTime(baseVersion, times, out baseVersionIndex, out baseVersionTime, out baseVersionId); - times.Remove(bt); - SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId); - - await storageKeeper - .AddFromDb(baseVersionId, false, filter) - .ConfigureAwait(false); - await storageKeeper - .AddFromDb(compareVersionId, true, filter) - .ConfigureAwait(false); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store"); - - var parsedlist = (await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - .Select(n => Volumes.VolumeBase.ParseFilename(n)) - .Where(p => p != null && p.FileType == RemoteVolumeType.Files) - .OrderByDescending(p => p.Time) - .ToArray(); - - var numberedList = parsedlist.Zip(Enumerable.Range(0, parsedlist.Length), (a, b) => new Tuple(b, a.Time, a)).ToList(); - if (numberedList.Count < 2) - throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", numberedList.Count), "NeedTwoBackupsToStartDiff"); - - Volumes.IParsedVolume baseFile; - Volumes.IParsedVolume compareFile; - - var bt = SelectTime(baseVersion, numberedList, out baseVersionIndex, out baseVersionTime, out baseFile); - numberedList.Remove(bt); - SelectTime(compareVersion, numberedList, out compareVersionIndex, out compareVersionTime, out compareFile); - - Func conv = (x) => - { - switch (x) - { - case FilelistEntryType.File: - return ListChangesElementType.File; - case FilelistEntryType.Folder: - return ListChangesElementType.Folder; - case FilelistEntryType.Symlink: - return ListChangesElementType.Symlink; - default: - return (ListChangesElementType)(-1); - } - }; - - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - using (var tmpfile = await backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options)) - foreach (var f in rd.Files) - if (FilterExpression.Matches(filter, f.Path)) - await storageKeeper - .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false) - .ConfigureAwait(false); - - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - using (var tmpfile = await backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options)) - foreach (var f in rd.Files) - if (FilterExpression.Matches(filter, f.Path)) - await storageKeeper - .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true) - .ConfigureAwait(false); - } - - var changes = await storageKeeper - .CreateChangeCountReport() + var dbtimes = await db + .FilesetTimes() + .ToListAsync() .ConfigureAwait(false); - var sizes = await storageKeeper - .CreateChangeSizeReport() + if (dbtimes.Count < 2) + throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", dbtimes.Count), "NeedTwoBackupsToStartDiff"); + + long baseVersionId; + long compareVersionId; + + var times = dbtimes.Zip(Enumerable.Range(0, dbtimes.Count), (a, b) => new Tuple(b, a.Value, a.Key)).ToList(); + var bt = SelectTime(baseVersion, times, out baseVersionIndex, out baseVersionTime, out baseVersionId); + times.Remove(bt); + SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId); + + await storageKeeper + .AddFromDb(baseVersionId, false, filter) + .ConfigureAwait(false); + await storageKeeper + .AddFromDb(compareVersionId, true, filter) .ConfigureAwait(false); - - var lst = (m_options.FullResult || callback != null) ? - (from n in storageKeeper.CreateChangedFileReport() - select n) : null; - - m_result.SetResult( - baseVersionTime, baseVersionIndex, compareVersionTime, compareVersionIndex, - changes.AddedFolders, changes.AddedSymlinks, changes.AddedFiles, - changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles, - changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles, - sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize, - (lst == null || callback == null) ? null : await lst.ToArrayAsync().ConfigureAwait(false) - ); - - if (callback != null) - callback(m_result, lst.ToEnumerable()); - - return; } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store"); + + var parsedlist = (await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + .Select(n => Volumes.VolumeBase.ParseFilename(n)) + .Where(p => p != null && p.FileType == RemoteVolumeType.Files) + .OrderByDescending(p => p.Time) + .ToArray(); + + var numberedList = parsedlist.Zip(Enumerable.Range(0, parsedlist.Length), (a, b) => new Tuple(b, a.Time, a)).ToList(); + if (numberedList.Count < 2) + throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", numberedList.Count), "NeedTwoBackupsToStartDiff"); + + Volumes.IParsedVolume baseFile; + Volumes.IParsedVolume compareFile; + + var bt = SelectTime(baseVersion, numberedList, out baseVersionIndex, out baseVersionTime, out baseFile); + numberedList.Remove(bt); + SelectTime(compareVersion, numberedList, out compareVersionIndex, out compareVersionTime, out compareFile); + + Func conv = (x) => + { + switch (x) + { + case FilelistEntryType.File: + return ListChangesElementType.File; + case FilelistEntryType.Folder: + return ListChangesElementType.Folder; + case FilelistEntryType.Symlink: + return ListChangesElementType.Symlink; + default: + return (ListChangesElementType)(-1); + } + }; + + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + using (var tmpfile = await backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options)) + foreach (var f in rd.Files) + if (FilterExpression.Matches(filter, f.Path)) + await storageKeeper + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false) + .ConfigureAwait(false); + + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + using (var tmpfile = await backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options)) + foreach (var f in rd.Files) + if (FilterExpression.Matches(filter, f.Path)) + await storageKeeper + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true) + .ConfigureAwait(false); + } + + var changes = await storageKeeper + .CreateChangeCountReport() + .ConfigureAwait(false); + + var sizes = await storageKeeper + .CreateChangeSizeReport() + .ConfigureAwait(false); + + var lst = (m_options.FullResult || callback != null) ? + (from n in storageKeeper.CreateChangedFileReport() + select n) : null; + + m_result.SetResult( + baseVersionTime, baseVersionIndex, compareVersionTime, compareVersionIndex, + changes.AddedFolders, changes.AddedSymlinks, changes.AddedFiles, + changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles, + changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles, + sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize, + (lst == null || callback == null) ? null : await lst.ToArrayAsync().ConfigureAwait(false) + ); + + if (callback != null) + callback(m_result, lst.ToEnumerable()); + + return; } } } diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index 96a311e70..f61de55dc 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -42,53 +42,51 @@ namespace Duplicati.Library.Main.Operation public async Task RunAsync(IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter) { var cancellationToken = CancellationToken.None; - using (var tmpdb = new TempFile()) - using (var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false)) + using var tmpdb = new TempFile(); + using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); + var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); + + try { - var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); + var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); + if (filteredList.Count == 0) + throw new Exception("No filesets found on remote target"); - try - { - var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); - if (filteredList.Count == 0) - throw new Exception("No filesets found on remote target"); + Exception lastEx = new Exception("No suitable files found on remote target"); - Exception lastEx = new Exception("No suitable files found on remote target"); + foreach (var fileversion in filteredList) + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; - foreach (var fileversion in filteredList) - try - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; + var file = fileversion.Value.File; + var entry = await db + .GetRemoteVolume(file.Name) + .ConfigureAwait(false); - var file = fileversion.Value.File; - var entry = await db - .GetRemoteVolume(file.Name) - .ConfigureAwait(false); + var files = new List(); + using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false)) + using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) + foreach (var cf in tmp.ControlFiles) + if (Library.Utility.FilterExpression.Matches(filter, cf.Key)) + files.Add(new ListResultFile(cf.Key, null)); - var files = new List(); - using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false)) - using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) - foreach (var cf in tmp.ControlFiles) - if (Library.Utility.FilterExpression.Matches(filter, cf.Key)) - files.Add(new ListResultFile(cf.Key, null)); + m_result.SetResult(new Library.Interface.IListResultFileset[] { new ListResultFileset(fileversion.Key, BackupType.PARTIAL_BACKUP, fileversion.Value.Time, -1, -1) }, files); + lastEx = null; + break; + } + catch (Exception ex) when (!ex.IsAbortException()) + { + lastEx = ex; + } - m_result.SetResult(new Library.Interface.IListResultFileset[] { new ListResultFileset(fileversion.Key, BackupType.PARTIAL_BACKUP, fileversion.Value.Time, -1, -1) }, files); - lastEx = null; - break; - } - catch (Exception ex) when (!ex.IsAbortException()) - { - lastEx = ex; - } - - if (lastEx != null) - throw lastEx; - } - finally - { - await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); - } + if (lastEx != null) + throw lastEx; + } + finally + { + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 4e4487aef..7f57aeb29 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -57,76 +57,74 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) { - using (var filesets = await db.SelectFileSets(m_options.Time, m_options.Version).ConfigureAwait(false)) + using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version).ConfigureAwait(false); + if (!filter.Empty) { - if (!filter.Empty) + if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) { - if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) - { - await filesets - .TakeFirst() - .ConfigureAwait(false); - } + await filesets + .TakeFirst() + .ConfigureAwait(false); } - - IAsyncEnumerable files; - if (m_options.ListFolderContents) - { - files = filesets.SelectFolderContents(filter); - } - else if (m_options.ListPrefixOnly) - { - files = filesets.GetLargestPrefix(filter); - } - else if (filter.Empty) - { - files = null; - } - else - { - files = filesets.SelectFiles(filter); - } - - if (m_options.ListSetsOnly) - { - m_result.SetResult( - await filesets - .QuickSets() - .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) - .ToArrayAsync() - .ConfigureAwait(false), - null - ); - } - else - { - m_result.SetResult( - await filesets - .Sets() - .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) - .ToArrayAsync() - .ConfigureAwait(false), - files == null - ? null - : - await files.Select(async n => - new ListResultFile( - n.Path, - await n - .Sizes() - .ToArrayAsync() - .ConfigureAwait(false) - ) - ) - .Select(x => x.Result) - .Cast() - .ToArrayAsync() - .ConfigureAwait(false) - ); - } - - return; } + + IAsyncEnumerable files; + if (m_options.ListFolderContents) + { + files = filesets.SelectFolderContents(filter); + } + else if (m_options.ListPrefixOnly) + { + files = filesets.GetLargestPrefix(filter); + } + else if (filter.Empty) + { + files = null; + } + else + { + files = filesets.SelectFiles(filter); + } + + if (m_options.ListSetsOnly) + { + m_result.SetResult( + await filesets + .QuickSets() + .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) + .ToArrayAsync() + .ConfigureAwait(false), + null + ); + } + else + { + m_result.SetResult( + await filesets + .Sets() + .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) + .ToArrayAsync() + .ConfigureAwait(false), + files == null + ? null + : + await files.Select(async n => + new ListResultFile( + n.Path, + await n + .Sizes() + .ToArrayAsync() + .ConfigureAwait(false) + ) + ) + .Select(x => x.Result) + .Cast() + .ToArrayAsync() + .ConfigureAwait(false) + ); + } + + return; } Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store"); diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 9b68834b9..816815888 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -54,174 +54,171 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - using (var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) + using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + if (await db.PartiallyRecreated().ConfigureAwait(false)) + throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); + + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); + + var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); + if (sets == null) + return; + + if (sets.Length == 0) { - if (await db.PartiallyRecreated().ConfigureAwait(false)) - throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - - await Utility.UpdateOptionsFromDb(db, m_options) - .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) - .ConfigureAwait(false); - - var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); - if (sets == null) - return; - - if (sets.Length == 0) - { - if (missing == null) - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets"); - else if (missing.Count == 0) - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files"); - else - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count)); - } + if (missing == null) + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets"); + else if (missing.Count == 0) + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files"); else + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count)); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.RemoveCount)); + + var pgoffset = 0.0f; + var pgspan = 0.95f / sets.Length; + + var filesets = await db + .FilesetTimes() + .ToListAsync() + .ConfigureAwait(false); + + var compare_list = sets.Select(async x => new { - Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.RemoveCount)); + FilesetID = x.FilesetID, + Timestamp = x.FilesetTime, + RemoveCount = x.RemoveCount, + Version = filesets.FindIndex(y => y.Key == x.FilesetID), + SetCount = await db + .GetFilesetFileCount(x.FilesetID) + .ConfigureAwait(false) + }) + .Select(x => x.Result) + .ToArray(); - var pgoffset = 0.0f; - var pgspan = 0.95f / sets.Length; - - var filesets = await db - .FilesetTimes() - .ToListAsync() + var replacementMetadataBlocksetId = -1L; + if (!m_options.DisableReplaceMissingMetadata) + { + var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); + replacementMetadataBlocksetId = await db + .GetEmptyMetadataBlocksetId( + (missing ?? []).Select(x => x.ID), + emptymetadata.FileHash, + emptymetadata.Blob.Length + ) .ConfigureAwait(false); + if (replacementMetadataBlocksetId < 0) + throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); + } - var compare_list = sets.Select(async x => new - { - FilesetID = x.FilesetID, - Timestamp = x.FilesetTime, - RemoveCount = x.RemoveCount, - Version = filesets.FindIndex(y => y.Key == x.FilesetID), - SetCount = await db - .GetFilesetFileCount(x.FilesetID) - .ConfigureAwait(false) - }) - .Select(x => x.Result) - .ToArray(); + var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); + var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); - var replacementMetadataBlocksetId = -1L; - if (!m_options.DisableReplaceMissingMetadata) + if (fully_emptied.Length == await db.FilesetTimes().CountAsync().ConfigureAwait(false)) + throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); + + if (!m_options.Dryrun) + await db.Transaction.CommitAsync().ConfigureAwait(false); + + + if (fully_emptied.Length != 0) + { + if (fully_emptied.Length == 1) + Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount); + else + Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); + + m_result.DeleteResults = new DeleteResults(m_result); + using (var rmdb = await LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) { - var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); - replacementMetadataBlocksetId = await db - .GetEmptyMetadataBlocksetId( - (missing ?? []).Select(x => x.ID), - emptymetadata.FileHash, - emptymetadata.Blob.Length - ) - .ConfigureAwait(false); - if (replacementMetadataBlocksetId < 0) - throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); + var opts = new Options(new Dictionary(m_options.RawOptions)); + opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); + opts.RawOptions.Remove("time"); + opts.RawOptions["no-auto-compact"] = "true"; + + await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) + .DoRunAsync(rmdb, true, false, backendManager).ConfigureAwait(false); + + if (!m_options.Dryrun) + await rmdb.Transaction + .CommitAsync("CommitDelete") + .ConfigureAwait(false); } - var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); - var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); + pgoffset += (pgspan * fully_emptied.Length); + m_result.OperationProgressUpdater.UpdateProgress(pgoffset); + } - if (fully_emptied.Length == await db.FilesetTimes().CountAsync().ConfigureAwait(false)) - throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); + if (to_purge.Length > 0) + { + m_result.PurgeResults = new PurgeFilesResults(m_result); - if (!m_options.Dryrun) - await db.Transaction.CommitAsync().ConfigureAwait(false); - - - if (fully_emptied.Length != 0) + foreach (var bs in to_purge) { - if (fully_emptied.Length == 1) - Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount); - else - Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); + Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); + var opts = new Options(new Dictionary(m_options.RawOptions)); - m_result.DeleteResults = new DeleteResults(m_result); - using (var rmdb = await LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) + using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db).ConfigureAwait(false)) { - var opts = new Options(new Dictionary(m_options.RawOptions)); - opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); + // Recompute the version number after we deleted the versions before + filesets = await pgdb + .FilesetTimes() + .ToListAsync() + .ConfigureAwait(false); + var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); + if (thisversion < 0) + throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); + + opts.RawOptions["version"] = thisversion.ToString(); opts.RawOptions.Remove("time"); opts.RawOptions["no-auto-compact"] = "true"; - await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) - .DoRunAsync(rmdb, true, false, backendManager).ConfigureAwait(false); + await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, async (cmd, filesetid, tablename) => + { + if (filesetid != bs.FilesetID) + throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); - if (!m_options.Dryrun) - await rmdb.Transaction - .CommitAsync("CommitDelete") - .ConfigureAwait(false); + // Update entries that would be removed because of missing metadata + var updatedEntries = 0; + if (!m_options.DisableReplaceMissingMetadata) + updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId); + + await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID"); + return updatedEntries; + }).ConfigureAwait(false); } - pgoffset += (pgspan * fully_emptied.Length); + pgoffset += pgspan; m_result.OperationProgressUpdater.UpdateProgress(pgoffset); } - - if (to_purge.Length > 0) - { - m_result.PurgeResults = new PurgeFilesResults(m_result); - - foreach (var bs in to_purge) - { - Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); - var opts = new Options(new Dictionary(m_options.RawOptions)); - - using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db).ConfigureAwait(false)) - { - // Recompute the version number after we deleted the versions before - filesets = await pgdb - .FilesetTimes() - .ToListAsync() - .ConfigureAwait(false); - var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); - if (thisversion < 0) - throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); - - opts.RawOptions["version"] = thisversion.ToString(); - opts.RawOptions.Remove("time"); - opts.RawOptions["no-auto-compact"] = "true"; - - await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, async (cmd, filesetid, tablename) => - { - if (filesetid != bs.FilesetID) - throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); - - // Update entries that would be removed because of missing metadata - var updatedEntries = 0; - if (!m_options.DisableReplaceMissingMetadata) - updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId); - - await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID"); - return updatedEntries; - }).ConfigureAwait(false); - } - - pgoffset += pgspan; - m_result.OperationProgressUpdater.UpdateProgress(pgoffset); - } - } } - - m_result.OperationProgressUpdater.UpdateProgress(0.95f); - - if (!m_options.Dryrun && await db.RepairInProgress().ConfigureAwait(false)) - { - Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); - await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) - .ConfigureAwait(false); - Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); - await db.RepairInProgress(false).ConfigureAwait(false); - } - else - { - await db.Transaction.RollBackAsync().ConfigureAwait(false); - await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) - .ConfigureAwait(false); - } - - m_result.OperationProgressUpdater.UpdateProgress(1.0f); - } + + m_result.OperationProgressUpdater.UpdateProgress(0.95f); + + if (!m_options.Dryrun && await db.RepairInProgress().ConfigureAwait(false)) + { + Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); + Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); + await db.RepairInProgress(false).ConfigureAwait(false); + } + else + { + await db.Transaction.RollBackAsync().ConfigureAwait(false); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); + } + + m_result.OperationProgressUpdater.UpdateProgress(1.0f); } } } diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index ccec9dc13..bd1340b3a 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -52,13 +52,11 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) - { - await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); - await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) - .ConfigureAwait(false); - } + using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .ConfigureAwait(false); } public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func> filtercommand) @@ -267,14 +265,12 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); m_result.CompactResults = new CompactResults(m_result); - using (var cdb = await Database.LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) - { - await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) - .DoCompactAsync(cdb, true, backendManager) - .ConfigureAwait(false); + using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false); + await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) + .DoCompactAsync(cdb, true, backendManager) + .ConfigureAwait(false); - await cdb.Transaction.CommitAsync("PostCompact"); - } + await cdb.Transaction.CommitAsync("PostCompact"); } m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 2b839856d..e07689d59 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -469,29 +469,25 @@ namespace Duplicati.Library.Main.Operation var remoteVolume = await db .GetRemoteVolume(volumename) .ConfigureAwait(false); - using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) - { - var parsed = VolumeBase.ParseFilename(remoteVolume.Name); - using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) - using (var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false)) - { - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + using var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false); + var parsed = VolumeBase.ParseFilename(remoteVolume.Name); + using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); + using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false); + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - var filesetid = await db - .CreateFileset(remoteVolume.ID, parsed.Time) - .ConfigureAwait(false); + var filesetid = await db + .CreateFileset(remoteVolume.ID, parsed.Time) + .ConfigureAwait(false); - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()) - .ConfigureAwait(false); + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()) + .ConfigureAwait(false); - if (!m_options.Dryrun) - await recreatedb.Transaction - .CommitAsync("CommitRecreateFilesetTransaction") - .ConfigureAwait(false); - } - } + if (!m_options.Dryrun) + await recreatedb.Transaction + .CommitAsync("CommitRecreateFilesetTransaction") + .ConfigureAwait(false); } if (!m_options.Dryrun && tp.MissingVolumes.Any()) diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index 6621b6702..2e47e557e 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -45,64 +45,62 @@ namespace Duplicati.Library.Main.Operation if (!Directory.Exists(m_options.Restorepath)) Directory.CreateDirectory(m_options.Restorepath); - using (var tmpdb = new TempFile()) - using (var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false)) + using var tmpdb = new TempFile(); + using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); + var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); + + try { - var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); + var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false), m_options); + if (filteredList.Count == 0) + throw new Exception("No filesets found on remote target"); - try - { - var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false), m_options); - if (filteredList.Count == 0) - throw new Exception("No filesets found on remote target"); + var lastEx = new Exception("No suitable files found on remote target"); - var lastEx = new Exception("No suitable files found on remote target"); - - foreach (var fileversion in filteredList) - try + foreach (var fileversion in filteredList) + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - return; - } - - var file = fileversion.Value.File; - var entry = await db - .GetRemoteVolume(file.Name) - .ConfigureAwait(false); - - var res = new List(); - using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) - foreach (var cf in tmp.ControlFiles) - if (FilterExpression.Matches(filter, cf.Key)) - { - var targetpath = Path.Combine(m_options.Restorepath, cf.Key); - using (var ts = File.Create(targetpath)) - await Library.Utility.Utility.CopyStreamAsync(cf.Value, ts, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - res.Add(targetpath); - } - - m_result.SetResult(res); - - lastEx = null; - break; - } - catch (Exception ex) - { - lastEx = ex; - if (ex.IsAbortException()) - throw; + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + return; } - if (lastEx != null) - throw lastEx; - } - finally - { - await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - } + var file = fileversion.Value.File; + var entry = await db + .GetRemoteVolume(file.Name) + .ConfigureAwait(false); + + var res = new List(); + using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) + foreach (var cf in tmp.ControlFiles) + if (FilterExpression.Matches(filter, cf.Key)) + { + var targetpath = Path.Combine(m_options.Restorepath, cf.Key); + using (var ts = File.Create(targetpath)) + await Library.Utility.Utility.CopyStreamAsync(cf.Value, ts, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + res.Add(targetpath); + } + + m_result.SetResult(res); + + lastEx = null; + break; + } + catch (Exception ex) + { + lastEx = ex; + if (ex.IsAbortException()) + throw; + } + + if (lastEx != null) + throw lastEx; + } + finally + { + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 3babe6ddd..266e2ee66 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -158,121 +158,119 @@ namespace Duplicati.Library.Main.Operation var updateCounter = 0L; var fullblockverification = options.FullBlockVerification; - using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false)) - using (var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize).ConfigureAwait(false)) + using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm); + using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize).ConfigureAwait(false); + await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks().ConfigureAwait(false)) { - await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks().ConfigureAwait(false)) + var targetpath = restorelist.Path; + + if (options.Dryrun) { - var targetpath = restorelist.Path; - - if (options.Dryrun) - { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchFile", "Would patch file with remote data: {0}", targetpath); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingFile", "Patching file with remote data: {0}", targetpath); - - try - { - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); - if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); - SystemIO.IO_OS.DirectoryCreate(folderpath); - } - - // TODO: Much faster if we iterate the volume and checks what blocks are used, - // because the compressors usually like sequential reading - using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) - { - file.Position = targetblock.Offset; - var size = blocks.ReadBlock(targetblock.Key, blockbuffer); - if (targetblock.Size == size) - { - var valid = !fullblockverification; - if (!valid) - { - var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - if (targetblock.Key == key) - valid = true; - else - Logging.Log.WriteWarningMessage(LOGTAG, "InvalidBlock", null, "Invalid block detected for {0}, expected hash: {1}, actual hash: {2}", targetpath, targetblock.Key, key); - } - - if (valid) - { - file.Write(blockbuffer, 0, size); - await blockmarker - .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false) - .ConfigureAwait(false); - } - } - else - { - Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlockSize", null, "Block with hash {0} should have size {1} but has size {2}", targetblock.Key, targetblock.Size, size); - } - } - - if ((++updateCounter) % 20 == 0) - await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) - .ConfigureAwait(false); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchFailed", ex, "Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } - } + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchFile", "Would patch file with remote data: {0}", targetpath); } - - if (!options.SkipMetadata) + else { - await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks().ConfigureAwait(false)) - { - var targetpath = restoremetadata.Path; - Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); + Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingFile", "Patching file with remote data: {0}", targetpath); - try + try + { + var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); + if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) { - // TODO: When we support multi-block metadata this needs to deal with it - using (var ms = new System.IO.MemoryStream()) + Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); + SystemIO.IO_OS.DirectoryCreate(folderpath); + } + + // TODO: Much faster if we iterate the volume and checks what blocks are used, + // because the compressors usually like sequential reading + using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) + await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) { - await foreach (var targetblock in restoremetadata.Blocks().ConfigureAwait(false)) + file.Position = targetblock.Offset; + var size = blocks.ReadBlock(targetblock.Key, blockbuffer); + if (targetblock.Size == size) { - ms.Position = targetblock.Offset; - var size = blocks.ReadBlock(targetblock.Key, blockbuffer); - if (targetblock.Size == size) + var valid = !fullblockverification; + if (!valid) { - ms.Write(blockbuffer, 0, size); + var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); + if (targetblock.Key == key) + valid = true; + else + Logging.Log.WriteWarningMessage(LOGTAG, "InvalidBlock", null, "Invalid block detected for {0}, expected hash: {1}, actual hash: {2}", targetpath, targetblock.Key, key); + } + + if (valid) + { + file.Write(blockbuffer, 0, size); await blockmarker - .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true) + .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false) .ConfigureAwait(false); } } - - ms.Position = 0; - metadatastorage.Add(targetpath, ms); - //blockmarker.RecordMetadata(restoremetadata.FileID, ms); + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlockSize", null, "Block with hash {0} should have size {1} but has size {2}", targetblock.Key, targetblock.Size, size); + } } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "MetatdataRecordFailed", ex, "Failed to record metadata for file: \"{0}\", message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } + + if ((++updateCounter) % 20 == 0) + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchFailed", ex, "Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; } } - await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) - .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); } + + if (!options.SkipMetadata) + { + await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks().ConfigureAwait(false)) + { + var targetpath = restoremetadata.Path; + Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); + + try + { + // TODO: When we support multi-block metadata this needs to deal with it + using (var ms = new System.IO.MemoryStream()) + { + await foreach (var targetblock in restoremetadata.Blocks().ConfigureAwait(false)) + { + ms.Position = targetblock.Offset; + var size = blocks.ReadBlock(targetblock.Key, blockbuffer); + if (targetblock.Size == size) + { + ms.Write(blockbuffer, 0, size); + await blockmarker + .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true) + .ConfigureAwait(false); + } + } + + ms.Position = 0; + metadatastorage.Add(targetpath, ms); + //blockmarker.RecordMetadata(restoremetadata.FileID, ms); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "MetatdataRecordFailed", ex, "Failed to record metadata for file: \"{0}\", message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; + } + } + } + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + await blockmarker.CommitAsync().ConfigureAwait(false); } private static void ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage) @@ -714,129 +712,19 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) { // Fill BLOCKS with data from known local source files - using (var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false)) + using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + var updateCount = 0L; + await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize).ConfigureAwait(false)) { - var updateCount = 0L; - await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize).ConfigureAwait(false)) + var targetpath = entry.TargetPath; + var targetfileid = entry.TargetFileID; + var sourcepath = entry.SourcePath; + var patched = false; + + try { - var targetpath = entry.TargetPath; - var targetfileid = entry.TargetFileID; - var sourcepath = entry.SourcePath; - var patched = false; - - try + if (SystemIO.IO_OS.FileExists(sourcepath)) { - if (SystemIO.IO_OS.FileExists(sourcepath)) - { - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); - if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); - SystemIO.IO_OS.DirectoryCreate(folderpath); - } - - using (var targetstream = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - { - try - { - using (var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath)) - { - await foreach (var block in entry.Blocks().ConfigureAwait(false)) - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - //TODO: Handle metadata - - if (sourcestream.Length > block.Offset) - { - sourcestream.Position = block.Offset; - - int size = Library.Utility.Utility.ForceStreamRead(sourcestream, blockbuffer, blockbuffer.Length); - if (size == block.Size) - { - var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); - if (key == block.Hash) - { - patched = true; - if (!options.Dryrun) - { - targetstream.Position = block.Offset; - targetstream.Write(blockbuffer, 0, size); - } - - await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false) - .ConfigureAwait(false); - } - } - } - } - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message); - if (ex.IsAbortException()) - throw; - } - } - - if ((++updateCount) % 20 == 0) - { - await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) - .ConfigureAwait(false); - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - } - - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); - if (ex.IsAbortException()) - throw; - if (options.UnittestMode) - throw; - } - - if (patched) - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); - else - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); - - if (patched && options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath); - } - - await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) - .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); - } - } - - private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) - { - // Fill BLOCKS with data from known local source files - using (var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false)) - { - var updateCount = 0L; - await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize).ConfigureAwait(false)) - { - var targetpath = restorelist.TargetPath; - var targetfileid = restorelist.TargetFileID; - var patched = false; - try - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) { @@ -844,93 +732,195 @@ namespace Duplicati.Library.Main.Operation SystemIO.IO_OS.DirectoryCreate(folderpath); } - using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + using (var targetstream = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) + { + try { - await foreach (var source in targetblock.BlockSources().ConfigureAwait(false)) + using var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath); + await foreach (var block in entry.Blocks().ConfigureAwait(false)) { - try + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + //TODO: Handle metadata + + if (sourcestream.Length > block.Offset) { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; + sourcestream.Position = block.Offset; - if (SystemIO.IO_OS.FileExists(source.Path)) + int size = Library.Utility.Utility.ForceStreamRead(sourcestream, blockbuffer, blockbuffer.Length); + if (size == block.Size) { - if (source.IsMetadata) + var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); + if (key == block.Hash) { - // TODO: Handle this by reconstructing - // metadata from file and checking the hash - - continue; - } - else - { - using (var sourcefile = SystemIO.IO_OS.FileOpenRead(source.Path)) + patched = true; + if (!options.Dryrun) { - sourcefile.Position = source.Offset; - int size = Library.Utility.Utility.ForceStreamRead(sourcefile, blockbuffer, blockbuffer.Length); - if (size == targetblock.Size) - { - var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); - if (key == targetblock.Hash) - { - if (!options.Dryrun) - { - if (targetblock.IsMetadata) - metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size)); - else - { - file.Position = targetblock.Offset; - file.Write(blockbuffer, 0, size); - } - } + targetstream.Position = block.Offset; + targetstream.Write(blockbuffer, 0, size); + } - await blockmarker - .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false) - .ConfigureAwait(false); - patched = true; - break; + await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false) + .ConfigureAwait(false); + } + } + } + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message); + if (ex.IsAbortException()) + throw; + } + } + + if ((++updateCount) % 20 == 0) + { + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + } + + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); + if (ex.IsAbortException()) + throw; + if (options.UnittestMode) + throw; + } + + if (patched) + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); + else + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); + + if (patched && options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath); + } + + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + await blockmarker.CommitAsync().ConfigureAwait(false); + } + + private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) + { + // Fill BLOCKS with data from known local source files + using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + var updateCount = 0L; + await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize).ConfigureAwait(false)) + { + var targetpath = restorelist.TargetPath; + var targetfileid = restorelist.TargetFileID; + var patched = false; + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); + if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) + { + Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); + SystemIO.IO_OS.DirectoryCreate(folderpath); + } + + using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) + await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + { + await foreach (var source in targetblock.BlockSources().ConfigureAwait(false)) + { + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + if (SystemIO.IO_OS.FileExists(source.Path)) + { + if (source.IsMetadata) + { + // TODO: Handle this by reconstructing + // metadata from file and checking the hash + + continue; + } + else + { + using var sourcefile = SystemIO.IO_OS.FileOpenRead(source.Path); + sourcefile.Position = source.Offset; + int size = Library.Utility.Utility.ForceStreamRead(sourcefile, blockbuffer, blockbuffer.Length); + if (size == targetblock.Size) + { + var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); + if (key == targetblock.Hash) + { + if (!options.Dryrun) + { + if (targetblock.IsMetadata) + metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size)); + else + { + file.Position = targetblock.Offset; + file.Write(blockbuffer, 0, size); } } + + await blockmarker + .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false) + .ConfigureAwait(false); + patched = true; + break; } } } } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, source.Path, ex.Message); - if (ex.IsAbortException()) - throw; - } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, source.Path, ex.Message); + if (ex.IsAbortException()) + throw; } } + } - if ((++updateCount) % 20 == 0) - await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) - .ConfigureAwait(false); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } - - if (patched) - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); - else - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); - - if (patched && options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath)); + if ((++updateCount) % 20 == 0) + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; } - await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) - .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); + if (patched) + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); + else + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); + + if (patched && options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath)); } + + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + await blockmarker.CommitAsync().ConfigureAwait(false); } private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) @@ -1024,226 +1014,224 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) { // Scan existing files for existing BLOCKS - using (var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false)) + using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + var updateCount = 0L; + await foreach (var restorelist in database.GetExistingFilesWithBlocks().ConfigureAwait(false)) { - var updateCount = 0L; - await foreach (var restorelist in database.GetExistingFilesWithBlocks().ConfigureAwait(false)) + var rename = !options.Overwrite; + var targetpath = restorelist.TargetPath; + var targetfileid = restorelist.TargetFileID; + var targetfilehash = restorelist.TargetHash; + var targetfilelength = restorelist.Length; + if (SystemIO.IO_OS.FileExists(targetpath)) { - var rename = !options.Overwrite; - var targetpath = restorelist.TargetPath; - var targetfileid = restorelist.TargetFileID; - var targetfilehash = restorelist.TargetHash; - var targetfilelength = restorelist.Length; - if (SystemIO.IO_OS.FileExists(targetpath)) + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + var currentfilelength = SystemIO.IO_OS.FileLength(targetpath); + var wasTruncated = false; + + // Adjust file length in overwrite mode if necessary (smaller is ok, will be extended during restore) + // We do it before scanning for blocks. This allows full verification on files that only needs to + // be truncated (i.e. forthwritten log files). + if (!rename && currentfilelength > targetfilelength) + { + var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); + if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) // clear readonly attribute + { + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribute", "Would reset read-only attribute on file: {0}", targetpath); + else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); + } + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldTruncateFile", "Would truncate file '{0}' to length of {1:N0} bytes", targetpath, targetfilelength); + else + { + using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) + file.SetLength(targetfilelength); + currentfilelength = targetfilelength; + } + wasTruncated = true; + } + + // If file size does not match and we have to rename on conflict, + // the whole scan can be skipped here because all blocks have to be restored anyway. + // For the other cases, we will check block and and file hashes and look for blocks + // to be restored and files that can already be verified. + if (!rename || currentfilelength == targetfilelength) + { + // a file hash for verification will only be necessary if the file has exactly + // the wanted size so we have a chance to already mark the file as data-verified. + bool calcFileHash = (currentfilelength == targetfilelength); + if (calcFileHash) filehasher.Initialize(); + + using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) + using (var block = new Blockprocessor(file, blockbuffer)) + await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + { + var size = block.Readblock(); + if (size <= 0) + break; + + //TODO: Handle Metadata + + bool blockhashmatch = false; + if (size == targetblock.Size) + { + // Parallelize file hash calculation on rename. Running read-only on same array should not cause conflicts or races. + // Actually, in future always calculate the file hash and mark the file data as already verified. + + System.Threading.Tasks.Task calcFileHashTask = null; + if (calcFileHash) + calcFileHashTask = System.Threading.Tasks.Task.Run( + () => filehasher.TransformBlock(blockbuffer, 0, size, blockbuffer, 0)); + + var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); + + if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. + + if (key == targetblock.Hash) + { + await blockmarker + .SetBlockRestored(targetfileid, targetblock.Index, key, size, false) + .ConfigureAwait(false); + blockhashmatch = true; + } + } + if (calcFileHash && !blockhashmatch) // will not be necessary anymore + { + filehasher.TransformFinalBlock(blockbuffer, 0, 0); // So a new initialize will not throw + calcFileHash = false; + if (rename) // file does not match. So break. + break; + } + } + + bool fullfilehashmatch = false; + if (calcFileHash) // now check if files are identical + { + filehasher.TransformFinalBlock(blockbuffer, 0, 0); + var filekey = Convert.ToBase64String(filehasher.Hash); + fullfilehashmatch = (filekey == targetfilehash); + } + + if (!rename && !fullfilehashmatch && !wasTruncated) // Reset read-only attribute (if set) to overwrite + { + var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); + if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) + { + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribyte", "Would reset read-only attribute on file: {0}", targetpath); + else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); + } + } + + if (fullfilehashmatch) + { + //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. + await blockmarker + .SetFileDataVerified(targetfileid) + .ConfigureAwait(false); + Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); + rename = false; + } + else if (rename) + { + // The new file will have none of the correct blocks, + // even if the scanned file had some + await blockmarker + .SetAllBlocksMissing(targetfileid) + .ConfigureAwait(false); + } + } + + if ((++updateCount) % 20 == 0) + { + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "TargetFileReadError", ex, "Failed to read target file: \"{0}\", message: {1}", targetpath, ex.Message); + if (ex.IsAbortException()) + throw; + if (options.UnittestMode) + throw; + } + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "MissingTargetFile", "Target file does not exist: {0}", targetpath); + rename = false; + } + + if (rename) + { + //Select a new filename + var ext = SystemIO.IO_OS.PathGetExtension(targetpath) ?? ""; + if (!string.IsNullOrEmpty(ext) && !ext.StartsWith(".", StringComparison.Ordinal)) + ext = "." + ext; + + // First we try with a simple date append, assuming that there are not many conflicts there + var newname = SystemIO.IO_OS.PathChangeExtension(targetpath, null) + "." + database.RestoreTime.ToLocalTime().ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); + var tr = newname + ext; + var c = 0; + while (SystemIO.IO_OS.FileExists(tr) && c < 1000) { try { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; + // If we have a file with the correct name, + // it is most likely the file we want + filehasher.Initialize(); - var currentfilelength = SystemIO.IO_OS.FileLength(targetpath); - var wasTruncated = false; + string key; + using (var file = SystemIO.IO_OS.FileOpenRead(tr)) + key = Convert.ToBase64String(filehasher.ComputeHash(file)); - // Adjust file length in overwrite mode if necessary (smaller is ok, will be extended during restore) - // We do it before scanning for blocks. This allows full verification on files that only needs to - // be truncated (i.e. forthwritten log files). - if (!rename && currentfilelength > targetfilelength) - { - var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); - if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) // clear readonly attribute - { - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribute", "Would reset read-only attribute on file: {0}", targetpath); - else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); - } - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldTruncateFile", "Would truncate file '{0}' to length of {1:N0} bytes", targetpath, targetfilelength); - else - { - using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - file.SetLength(targetfilelength); - currentfilelength = targetfilelength; - } - wasTruncated = true; - } - - // If file size does not match and we have to rename on conflict, - // the whole scan can be skipped here because all blocks have to be restored anyway. - // For the other cases, we will check block and and file hashes and look for blocks - // to be restored and files that can already be verified. - if (!rename || currentfilelength == targetfilelength) - { - // a file hash for verification will only be necessary if the file has exactly - // the wanted size so we have a chance to already mark the file as data-verified. - bool calcFileHash = (currentfilelength == targetfilelength); - if (calcFileHash) filehasher.Initialize(); - - using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) - using (var block = new Blockprocessor(file, blockbuffer)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) - { - var size = block.Readblock(); - if (size <= 0) - break; - - //TODO: Handle Metadata - - bool blockhashmatch = false; - if (size == targetblock.Size) - { - // Parallelize file hash calculation on rename. Running read-only on same array should not cause conflicts or races. - // Actually, in future always calculate the file hash and mark the file data as already verified. - - System.Threading.Tasks.Task calcFileHashTask = null; - if (calcFileHash) - calcFileHashTask = System.Threading.Tasks.Task.Run( - () => filehasher.TransformBlock(blockbuffer, 0, size, blockbuffer, 0)); - - var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - - if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. - - if (key == targetblock.Hash) - { - await blockmarker - .SetBlockRestored(targetfileid, targetblock.Index, key, size, false) - .ConfigureAwait(false); - blockhashmatch = true; - } - } - if (calcFileHash && !blockhashmatch) // will not be necessary anymore - { - filehasher.TransformFinalBlock(blockbuffer, 0, 0); // So a new initialize will not throw - calcFileHash = false; - if (rename) // file does not match. So break. - break; - } - } - - bool fullfilehashmatch = false; - if (calcFileHash) // now check if files are identical - { - filehasher.TransformFinalBlock(blockbuffer, 0, 0); - var filekey = Convert.ToBase64String(filehasher.Hash); - fullfilehashmatch = (filekey == targetfilehash); - } - - if (!rename && !fullfilehashmatch && !wasTruncated) // Reset read-only attribute (if set) to overwrite - { - var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); - if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) - { - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribyte", "Would reset read-only attribute on file: {0}", targetpath); - else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); - } - } - - if (fullfilehashmatch) - { - //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. - await blockmarker - .SetFileDataVerified(targetfileid) - .ConfigureAwait(false); - Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); - rename = false; - } - else if (rename) - { - // The new file will have none of the correct blocks, - // even if the scanned file had some - await blockmarker - .SetAllBlocksMissing(targetfileid) - .ConfigureAwait(false); - } - } - - if ((++updateCount) % 20 == 0) + if (key == targetfilehash) { + //TODO: Also needs metadata check to make correct decision. + // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .SetAllBlocksRestored(targetfileid, false) .ConfigureAwait(false); - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; + await blockmarker + .SetFileDataVerified(targetfileid) + .ConfigureAwait(false); + break; } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(LOGTAG, "TargetFileReadError", ex, "Failed to read target file: \"{0}\", message: {1}", targetpath, ex.Message); - if (ex.IsAbortException()) - throw; + Logging.Log.WriteWarningMessage(LOGTAG, "FailedToReadRestoreTarget", ex, "Failed to read candidate restore target {0}", tr); if (options.UnittestMode) throw; } - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "MissingTargetFile", "Target file does not exist: {0}", targetpath); - rename = false; + tr = newname + " (" + (c++).ToString() + ")" + ext; } - if (rename) - { - //Select a new filename - var ext = SystemIO.IO_OS.PathGetExtension(targetpath) ?? ""; - if (!string.IsNullOrEmpty(ext) && !ext.StartsWith(".", StringComparison.Ordinal)) - ext = "." + ext; - - // First we try with a simple date append, assuming that there are not many conflicts there - var newname = SystemIO.IO_OS.PathChangeExtension(targetpath, null) + "." + database.RestoreTime.ToLocalTime().ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); - var tr = newname + ext; - var c = 0; - while (SystemIO.IO_OS.FileExists(tr) && c < 1000) - { - try - { - // If we have a file with the correct name, - // it is most likely the file we want - filehasher.Initialize(); - - string key; - using (var file = SystemIO.IO_OS.FileOpenRead(tr)) - key = Convert.ToBase64String(filehasher.ComputeHash(file)); - - if (key == targetfilehash) - { - //TODO: Also needs metadata check to make correct decision. - // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. - await blockmarker - .SetAllBlocksRestored(targetfileid, false) - .ConfigureAwait(false); - await blockmarker - .SetFileDataVerified(targetfileid) - .ConfigureAwait(false); - break; - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FailedToReadRestoreTarget", ex, "Failed to read candidate restore target {0}", tr); - if (options.UnittestMode) - throw; - } - tr = newname + " (" + (c++).ToString() + ")" + ext; - } - - newname = tr; - - Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); - await database - .UpdateTargetPath(targetfileid, newname) - .ConfigureAwait(false); - } + newname = tr; + Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); + await database + .UpdateTargetPath(targetfileid, newname) + .ConfigureAwait(false); } - await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) - .ConfigureAwait(false); - await blockmarker - .CommitAsync() - .ConfigureAwait(false); } + + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater) + .ConfigureAwait(false); + await blockmarker + .CommitAsync() + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index d58e4dfe0..90b0e06d9 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -52,23 +52,21 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) - { - await Utility.UpdateOptionsFromDb(db, m_options) - .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) - .ConfigureAwait(false); + using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + .ConfigureAwait(false); - await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks) - .ConfigureAwait(false); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks) + .ConfigureAwait(false); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); - await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); - await db.Transaction - .CommitAsync("TestHandlerComplete") - .ConfigureAwait(false); - } + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); + await db.Transaction + .CommitAsync("TestHandlerComplete") + .ConfigureAwait(false); } public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend) From 9ac583db94b90c054989ee7c60e2ec549e4fb212 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 11:53:11 +0200 Subject: [PATCH 318/362] Made all relevant using statements use await to use DisposeAsync --- .../DatabaseTool/Commands/Downgrade.cs | 6 +- .../DatabaseTool/Commands/Upgrade.cs | 6 +- Duplicati/CommandLine/DatabaseTool/Helper.cs | 21 ++-- Duplicati/Library/Main/Controller.cs | 8 +- .../Library/Main/Database/ExtensionMethods.cs | 4 +- .../Main/Database/LocalBackupDatabase.cs | 52 ++++---- .../Main/Database/LocalBugReportDatabase.cs | 4 +- .../Library/Main/Database/LocalDatabase.cs | 114 +++++++++--------- .../Main/Database/LocalDeleteDatabase.cs | 22 ++-- .../Database/LocalListAffectedDatabase.cs | 20 +-- .../Database/LocalListBrokenFilesDatabase.cs | 18 +-- .../Main/Database/LocalListChangesDatabase.cs | 16 +-- .../Main/Database/LocalListDatabase.cs | 73 ++++++----- .../Main/Database/LocalPurgeDatabase.cs | 30 ++--- .../Main/Database/LocalRecreateDatabase.cs | 24 ++-- .../Main/Database/LocalRepairDatabase.cs | 68 +++++------ .../Main/Database/LocalRestoreDatabase.cs | 94 +++++++-------- .../Main/Database/LocalTestDatabase.cs | 32 ++--- .../Library/Main/Operation/BackupHandler.cs | 8 +- .../Library/Main/Operation/CompactHandler.cs | 2 +- .../Main/Operation/CreateBugReportHandler.cs | 2 +- .../Library/Main/Operation/DeleteHandler.cs | 2 +- .../Library/Main/Operation/ListAffected.cs | 2 +- .../Main/Operation/ListBrokenFilesHandler.cs | 2 +- .../Main/Operation/ListChangesHandler.cs | 4 +- .../Main/Operation/ListControlFilesHandler.cs | 2 +- .../Main/Operation/ListFileVersionsHandler.cs | 2 +- .../Main/Operation/ListFilesHandler.cs | 6 +- .../Main/Operation/ListFilesetsHandler.cs | 2 +- .../Main/Operation/ListFolderHandler.cs | 2 +- .../Main/Operation/PurgeBrokenFilesHandler.cs | 6 +- .../Main/Operation/PurgeFilesHandler.cs | 6 +- .../Main/Operation/RecreateDatabaseHandler.cs | 8 +- .../Library/Main/Operation/RepairHandler.cs | 16 +-- .../Operation/RestoreControlFilesHandler.cs | 2 +- .../Library/Main/Operation/RestoreHandler.cs | 10 +- .../Main/Operation/SearchEntriesHandler.cs | 2 +- .../Library/Main/Operation/TestHandler.cs | 14 +-- .../Library/Main/Operation/VacuumHandler.cs | 2 +- 39 files changed, 360 insertions(+), 354 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs index af5c58841..f2de7e23d 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs @@ -161,10 +161,10 @@ public static class Downgrade if (!nobackups) Helper.CreateFileBackup(db); - using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) + await using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) .ConfigureAwait(false); - using var tr = con.BeginTransaction(); - using var cmd = con.CreateCommand(tr); + await using var tr = con.BeginTransaction(); + await using var cmd = con.CreateCommand(tr); foreach (var script in downgradeScripts) { Console.WriteLine($"Applying downgrade script {script.Filename} ..."); diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs index 437c583bf..2e1cd7aeb 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs @@ -173,10 +173,10 @@ public static class Upgrade if (!nobackups) Helper.CreateFileBackup(db); - using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) + await using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) .ConfigureAwait(false); - using var tr = con.BeginTransaction(); - using var cmd = con.CreateCommand(tr); + await using var tr = con.BeginTransaction(); + await using var cmd = con.CreateCommand(tr); foreach (var script in upgradeScripts) { Console.WriteLine($"Applying upgrade script {script.Filename} ..."); diff --git a/Duplicati/CommandLine/DatabaseTool/Helper.cs b/Duplicati/CommandLine/DatabaseTool/Helper.cs index edb258025..c3ed85113 100644 --- a/Duplicati/CommandLine/DatabaseTool/Helper.cs +++ b/Duplicati/CommandLine/DatabaseTool/Helper.cs @@ -60,9 +60,9 @@ public static class Helper { try { - using var con = await SQLiteLoader.LoadConnectionAsync(serverdb, 0) + await using var con = await SQLiteLoader.LoadConnectionAsync(serverdb, 0) .ConfigureAwait(false); - using var cmd = con.CreateCommand(@" + await using var cmd = con.CreateCommand(@" SELECT ""DBPath"" FROM ""Backup"" "); @@ -93,11 +93,11 @@ public static class Helper /// A task that when awaited contains a tuple with the version and whether it is a server database public static async Task<(int Version, bool isserver)> ExamineDatabase(string db) { - using (var con = await SQLiteLoader.LoadConnectionAsync(db, 0).ConfigureAwait(false)) - { - using var cmd = con.CreateCommand(); + await using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) + .ConfigureAwait(false); + await using var cmd = con.CreateCommand(); - var isserverdb = await cmd.ExecuteScalarInt64Async(@" + var isserverdb = await cmd.ExecuteScalarInt64Async(@" SELECT COUNT(*) FROM sqlite_master WHERE @@ -105,16 +105,15 @@ public static class Helper AND name='Backup' OR name='Schedule' ") - .ConfigureAwait(false) == 2; + .ConfigureAwait(false) == 2; - var version = (int)await cmd.ExecuteScalarInt64Async(@" + var version = (int)await cmd.ExecuteScalarInt64Async(@" SELECT MAX(Version) FROM Version ") - .ConfigureAwait(false); + .ConfigureAwait(false); - return (version, isserverdb); - } + return (version, isserverdb); } /// diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index b16d9b920..f714fea60 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -252,9 +252,9 @@ namespace Duplicati.Library.Main { return RunAction(new ListRemoteResults(), async (result, backendManager) => { - using (var tf = File.Exists(m_options.Dbpath) ? null : new TempFile()) - using (var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false)) - result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); + using var tf = File.Exists(m_options.Dbpath) ? null : new TempFile(); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false); + result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); }); } @@ -277,7 +277,7 @@ namespace Duplicati.Library.Main // and uses the same prefix (see issues #2678, #3845, and #4244). if (File.Exists(m_options.Dbpath)) { - using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false); + await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false); var dbRemoteVolumes = db.GetRemoteVolumes(); var dbRemoteFiles = await dbRemoteVolumes .Select(x => x.Name) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index cae4e2e05..0973ea558 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -262,7 +262,7 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReaderEnumerableAsync", $"ExecuteReaderEnumerableAsync: {self.GetPrintableCommandText()}") : null) - using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) yield return rd; } @@ -391,7 +391,7 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64Async", string.Format("ExecuteScalarInt64Async: {0}", self.GetPrintableCommandText())) : null) - using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) if (await rd.ReadAsync().ConfigureAwait(false)) return ConvertValueToInt64(rd, 0, defaultvalue); diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 281bdcec7..c594960c2 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -401,7 +401,7 @@ namespace Duplicati.Library.Main.Database if (deletedBlockCacheSizeLong >= 0) { - using var cmd = dbnew.Connection.CreateCommand(); + await using var cmd = dbnew.Connection.CreateCommand(); dbnew.m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); await cmd.SetCommandAndParameters($@" CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS @@ -453,7 +453,7 @@ namespace Duplicati.Library.Main.Database ") .ConfigureAwait(false); - using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await reader.ReadAsync().ConfigureAwait(false)) { var id = reader.ConvertValueToInt64(0); @@ -888,7 +888,7 @@ namespace Duplicati.Library.Main.Database if (c != 1) { Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size); - using (var cmd = m_connection.CreateCommand(m_rtr)) + await using (var cmd = m_connection.CreateCommand(m_rtr)) { var bid = await cmd.SetCommandAndParameters(@" SELECT ""ID"" @@ -1097,7 +1097,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null).ConfigureAwait(false); + await using var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null).ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) { oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); @@ -1112,7 +1112,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries).ConfigureAwait(false); + await using var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries).ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) { length = -1; @@ -1140,7 +1140,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using var rd = await m_findfileCommand.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await m_findfileCommand.ExecuteReaderAsync().ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) { oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); @@ -1184,7 +1184,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid); - using var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync().ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) return ( rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null"), @@ -1223,7 +1223,7 @@ namespace Duplicati.Library.Main.Database if (!string.IsNullOrWhiteSpace(m_tempDeletedBlockTable)) try { - using (var cmd = m_connection.CreateCommand(m_rtr)) + await using (var cmd = m_connection.CreateCommand(m_rtr)) await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}""") .ConfigureAwait(false); await m_rtr.CommitAsync().ConfigureAwait(false); @@ -1242,7 +1242,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the size of the last written DBlock volume, or -1 if no such volume exists. public async Task GetLastWrittenDBlockVolumeSize() { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); return await cmd.SetCommandAndParameters(@" SELECT ""Size"" FROM ""RemoteVolume"" @@ -1300,7 +1300,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains a tuple with the count of files and the total size of files in the last backup fileset. internal async Task> GetLastBackupFileCountAndSize() { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var lastFilesetId = await cmd.ExecuteScalarInt64Async(@" SELECT ""ID"" FROM ""Fileset"" @@ -1358,7 +1358,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the change statistics are updated. internal async Task UpdateChangeStatistics(BackupResults results) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var prevFileSetId = await GetPreviousFilesetID(cmd) .ConfigureAwait(false); await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId) @@ -1388,8 +1388,8 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the files are appended. public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) { - using var cmd = m_connection.CreateCommand(); - using var cmdDelete = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); + await using var cmdDelete = m_connection.CreateCommand(); long lastFilesetId = prevId < 0 ? await GetPreviousFilesetID(cmd, timestamp, filesetid) .ConfigureAwait(false) @@ -1429,7 +1429,7 @@ namespace Duplicati.Library.Main.Database if (deleted != null) { - using var tmplist = await TemporaryDbValueList + await using var tmplist = await TemporaryDbValueList .CreateAsync(this, deleted) .ConfigureAwait(false); @@ -1488,8 +1488,8 @@ namespace Duplicati.Library.Main.Database return; } - using var cmd = m_connection.CreateCommand(); - using var cmdDelete = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); + await using var cmdDelete = m_connection.CreateCommand(); long lastFilesetId = prevFileSetId < 0 ? await GetPreviousFilesetID(cmd, timestamp, fileSetId) .ConfigureAwait(false) @@ -1629,7 +1629,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of volume names that are missing index files. public async IAsyncEnumerable GetMissingIndexFiles() { - using var cmd = m_connection.CreateCommand(m_rtr) + await using var cmd = m_connection.CreateCommand(m_rtr) .SetCommandAndParameters(@" SELECT ""Name"" FROM ""RemoteVolume"" @@ -1647,7 +1647,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Verified.ToString() ]); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } @@ -1662,7 +1662,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the block is moved. public async Task MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var c = await cmd.SetCommandAndParameters(@" UPDATE ""Block"" SET ""VolumeID"" = @NewVolumeId @@ -1693,7 +1693,7 @@ namespace Duplicati.Library.Main.Database { var volumeid = await GetRemoteVolumeID(name).ConfigureAwait(false); - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var c = await cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""Block"" @@ -1717,7 +1717,7 @@ namespace Duplicati.Library.Main.Database public async Task GetBlocklistHashes(string name) { var volumeid = GetRemoteVolumeID(name); - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); // Grab the strings and return as array to avoid concurrent access to the IEnumerable cmd.SetCommandAndParameters(@" SELECT DISTINCT ""Block"".""Hash"" @@ -1743,7 +1743,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the first path, or null if no paths exist. public async Task GetFirstPath() { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var v0 = await cmd.ExecuteScalarAsync(@" SELECT ""Path"" FROM ""File"" @@ -1767,7 +1767,7 @@ namespace Duplicati.Library.Main.Database { var data = new List(); - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""VolumeName"", ""JournalID"", @@ -1779,7 +1779,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) { yield return new Interface.USNJournalDataEntry @@ -1802,7 +1802,7 @@ namespace Duplicati.Library.Main.Database { foreach (var entry in data) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var c = await cmd.SetCommandAndParameters(@" INSERT INTO ""ChangeJournalData"" ( ""FilesetID"", @@ -1844,7 +1844,7 @@ namespace Duplicati.Library.Main.Database { foreach (var entry in data) { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); await cmd.SetCommandAndParameters(@" UPDATE ""ChangeJournalData"" SET ""NextUSN"" = @NextUsn diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index ff73c0147..d1610487e 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -63,12 +63,12 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the obfuscation is finished. public async Task Fix() { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); // TODO: Rewrite this to use PathPrefix // TODO: Needs to be much faster - using (var upcmd = m_connection.CreateCommand(m_rtr)) + await using (var upcmd = m_connection.CreateCommand(m_rtr)) { await upcmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tablename}"" ( diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 1afb6f8f8..828dcc805 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -274,7 +274,7 @@ namespace Duplicati.Library.Main.Database if (dbnew.m_connection.State != ConnectionState.Open) await dbnew.m_connection.OpenAsync().ConfigureAwait(false); - using var cmd = dbnew.m_connection.CreateCommand() + await using var cmd = dbnew.m_connection.CreateCommand() .SetTransaction(dbnew.m_rtr); if (operation != null) { @@ -297,7 +297,7 @@ namespace Duplicati.Library.Main.Database else { // Get last operation - using var rd = await cmd.ExecuteReaderAsync(@" + await using var rd = await cmd.ExecuteReaderAsync(@" SELECT ""ID"", ""Timestamp"" @@ -567,7 +567,7 @@ namespace Duplicati.Library.Main.Database if (deleteGraceTime.Ticks > 0) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = @DeleteGraceTime WHERE ""Name"" = @Name @@ -586,7 +586,7 @@ namespace Duplicati.Library.Main.Database if (setArchived.HasValue) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""RemoteVolume"" SET ""ArchiveTime"" = @ArchiveTime WHERE ""Name"" = @Name @@ -615,7 +615,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of key-value pairs, where each pair contains the fileset ID and its timestamp. public async IAsyncEnumerable> FilesetTimes() { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" SELECT ""ID"", ""Timestamp"" @@ -624,7 +624,7 @@ namespace Duplicati.Library.Main.Database ") .ConfigureAwait(false); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( rd.ConvertValueToInt64(0), @@ -722,7 +722,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of key-value pairs, where each pair contains the file name and its corresponding remote volume ID. public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" SELECT ""Name"", ""ID"" @@ -732,12 +732,12 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); cmd.SetTransaction(m_rtr); - using var tmptable = await TemporaryDbValueList.CreateAsync(this, files) + await using var tmptable = await TemporaryDbValueList.CreateAsync(this, files) .ConfigureAwait(false); await cmd.ExpandInClauseParameterMssqliteAsync("@Name", tmptable) .ConfigureAwait(false); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } @@ -753,7 +753,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Name", file); - using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync().ConfigureAwait(false)) if (await rd.ReadAsync().ConfigureAwait(false)) return new RemoteVolumeEntry( rd.ConvertValueToInt64(0), @@ -795,7 +795,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable GetRemoteVolumes() { m_selectremotevolumesCommand.SetTransaction(m_rtr); - using var rd = await m_selectremotevolumesCommand + await using var rd = await m_selectremotevolumesCommand .ExecuteReaderAsync() .ConfigureAwait(false); @@ -864,7 +864,7 @@ namespace Duplicati.Library.Main.Database /// Thrown if the number of unlinked remote volumes is not equal to 1. public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" DELETE FROM ""RemoteVolume"" WHERE ""Name"" = @Name @@ -901,7 +901,7 @@ namespace Duplicati.Library.Main.Database { if (names == null || !names.Any()) return; - using var deletecmd = m_connection.CreateCommand(m_rtr); + await using var deletecmd = m_connection.CreateCommand(m_rtr); string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var volidstable = "DelVolSetIds-" + temptransguid; var blocksetidstable = "DelBlockSetIds-" + temptransguid; @@ -1244,7 +1244,7 @@ namespace Duplicati.Library.Main.Database public async Task Vacuum() { m_hasExecutedVacuum = true; - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); await cmd.ExecuteNonQueryAsync("VACUUM").ConfigureAwait(false); } @@ -1336,7 +1336,7 @@ namespace Duplicati.Library.Main.Database await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch) .ConfigureAwait(false); var res = new List(); - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters($@" SELECT ""ID"" FROM ""Fileset"" @@ -1345,7 +1345,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValues(values); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); @@ -1356,7 +1356,7 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" ORDER BY ""Timestamp"" DESC "); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); @@ -1386,7 +1386,7 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); var res = new List(); - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters(@$" SELECT ""ID"" FROM ""Fileset"" @@ -1394,7 +1394,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC ") .SetParameterValues(args); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); @@ -1409,7 +1409,7 @@ namespace Duplicati.Library.Main.Database /// A task that, when awaited, returns true if the fileset is a full backup, otherwise false. public async Task IsFilesetFullBackup(DateTime filesetTime) { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters($@" SELECT ""IsFullBackup"" FROM ""Fileset"" @@ -1418,7 +1418,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); if (!await rd.ReadAsync().ConfigureAwait(false)) return false; var isFullBackup = rd.GetInt32(0); @@ -1431,7 +1431,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of key-value pairs representing the database options. private async IAsyncEnumerable> GetDbOptionList() { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""Key"", ""Value"" @@ -1439,7 +1439,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( rd.ConvertValueToString(0) ?? "", @@ -1542,7 +1542,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the options have been set. public async Task SetDbOptions(IDictionary options) { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Configuration"" ") @@ -1574,7 +1574,7 @@ namespace Duplicati.Library.Main.Database /// A task that, when awaited, returns the count of blocks larger than the specified size. public async Task GetBlocksLargerThan(long fhblocksize) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size @@ -1619,7 +1619,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the consistency check is finished. private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair) { - using var cmd = m_connection.CreateCommand() + await using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); // Calculate the lengths for each blockset var combinedLengths = @" @@ -1654,7 +1654,7 @@ namespace Duplicati.Library.Main.Database AND ""A"".""CalcLen"" != ""A"".""Length"" "; - using (var rd = await cmd.ExecuteReaderAsync(reportDetails).ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(reportDetails).ConfigureAwait(false)) if (await rd.ReadAsync().ConfigureAwait(false)) { var sb = new StringBuilder(); @@ -1802,7 +1802,7 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync() .ConfigureAwait(false); if (await reader.ReadAsync().ConfigureAwait(false)) throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); @@ -1847,7 +1847,7 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync() .ConfigureAwait(false); if (await reader.ReadAsync().ConfigureAwait(false)) throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); @@ -1870,7 +1870,7 @@ namespace Duplicati.Library.Main.Database if (nonAttachedFiles != 0) { // Attempt to create a better error message by finding the first 10 fileset ids with the issue - using var filesetIdReader = await cmd.ExecuteReaderAsync(@" + await using var filesetIdReader = await cmd.ExecuteReaderAsync(@" SELECT DISTINCT(FilesetID) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN ( @@ -1911,7 +1911,7 @@ namespace Duplicati.Library.Main.Database if (verifyfilelists) { var anyError = new List(); - using (var cmd2 = m_connection.CreateCommand()) + await using (var cmd2 = m_connection.CreateCommand()) { cmd2.SetTransaction(m_rtr); cmd.SetCommandAndParameters(@" @@ -2008,7 +2008,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of blocks associated with the specified volume ID. public async IAsyncEnumerable GetBlocks(long volumeid) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" SELECT DISTINCT ""Hash"", ""Size"" @@ -2020,7 +2020,7 @@ namespace Duplicati.Library.Main.Database cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new Block( rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), @@ -2302,7 +2302,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the fileset has been written. public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId) { - using var cmd = m_connection.CreateCommand() + await using var cmd = m_connection.CreateCommand() .SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS) .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) @@ -2310,7 +2310,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); string? lastpath = null; - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) { var blocksetID = rd.ConvertValueToInt64(0, -1); @@ -2336,7 +2336,7 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(LIST_FILESETS); cmd.SetParameterValue("@FilesetId", filesetId); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) if (await rd.ReadAsync().ConfigureAwait(false)) { var more = false; @@ -2394,7 +2394,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the link operation is finished. public async Task LinkFilesetToVolume(long filesetid, long volumeid) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""Fileset"" SET ""VolumeID"" = @VolumeId WHERE ""ID"" = @FilesetId @@ -2434,7 +2434,7 @@ namespace Duplicati.Library.Main.Database ) "; - using var cmd = await m_connection.CreateCommandAsync(query) + await using var cmd = await m_connection.CreateCommandAsync(query) .ConfigureAwait(false); await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) @@ -2492,7 +2492,7 @@ namespace Duplicati.Library.Main.Database if (filter.Empty) { - using var cmd = await db.Connection.CreateCommandAsync($@" + await using var cmd = await db.Connection.CreateCommandAsync($@" CREATE TEMPORARY TABLE ""{ftt.Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" @@ -2509,7 +2509,7 @@ namespace Duplicati.Library.Main.Database if (type == FilterType.Regexp || type == FilterType.Group) { // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full - using var cmd = await db.Connection.CreateCommandAsync($@" + await using var cmd = await db.Connection.CreateCommandAsync($@" CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( ""Path"" TEXT NOT NULL ) @@ -2525,14 +2525,14 @@ namespace Duplicati.Library.Main.Database VALUES (@Path) "); - using var c2 = await db.Connection.CreateCommandAsync(@" + await using var c2 = await db.Connection.CreateCommandAsync(@" SELECT DISTINCT ""Path"" FROM ""File"" ") .ConfigureAwait(false); c2.SetTransaction(db.Transaction); - using (var rd = await c2.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await c2.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) { var p = rd.ConvertValueToString(0) ?? ""; @@ -2569,7 +2569,7 @@ namespace Duplicati.Library.Main.Database } } - using var cmd = db.Connection.CreateCommand(); + await using var cmd = db.Connection.CreateCommand(); cmd.SetTransaction(db.Transaction); await cmd.ExecuteNonQueryAsync($@" @@ -2603,7 +2603,7 @@ namespace Duplicati.Library.Main.Database if (Tablename != null) try { - using var cmd = m_db.Connection.CreateCommand(); + await using var cmd = m_db.Connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@$" DROP TABLE IF EXISTS ""{Tablename}"" ") @@ -2624,7 +2624,7 @@ namespace Duplicati.Library.Main.Database public async Task RenameRemoteFile(string oldname, string newname) { //Rename the old entry, to preserve ID links - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""Remotevolume"" SET ""Name"" = @Newname WHERE ""Name"" = @Oldname @@ -2678,7 +2678,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the ID of the newly created fileset. public virtual async Task CreateFileset(long volumeid, DateTime timestamp) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" INSERT INTO ""Fileset"" ( ""OperationID"", ""Timestamp"", @@ -2738,7 +2738,7 @@ namespace Duplicati.Library.Main.Database /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data public async IAsyncEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); // Group subquery by hash to ensure that each blocklist hash appears only once in the result // The AllBlocks CTE is used to map both active and duplicate blocks from the volume, // because the new volume is initially registered with only duplicate blocks. @@ -2804,7 +2804,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@HashesPerBlock", blocksize / hashsize) .ConfigureAwait(false); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) { var blockhash = rd.ConvertValueToString(0); @@ -2833,7 +2833,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the update is finished. public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""Fileset"" SET ""IsFullBackup"" = @IsFullBackup WHERE ""ID"" = @FilesetId @@ -2854,7 +2854,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the entries have been cleared. public async Task ClearFilesetEntries(long filesetId) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId ") @@ -2893,7 +2893,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of key-value pairs where the key is the fileset ID and the value is the timestamp of the fileset. public async IAsyncEnumerable> GetIncompleteFilesets() { - using var cmd = await m_connection.CreateCommandAsync(@$" + await using var cmd = await m_connection.CreateCommandAsync(@$" SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" @@ -2913,7 +2913,7 @@ namespace Duplicati.Library.Main.Database ") .ConfigureAwait(false); - using var rd = await cmd.SetTransaction(m_rtr) + await using var rd = await cmd.SetTransaction(m_rtr) .ExecuteReaderAsync() .ConfigureAwait(false); @@ -2934,7 +2934,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited returns the remote volume entry associated with the fileset ID, or default if not found. public async Task GetRemoteVolumeFromFilesetID(long filesetID) { - using var cmd = await m_connection.CreateCommandAsync(@" + await using var cmd = await m_connection.CreateCommandAsync(@" SELECT ""RemoteVolume"".""ID"", ""Name"", @@ -2953,7 +2953,7 @@ namespace Duplicati.Library.Main.Database ") .ConfigureAwait(false); - using var rd = await cmd.SetTransaction(m_rtr) + await using var rd = await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetID) .ExecuteReaderAsync() .ConfigureAwait(false); @@ -2980,7 +2980,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the purge operation is finished. public async Task PurgeLogData(DateTime threshold) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold); await cmd.SetCommandAndParameters(@" @@ -3035,8 +3035,8 @@ namespace Duplicati.Library.Main.Database await m_rtr.DisposeAsync().ConfigureAwait(false); if (m_connection.State == ConnectionState.Open && !m_hasExecutedVacuum) { - using (var command = m_connection.CreateCommand()) - using (var tr = m_connection.BeginTransaction()) + await using (var command = m_connection.CreateCommand()) + await using (var tr = m_connection.BeginTransaction()) { // SQLite recommends that PRAGMA optimize is run just before closing each database connection. await command diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index a381491f7..83badd292 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -118,10 +118,10 @@ namespace Duplicati.Library.Main.Database /// An async enumerable of key-value pairs, where the key is the fileset name and the value is the size of the fileset. public async IAsyncEnumerable> DropFilesetsFromTable(DateTime[] toDelete) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var deleted = 0; - using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds)).ConfigureAwait(false)) + await using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds)).ConfigureAwait(false)) deleted += await ( await cmd.SetCommandAndParameters(@" DELETE FROM ""Fileset"" @@ -281,7 +281,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) .SetParameterValue("@State", RemoteVolumeState.Deleting.ToString()); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( rd.ConvertValueToString(0) ?? "", @@ -297,7 +297,7 @@ namespace Duplicati.Library.Main.Database /// An async enumerable of IListResultFileset. internal async IAsyncEnumerable FilesetsWithBackupVersion() { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); // TODO check if this is still the case? (shouldn't be with new sqlite driver): // We can also use the ROW_NUMBER() window function to generate the backup versions, // but this requires at least SQLite 3.25, which is not available in some common @@ -310,7 +310,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC "); - using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); int version = 0; while (await reader.ReadAsync().ConfigureAwait(false)) { @@ -497,7 +497,7 @@ namespace Duplicati.Library.Main.Database var createtable = $"{@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS "}{collected}"; - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); try { await cmd @@ -523,7 +523,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""B"".""Sorttime"" ASC "); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new VolumeUsage( rd.ConvertValueToString(0) ?? "", @@ -881,7 +881,7 @@ namespace Duplicati.Library.Main.Database if (deletedVolume.Type != RemoteVolumeType.Blocks) return; - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var updatedBlocks = "BlocksToUpdate-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var replacementBlocks = "ReplacementBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try @@ -896,7 +896,7 @@ namespace Duplicati.Library.Main.Database .ExecuteNonQueryAsync() .ConfigureAwait(false); - using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved).ConfigureAwait(false)) + await using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved).ConfigureAwait(false)) await ( await cmd.SetCommandAndParameters($@" CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS @@ -1005,7 +1005,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of that represents the order in which volumes should be deleted. public async IAsyncEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); // Although the generated index volumes are always in pairs, // this code handles many-to-many relations between // index files and block volumes, should this be added later @@ -1029,7 +1029,7 @@ namespace Duplicati.Library.Main.Database AND ""B"".""Size"" IS NOT NULL "); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) { var name = rd.ConvertValueToString(0) ?? ""; diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 62f5676ec..97f5d149c 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -170,9 +170,9 @@ namespace Duplicati.Library.Main.Database ) "; - using var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false); - using var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false); + await using var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) { var v = dict[rd.ConvertValueToInt64(0)]; @@ -249,9 +249,9 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Path"" "; - using var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false); - using var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false); + await using var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new ListResultFile() { @@ -268,7 +268,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of log line results. public async IAsyncEnumerable GetLogLines(IEnumerable items) { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); foreach (var slice in items.Chunk(CHUNK_SIZE / 2)) { @@ -299,7 +299,7 @@ namespace Duplicati.Library.Main.Database foreach ((var x, var i) in items.Select((x, i) => (x, i))) cmd.SetParameterValue($"@Message{i}", "%" + x + "%"); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new ListResultRemoteLog() { @@ -382,8 +382,8 @@ namespace Duplicati.Library.Main.Database ) "; - using var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterMssqlite("@Names", items.ToArray()); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterMssqlite("@Names", items.ToArray()); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new ListResultRemoteVolume() { diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 474abac6f..aeacf2280 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -219,7 +219,7 @@ namespace Duplicati.Library.Main.Database query += @" GROUP BY ""A"".""FilesetID"""; - using var cmd = Connection.CreateCommand(m_rtr) + await using var cmd = Connection.CreateCommand(m_rtr) .SetCommandAndParameters(query) .SetParameterValues(clause.Values); @@ -239,7 +239,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of broken file IDs. public async IAsyncEnumerable> GetBrokenFilenames(long filesetid) { - using var cmd = Connection.CreateCommand(m_rtr) + await using var cmd = Connection.CreateCommand(m_rtr) .SetCommandAndParameters(BROKEN_FILE_NAMES) .SetParameterValue("@FilesetId", filesetid); @@ -257,7 +257,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of representing the orphaned index files. public async IAsyncEnumerable GetOrphanedIndexFiles() { - using var cmd = Connection.CreateCommand($@" + await using var cmd = Connection.CreateCommand($@" SELECT ""Name"", ""Hash"", @@ -289,7 +289,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the insertion is finished. public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname) { - using var cmd = Connection.CreateCommand(m_rtr) + await using var cmd = Connection.CreateCommand(m_rtr) .SetCommandAndParameters(INSERT_BROKEN_IDS(tablename, IDfieldname)) .SetParameterValue("@FilesetId", filesetid); @@ -307,7 +307,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the ID of the empty metadata blockset, or -1 if no suitable blockset is found public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize) { - using var cmd = Connection.CreateCommand(@" + await using var cmd = Connection.CreateCommand(@" SELECT ""ID"" FROM ""Blockset"" WHERE @@ -386,7 +386,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the number of rows affected public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" UPDATE ""Metadataset"" SET ""BlocksetID"" = @EmptyBlocksetID WHERE @@ -423,7 +423,7 @@ namespace Duplicati.Library.Main.Database { if (names == null || !names.Any()) return; - using var deletecmd = m_connection.CreateCommand(m_rtr); + await using var deletecmd = m_connection.CreateCommand(m_rtr); var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var volidstable = "DelVolSetIds-" + temptransguid; @@ -435,7 +435,7 @@ namespace Duplicati.Library.Main.Database ") .ConfigureAwait(false); - using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names).ConfigureAwait(false)) + await using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names).ConfigureAwait(false)) await ( await deletecmd.SetCommandAndParameters($@" INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") @@ -505,7 +505,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the count of files in the specified fileset. public async Task GetFilesetFileCount(long filesetid) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 14ff354db..bfb12d220 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -255,7 +255,7 @@ namespace Duplicati.Library.Main.Database m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) }; - using (var cmd = sh.m_db.Connection.CreateCommand(db.Transaction)) + await using (var cmd = sh.m_db.Connection.CreateCommand(db.Transaction)) { await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" ( @@ -388,7 +388,7 @@ namespace Duplicati.Library.Main.Database var combined = $"({folders} UNION {symlinks} UNION {files})"; - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); if (filter == null || filter.Empty) { // Simple case, select everything @@ -506,7 +506,7 @@ namespace Duplicati.Library.Main.Database .PrepareAsync() .ConfigureAwait(false); - using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) + await using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) .SetCommandAndParameters($@" INSERT INTO ""{tablename}"" ( ""Path"", @@ -525,7 +525,7 @@ namespace Duplicati.Library.Main.Database "); await cmd2.PrepareAsync().ConfigureAwait(false); - using var rd = await cmd + await using var rd = await cmd .ExecuteReaderAsync() .ConfigureAwait(false); @@ -568,7 +568,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of strings, where each string is a value from the first column of the reader. private static async IAsyncEnumerable ReaderToStringList(SqliteDataReader rd) { - using (rd) + await using (rd) while (await rd.ReadAsync().ConfigureAwait(false)) { var v = rd.GetValue(0); @@ -632,7 +632,7 @@ namespace Duplicati.Library.Main.Database { var (Added, Deleted, Modified) = GetSqls(true); - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); var result = new ChangeSizeReport { PreviousSize = await cmd.ExecuteScalarInt64Async($@" @@ -688,7 +688,7 @@ namespace Duplicati.Library.Main.Database FROM ({Modified}) "; - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); var result = new ChangeCountReport { @@ -750,7 +750,7 @@ namespace Duplicati.Library.Main.Database { var (Added, Deleted, Modified) = GetSqls(false); - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + await using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { var elTypes = new[] { Interface.ListChangesElementType.Folder, diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 2e6da5087..d9fd7fa30 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -182,7 +182,7 @@ namespace Duplicati.Library.Main.Database var query = tmp.Item1; var args = tmp.Item2; - using (var cmd = fs.m_db.Connection.CreateCommand()) + await using (var cmd = fs.m_db.Connection.CreateCommand()) { await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{fs.m_tablename}"" AS @@ -312,8 +312,8 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of file versions representing the largest prefix. private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) { - using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false); - using var cmd = m_db.Connection.CreateCommand(); + await using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false); + await using var cmd = m_db.Connection.CreateCommand(); //First we trim the filelist to exclude filenames not found in any of the filesets await cmd.ExecuteNonQueryAsync($@" DELETE FROM ""{tmpnames.Tablename}"" @@ -449,7 +449,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""Path"" FROM ""{table}"" "); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) { var s = rd.ConvertValueToString(0) ?? ""; @@ -487,8 +487,8 @@ namespace Duplicati.Library.Main.Database if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); - using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true)).ConfigureAwait(false); - using var cmd = m_db.Connection.CreateCommand(); + await using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true)).ConfigureAwait(false); + await using var cmd = m_db.Connection.CreateCommand(); //First we trim the filelist to exclude filenames not found in any of the filesets await cmd.ExecuteNonQueryAsync($@" DELETE FROM ""{tmpnames.Tablename}"" @@ -523,7 +523,7 @@ namespace Duplicati.Library.Main.Database ") .ConfigureAwait(false); - using (var c2 = m_db.Connection.CreateCommand()) + await using (var c2 = m_db.Connection.CreateCommand()) { c2.SetCommandAndParameters($@" INSERT INTO ""{tbname}"" (""Path"") @@ -593,7 +593,7 @@ namespace Duplicati.Library.Main.Database AND ""C"".""Path"" = ""D"".""Path"" "; - using var rd = await cmd + await using var rd = await cmd .ExecuteReaderAsync(query) .ConfigureAwait(false); @@ -621,7 +621,7 @@ namespace Duplicati.Library.Main.Database { try { - using var c = m_db.Connection.CreateCommand(); + await using var c = m_db.Connection.CreateCommand(); await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}""") .ConfigureAwait(false); } @@ -633,8 +633,8 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable SelectFiles(IFilter filter) { - using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false); - using var cmd = m_db.Connection.CreateCommand(); + await using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false); + await using var cmd = m_db.Connection.CreateCommand(); //First we trim the filelist to exclude filenames not found in any of the filesets await cmd.ExecuteNonQueryAsync($@" DELETE FROM ""{tmpnames.Tablename}"" @@ -703,7 +703,7 @@ namespace Duplicati.Library.Main.Database AND ""C"".""Path"" = ""D"".""Path"" "; - using var rd = await cmd.ExecuteReaderAsync(query) + await using var rd = await cmd.ExecuteReaderAsync(query) .ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) { @@ -722,7 +722,7 @@ namespace Duplicati.Library.Main.Database /// public async Task TakeFirst() { - using var cmd = m_db.Connection.CreateCommand(); + await using var cmd = m_db.Connection.CreateCommand(); await cmd.ExecuteNonQueryAsync($@" DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN ( @@ -741,8 +741,8 @@ namespace Duplicati.Library.Main.Database for (var i = 0; i < m_filesets.Length; i++) dict[m_filesets[i].Key] = i; - using var cmd = m_db.Connection.CreateCommand(); - using var rd = await cmd.ExecuteReaderAsync(@" + await using var cmd = m_db.Connection.CreateCommand(); + await using var rd = await cmd.ExecuteReaderAsync(@" SELECT DISTINCT ""ID"", ""IsFullBackup"" @@ -789,8 +789,8 @@ namespace Duplicati.Library.Main.Database GROUP BY ""A"".""FilesetID"" "; - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); - using var rd = await cmd.ExecuteReaderAsync($@" + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var rd = await cmd.ExecuteReaderAsync($@" SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", @@ -832,7 +832,7 @@ namespace Duplicati.Library.Main.Database { try { - using var cmd = m_db.Connection.CreateCommand(); + await using var cmd = m_db.Connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ") .ConfigureAwait(false); } @@ -888,7 +888,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""f"".""Timestamp"" DESC; "; - using var cmd = m_connection.CreateCommand(query); + await using var cmd = m_connection.CreateCommand(query); await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) { var id = rd.ConvertValueToInt64(0); @@ -932,7 +932,7 @@ namespace Duplicati.Library.Main.Database if (limit <= 0) limit = long.MaxValue; - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); if (prefixIds.Count() == 0) return new PaginatedResults(0, (int)limit, 0, 0, Enumerable.Empty()); @@ -1043,7 +1043,7 @@ namespace Duplicati.Library.Main.Database if (limit <= 0) limit = long.MaxValue; - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); if (!prefixIds.Any()) return new PaginatedResults(0, (int)limit, 0, 0, Enumerable.Empty()); @@ -1136,9 +1136,9 @@ namespace Duplicati.Library.Main.Database /// /// The prefixes to get the IDs for. /// An asynchronous enumerable of prefix IDs. - public IAsyncEnumerable GetPrefixIds(IEnumerable prefixes) + public async IAsyncEnumerable GetPrefixIds(IEnumerable prefixes) { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); // Map folders to prefix IDs cmd.SetCommandAndParameters($@" @@ -1148,8 +1148,10 @@ namespace Duplicati.Library.Main.Database ") .ExpandInClauseParameter("@Prefixes", prefixes); - return cmd.ExecuteReaderEnumerableAsync() - .Select(x => x.ConvertValueToInt64(0)); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + { + yield return rd.ConvertValueToInt64(0); + } } /// @@ -1157,9 +1159,9 @@ namespace Duplicati.Library.Main.Database /// /// The filesetId to get the prefixes for. /// An asynchronous enumerable of root prefixes. - public IAsyncEnumerable GetRootPrefixes(long filesetid) + public async IAsyncEnumerable GetRootPrefixes(long filesetid) { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters(@" WITH Prefixes AS ( @@ -1187,9 +1189,14 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@filesetid", filesetid); - return cmd.ExecuteReaderEnumerableAsync() - .Select(x => x.ConvertValueToInt64(0, -1)) - .Where(x => x != -1); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + { + var prefixId = rd.ConvertValueToInt64(0, -1); + if (prefixId != -1) + { + yield return prefixId; + } + } } /// @@ -1199,7 +1206,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited returns an asynchronous enumerable of (PrefixID, ParentPrefixID) pairs. public async Task> GetRootPrefixesWithParents(long filesetid) { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); var dirSeparator = Util.GuessDirSeparator(( await cmd.SetCommandAndParameters(@" @@ -1274,7 +1281,7 @@ namespace Duplicati.Library.Main.Database if (limit <= 0) limit = long.MaxValue; - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); var countWhere = "WHERE fl.\"Path\" IN (@Paths)"; if (filesetIds != null && filesetIds.Length > 0) @@ -1416,7 +1423,7 @@ namespace Duplicati.Library.Main.Database if (limit <= 0) limit = long.MaxValue; - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); // Analyze the filter to determine the default behavior FilterExpression.AnalyzeFilters(filter, out var includes, out var excludes); diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index bbaad66fd..50cdb25e3 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -89,7 +89,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the name of the remote volume. public async Task GetRemoteVolumeNameForFileset(long id) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""B"".""Name"" FROM @@ -101,7 +101,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", id); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); if (!await rd.ReadAsync().ConfigureAwait(false)) throw new Exception($"No remote volume found for fileset with id {id}"); else @@ -114,7 +114,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited contains the count of orphan files. internal async Task CountOrphanFiles() { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""FileLookup"" @@ -123,7 +123,7 @@ namespace Duplicati.Library.Main.Database FROM ""FilesetEntry"" ) "); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) return rd.ConvertValueToInt64(0, 0); else @@ -227,7 +227,7 @@ namespace Duplicati.Library.Main.Database m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) }; - using (var cmd = db.Connection.CreateCommand()) + await using (var cmd = db.Connection.CreateCommand()) await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tempf.m_tablename}"" ( ""FileID"" INTEGER PRIMARY KEY @@ -246,7 +246,7 @@ namespace Duplicati.Library.Main.Database public async Task ApplyFilter(Func> filtercommand) { int updated; - using (var cmd = m_db.Connection.CreateCommand()) + await using (var cmd = m_db.Connection.CreateCommand()) updated = await filtercommand(cmd, ParentID, m_tablename) .ConfigureAwait(false); @@ -267,7 +267,7 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using var cmd = m_db.Connection.CreateCommand(); + await using var cmd = m_db.Connection.CreateCommand(); await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{filenamestable}"" ( ""Path"" TEXT NOT NULL @@ -310,8 +310,8 @@ namespace Duplicati.Library.Main.Database { // Do row-wise iteration var values = new object[2]; - using var cmd = m_db.Connection.CreateCommand(); - using var cmd2 = m_db.Connection.CreateCommand(); + await using var cmd = m_db.Connection.CreateCommand(); + await using var cmd2 = m_db.Connection.CreateCommand(); await cmd2.SetCommandAndParameters($@" INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId) @@ -332,7 +332,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", ParentID); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) { rd.GetValues(values); @@ -357,7 +357,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the checks have been applied. private async Task PostFilterChecks(int updated) { - using var cmd = m_db.Connection.CreateCommand(); + await using var cmd = m_db.Connection.CreateCommand(); UpdatedFileCount = updated; RemovedFileCount = await cmd.ExecuteScalarInt64Async($@" @@ -410,7 +410,7 @@ namespace Duplicati.Library.Main.Database await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup) .ConfigureAwait(false); - using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + await using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) await cmd.SetCommandAndParameters($@" INSERT INTO ""FilesetEntry"" ( ""FilesetID"", @@ -436,8 +436,8 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable> ListAllDeletedFiles() { - using var cmd = m_db.Connection.CreateCommand(); - using var rd = await cmd.ExecuteReaderAsync($@" + await using var cmd = m_db.Connection.CreateCommand(); + await using var rd = await cmd.ExecuteReaderAsync($@" SELECT ""B"".""Path"", ""C"".""Length"" @@ -465,7 +465,7 @@ namespace Duplicati.Library.Main.Database { try { - using var cmd = m_db.Connection.CreateCommand(); + await using var cmd = m_db.Connection.CreateCommand(); await cmd .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 836ffe44a..56416a8cd 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -203,7 +203,7 @@ namespace Duplicati.Library.Main.Database if (dbnew.m_connection == null) throw new Exception("Connection is null"); - using var cmd = dbnew.m_connection.CreateCommand(); + await using var cmd = dbnew.m_connection.CreateCommand(); await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" ( @@ -440,7 +440,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the operation is finished. public async Task FindMissingBlocklistHashes(long hashsize, long blocksize) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); //Update all small blocklists and matching blocks var selectSmallBlocks = $@" @@ -594,7 +594,7 @@ namespace Duplicati.Library.Main.Database { Logging.Log.WriteErrorMessage(LOGTAG, "BlocksetInsertFailed", ex, "Blockset insert failed, committing temporary tables for trace purposes"); - using (var fixcmd = m_connection.CreateCommand(m_rtr)) + await using (var fixcmd = m_connection.CreateCommand(m_rtr)) { await fixcmd.ExecuteNonQueryAsync($@" CREATE TABLE ""{m_tempblocklist}-Failure"" AS @@ -640,7 +640,7 @@ namespace Duplicati.Library.Main.Database public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, bool hashOnly = false) { // TODO should values be parameters, rather than hardcoded into the SQL queries? - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var extra = hashOnly ? "" : $@" UNION SELECT @@ -749,7 +749,7 @@ namespace Duplicati.Library.Main.Database { Logging.Log.WriteErrorMessage(LOGTAG, "BlockOrBlocksetInsertFailed", ex, "Block or Blockset insert failed, committing temporary tables for trace purposes"); - using (var fixcmd = m_connection.CreateCommand(m_rtr)) + await using (var fixcmd = m_connection.CreateCommand(m_rtr)) { await fixcmd.ExecuteNonQueryAsync($@" CREATE TABLE ""{m_tempblocklist}_Failure"" AS @@ -1086,7 +1086,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable collection of block hashes associated with the specified volume ID. public async IAsyncEnumerable GetBlockLists(long volumeid) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", @@ -1098,7 +1098,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? ""; } @@ -1114,7 +1114,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable collection of remote volumes that are missing blocks. public async IAsyncEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var selectCommand = @" SELECT DISTINCT ""RemoteVolume"".""Name"", @@ -1216,7 +1216,7 @@ namespace Duplicati.Library.Main.Database } } - using var rd = await cmd + await using var rd = await cmd .ExecuteReaderAsync() .ConfigureAwait(false); @@ -1365,7 +1365,7 @@ namespace Duplicati.Library.Main.Database AND ""Type"" = '{RemoteVolumeType.Blocks}' "; - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var cnt = await cmd.ExecuteScalarInt64Async(countsql) .ConfigureAwait(false); @@ -1401,7 +1401,7 @@ namespace Duplicati.Library.Main.Database var tmptablename = "DeletedBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); // 1. Select blocks not used by any file and not as a blocklist into temporary table await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tmptablename}"" AS @@ -1463,7 +1463,7 @@ namespace Duplicati.Library.Main.Database public async override ValueTask DisposeAsync() { - using (var cmd = m_connection.CreateCommand(m_rtr)) + await using (var cmd = m_connection.CreateCommand(m_rtr)) { if (m_tempblocklist != null) try diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 2e1a252c7..035f116c6 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -87,7 +87,7 @@ namespace Duplicati.Library.Main.Database /// Thrown if the remote file does not exist. public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", @@ -129,7 +129,7 @@ namespace Duplicati.Library.Main.Database if (prevFilesetId <= 0) throw new ArgumentException("prevId must be > 0"); - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId @@ -148,7 +148,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of index file names that reference the specified block file. public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", @@ -172,7 +172,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of key-value pairs where the key is the fileset ID and the value is the timestamp of the fileset. public async IAsyncEnumerable> GetFilesetsWithMissingFiles() { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""ID"", ""Timestamp"" @@ -188,7 +188,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) { yield return new KeyValuePair( @@ -205,7 +205,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited returns the number of deleted entries. public async Task DeleteFilesetEntries(long filesetid) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId ") @@ -254,7 +254,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of representing the block volumes. public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""Name"", ""Hash"", @@ -435,7 +435,7 @@ namespace Duplicati.Library.Main.Database public static async Task CreateMissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) { var blocklist = new MissingBlockList(volumename, connection, transaction); - using (var cmd = connection.CreateCommand(transaction.Transaction)) + await using (var cmd = connection.CreateCommand(transaction.Transaction)) { await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{blocklist.m_tablename}"" ( @@ -572,7 +572,7 @@ namespace Duplicati.Library.Main.Database /// public async IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize) { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", @@ -612,7 +612,7 @@ namespace Duplicati.Library.Main.Database /// public async IAsyncEnumerable GetSourceItemsWithMetadataBlocks() { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", @@ -651,7 +651,7 @@ namespace Duplicati.Library.Main.Database /// public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; @@ -775,7 +775,7 @@ namespace Duplicati.Library.Main.Database throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); // Move the source blocks into the DuplicateBlock table - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" INSERT OR IGNORE INTO ""DuplicateBlock"" ( ""BlockID"", ""VolumeID"" @@ -880,7 +880,7 @@ namespace Duplicati.Library.Main.Database ) "; - using var cmd = m_connection.CreateCommand(cmdtxt) + await using var cmd = m_connection.CreateCommand(cmdtxt) .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); @@ -895,7 +895,7 @@ namespace Duplicati.Library.Main.Database /// public async IAsyncEnumerable GetMissingBlockSources() { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", @@ -942,7 +942,7 @@ namespace Duplicati.Library.Main.Database if (m_tablename != null) { - using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + await using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") .ConfigureAwait(false); await cmd.SetTransaction(m_rtr) @@ -974,7 +974,7 @@ namespace Duplicati.Library.Main.Database /// A task that when completed indicates that the repair has been attempted. public async Task FixDuplicateMetahash() { - using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); var sql_count = @" SELECT COUNT(*) @@ -1025,7 +1025,7 @@ namespace Duplicati.Library.Main.Database AND ""A"".""ID"" = ""B"".""ID"" "; - using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) { c2.SetCommandAndParameters($@" UPDATE ""{tablename}"" @@ -1041,7 +1041,7 @@ namespace Duplicati.Library.Main.Database AND ""ID"" != @MetadataId "); - using var rd = await cmd.ExecuteReaderAsync(sql) + await using var rd = await cmd.ExecuteReaderAsync(sql) .ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) @@ -1078,7 +1078,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""ID"" "; - using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) { c2.SetCommandAndParameters($@" UPDATE ""FilesetEntry"" @@ -1160,7 +1160,7 @@ namespace Duplicati.Library.Main.Database /// A task that when completed indicates that the repair has been attempted. public async Task FixDuplicateFileentries() { - using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); var sql_count = @" SELECT COUNT(*) @@ -1215,7 +1215,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""ID"" "; - using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) { c2.SetCommandAndParameters(@" UPDATE ""FilesetEntry"" @@ -1276,7 +1276,7 @@ namespace Duplicati.Library.Main.Database { var blocklistbuffer = new byte[blocksize]; - using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); using var blockhasher = HashFactory.CreateHasher(blockhashalgorithm); var hashsize = blockhasher.HashSize / 8; @@ -1324,11 +1324,11 @@ namespace Duplicati.Library.Main.Database if (itemswithnoblocklisthash != 0) { Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); - using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) - using (var c3 = m_connection.CreateCommand(m_rtr.Transaction)) - using (var c4 = m_connection.CreateCommand(m_rtr.Transaction)) - using (var c5 = m_connection.CreateCommand(m_rtr.Transaction)) - using (var c6 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c3 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c4 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c5 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c6 = m_connection.CreateCommand(m_rtr.Transaction)) { c3.SetCommandAndParameters(@" INSERT INTO ""BlocklistHash"" ( @@ -1537,7 +1537,7 @@ namespace Duplicati.Library.Main.Database /// A task that when completed indicates that the repair has been attempted. public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize) { - using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); var dup_sql = @" SELECT * @@ -1577,7 +1577,7 @@ namespace Duplicati.Library.Main.Database ", 0) .ConfigureAwait(false); - using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) { c2.SetCommandAndParameters(@" DELETE FROM ""BlocklistHash"" @@ -1646,7 +1646,7 @@ namespace Duplicati.Library.Main.Database /// A task that when awaited indicates the completion of the check. public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks) { - using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try @@ -1725,7 +1725,7 @@ namespace Duplicati.Library.Main.Database /// Thrown if the blocklist does not match the expected entries. public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength) { - using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); var query = $@" SELECT @@ -1782,7 +1782,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of missing local fileset names. public async IAsyncEnumerable MissingLocalFilesets() { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""Name"" FROM ""RemoteVolume"" WHERE @@ -1810,7 +1810,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of tuples containing the fileset ID, timestamp, and whether it is a full backup. public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets() { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""ID"", ""Timestamp"", @@ -1845,7 +1845,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of remote volumes that are empty index files. public async IAsyncEnumerable EmptyIndexFiles() { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""Name"", ""Hash"", diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 63bab97c6..94a2e3311 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -149,7 +149,7 @@ namespace Duplicati.Library.Main.Database m_totalprogtable = "TotalProgress-" + m_temptabsetguid; m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; - using var cmd = m_connection.CreateCommand() + await using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); // How to handle METADATA? @@ -404,7 +404,7 @@ namespace Duplicati.Library.Main.Database m_tempfiletable = "Fileset-" + m_temptabsetguid; m_tempblocktable = "Blocks-" + m_temptabsetguid; - using (var cmd = m_connection.CreateCommand()) + await using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); var filesetIds = @@ -575,7 +575,7 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tempfiletable}"" ) "); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) sb.AppendLine(rd.ConvertValueToString(0)); @@ -618,7 +618,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", filesetId); object[] values = new object[4]; - using var cmd2 = m_connection.CreateCommand($@" + await using var cmd2 = m_connection.CreateCommand($@" INSERT INTO ""{m_tempfiletable}"" ( ""ID"", ""Path"", @@ -634,7 +634,7 @@ namespace Duplicati.Library.Main.Database 0 ) "); - using var rd = await cmd.ExecuteReaderAsync() + await using var rd = await cmd.ExecuteReaderAsync() .ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) { @@ -681,7 +681,7 @@ namespace Duplicati.Library.Main.Database ""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" "); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) { var filecount = 0L; var filesize = 0L; @@ -712,7 +712,7 @@ namespace Duplicati.Library.Main.Database /// A task that, when awaited, returns the first path found in the temporary file table, or null if no paths are found. public async Task GetFirstPath() { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC @@ -733,7 +733,7 @@ namespace Duplicati.Library.Main.Database /// A task that, when awaited, returns the largest prefix path found in the temporary file table. public async Task GetLargestPrefix() { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC @@ -800,7 +800,7 @@ namespace Duplicati.Library.Main.Database : largest_prefix); - using var cmd = m_connection.CreateCommand() + await using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); if (string.IsNullOrEmpty(destination)) { @@ -942,7 +942,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the missing blocks have been found and inserted into the temporary block table. public async Task FindMissingBlocks(bool skipMetadata) { - using var cmd = m_connection.CreateCommand() + await using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); var p1 = await cmd.ExecuteNonQueryAsync($@" INSERT INTO ""{m_tempblocktable}"" ( @@ -1041,7 +1041,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the target path has been updated. public async Task UpdateTargetPath(long ID, string newname) { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @TargetPath WHERE ""ID"" = @ID @@ -1303,7 +1303,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of IExistingFile objects representing the existing files with their blocks. public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename) { - using var cmd = db.Connection.CreateCommand($@" + await using var cmd = db.Connection.CreateCommand($@" SELECT ""{tablename}"".""TargetPath"", ""Blockset"".""FullHash"", @@ -1326,7 +1326,7 @@ namespace Duplicati.Library.Main.Database ""BlocksetEntry"".""Index"" ") .SetTransaction(db.Transaction); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) { var more = true; @@ -1488,7 +1488,7 @@ namespace Duplicati.Library.Main.Database { // TODO: Skip metadata as required // Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows - using var cmd = db.Connection.CreateCommand($@" + await using var cmd = db.Connection.CreateCommand($@" SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", @@ -1518,7 +1518,7 @@ namespace Duplicati.Library.Main.Database ""B"".""Index"" ") .SetTransaction(db.Transaction); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); var more = await rd.ReadAsync().ConfigureAwait(false); while (more) @@ -1553,7 +1553,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of IRemoteVolume objects representing the missing remote volumes. public async IAsyncEnumerable GetMissingVolumes() { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT ""RV"".""Name"", ""RV"".""Hash"", @@ -1583,7 +1583,7 @@ namespace Duplicati.Library.Main.Database // Now it is likely to restore all files from front to back. Large files will always be done last. // One could also use like the average block number in a volume, that needs to be measured. - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) { yield return new RemoteVolume( @@ -1632,7 +1632,7 @@ namespace Duplicati.Library.Main.Database m_blocksize = blocksize, }; - using var c = db.Connection.CreateCommand() + await using var c = db.Connection.CreateCommand() .SetTransaction(db.Transaction); fam.m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); await c.ExecuteNonQueryAsync($@" @@ -1677,7 +1677,7 @@ namespace Duplicati.Library.Main.Database { if (m_tmptable != null) { - using var c = m_db.Connection.CreateCommand(@$"DROP TABLE IF EXISTS ""{m_tmptable}""") + await using var c = m_db.Connection.CreateCommand(@$"DROP TABLE IF EXISTS ""{m_tmptable}""") .SetTransaction(m_db.Transaction); await c.ExecuteNonQueryAsync().ConfigureAwait(false); await m_db.Transaction.CommitAsync().ConfigureAwait(false); @@ -1746,7 +1746,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable FilesWithMissingBlocks() { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - using var cmd = m_db.Connection.CreateCommand($@" + await using var cmd = m_db.Connection.CreateCommand($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", @@ -1775,7 +1775,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_db.Transaction); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) { var more = true; @@ -1799,7 +1799,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable MetadataWithMissingBlocks() { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - using var cmd = m_db.Connection.CreateCommand($@" + await using var cmd = m_db.Connection.CreateCommand($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", @@ -1828,7 +1828,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_db.Transaction); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) { var more = true; @@ -1915,7 +1915,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of IFileToRestore objects representing the files to restore. public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified) { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT ""{m_tempfiletable}"".""ID"", ""{m_tempfiletable}"".""TargetPath"", @@ -1931,7 +1931,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Verified", !onlyNonVerified); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new FileToRestore( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); @@ -1944,7 +1944,7 @@ namespace Duplicati.Library.Main.Database public async IAsyncEnumerable GetFilesAndSymlinksToRestore() { // Order by length descending, so that larger files are restored first. - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT ""F"".""ID"", ""F"".""Path"", @@ -1959,7 +1959,7 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Length"" DESC ") .SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new FileRequest( rd.ConvertValueToInt64(0), @@ -1977,9 +1977,9 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of FileRequest objects representing the folders to restore. public async IAsyncEnumerable GetFolderMetadataToRestore() { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); cmd.SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync($@" + await using var rd = await cmd.ExecuteReaderAsync($@" SELECT ""F"".""ID"", '', @@ -2027,7 +2027,7 @@ namespace Duplicati.Library.Main.Database ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "; - using var cmd = Connection.CreateCommand($@" + await using var cmd = Connection.CreateCommand($@" SELECT ""Block"".""ID"", ""Block"".""VolumeID"" @@ -2040,7 +2040,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await reader.ReadAsync().ConfigureAwait(false)) yield return ( reader.ConvertValueToInt64(0), @@ -2061,7 +2061,7 @@ namespace Duplicati.Library.Main.Database try { - using var cmd = connection.CreateCommand(@$" + await using var cmd = connection.CreateCommand(@$" SELECT ""Block"".""ID"", ""Block"".""Hash"", @@ -2076,7 +2076,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(transaction) .SetParameterValue("@BlocksetID", blocksetID); - using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync() .ConfigureAwait(false); for (long i = 0; await reader.ReadAsync().ConfigureAwait(false); i++) yield return new BlockRequest( @@ -2106,7 +2106,7 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); try { - using var cmd = connection.CreateCommand($@" + await using var cmd = connection.CreateCommand($@" SELECT ""Block"".""ID"", ""Block"".""Hash"", @@ -2124,7 +2124,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(transaction) .SetParameterValue("@FileID", fileID); - using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync() .ConfigureAwait(false); for (long i = 0; await reader.ReadAsync().ConfigureAwait(false); i++) { @@ -2152,7 +2152,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of tuples containing the volume name, size, and hash. public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) { - using var cmd = m_connection.CreateCommand(@" + await using var cmd = m_connection.CreateCommand(@" SELECT ""Name"", ""Size"", @@ -2163,7 +2163,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@VolumeID", VolumeID); - using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync() .ConfigureAwait(false); while (await reader.ReadAsync().ConfigureAwait(false)) yield return ( @@ -2179,7 +2179,7 @@ namespace Duplicati.Library.Main.Database /// A task that completes when the temporary tables are dropped and the transaction is committed. public async Task DropRestoreTable() { - using var cmd = m_connection.CreateCommand() + await using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); if (m_tempfiletable != null) @@ -2438,7 +2438,7 @@ namespace Duplicati.Library.Main.Database return; m_hasUpdates = false; - using var rd = await m_statUpdateCommand + await using var rd = await m_statUpdateCommand .SetTransaction(m_db.Transaction) .ExecuteReaderAsync() .ConfigureAwait(false); @@ -2584,7 +2584,7 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of strings representing the target folders. public async IAsyncEnumerable GetTargetFolders() { - using var cmd = m_connection.CreateCommand($@" + await using var cmd = m_connection.CreateCommand($@" SELECT ""TargetPath"" FROM ""{m_tempfiletable}"" WHERE ""BlocksetID"" == @BlocksetID @@ -2592,7 +2592,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? ""; } @@ -2727,8 +2727,8 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of representing the files and their source blocks. public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize) { - using (var cmdReader = m_connection.CreateCommand()) - using (var cmd = m_connection.CreateCommand()) + await using (var cmdReader = m_connection.CreateCommand()) + await using (var cmd = m_connection.CreateCommand()) { cmdReader.SetTransaction(m_rtr); cmd.SetTransaction(m_rtr); @@ -2741,7 +2741,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"" "); - using (var rd = await cmdReader.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmdReader.ExecuteReaderAsync().ConfigureAwait(false)) { while (await rd.ReadAsync().ConfigureAwait(false)) { @@ -2803,7 +2803,7 @@ namespace Duplicati.Library.Main.Database GROUP BY ""File"".""Path"" "; - using (var cmd = m_connection.CreateCommand()) + await using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ") @@ -2852,10 +2852,10 @@ namespace Duplicati.Library.Main.Database ""{m_tempblocktable}"".""Index"" "; - using (var cmd = m_connection.CreateCommand()) + await using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); - using var rd = await cmd.ExecuteReaderAsync(sources) + await using var rd = await cmd.ExecuteReaderAsync(sources) .ConfigureAwait(false); if (await rd.ReadAsync().ConfigureAwait(false)) { diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 47f6a4b90..4eddb114e 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -80,7 +80,7 @@ namespace Duplicati.Library.Main.Database /// A task that represents the asynchronous operation. public async Task UpdateVerificationCount(string name) { - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); await cmd.SetCommandAndParameters(@" UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, @@ -204,7 +204,7 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); samples = Math.Max(1, samples); - using var cmd = m_connection.CreateCommand(m_rtr); + await using var cmd = m_connection.CreateCommand(m_rtr); var files = new List(); var max = cmd.ExecuteScalarInt64(@" SELECT MAX(""VerificationCount"") @@ -237,13 +237,13 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Uploaded.ToString() ]); - using (var rd = cmd.ExecuteReader()) + await using (var rd = cmd.ExecuteReader()) while (rd.Read()) yield return new RemoteVolume(rd); //First we select some filesets var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); - using (var rd = cmd.SetCommandAndParameters(@$" + await using (var rd = cmd.SetCommandAndParameters(@$" SELECT ""A"".""VolumeID"", ""A"".""Name"", @@ -307,7 +307,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) files.Add(new RemoteVolume(rd)); @@ -336,7 +336,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) files.Add(new RemoteVolume(rd)); @@ -397,7 +397,7 @@ namespace Duplicati.Library.Main.Database bl.m_volumename = volumename; var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction)) + await using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction)) { await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tablename}"" @@ -427,7 +427,7 @@ namespace Duplicati.Library.Main.Database if (m_tablename != null) try { - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction); await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") .ConfigureAwait(false); } @@ -629,7 +629,7 @@ namespace Duplicati.Library.Main.Database var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); try { await cmd @@ -644,7 +644,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( (Interface.TestEntryStatus)rd.ConvertValueToInt64(0), @@ -822,7 +822,7 @@ namespace Duplicati.Library.Main.Database var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); try { await cmd @@ -837,7 +837,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); @@ -1054,7 +1054,7 @@ namespace Duplicati.Library.Main.Database var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); try { await cmd @@ -1073,7 +1073,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); @@ -1266,7 +1266,7 @@ namespace Duplicati.Library.Main.Database var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"""; - using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); try { // Create expected hash+size table filtered by volume @@ -1283,7 +1283,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 2f7f6b9be..b00e3207a 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -540,7 +540,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); - using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) .DoRunAsync(db, true, currentIsSmall, backendManager) .ConfigureAwait(false); @@ -550,7 +550,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); - using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) .DoCompactAsync(db, true, backendManager) .ConfigureAwait(false); @@ -582,7 +582,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest); m_result.TestResults = new TestResults(m_result); - using var testdb = await LocalTestDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var testdb = await LocalTestDatabase.CreateAsync(m_database).ConfigureAwait(false); await new TestHandler(m_options, (TestResults)m_result.TestResults) .DoRunAsync(samplesToTest, testdb, backendManager) .ConfigureAwait(false); @@ -714,7 +714,7 @@ namespace Duplicati.Library.Main.Operation .TerminatedWithActiveUploads(true) .ConfigureAwait(false); - using (m_database = database) + await using (m_database = database) using (var db = new Backup.BackupDatabase(m_database, m_options)) // Setup runners and instances here using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp)) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index e7c642410..781ca99f2 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache).ConfigureAwait(false)) { await Utility.UpdateOptionsFromDb(db, m_options) .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index deca98811..69492ae26 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation using (var tmp = new TempFile()) { File.Copy(m_options.Dbpath, tmp, true); - using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache).ConfigureAwait(false)) { await db.Fix().ConfigureAwait(false); if (m_options.AutoVacuum) diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 3b34d0b66..19498295f 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -51,7 +51,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache).ConfigureAwait(false); await Utility.UpdateOptionsFromDb(db, m_options) .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 006e91f94..33b0fa1bc 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -44,7 +44,7 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); if (callback == null) { m_result.SetResult( diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index b126115d4..13dc5502c 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); await Utility.UpdateOptionsFromDb(db, m_options) .ConfigureAwait(false); await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index 3043fe088..8ee6cc459 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -80,8 +80,8 @@ namespace Duplicati.Library.Main.Operation DateTime compareVersionTime; using var tmpdb = useLocalDb ? null : new TempFile(); - using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache).ConfigureAwait(false); - using var storageKeeper = await db.CreateStorageHelper().ConfigureAwait(false); + await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache).ConfigureAwait(false); + await using var storageKeeper = await db.CreateStorageHelper().ConfigureAwait(false); if (useLocalDb) { var dbtimes = await db diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index f61de55dc..ee5d229a3 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.Operation { var cancellationToken = CancellationToken.None; using var tmpdb = new TempFile(); - using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); try diff --git a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs index 05447a7ef..90533677b 100644 --- a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs @@ -47,7 +47,7 @@ internal static class ListFileVersionsHandler if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) .ConfigureAwait(false); long[]? filesetIds = null; diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 7f57aeb29..77479350c 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -55,9 +55,9 @@ namespace Duplicati.Library.Main.Operation //Use a speedy local query if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) - using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) { - using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version).ConfigureAwait(false); + await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version).ConfigureAwait(false); if (!filter.Empty) { if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) @@ -137,7 +137,7 @@ namespace Duplicati.Library.Main.Operation // Otherwise, grab info from remote location using (var tmpdb = new TempFile()) - using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache).ConfigureAwait(false)) { var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) diff --git a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs index 32e28cb6d..b1169b1a2 100644 --- a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs @@ -49,7 +49,7 @@ internal static class ListFilesetsHandler // Use a speedy local query if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb) { - using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) .ConfigureAwait(false); result.Filesets = await db diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index 9562cb98e..b71e0a81c 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -46,7 +46,7 @@ internal static class ListFolderHandler if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) .ConfigureAwait(false); var filesetIds = await db diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 816815888..47afcb0eb 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -54,7 +54,7 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); if (await db.PartiallyRecreated().ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); @@ -134,7 +134,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); m_result.DeleteResults = new DeleteResults(m_result); - using (var rmdb = await LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) + await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) { var opts = new Options(new Dictionary(m_options.RawOptions)); opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); @@ -163,7 +163,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); var opts = new Options(new Dictionary(m_options.RawOptions)); - using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db).ConfigureAwait(false)) + await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db).ConfigureAwait(false)) { // Recompute the version number after we deleted the versions before filesets = await pgdb diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index bd1340b3a..dd7aece5a 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); await db .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) @@ -138,7 +138,7 @@ namespace Duplicati.Library.Main.Operation if (ix != 0 && filesets[ix - 1].Value <= ts) throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); - using (var tempset = await db.CreateTemporaryFileset(versionid).ConfigureAwait(false)) + await using (var tempset = await db.CreateTemporaryFileset(versionid).ConfigureAwait(false)) { if (filtercommand == null) await tempset.ApplyFilter(filter).ConfigureAwait(false); @@ -265,7 +265,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); m_result.CompactResults = new CompactResults(m_result); - using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false); + await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false); await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) .DoCompactAsync(cdb, true, backendManager) .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 660f0c6f9..3c00e6c08 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -64,7 +64,7 @@ namespace Duplicati.Library.Main.Operation if (File.Exists(path)) throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); - using var db = + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache) .ConfigureAwait(false); @@ -88,7 +88,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.RepairOnlyPaths) throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); - using var db = + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache) .ConfigureAwait(false); if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version).ConfigureAwait(false)).Any()) @@ -134,7 +134,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running); //We build a local database in steps. - using var restoredb = + await using var restoredb = await LocalRecreateDatabase.CreateAsync(dbparent, m_options) .ConfigureAwait(false); await restoredb.RepairInProgress(true).ConfigureAwait(false); @@ -652,7 +652,7 @@ namespace Duplicati.Library.Main.Operation // except to continue a backup m_result.EndTime = DateTime.UtcNow; - using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb).ConfigureAwait(false)) + await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb).ConfigureAwait(false)) { var broken = await lbfdb .GetBrokenFilesets(new DateTime(0), null) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index e07689d59..1383e1fda 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -65,7 +65,7 @@ namespace Duplicati.Library.Main.Operation long knownRemotes = -1; try { - using var db = + await using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) .ConfigureAwait(false); knownRemotes = await db @@ -133,7 +133,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using var db = + await using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) .ConfigureAwait(false); @@ -230,7 +230,7 @@ namespace Duplicati.Library.Main.Operation { if (tp.VerificationRequiredVolumes.Any()) { - using var testdb = + await using var testdb = await LocalTestDatabase.CreateAsync(db) .ConfigureAwait(false); @@ -473,7 +473,7 @@ namespace Duplicati.Library.Main.Operation var parsed = VolumeBase.ParseFilename(remoteVolume.Name); using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); - using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false); + await using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false); if (compressor == null) throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); @@ -1063,7 +1063,7 @@ namespace Duplicati.Library.Main.Operation // The dblock files are the most complex to recreate // as data can be either file contents, metadata or blocklist hashes // We attempt to recover all three source parts in the steps below - using var mbl = await db + await using var mbl = await db .CreateBlockList(originalVolume.Name) .ConfigureAwait(false); @@ -1321,7 +1321,7 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using var db = + await using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) .ConfigureAwait(false); @@ -1351,7 +1351,7 @@ namespace Duplicati.Library.Main.Operation // Clear out the old fileset await db.DeleteFilesetEntries(entry.Key).ConfigureAwait(false); - using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false)) + await using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false)) { await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression()) .ConfigureAwait(false); @@ -1376,7 +1376,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using var db = + await using var db = await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index 2e47e557e..d3d18639e 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Operation Directory.CreateDirectory(m_options.Restorepath); using var tmpdb = new TempFile(); - using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); try diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 266e2ee66..c6c9af53e 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -159,8 +159,8 @@ namespace Duplicati.Library.Main.Operation var fullblockverification = options.FullBlockVerification; using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm); - using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); - using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize).ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize).ConfigureAwait(false); await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks().ConfigureAwait(false)) { var targetpath = restorelist.Path; @@ -712,7 +712,7 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) { // Fill BLOCKS with data from known local source files - using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); var updateCount = 0L; await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize).ConfigureAwait(false)) { @@ -818,7 +818,7 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) { // Fill BLOCKS with data from known local source files - using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); var updateCount = 0L; await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize).ConfigureAwait(false)) { @@ -1014,7 +1014,7 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) { // Scan existing files for existing BLOCKS - using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); var updateCount = 0L; await foreach (var restorelist in database.GetExistingFilesWithBlocks().ConfigureAwait(false)) { diff --git a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs index 3c55ba2ba..6f09c175b 100644 --- a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs +++ b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs @@ -49,7 +49,7 @@ internal static class SearchEntriesHandler if (!System.IO.File.Exists(options.Dbpath)) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) .ConfigureAwait(false); long[]? filesetIds = null; diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 90b0e06d9..58852c7a6 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); await Utility.UpdateOptionsFromDb(db, m_options) .ConfigureAwait(false); await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) @@ -280,7 +280,7 @@ namespace Duplicati.Library.Main.Operation case RemoteVolumeType.Files: //Compare with db and see if all files are accounted for // with correct file hashes and blocklist hashes - using (var fl = await db.CreateFilelist(vol.Name).ConfigureAwait(false)) + await using (var fl = await db.CreateFilelist(vol.Name).ConfigureAwait(false)) { using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options)) foreach (var f in rd.Files) @@ -301,7 +301,7 @@ namespace Duplicati.Library.Main.Operation foreach (var v in rd.Volumes) { blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); - using (var bl = await db.CreateBlocklist(v.Filename).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklist(v.Filename).ConfigureAwait(false)) { foreach (var h in v.Blocks) await bl @@ -320,7 +320,7 @@ namespace Duplicati.Library.Main.Operation if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) { var hashesPerBlock = options.Blocksize / options.BlockhashSize; - using (var bl = await db.CreateBlocklistHashList(vol.Name).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklistHashList(vol.Name).ConfigureAwait(false)) { foreach (var b in rd.BlockLists) await bl @@ -338,7 +338,7 @@ namespace Duplicati.Library.Main.Operation } // Compare with db and see that all blocklists are listed - using (var il = await db.CreateIndexlist(vol.Name).ConfigureAwait(false)) + await using (var il = await db.CreateIndexlist(vol.Name).ConfigureAwait(false)) { foreach (var t in blocklinks) await il @@ -356,7 +356,7 @@ namespace Duplicati.Library.Main.Operation return new KeyValuePair>>(vol.Name, combined); case RemoteVolumeType.Blocks: using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var bl = await db.CreateBlocklist(vol.Name).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklist(vol.Name).ConfigureAwait(false)) using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options)) { //Verify that all blocks are in the file @@ -406,7 +406,7 @@ namespace Duplicati.Library.Main.Operation private async Task ReplaceFaultyIndexFilesAsync(List faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, CancellationToken cancellationToken) { - using var repairdb = + await using var repairdb = await LocalRepairDatabase.CreateAsync(db) .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index d5846b943..dfc62247c 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -37,7 +37,7 @@ namespace Duplicati.Library.Main.Operation public virtual async Task RunAsync() { - using var db = + await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache) .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); From bc97eaf71b5b0a1d0df1add6c7e5c308d78b5654 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 12:06:20 +0200 Subject: [PATCH 319/362] Removed redundant calls to a SqliteCommand.Parameters.Clear() --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 -- .../Library/Main/Database/LocalListBrokenFilesDatabase.cs | 1 - Duplicati/Library/Main/Database/LocalRestoreDatabase.cs | 3 --- 3 files changed, 6 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 828dcc805..991a40c6f 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -929,7 +929,6 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""{volidstable}"" "; - deletecmd.Parameters.Clear(); var bsIdsSubQuery = @$" SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" @@ -967,7 +966,6 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" "; - deletecmd.Parameters.Clear(); // Create a temp table to associate metadata that is being deleted to a fileset var metadataFilesetQuery = $@" diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index aeacf2280..d3e59fae9 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -453,7 +453,6 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""{volidstable}"" "; - deletecmd.Parameters.Clear(); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""IndexBlockLink"" diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 94a2e3311..c7c1d9083 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -434,8 +434,6 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime); - cmd.Parameters.Clear(); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ") .ConfigureAwait(false); @@ -588,7 +586,6 @@ namespace Duplicati.Library.Main.Database var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0).ConfigureAwait(false)); Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); - cmd.Parameters.Clear(); } await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ") From b9555cdcd421ff9119742df1153de3c23ead98b1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 14:21:35 +0200 Subject: [PATCH 320/362] Made all the async database extension methods accept a cancellation token --- .../Library/Main/Database/ExtensionMethods.cs | 181 +++++++++++++----- 1 file changed, 136 insertions(+), 45 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 0973ea558..011b8586f 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -23,6 +23,7 @@ using System.Collections.Generic; using System.Data; using System.Linq; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -79,7 +80,7 @@ public static partial class ExtensionMethods /// A new with the command text set. public static SqliteCommand CreateCommand(this SqliteConnection self, string cmdtext) { - return CreateCommandAsync(self, cmdtext).Await(); + return CreateCommandAsync(self, cmdtext, default).Await(); } /// @@ -87,12 +88,14 @@ public static partial class ExtensionMethods /// /// The to create the command for. /// The command text to set for the command. + /// Cancellation token to cancel the operation. /// A new with the command text set and prepared asynchronously. - public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext) + public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext, CancellationToken cancellationToken) { var cmd = self.CreateCommand() .SetCommandAndParameters(cmdtext); - await cmd.PrepareAsync().ConfigureAwait(false); + + await cmd.PrepareAsync(cancellationToken).ConfigureAwait(false); return cmd; } @@ -127,10 +130,11 @@ public static partial class ExtensionMethods /// /// The instance to execute on. /// The command text to execute. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the number of rows affected. - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string? cmdtext) + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string? cmdtext, CancellationToken cancellationToken) { - return await ExecuteNonQueryAsync(self, true, cmdtext, null) + return await ExecuteNonQueryAsync(self, true, cmdtext, null, cancellationToken) .ConfigureAwait(false); } @@ -140,10 +144,24 @@ public static partial class ExtensionMethods /// The instance to execute on. /// The command text to execute. /// The values to use as parameters. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the number of rows affected. - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary values) + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary values, CancellationToken cancellationToken) { - return await ExecuteNonQueryAsync(self, true, cmd, values) + return await ExecuteNonQueryAsync(self, true, cmd, values, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the number of rows affected. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the number of rows affected. + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken) + { + return await ExecuteNonQueryAsync(self, writeLog, null, null, cancellationToken) .ConfigureAwait(false); } @@ -154,8 +172,9 @@ public static partial class ExtensionMethods /// Whether to write a log entry. /// The command text to execute. /// The values to set as parameters. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the number of rows affected. - public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null) + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values, CancellationToken cancellationToken) { if (cmd != null) self.SetCommandAndParameters(cmd); @@ -164,7 +183,7 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", string.Format("ExecuteNonQueryAsync: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteNonQueryAsync().ConfigureAwait(false); + return await self.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } /// @@ -172,10 +191,11 @@ public static partial class ExtensionMethods /// /// The instance to execute on. /// The command text to execute. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the . - public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext) + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken) { - return await ExecuteReaderAsync(self, true, cmdtext, null) + return await ExecuteReaderAsync(self, true, cmdtext, null, cancellationToken) .ConfigureAwait(false); } @@ -185,10 +205,11 @@ public static partial class ExtensionMethods /// The instance to execute on. /// The command text to execute. /// The values to use as parameters. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the . - public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary? values) + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary? values, CancellationToken cancellationToken) { - return await ExecuteReaderAsync(self, true, cmdtext, values) + return await ExecuteReaderAsync(self, true, cmdtext, values, cancellationToken) .ConfigureAwait(false); } @@ -199,8 +220,9 @@ public static partial class ExtensionMethods /// Whether to write a log entry. /// The command text to execute. /// The values to set as parameters. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the . - public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext = null, Dictionary? values = null) + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext, Dictionary? values, CancellationToken cancellationToken) { if (cmdtext != null) self.SetCommandAndParameters(cmdtext); @@ -209,17 +231,18 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteReaderAsync().ConfigureAwait(false); + return await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); } /// /// Executes the command asynchronously and returns an enumerable of . /// /// The instance to execute on. + /// Cancellation token to cancel the operation. /// An asynchronous enumerable of . - public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self) + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, CancellationToken cancellationToken) { - return ExecuteReaderEnumerableAsync(self, true, null, null); + return ExecuteReaderEnumerableAsync(self, true, null, null, cancellationToken); } /// @@ -227,10 +250,11 @@ public static partial class ExtensionMethods /// /// The instance to execute on. /// The command text to execute. + /// Cancellation token to cancel the operation. /// An asynchronous enumerable of . - public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext) + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken) { - return ExecuteReaderEnumerableAsync(self, true, cmdtext, null); + return ExecuteReaderEnumerableAsync(self, true, cmdtext, null, cancellationToken); } /// @@ -239,10 +263,11 @@ public static partial class ExtensionMethods /// The instance to execute on. /// The command text to execute. /// The values to use as parameters. + /// Cancellation token to cancel the operation. /// An asynchronous enumerable of . - public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, Dictionary? values) + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, Dictionary? values, CancellationToken cancellationToken) { - return ExecuteReaderEnumerableAsync(self, true, cmdtext, values); + return ExecuteReaderEnumerableAsync(self, true, cmdtext, values, cancellationToken); } /// @@ -253,7 +278,7 @@ public static partial class ExtensionMethods /// The command text to execute. /// The values to set as parameters. /// An asynchronous enumerable of . - public static async IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, bool writeLog, string? cmdtext = null, Dictionary? values = null) + public static async IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, bool writeLog, string? cmdtext, Dictionary? values, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { if (cmdtext != null) self.SetCommandAndParameters(cmdtext); @@ -262,8 +287,8 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReaderEnumerableAsync", $"ExecuteReaderEnumerableAsync: {self.GetPrintableCommandText()}") : null) - await using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)) + while (await rd.ReadAsync(cancellationToken).ConfigureAwait(false)) yield return rd; } @@ -271,10 +296,11 @@ public static partial class ExtensionMethods /// Executes the command asynchronously and returns the first column of the first row in the result set. /// /// The instance to execute on. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. - public static async Task ExecuteScalarAsync(this SqliteCommand self) + public static async Task ExecuteScalarAsync(this SqliteCommand self, CancellationToken cancellationToken) { - return await self.ExecuteScalarAsync(true, null, null) + return await self.ExecuteScalarAsync(true, null, null, cancellationToken) .ConfigureAwait(false); } @@ -283,10 +309,11 @@ public static partial class ExtensionMethods /// /// The instance to execute on. /// The command text to execute. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. - public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext) + public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken) { - return await self.ExecuteScalarAsync(true, cmdtext, null) + return await self.ExecuteScalarAsync(true, cmdtext, null, cancellationToken) .ConfigureAwait(false); } @@ -296,10 +323,11 @@ public static partial class ExtensionMethods /// The instance to execute on. /// The command text to execute. /// The values to use as parameters. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. - public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext, Dictionary? values) + public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext, Dictionary? values, CancellationToken cancellationToken) { - return await self.ExecuteScalarAsync(true, cmdtext, values) + return await self.ExecuteScalarAsync(true, cmdtext, values, cancellationToken) .ConfigureAwait(false); } @@ -310,8 +338,9 @@ public static partial class ExtensionMethods /// Whether to write a log entry. /// The command text to execute. /// The values to set as parameters. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. - public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null) + public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values, CancellationToken cancellationToken) { if (cmd != null) self.SetCommandAndParameters(cmd); @@ -320,16 +349,31 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarAsync", string.Format("ExecuteScalarAsync: {0}", self.GetPrintableCommandText())) : null) - return await self.ExecuteScalarAsync().ConfigureAwait(false); + return await self.ExecuteScalarAsync(cancellationToken) + .ConfigureAwait(false); } /// /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. /// /// The instance to execute on. - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue = -1) + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, CancellationToken cancellationToken) { - return await ExecuteScalarInt64Async(self, true, null, null, defaultvalue) + return await ExecuteScalarInt64Async(self, true, null, null, -1, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, null, null, defaultvalue, cancellationToken) .ConfigureAwait(false); } @@ -339,10 +383,24 @@ public static partial class ExtensionMethods /// The instance to execute on. /// Whether to write a log entry. /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue) + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue, CancellationToken cancellationToken) { - return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue) + return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The command text to execute. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, cmd, null, -1, cancellationToken) .ConfigureAwait(false); } @@ -352,10 +410,25 @@ public static partial class ExtensionMethods /// The instance to execute on. /// The command text to execute. /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, long defaultvalue = -1) + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, long defaultvalue, CancellationToken cancellationToken) { - return await ExecuteScalarInt64Async(self, true, cmd, null, defaultvalue) + return await ExecuteScalarInt64Async(self, true, cmd, null, defaultvalue, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary? values, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, cmd, values, -1, cancellationToken) .ConfigureAwait(false); } @@ -366,10 +439,26 @@ public static partial class ExtensionMethods /// The command text to execute. /// The values to use as parameters. /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary? values, long defaultvalue = -1) + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary? values, long defaultvalue, CancellationToken cancellationToken) { - return await ExecuteScalarInt64Async(self, true, cmd, values, defaultvalue) + return await ExecuteScalarInt64Async(self, true, cmd, values, defaultvalue, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, writeLog, cmd, values, -1, cancellationToken) .ConfigureAwait(false); } @@ -381,8 +470,9 @@ public static partial class ExtensionMethods /// The command text to execute. /// The values to set as parameters. /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. - public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd = null, Dictionary? values = null, long defaultvalue = -1) + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values, long defaultvalue, CancellationToken cancellationToken) { if (cmd != null) self.SetCommandAndParameters(cmd); @@ -391,8 +481,8 @@ public static partial class ExtensionMethods self.SetParameterValues(values); using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64Async", string.Format("ExecuteScalarInt64Async: {0}", self.GetPrintableCommandText())) : null) - await using (var rd = await self.ExecuteReaderAsync().ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)) + if (await rd.ReadAsync(cancellationToken).ConfigureAwait(false)) return ConvertValueToInt64(rd, 0, defaultvalue); return defaultvalue; @@ -441,8 +531,9 @@ public static partial class ExtensionMethods /// The to expand the IN clause parameter for. /// The name of the original parameter to replace, must start with '@'. /// The collection of values to expand the IN clause with. + /// Cancellation token to cancel the operation. /// A task that when awaited contains the modified with the IN clause expanded. - internal static async Task ExpandInClauseParameterMssqliteAsync(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values) + internal static async Task ExpandInClauseParameterMssqliteAsync(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); @@ -453,7 +544,7 @@ public static partial class ExtensionMethods // We have a temporary table, so we need to replace the parameter with the table name cmd.CommandText = cmd.CommandText.Replace( originalParamName, - await values.GetInClause().ConfigureAwait(false), + await values.GetInClause(cancellationToken).ConfigureAwait(false), StringComparison.OrdinalIgnoreCase ); From da57bbae33e1556b95564c015d60cf4608b54c2e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 14:21:57 +0200 Subject: [PATCH 321/362] Made the TemporaryDbValueList methods accept a cancellation token --- .../Library/Main/Database/ExtensionMethods.cs | 2 +- .../Main/Database/TemporaryDbValueList.cs | 38 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 011b8586f..820d03f44 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -1225,7 +1225,7 @@ public static partial class ExtensionMethods return ExpandInClauseParameter(cmd, originalParamName, values.Values); // We have a temporary table, so we need to replace the parameter with the table name - cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause().Await(), StringComparison.OrdinalIgnoreCase); + cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(default).Await(), StringComparison.OrdinalIgnoreCase); return cmd; } diff --git a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs index 644410517..d1acad234 100644 --- a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs +++ b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs @@ -22,6 +22,7 @@ using System; using System.Collections.Generic; using System.Data; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -96,10 +97,11 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable /// /// The database to use. /// The values to use. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the created TemporaryDbValueList. - internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) + internal static async Task CreateAsync(LocalDatabase db, IEnumerable values, CancellationToken token) { - return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "INTEGER")) + return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "INTEGER"), token) .ConfigureAwait(false); } @@ -108,10 +110,11 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable /// /// The database to use. /// The values to use. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the created TemporaryDbValueList. - internal static async Task CreateAsync(LocalDatabase db, IEnumerable values) + internal static async Task CreateAsync(LocalDatabase db, IEnumerable values, CancellationToken token) { - return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "TEXT")) + return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "TEXT"), token) .ConfigureAwait(false); } @@ -121,10 +124,11 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable /// The connection to use. /// The reusable transaction to use. /// The values to use. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the created TemporaryDbValueList. - internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values) + internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values, CancellationToken token) { - return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "INTEGER")) + return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "INTEGER"), token) .ConfigureAwait(false); } @@ -134,10 +138,11 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable /// The connection to use. /// The reusable transaction to use. /// The values to use. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the created TemporaryDbValueList. - internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values) + internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values, CancellationToken token) { - return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "TEXT")) + return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "TEXT"), token) .ConfigureAwait(false); } @@ -146,11 +151,12 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable /// the values are forced into a table. /// /// The TemporaryDbValueList to create. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the created TemporaryDbValueList. - private static async Task DoCreateAsync(TemporaryDbValueList valueList) + private static async Task DoCreateAsync(TemporaryDbValueList valueList, CancellationToken token) { if (valueList._values.Count() > LocalDatabase.CHUNK_SIZE) - await valueList.ForceToTable().ConfigureAwait(false); + await valueList.ForceToTable(token).ConfigureAwait(false); return valueList; } @@ -168,13 +174,14 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable /// /// Get the in clause for the values, creating the table if needed. /// + /// A cancellation token to cancel the operation. /// A task that when awaited contains the SQL in clause string. - public async Task GetInClause() + public async Task GetInClause(CancellationToken token) { if (_disposed) throw new ObjectDisposedException(nameof(TemporaryDbValueList)); - await ForceToTable().ConfigureAwait(false); + await ForceToTable(token).ConfigureAwait(false); return $@" SELECT ""Value"" @@ -185,8 +192,9 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable /// /// Force the values to be written to a table, if not already done. /// + /// A cancellation token to cancel the operation. /// A task that completes when the table has been created and the values inserted. - public async Task ForceToTable() + public async Task ForceToTable(CancellationToken token) { if (_disposed) throw new ObjectDisposedException(nameof(TemporaryDbValueList)); @@ -197,7 +205,7 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable _cmd = _con.CreateCommand(_rtr); await _cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType}) - ") + ", token) .ConfigureAwait(false); _isTable = true; @@ -241,7 +249,7 @@ internal class TemporaryDbValueList : IDisposable, IAsyncDisposable { if (_cmd != null) { - await _cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{_tableName}""") + await _cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{_tableName}""", default) .ConfigureAwait(false); await _cmd.DisposeAsync().ConfigureAwait(false); } From 87dc97842573dd894ab015ca950f69ad7515a33e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 14:37:13 +0200 Subject: [PATCH 322/362] Made the ChangeStatistics methods accept a cancellation token --- .../Library/Main/Database/ChangeStatistics.cs | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Database/ChangeStatistics.cs b/Duplicati/Library/Main/Database/ChangeStatistics.cs index 47f80b371..b4a79a5e4 100644 --- a/Duplicati/Library/Main/Database/ChangeStatistics.cs +++ b/Duplicati/Library/Main/Database/ChangeStatistics.cs @@ -21,6 +21,7 @@ #nullable enable using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Data.Sqlite; @@ -43,8 +44,9 @@ public static class ChangeStatistics /// The results object to update with the statistics. /// The ID of the current fileset. /// The ID of the previous fileset. + /// The cancellation token to monitor for cancellation requests. /// A task that completes when the statistics are updated. - internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId) + internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId, CancellationToken token) { var tmpName = $"TmpFileState_{Guid.NewGuid():N}"; @@ -78,7 +80,7 @@ public static class ChangeStatistics ") .SetParameterValue("@LastFilesetId", previousFilesetId) .SetParameterValue("@CurrentFilesetId", currentFilesetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); // Index for fast comparison @@ -88,47 +90,47 @@ public static class ChangeStatistics ""PrefixID"", ""Path"" ) - ") + ", token) .ConfigureAwait(false); // Added - results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID) + results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) .ConfigureAwait(false); - results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID) + results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.AddedFiles = await CountAdded(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID - ]) + ], token) .ConfigureAwait(false); // Deleted - results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID) + results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) .ConfigureAwait(false); - results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID) + results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.DeletedFiles = await CountDeleted(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID - ]) + ], token) .ConfigureAwait(false); // Modified - results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID) + results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) .ConfigureAwait(false); - results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID) + results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.ModifiedFiles = await CountModified(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID - ]) + ], token) .ConfigureAwait(false); } finally { try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";", token) .ConfigureAwait(false); } catch (Exception ex) @@ -145,8 +147,9 @@ public static class ChangeStatistics /// The name of the temporary table. /// The ID of the blockset to count, or null for all. /// The blocksets to exclude from the count. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the number of added files or folders. - private static async Task CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + private static async Task CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { var conditions = $@" ""Source"" = 1 @@ -160,7 +163,7 @@ public static class ChangeStatistics ) "; - return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets) + return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token) .ConfigureAwait(false); } @@ -171,8 +174,9 @@ public static class ChangeStatistics /// The name of the temporary table. /// The ID of the blockset to count, or null for all. /// The blocksets to exclude from the count. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the number of deleted files or folders. - private static async Task CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + private static async Task CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { var conditions = @$" ""Source"" = 0 @@ -186,7 +190,7 @@ public static class ChangeStatistics ) "; - return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets) + return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token) .ConfigureAwait(false); } @@ -197,8 +201,9 @@ public static class ChangeStatistics /// The name of the temporary table. /// The ID of the blockset to count, or null for all. /// The blocksets to exclude from the count. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the number of modified files or folders. - private static async Task CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + private static async Task CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { string conditions; @@ -241,7 +246,7 @@ public static class ChangeStatistics return await cmd .SetCommandAndParameters(sql) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } @@ -254,8 +259,9 @@ public static class ChangeStatistics /// The base condition for the count. /// The ID of the blockset to count, or null for all. /// The blocksets to exclude from the count. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the number of files or folders that match the condition. - private static async Task CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets) + private static async Task CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { var fullCondition = baseCondition; var blocksetCondition = GetBlocksetCondition(alias, blocksetId, excludeBlocksets); @@ -270,7 +276,7 @@ public static class ChangeStatistics return await cmd .SetCommandAndParameters(sql) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } From 3b3a7801b854fa4d55d6bc85da59621051aed36e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 15:36:10 +0200 Subject: [PATCH 323/362] Made the LocalDatabase methods accept a cancellation token --- .../Library/Main/Database/LocalDatabase.cs | 601 ++++++++++-------- .../Main/Database/ReusableTransaction.cs | 8 +- Duplicati/Library/Main/ResultClasses.cs | 14 +- 3 files changed, 351 insertions(+), 272 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 991a40c6f..8571f3c4f 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -217,14 +217,15 @@ namespace Duplicati.Library.Main.Database /// Indicates whether the connection should be closed when the database is disposed. /// The size of the SQLite page cache in bytes. /// An optional existing instance to use. If not provided, a new instance will be created. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a new instance of . - public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize, LocalDatabase? db = null) + public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize, LocalDatabase? db, CancellationToken token) { db ??= new LocalDatabase(); var connection = await CreateConnectionAsync(path, pagecachesize) .ConfigureAwait(false); - db = await CreateLocalDatabaseAsync(connection, operation, db) + db = await CreateLocalDatabaseAsync(connection, operation, db, token) .ConfigureAwait(false); db.ShouldCloseConnection = shouldclose; @@ -239,13 +240,15 @@ namespace Duplicati.Library.Main.Database /// /// The parent instance to copy from. /// An optional existing instance to use. If not provided, a new instance will be created. - public static async Task CreateLocalDatabaseAsync(LocalDatabase dbparent, LocalDatabase? dbnew = null) + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a new instance of . + public static async Task CreateLocalDatabaseAsync(LocalDatabase dbparent, LocalDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalDatabase(); dbnew.m_connection = dbparent.m_connection; dbnew.m_rtr = dbparent.m_rtr; - dbnew = await CreateLocalDatabaseAsync(dbparent.m_connection, dbnew) + dbnew = await CreateLocalDatabaseAsync(dbparent.m_connection, dbnew, token) .ConfigureAwait(false); dbnew.OperationTimestamp = dbparent.OperationTimestamp; @@ -262,17 +265,18 @@ namespace Duplicati.Library.Main.Database /// The SQLite connection to use for the database operations. /// The description of the operation being performed. /// An optional existing instance to use. If not provided, a new instance will be created. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a new instance of . - public static async Task CreateLocalDatabaseAsync(SqliteConnection connection, string operation, LocalDatabase? dbnew = null) + public static async Task CreateLocalDatabaseAsync(SqliteConnection connection, string operation, LocalDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalDatabase(); - dbnew = await CreateLocalDatabaseAsync(connection, dbnew) + dbnew = await CreateLocalDatabaseAsync(connection, dbnew, token) .ConfigureAwait(false); dbnew.OperationTimestamp = DateTime.UtcNow; if (dbnew.m_connection.State != ConnectionState.Open) - await dbnew.m_connection.OpenAsync().ConfigureAwait(false); + await dbnew.m_connection.OpenAsync(token).ConfigureAwait(false); await using var cmd = dbnew.m_connection.CreateCommand() .SetTransaction(dbnew.m_rtr); @@ -291,7 +295,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Description", operation) .SetParameterValue("@Timestamp", dbnew.OperationTimestamp) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); } else @@ -304,10 +308,10 @@ namespace Duplicati.Library.Main.Database FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1 - ") + ", token) .ConfigureAwait(false); - if (!await rd.ReadAsync().ConfigureAwait(false)) + if (!await rd.ReadAsync(token).ConfigureAwait(false)) throw new Exception("LocalDatabase does not contain a previous operation."); dbnew.m_operationid = rd.ConvertValueToInt64(0); @@ -323,8 +327,9 @@ namespace Duplicati.Library.Main.Database /// /// The SQLite connection to use for the database operations. /// An optional existing instance to use. If not provided, a new instance will be created. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a new instance of . - private static async Task CreateLocalDatabaseAsync(SqliteConnection connection, LocalDatabase? dbnew = null) + private static async Task CreateLocalDatabaseAsync(SqliteConnection connection, LocalDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalDatabase(); dbnew.m_connection = connection; @@ -358,7 +363,7 @@ namespace Duplicati.Library.Main.Database @Message, @Exception ) - ") + ", token) .ConfigureAwait(false); dbnew.m_insertremotelogCommand = await connection.CreateCommandAsync(@" @@ -376,7 +381,7 @@ namespace Duplicati.Library.Main.Database @Path, @Data ) - ") + ", token) .ConfigureAwait(false); dbnew.m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" @@ -387,17 +392,17 @@ namespace Duplicati.Library.Main.Database ""Hash"" = @Hash, ""Size"" = @Size WHERE ""Name"" = @Name - ") + ", token) .ConfigureAwait(false); dbnew.m_selectremotevolumesCommand = - await connection.CreateCommandAsync(selectremotevolumes_sql) + await connection.CreateCommandAsync(selectremotevolumes_sql, token) .ConfigureAwait(false); dbnew.m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$" {selectremotevolumes_sql} WHERE ""Name"" = @Name - ") + ", token) .ConfigureAwait(false); dbnew.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" @@ -412,7 +417,7 @@ namespace Duplicati.Library.Main.Database WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') ) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') - ") + ", token) .ConfigureAwait(false); dbnew.m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" @@ -423,7 +428,7 @@ namespace Duplicati.Library.Main.Database ""DeleteGraceTime"" < @Now OR ""State"" != @State ) - ") + ", token) .ConfigureAwait(false); // >12 is to handle removal of old records that were in ticks @@ -435,14 +440,14 @@ namespace Duplicati.Library.Main.Database ""DeleteGraceTime"" < @Now OR LENGTH(""DeleteGraceTime"") > 12 ) - ") + ", token) .ConfigureAwait(false); dbnew.m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = @Name - ") + ", token) .ConfigureAwait(false); dbnew.m_createremotevolumeCommand = await connection.CreateCommandAsync(@" @@ -467,7 +472,7 @@ namespace Duplicati.Library.Main.Database @ArchiveTime ); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); dbnew.m_insertIndexBlockLink = await connection.CreateCommandAsync(@" @@ -479,21 +484,21 @@ namespace Duplicati.Library.Main.Database @IndexVolumeId, @BlockVolumeId ) - ") + ", token) .ConfigureAwait(false); dbnew.m_findpathprefixCommand = await connection.CreateCommandAsync(@" SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = @Prefix - ") + ", token) .ConfigureAwait(false); dbnew.m_insertpathprefixCommand = await connection.CreateCommandAsync(@" INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (@Prefix); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); return dbnew; @@ -516,10 +521,11 @@ namespace Duplicati.Library.Main.Database /// The new state of the remote volume. /// The size of the remote volume in bytes. /// The hash of the remote volume, or null if not applicable. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the remote volume has been updated. - public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, CancellationToken token) { - await UpdateRemoteVolume(name, state, size, hash, false) + await UpdateRemoteVolume(name, state, size, hash, false, token) .ConfigureAwait(false); } @@ -531,10 +537,11 @@ namespace Duplicati.Library.Main.Database /// The size of the remote volume in bytes. /// The hash of the remote volume, or null if not applicable. /// If true, suppresses cleanup of the remote volume after updating. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the remote volume has been updated. - public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, CancellationToken token) { - await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null) + await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null, token) .ConfigureAwait(false); } @@ -548,8 +555,9 @@ namespace Duplicati.Library.Main.Database /// If true, suppresses cleanup of the remote volume after updating. /// The time after which the remote volume can be deleted. /// If true, sets the remote volume as archived. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the remote volume has been updated. - public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived) + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, CancellationToken token) { var c = await m_updateremotevolumeCommand.SetTransaction(m_rtr) .SetParameterValue("@OperationID", m_operationid) @@ -557,7 +565,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) @@ -571,13 +579,13 @@ namespace Duplicati.Library.Main.Database UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = @DeleteGraceTime WHERE ""Name"" = @Name - ") + ", token) .ConfigureAwait(false); c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)) .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) @@ -590,13 +598,13 @@ namespace Duplicati.Library.Main.Database UPDATE ""RemoteVolume"" SET ""ArchiveTime"" = @ArchiveTime WHERE ""Name"" = @Name - ") + ", token) .ConfigureAwait(false); c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0) .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) @@ -605,15 +613,16 @@ namespace Duplicati.Library.Main.Database if (!suppressCleanup && state == RemoteVolumeState.Deleted) { - await RemoveRemoteVolume(name).ConfigureAwait(false); + await RemoveRemoteVolume(name, token).ConfigureAwait(false); } } /// /// Gets the ID and timestamp of all filesets in the database, ordered by timestamp in descending order. /// + /// Cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of key-value pairs, where each pair contains the fileset ID and its timestamp. - public async IAsyncEnumerable> FilesetTimes() + public async IAsyncEnumerable> FilesetTimes([EnumeratorCancellation] CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" SELECT @@ -621,11 +630,11 @@ namespace Duplicati.Library.Main.Database ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC - ") + ", token) .ConfigureAwait(false); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair( rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() @@ -639,16 +648,17 @@ namespace Duplicati.Library.Main.Database /// An array of versions to filter files by. If null or empty, it will not be used in the query. /// An optional list of filesets to use for filtering. If null, it will fetch all filesets from the database. /// If true, matches files with the exact timestamp; otherwise, matches files with timestamps less than or equal to the specified time. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a tuple containing the SQL WHERE clause and a dictionary of parameter values. /// Thrown if the provided time is unspecified. - public async Task<(string Query, Dictionary Values)> GetFilelistWhereClause(DateTime time, long[]? versions, IEnumerable>? filesetslist = null, bool singleTimeMatch = false) + public async Task<(string Query, Dictionary Values)> GetFilelistWhereClause(DateTime time, long[]? versions, IEnumerable>? filesetslist, bool singleTimeMatch, CancellationToken token) { KeyValuePair[] filesets; if (filesetslist != null) filesets = [.. filesetslist]; else - filesets = await FilesetTimes() - .ToArrayAsync() + filesets = await FilesetTimes(token) + .ToArrayAsync(cancellationToken: token) .ConfigureAwait(false); var query = new StringBuilder(); @@ -705,13 +715,14 @@ namespace Duplicati.Library.Main.Database /// Gets the ID of a remote volume by its name. /// /// The name of the remote volume. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns the ID of the remote volume. If the volume does not exist, it returns -1. - public async Task GetRemoteVolumeID(string file) + public async Task GetRemoteVolumeID(string file, CancellationToken token) { return await m_selectremotevolumeIdCommand .SetTransaction(m_rtr) .SetParameterValue("@Name", file) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); } @@ -719,8 +730,9 @@ namespace Duplicati.Library.Main.Database /// Gets the IDs of remote volumes for a list of files. /// /// An enumerable collection of file names. + /// Cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of key-value pairs, where each pair contains the file name and its corresponding remote volume ID. - public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files) + public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files, [EnumeratorCancellation] CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" SELECT @@ -728,17 +740,17 @@ namespace Duplicati.Library.Main.Database ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name) - ") + ", token) .ConfigureAwait(false); cmd.SetTransaction(m_rtr); - await using var tmptable = await TemporaryDbValueList.CreateAsync(this, files) + await using var tmptable = await TemporaryDbValueList.CreateAsync(this, files, token) .ConfigureAwait(false); - await cmd.ExpandInClauseParameterMssqliteAsync("@Name", tmptable) + await cmd.ExpandInClauseParameterMssqliteAsync("@Name", tmptable, token) .ConfigureAwait(false); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } @@ -746,15 +758,16 @@ namespace Duplicati.Library.Main.Database /// Gets a remote volume entry by its file name. /// /// The name of the remote volume file. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a representing the remote volume. If the volume does not exist, it returns an empty entry. - public async Task GetRemoteVolume(string file) + public async Task GetRemoteVolume(string file, CancellationToken token) { m_selectremotevolumeCommand .SetTransaction(m_rtr) .SetParameterValue("@Name", file); - await using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync().ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync(token).ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) return new RemoteVolumeEntry( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), @@ -772,12 +785,13 @@ namespace Duplicati.Library.Main.Database /// /// Gets remote volumes that are duplicates, meaning they have the same name but different states. /// + /// Cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of key-value pairs, where each pair contains the name of the remote volume and its state. - public async IAsyncEnumerable> DuplicateRemoteVolumes() + public async IAsyncEnumerable> DuplicateRemoteVolumes([EnumeratorCancellation] CancellationToken token) { m_selectduplicateRemoteVolumesCommand.SetTransaction(m_rtr); - await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { yield return new KeyValuePair( rd.ConvertValueToString(0) ?? throw new Exception("Name was null"), @@ -791,15 +805,16 @@ namespace Duplicati.Library.Main.Database /// /// Gets all remote volumes from the database. /// + /// Cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of representing all remote volumes. - public async IAsyncEnumerable GetRemoteVolumes() + public async IAsyncEnumerable GetRemoteVolumes([EnumeratorCancellation] CancellationToken token) { m_selectremotevolumesCommand.SetTransaction(m_rtr); await using var rd = await m_selectremotevolumesCommand - .ExecuteReaderAsync() + .ExecuteReaderAsync(token) .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { yield return new RemoteVolumeEntry( rd.ConvertValueToInt64(0), @@ -820,8 +835,9 @@ namespace Duplicati.Library.Main.Database /// The operation performed. /// The path involved. /// Any data relating to the operation. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the operation log has been recorded. - public async Task LogRemoteOperation(string operation, string path, string? data) + public async Task LogRemoteOperation(string operation, string path, string? data, CancellationToken token) { await m_insertremotelogCommand .SetTransaction(m_rtr) @@ -830,7 +846,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Operation", operation) .SetParameterValue("@Path", path) .SetParameterValue("@Data", data) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -840,8 +856,9 @@ namespace Duplicati.Library.Main.Database /// The message type. /// The message. /// An optional exception. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the log message has been recorded. - public async Task LogMessage(string type, string message, Exception? exception) + public async Task LogMessage(string type, string message, Exception? exception, CancellationToken token) { await m_insertlogCommand .SetTransaction(m_rtr) @@ -850,7 +867,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", type) .SetParameterValue("@Message", message) .SetParameterValue("@Exception", exception?.ToString()) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -860,22 +877,23 @@ namespace Duplicati.Library.Main.Database /// /// The name of the remote volume to unlink. /// The state of the remote volume to unlink. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the remote volume has been unlinked. /// Thrown if the number of unlinked remote volumes is not equal to 1. - public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state) + public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state, CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" DELETE FROM ""RemoteVolume"" WHERE ""Name"" = @Name AND ""State"" = @State - ") + ", token) .ConfigureAwait(false); var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@Name", name) .SetParameterValue("@State", state.ToString()) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) @@ -886,18 +904,20 @@ namespace Duplicati.Library.Main.Database /// Removes a remote volume from the database. /// /// The name of the remote volume to remove. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the remote volume has been removed. - public async Task RemoveRemoteVolume(string name) + public async Task RemoveRemoteVolume(string name, CancellationToken token) { - await RemoveRemoteVolumes([name]).ConfigureAwait(false); + await RemoveRemoteVolumes([name], token).ConfigureAwait(false); } /// /// Removes multiple remote volumes from the database. /// /// An enumerable collection of names of remote volumes to remove. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the remote volumes have been removed. - public async Task RemoveRemoteVolumes(IEnumerable names) + public async Task RemoveRemoteVolumes(IEnumerable names, CancellationToken token) { if (names == null || !names.Any()) return; @@ -912,7 +932,7 @@ namespace Duplicati.Library.Main.Database CREATE TEMP TABLE ""{volidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) - ") + ", token) .ConfigureAwait(false); await deletecmd.SetCommandAndParameters($@" @@ -922,7 +942,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" IN (@VolumeNames) ") .ExpandInClauseParameterMssqlite("@VolumeNames", [.. names]) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); var volIdsSubQuery = $@" @@ -953,13 +973,13 @@ namespace Duplicati.Library.Main.Database CREATE TEMP TABLE ""{blocksetidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery} - ") + ", token) .ConfigureAwait(false); bsIdsSubQuery = $@" @@ -991,7 +1011,7 @@ namespace Duplicati.Library.Main.Database ""MetadataID"" INTEGER PRIMARY KEY, ""FilesetID"" INTEGER ) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" @@ -1000,7 +1020,7 @@ namespace Duplicati.Library.Main.Database ""FilesetID"" ) {metadataFilesetQuery} - ") + ", token) .ConfigureAwait(false); // Delete FilesetEntry rows that had their metadata deleted @@ -1021,7 +1041,7 @@ namespace Duplicati.Library.Main.Database FROM ""{metadataFilesetTable}"" ) ) - ") + ", token) .ConfigureAwait(false); // Delete FilesetEntry rows that had their blocks deleted @@ -1032,7 +1052,7 @@ namespace Duplicati.Library.Main.Database FROM ""FileLookup"" WHERE ""FileLookup"".""BlocksetID"" IN ({bsIdsSubQuery}) ) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" @@ -1041,37 +1061,37 @@ namespace Duplicati.Library.Main.Database SELECT ""MetadataID"" FROM ""{metadataFilesetTable}"" ) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen @@ -1079,19 +1099,19 @@ namespace Duplicati.Library.Main.Database DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" @@ -1101,7 +1121,7 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) ) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" @@ -1111,14 +1131,14 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) ) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" CREATE TABLE ""{filesetidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" @@ -1126,7 +1146,7 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); // Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore @@ -1143,7 +1163,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""FilesetID"" FROM ""FilesetEntry"" ) - ") + ", token) .ConfigureAwait(false); // Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later @@ -1168,7 +1188,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString() ]) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" @@ -1177,30 +1197,30 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""{filesetidstable}"" ) - ") + ", token) .ConfigureAwait(false); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // SQLite.SQLiteException (0x80004005): database table is locked - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{blocksetidstable}"" ") + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{blocksetidstable}"" ", token) .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ") + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ", token) .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{metadataFilesetTable}"" ") + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{metadataFilesetTable}"" ", token) .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{filesetidstable}"" ") + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{filesetidstable}"" ", token) .ConfigureAwait(false); try { deletecmd.CommandTimeout = 2; - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ") + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ", token) .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ") + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ", token) .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ") + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ", token) .ConfigureAwait(false); - await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ") + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ", token) .ConfigureAwait(false); } catch { /* Ignore, will be deleted on close anyway. */ } @@ -1214,7 +1234,7 @@ namespace Duplicati.Library.Main.Database await m_removeremotevolumeCommand .SetTransaction(m_rtr) .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -1226,7 +1246,7 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""FileLookup"" ) - ") + ", token) .ConfigureAwait(false); if (nonAttachedFiles > 0) @@ -1238,13 +1258,15 @@ namespace Duplicati.Library.Main.Database /// This operation can help optimize the database performance by defragmenting it. /// Note: This operation can take a significant amount of time depending on the size of the database. /// + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the VACUUM operation has finished. - public async Task Vacuum() + public async Task Vacuum(CancellationToken token) { m_hasExecutedVacuum = true; await using var cmd = m_connection.CreateCommand(); - await cmd.ExecuteNonQueryAsync("VACUUM").ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync("VACUUM", token) + .ConfigureAwait(false); } /// @@ -1254,10 +1276,11 @@ namespace Duplicati.Library.Main.Database /// The type of the remote volume. /// The size of the remote volume in bytes. Use -1 for unknown size. /// The state of the remote volume. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns the ID of the newly registered remote volume. - public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state, CancellationToken token) { - return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0)) + return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), token) .ConfigureAwait(false); } @@ -1267,10 +1290,11 @@ namespace Duplicati.Library.Main.Database /// The name of the remote volume. /// The type of the remote volume. /// The state of the remote volume. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns the ID of the newly registered remote volume. - public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, CancellationToken token) { - return await RegisterRemoteVolume(name, type, state, new TimeSpan(0)) + return await RegisterRemoteVolume(name, type, state, new TimeSpan(0), token) .ConfigureAwait(false); } @@ -1281,10 +1305,11 @@ namespace Duplicati.Library.Main.Database /// The type of the remote volume. /// The state of the remote volume. /// The time after which the remote volume can be deleted. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns the ID of the newly registered remote volume. - public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, CancellationToken token) { - return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime) + return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime, token) .ConfigureAwait(false); } @@ -1296,8 +1321,9 @@ namespace Duplicati.Library.Main.Database /// The state of the remote volume. /// The size of the remote volume in bytes. Use -1 for unknown size. /// The time after which the remote volume can be deleted. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns the ID of the newly registered remote volume. - public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime) + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, CancellationToken token) { var r = await m_createremotevolumeCommand .SetTransaction(m_rtr) @@ -1309,7 +1335,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@VerificationCount", 0) .SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks) .SetParameterValue("@ArchiveTime", 0) - .ExecuteScalarInt64Async() + .ExecuteScalarInt64Async(token) .ConfigureAwait(false); return r; @@ -1322,16 +1348,17 @@ namespace Duplicati.Library.Main.Database /// The time to restore from. /// Optional array of versions to match against the filesets. /// If true, only match filesets that exactly match the restore time. + /// Cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of fileset IDs that match the criteria. /// Thrown if the provided DateTime is unspecified. /// Thrown if no backups are found at the specified date. - public async IAsyncEnumerable GetFilesetIDs(DateTime restoretime, long[]? versions, bool singleTimeMatch = false) + public async IAsyncEnumerable GetFilesetIDs(DateTime restoretime, long[]? versions, bool singleTimeMatch, [EnumeratorCancellation] CancellationToken token) { if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); (var wherequery, var values) = - await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch) + await GetFilelistWhereClause(restoretime, versions, null, singleTimeMatch, token) .ConfigureAwait(false); var res = new List(); await using var cmd = m_connection.CreateCommand(); @@ -1343,8 +1370,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValues(values); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); if (res.Count == 0) @@ -1354,8 +1381,8 @@ namespace Duplicati.Library.Main.Database FROM ""Fileset"" ORDER BY ""Timestamp"" DESC "); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); if (res.Count == 0) @@ -1373,14 +1400,15 @@ namespace Duplicati.Library.Main.Database /// /// The time to restore from. /// Optional array of versions to match against the filesets. + /// Cancellation token to monitor for cancellation requests. /// An asynchronous task that returns a collection of fileset IDs that match the criteria. - public async Task> FindMatchingFilesets(DateTime restoretime, long[]? versions) + public async Task> FindMatchingFilesets(DateTime restoretime, long[]? versions, CancellationToken token) { if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); var (wherequery, args) = - await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true) + await GetFilelistWhereClause(restoretime, versions, null, true, token) .ConfigureAwait(false); var res = new List(); @@ -1392,8 +1420,8 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC ") .SetParameterValues(args); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); return res; @@ -1404,8 +1432,9 @@ namespace Duplicati.Library.Main.Database /// A fileset is considered a full backup if its "IsFullBackup" field is set to FULL_BACKUP. /// /// The timestamp of the fileset to check. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns true if the fileset is a full backup, otherwise false. - public async Task IsFilesetFullBackup(DateTime filesetTime) + public async Task IsFilesetFullBackup(DateTime filesetTime, CancellationToken token) { await using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters($@" @@ -1416,8 +1445,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - if (!await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (!await rd.ReadAsync(token).ConfigureAwait(false)) return false; var isFullBackup = rd.GetInt32(0); return isFullBackup == BackupType.FULL_BACKUP; @@ -1426,8 +1455,9 @@ namespace Duplicati.Library.Main.Database /// /// Retrieves a list of database options from the "Configuration" table. /// + /// Cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of key-value pairs representing the database options. - private async IAsyncEnumerable> GetDbOptionList() + private async IAsyncEnumerable> GetDbOptionList([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT @@ -1437,8 +1467,8 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "" @@ -1448,11 +1478,12 @@ namespace Duplicati.Library.Main.Database /// /// Retrieves all database options as a dictionary. /// + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a dictionary containing all database options. - public async Task> GetDbOptions() + public async Task> GetDbOptions(CancellationToken token) { - var res = await GetDbOptionList() - .ToDictionaryAsync(x => x.Key, x => x.Value) + var res = await GetDbOptionList(token) + .ToDictionaryAsync(x => x.Key, x => x.Value, cancellationToken: token) .ConfigureAwait(false); return res; @@ -1463,73 +1494,78 @@ namespace Duplicati.Library.Main.Database /// /// The key to update. /// The value to set. - private async Task UpdateDbOption(string key, bool value) + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the option has been updated. + private async Task UpdateDbOption(string key, bool value, CancellationToken token) { - var opts = await GetDbOptions().ConfigureAwait(false); + var opts = await GetDbOptions(token).ConfigureAwait(false); if (value) opts[key] = "true"; else opts.Remove(key); - await SetDbOptions(opts).ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await SetDbOptions(opts, token).ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// /// Flag indicating if a repair is in progress. /// + /// Cancellation token to monitor for cancellation requests. /// Optional value to set the flag to. If null, the current value in the database is returned. /// A task that, when awaited, returns true if a repair is in progress, otherwise false. - public async Task RepairInProgress(bool? value = null) + public async Task RepairInProgress(CancellationToken token, bool? value = null) { if (value is bool v) { - await UpdateDbOption("repair-in-progress", v) + await UpdateDbOption("repair-in-progress", v, token) .ConfigureAwait(false); return v; } - var opts = await GetDbOptions().ConfigureAwait(false); + var opts = await GetDbOptions(token).ConfigureAwait(false); return opts.ContainsKey("repair-in-progress"); } /// /// Flag indicating if the database has been partially recreated. /// + /// Cancellation token to monitor for cancellation requests. /// Optional value to set the flag to. If null, the current value in the database is returned. /// A task that, when awaited, returns true if the database has been partially recreated, otherwise false. - public async Task PartiallyRecreated(bool? value = null) + public async Task PartiallyRecreated(CancellationToken token, bool? value = null) { if (value is bool v) { - await UpdateDbOption("partially-recreated", v) + await UpdateDbOption("partially-recreated", v, token) .ConfigureAwait(false); return v; } - var opts = await GetDbOptions().ConfigureAwait(false); + var opts = await GetDbOptions(token).ConfigureAwait(false); return opts.ContainsKey("partially-recreated"); } /// /// Flag indicating if the database has been terminated with active uploads. /// + /// Cancellation token to monitor for cancellation requests. /// Optional value to set the flag to. If null, the current value in the database is returned. /// A task that, when awaited, returns true if the database has been terminated with active uploads, otherwise false. - public async Task TerminatedWithActiveUploads(bool? value = null) + public async Task TerminatedWithActiveUploads(CancellationToken token, bool? value = null) { if (value is bool v) { - await UpdateDbOption("terminated-with-active-uploads", v) + await UpdateDbOption("terminated-with-active-uploads", v, token) .ConfigureAwait(false); return v; } - var opts = await GetDbOptions().ConfigureAwait(false); + var opts = await GetDbOptions(token).ConfigureAwait(false); return opts.ContainsKey("terminated-with-active-uploads"); } @@ -1537,13 +1573,14 @@ namespace Duplicati.Library.Main.Database /// Sets the database options. /// /// The options to set. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the options have been set. - public async Task SetDbOptions(IDictionary options) + public async Task SetDbOptions(IDictionary options, CancellationToken token) { await using var cmd = m_connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@" DELETE FROM ""Configuration"" - ") + ", token) .ConfigureAwait(false); foreach (var kp in options) @@ -1560,7 +1597,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Key", kp.Key) .SetParameterValue("@Value", kp.Value) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } @@ -1569,19 +1606,20 @@ namespace Duplicati.Library.Main.Database /// Counts the number of blocks larger than a specified size. /// /// The size in bytes to compare against. + /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns the count of blocks larger than the specified size. - public async Task GetBlocksLargerThan(long fhblocksize) + public async Task GetBlocksLargerThan(long fhblocksize, CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size - ") + ", token) .ConfigureAwait(false); return await cmd .SetParameterValue("@Size", fhblocksize) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); } @@ -1591,9 +1629,10 @@ namespace Duplicati.Library.Main.Database /// The block size in bytes. /// The hash size in bytes. /// Also verify filelists (can be slow). + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the consistency check is finished. - public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists) - => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false) + public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists, CancellationToken token) + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false, token) .ConfigureAwait(false); /// @@ -1602,9 +1641,10 @@ namespace Duplicati.Library.Main.Database /// The block size in bytes. /// The hash size in bytes. /// Also verify filelists (can be slow). + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the consistency check is finished. - public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists) - => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true) + public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists, CancellationToken token) + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true, token) .ConfigureAwait(false); /// @@ -1614,8 +1654,9 @@ namespace Duplicati.Library.Main.Database /// The hash size in bytes. /// Also verify filelists (can be slow). /// Disable verify for errors that will be fixed by repair. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the consistency check is finished. - private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair) + private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair, CancellationToken token) { await using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); @@ -1652,8 +1693,8 @@ namespace Duplicati.Library.Main.Database AND ""A"".""CalcLen"" != ""A"".""Length"" "; - await using (var rd = await cmd.ExecuteReaderAsync(reportDetails).ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(reportDetails, token).ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) { var sb = new StringBuilder(); sb.AppendLine("Found inconsistency in the following files while validating database: "); @@ -1663,7 +1704,7 @@ namespace Duplicati.Library.Main.Database if (c < 5) sb.AppendFormat("{0}, actual size {1}, dbsize {2}, blocksetid: {3}{4}", rd.GetValue(3), rd.GetValue(1), rd.GetValue(0), rd.GetValue(2), Environment.NewLine); c++; - } while (await rd.ReadAsync().ConfigureAwait(false)); + } while (await rd.ReadAsync(token).ConfigureAwait(false)); c -= 5; if (c > 0) @@ -1676,7 +1717,7 @@ namespace Duplicati.Library.Main.Database var real_count = await cmd.ExecuteScalarInt64Async(@" SELECT COUNT(*) FROM ""BlocklistHash"" - ", 0) + ", 0, token) .ConfigureAwait(false); var unique_count = await cmd.ExecuteScalarInt64Async(@" @@ -1687,7 +1728,7 @@ namespace Duplicati.Library.Main.Database ""Index"" FROM ""BlocklistHash"" ) - ", 0) + ", 0, token) .ConfigureAwait(false); if (real_count != unique_count) @@ -1725,7 +1766,7 @@ namespace Duplicati.Library.Main.Database ) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"" ) - ", 0) + ", 0, token) .ConfigureAwait(false); if (itemswithnoblocklisthash != 0) @@ -1742,7 +1783,7 @@ namespace Duplicati.Library.Main.Database ) "); - if (await cmd.ExecuteScalarInt64Async().ConfigureAwait(false) != 0) + if (await cmd.ExecuteScalarInt64Async(token).ConfigureAwait(false) != 0) throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); cmd.SetCommandAndParameters(@" @@ -1758,7 +1799,7 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); - if (await cmd.ExecuteScalarInt64Async(0).ConfigureAwait(false) != 0) + if (await cmd.ExecuteScalarInt64Async(0, token).ConfigureAwait(false) != 0) throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!"); if (!laxVerifyForRepair) @@ -1777,7 +1818,7 @@ namespace Duplicati.Library.Main.Database cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); var filesetsMissingVolumes = - await cmd.ExecuteScalarInt64Async(0) + await cmd.ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); if (filesetsMissingVolumes != 0) @@ -1800,9 +1841,9 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - await using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync(token) .ConfigureAwait(false); - if (await reader.ReadAsync().ConfigureAwait(false)) + if (await reader.ReadAsync(token).ConfigureAwait(false)) throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); } @@ -1822,7 +1863,7 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - var volumesMissingFilests = await cmd.ExecuteScalarInt64Async(0) + var volumesMissingFilests = await cmd.ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); if (volumesMissingFilests != 0) @@ -1845,9 +1886,9 @@ namespace Duplicati.Library.Main.Database "); cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - await using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync(token) .ConfigureAwait(false); - if (await reader.ReadAsync().ConfigureAwait(false)) + if (await reader.ReadAsync(token).ConfigureAwait(false)) throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); } @@ -1862,7 +1903,7 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""FileLookup"" ) - ") + ", token) .ConfigureAwait(false); if (nonAttachedFiles != 0) @@ -1876,12 +1917,12 @@ namespace Duplicati.Library.Main.Database FROM ""FileLookup"" ) LIMIT 11 - ") + ", token) .ConfigureAwait(false); var filesetIds = new HashSet(); var overflow = false; - while (await filesetIdReader.ReadAsync().ConfigureAwait(false)) + while (await filesetIdReader.ReadAsync(token).ConfigureAwait(false)) { if (filesetIds.Count >= 10) { @@ -1891,13 +1932,13 @@ namespace Duplicati.Library.Main.Database filesetIds.Add(filesetIdReader.ConvertValueToInt64(0)); } - var pairs = FilesetTimes() + var pairs = FilesetTimes(token) .Select((x, i) => new { FilesetId = x.Key, Version = i, Time = x.Value }) .Where(x => filesetIds.Contains(x.FilesetId)) .Select(x => $"Fileset {x.Version}: {x.Time} (id = {x.FilesetId})"); // Fall back to a generic error message if we can't find the fileset ids - if (!await pairs.AnyAsync().ConfigureAwait(false)) + if (!await pairs.AnyAsync(cancellationToken: token).ConfigureAwait(false)) throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); if (overflow) @@ -1916,7 +1957,7 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""Fileset"" "); - await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync().Select(x => x.ConvertValueToInt64(0, -1)).ConfigureAwait(false)) + await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync(token).Select(x => x.ConvertValueToInt64(0, -1)).ConfigureAwait(false)) { var expandedlist = await cmd2.SetCommandAndParameters($@" SELECT COUNT(*) @@ -1931,7 +1972,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", filesetid) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != @FolderBlocksetId AND ""FileLookup"".""BlocksetID"" != @SymlinkBlocksetId"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); @@ -1942,20 +1983,20 @@ namespace Duplicati.Library.Main.Database WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetid) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); if (expandedlist != storedlist) { var filesetname = filesetid.ToString(); - var fileset = await FilesetTimes() + var fileset = await FilesetTimes(token) .Zip( - AsyncEnumerable.Range(0, await FilesetTimes() - .CountAsync() + AsyncEnumerable.Range(0, await FilesetTimes(token) + .CountAsync(cancellationToken: token) .ConfigureAwait(false) ), (a, b) => new Tuple(b, a.Key, a.Value) ) - .FirstOrDefaultAsync(x => x.Item2 == filesetid) + .FirstOrDefaultAsync(x => x.Item2 == filesetid, cancellationToken: token) .ConfigureAwait(false); if (fileset != null) @@ -2003,8 +2044,9 @@ namespace Duplicati.Library.Main.Database /// Retrieves all blocks associated with a specific volume ID. /// /// The ID of the volume to retrieve blocks for. + /// Cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of blocks associated with the specified volume ID. - public async IAsyncEnumerable GetBlocks(long volumeid) + public async IAsyncEnumerable GetBlocks(long volumeid, [EnumeratorCancellation] CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" SELECT DISTINCT @@ -2012,14 +2054,14 @@ namespace Duplicati.Library.Main.Database ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId - ") + ", token) .ConfigureAwait(false); cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new Block( rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), rd.ConvertValueToInt64(1) @@ -2056,16 +2098,22 @@ namespace Duplicati.Library.Main.Database /// The current blocklist hash being processed. /// private string? m_current = null; + /// + /// The cancellation token for this enumerator. + /// + private readonly CancellationToken m_token; /// /// Initializes a new instance of the class. /// /// The parent enumerable that this enumerator belongs to. /// The data reader used to read blocklist hashes. - public BlocklistHashEnumerator(BlocklistHashEnumerable parent, SqliteDataReader reader) + /// Cancellation token to monitor for cancellation requests. + public BlocklistHashEnumerator(BlocklistHashEnumerable parent, SqliteDataReader reader, CancellationToken token) { m_reader = reader; m_parent = parent; + m_token = token; } /// @@ -2094,7 +2142,7 @@ namespace Duplicati.Library.Main.Database if (m_current == null) return false; - if (!await m_reader.ReadAsync().ConfigureAwait(false)) + if (!await m_reader.ReadAsync(m_token).ConfigureAwait(false)) { m_current = null; m_parent.MoreData = false; @@ -2147,14 +2195,15 @@ namespace Duplicati.Library.Main.Database /// /// Gets an enumerator that iterates through the blocklist hashes. /// - public IAsyncEnumerator GetEnumerator() + /// Cancellation token to monitor for cancellation requests. + public IAsyncEnumerator GetEnumerator(CancellationToken token) { - return new BlocklistHashEnumerator(this, m_reader); + return new BlocklistHashEnumerator(this, m_reader, token); } IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken token) { - return GetEnumerator(); + return GetEnumerator(token); } } @@ -2297,8 +2346,9 @@ namespace Duplicati.Library.Main.Database /// /// The volume writer to which the fileset will be written. /// The ID of the fileset to write. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the fileset has been written. - public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId) + public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, CancellationToken token) { await using var cmd = m_connection.CreateCommand() .SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS) @@ -2308,8 +2358,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); string? lastpath = null; - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var blocksetID = rd.ConvertValueToInt64(0, -1); var path = rd.ConvertValueToString(2); @@ -2334,8 +2384,8 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(LIST_FILESETS); cmd.SetParameterValue("@FilesetId", filesetId); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - if (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) { var more = false; do @@ -2376,7 +2426,7 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); if (blrd == null) - more = await rd.ReadAsync().ConfigureAwait(false); + more = await rd.ReadAsync(token).ConfigureAwait(false); else more = blrd.MoreData; @@ -2389,20 +2439,21 @@ namespace Duplicati.Library.Main.Database /// /// The ID of the fileset to link. /// The ID of the volume to link the fileset to. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the link operation is finished. - public async Task LinkFilesetToVolume(long filesetid, long volumeid) + public async Task LinkFilesetToVolume(long filesetid, long volumeid, CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""Fileset"" SET ""VolumeID"" = @VolumeId WHERE ""ID"" = @FilesetId - ") + ", token) .ConfigureAwait(false); var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@FilesetId", filesetid) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) @@ -2413,8 +2464,9 @@ namespace Duplicati.Library.Main.Database /// Pushes timestamp changes from the latest version of a fileset to the previous version. /// /// The ID of the fileset whose timestamp changes will be pushed. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the timestamp changes have been pushed. - public async Task PushTimestampChangesToPreviousVersion(long filesetId) + public async Task PushTimestampChangesToPreviousVersion(long filesetId, CancellationToken token) { var query = @" UPDATE ""FilesetEntry"" AS ""oldVersion"" @@ -2432,11 +2484,13 @@ namespace Duplicati.Library.Main.Database ) "; - await using var cmd = await m_connection.CreateCommandAsync(query) + await using var cmd = await m_connection + .CreateCommandAsync(query, token) .ConfigureAwait(false); + await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -2475,8 +2529,9 @@ namespace Duplicati.Library.Main.Database /// /// The database to use for creating the temporary table. /// The filter to apply to the filenames. + /// Cancellation token to monitor for cancellation requests. /// A task that represents the asynchronous operation, with a as the result. - public static async Task CreateFilteredFilenameTableAsync(LocalDatabase db, IFilter filter) + public static async Task CreateFilteredFilenameTableAsync(LocalDatabase db, IFilter filter, CancellationToken token) { var ftt = new FilteredFilenameTable(db); @@ -2494,11 +2549,11 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{ftt.Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" - ") + ", token) .ConfigureAwait(false); await cmd.SetTransaction(db.Transaction) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); return ftt; @@ -2511,11 +2566,11 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( ""Path"" TEXT NOT NULL ) - ") + ", token) .ConfigureAwait(false); await cmd.SetTransaction(db.Transaction) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); cmd.SetCommandAndParameters($@" @@ -2526,24 +2581,24 @@ namespace Duplicati.Library.Main.Database await using var c2 = await db.Connection.CreateCommandAsync(@" SELECT DISTINCT ""Path"" FROM ""File"" - ") + ", token) .ConfigureAwait(false); c2.SetTransaction(db.Transaction); - await using (var rd = await c2.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await c2.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var p = rd.ConvertValueToString(0) ?? ""; if (FilterExpression.Matches(filter, p)) { await cmd .SetParameterValue("@Path", p) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } - await db.Transaction.CommitAsync().ConfigureAwait(false); + await db.Transaction.CommitAsync(token: token).ConfigureAwait(false); } else { @@ -2574,7 +2629,7 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( ""Path"" TEXT NOT NULL ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -2582,10 +2637,10 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb} - ", args) + ", args, token) .ConfigureAwait(false); - await db.Transaction.CommitAsync().ConfigureAwait(false); + await db.Transaction.CommitAsync(token: token).ConfigureAwait(false); } return ftt; @@ -2604,7 +2659,7 @@ namespace Duplicati.Library.Main.Database await using var cmd = m_db.Connection.CreateCommand(); await cmd.ExecuteNonQueryAsync(@$" DROP TABLE IF EXISTS ""{Tablename}"" - ") + ", default) .ConfigureAwait(false); } catch { } @@ -2617,22 +2672,23 @@ namespace Duplicati.Library.Main.Database /// /// The current name of the remote file. /// The new name for the remote file. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the renaming operation is finished. /// Thrown if the renaming operation does not affect exactly one row. - public async Task RenameRemoteFile(string oldname, string newname) + public async Task RenameRemoteFile(string oldname, string newname, CancellationToken token) { //Rename the old entry, to preserve ID links await using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""Remotevolume"" SET ""Name"" = @Newname WHERE ""Name"" = @Oldname - ") + ", token) .ConfigureAwait(false); var c = await cmd.SetTransaction(m_rtr) .SetParameterValue("@Newname", newname) .SetParameterValue("@Oldname", oldname) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) @@ -2647,7 +2703,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name ") .SetParameterValue("@Name", newname) - .ExecuteScalarAsync() + .ExecuteScalarAsync(token) .ConfigureAwait(false) )?.ToString() ?? "", true @@ -2656,15 +2712,15 @@ namespace Duplicati.Library.Main.Database //Create a fake new entry with the old name and mark as deleting // as this ensures we will remove it, if it shows up in some later listing var newvolId = - await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting) - .ConfigureAwait(false); + await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, token) + .ConfigureAwait(false); // IF needed, also create an empty fileset, so the validation works if (type == RemoteVolumeType.Files) { - await CreateFileset(newvolId, DateTime.UnixEpoch) + await CreateFileset(newvolId, DateTime.UnixEpoch, token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } @@ -2673,8 +2729,9 @@ namespace Duplicati.Library.Main.Database /// /// The ID of the fileset volume to update. /// The timestamp of the operation to create. + /// Cancellation token to monitor for cancellation requests. /// A task that when awaited contains the ID of the newly created fileset. - public virtual async Task CreateFileset(long volumeid, DateTime timestamp) + public virtual async Task CreateFileset(long volumeid, DateTime timestamp, CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" INSERT INTO ""Fileset"" ( @@ -2690,7 +2747,7 @@ namespace Duplicati.Library.Main.Database @IsFullBackup ); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); var id = await cmd @@ -2699,7 +2756,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); return id; @@ -2710,9 +2767,10 @@ namespace Duplicati.Library.Main.Database /// /// The ID of the index volume. /// The ID of the block volume. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the link has been added. /// Thrown if either volume ID is less than or equal to 0. - public async Task AddIndexBlockLink(long indexVolumeID, long blockVolumeID) + public async Task AddIndexBlockLink(long indexVolumeID, long blockVolumeID, CancellationToken token) { if (indexVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(indexVolumeID), "Index volume ID must be greater than 0."); @@ -2723,7 +2781,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@IndexVolumeId", indexVolumeID) .SetParameterValue("@BlockVolumeId", blockVolumeID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -2733,8 +2791,9 @@ namespace Duplicati.Library.Main.Database /// The volume ID to get blocklists for. /// The blocksize. /// The size of the hash. + /// Cancellation token to monitor for cancellation requests. /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data - public async IAsyncEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize) + public async IAsyncEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); // Group subquery by hash to ensure that each blocklist hash appears only once in the result @@ -2799,11 +2858,10 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(sql) .SetParameterValue("@VolumeId", volumeid) - .SetParameterValue("@HashesPerBlock", blocksize / hashsize) - .ConfigureAwait(false); + .SetParameterValue("@HashesPerBlock", blocksize / hashsize); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var blockhash = rd.ConvertValueToString(0); if (curHash != null && (blockhash != curHash || count + hashsize > buffer.Length)) @@ -2828,20 +2886,21 @@ namespace Duplicati.Library.Main.Database /// /// Existing file set to update. /// Full backup state. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the update is finished. - public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup) + public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup, CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" UPDATE ""Fileset"" SET ""IsFullBackup"" = @IsFullBackup WHERE ""ID"" = @FilesetId - ") + ", token) .ConfigureAwait(false); await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId) .SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -2849,18 +2908,19 @@ namespace Duplicati.Library.Main.Database /// Removes all entries in the fileset entry table for a given fileset ID. /// /// The fileset ID to clear. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the entries have been cleared. - public async Task ClearFilesetEntries(long filesetId) + public async Task ClearFilesetEntries(long filesetId, CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId - ") + ", token) .ConfigureAwait(false); await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -2868,18 +2928,19 @@ namespace Duplicati.Library.Main.Database /// Gets the last previous fileset that was incomplete. /// /// The transaction to use. + /// Cancellation token to monitor for cancellation requests. /// A task that when awaited returns the last incomplete fileset or default - public async Task GetLastIncompleteFilesetVolume() + public async Task GetLastIncompleteFilesetVolume(CancellationToken token) { - var candidates = GetIncompleteFilesets() + var candidates = GetIncompleteFilesets(token) .OrderBy(x => x.Value); - if (await candidates.AnyAsync().ConfigureAwait(false)) + if (await candidates.AnyAsync(cancellationToken: token).ConfigureAwait(false)) return await GetRemoteVolumeFromFilesetID( ( - await candidates.LastAsync().ConfigureAwait(false) + await candidates.LastAsync(cancellationToken: token).ConfigureAwait(false) ).Key - ) + , token) .ConfigureAwait(false); return default; @@ -2888,8 +2949,9 @@ namespace Duplicati.Library.Main.Database /// /// Gets a list of incomplete filesets. /// + /// Cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of key-value pairs where the key is the fileset ID and the value is the timestamp of the fileset. - public async IAsyncEnumerable> GetIncompleteFilesets() + public async IAsyncEnumerable> GetIncompleteFilesets([EnumeratorCancellation] CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@$" SELECT DISTINCT @@ -2908,14 +2970,14 @@ namespace Duplicati.Library.Main.Database ""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}' ) - ") + ", token) .ConfigureAwait(false); await using var rd = await cmd.SetTransaction(m_rtr) - .ExecuteReaderAsync() + .ExecuteReaderAsync(token) .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { yield return new KeyValuePair( rd.ConvertValueToInt64(0), @@ -2929,8 +2991,9 @@ namespace Duplicati.Library.Main.Database /// Gets the remote volume entry from the fileset ID. /// /// The fileset ID. + /// Cancellation token to monitor for cancellation requests. /// A task that when awaited returns the remote volume entry associated with the fileset ID, or default if not found. - public async Task GetRemoteVolumeFromFilesetID(long filesetID) + public async Task GetRemoteVolumeFromFilesetID(long filesetID, CancellationToken token) { await using var cmd = await m_connection.CreateCommandAsync(@" SELECT @@ -2948,15 +3011,15 @@ namespace Duplicati.Library.Main.Database WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Fileset"".""ID"" = @FilesetId - ") + ", token) .ConfigureAwait(false); await using var rd = await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetID) - .ExecuteReaderAsync() + .ExecuteReaderAsync(token) .ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) return new RemoteVolumeEntry( rd.ConvertValueToInt64(0, -1), rd.ConvertValueToString(1), @@ -2975,8 +3038,9 @@ namespace Duplicati.Library.Main.Database /// Purges log data and remote operations older than the specified threshold. /// /// The threshold date and time; all log data and remote operations older than this will be purged. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the purge operation is finished. - public async Task PurgeLogData(DateTime threshold) + public async Task PurgeLogData(DateTime threshold, CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold); @@ -2986,7 +3050,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Timestamp"" < @Timestamp ") .SetParameterValue("@Timestamp", t) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); await cmd.SetCommandAndParameters(@" @@ -2994,10 +3058,10 @@ namespace Duplicati.Library.Main.Database WHERE ""Timestamp"" < @Timestamp ") .SetParameterValue("@Timestamp", t) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// @@ -3005,15 +3069,15 @@ namespace Duplicati.Library.Main.Database /// /// The threshold date and time; all deleted remote volumes older than this will be purged. /// A task that completes when the purge operation is finished. - public async Task PurgeDeletedVolumes(DateTime threshold) + public async Task PurgeDeletedVolumes(DateTime threshold, CancellationToken token) { await m_removedeletedremotevolumeCommand .SetTransaction(m_rtr) .SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } public virtual void Dispose() @@ -3039,7 +3103,7 @@ namespace Duplicati.Library.Main.Database // SQLite recommends that PRAGMA optimize is run just before closing each database connection. await command .SetTransaction(tr) - .ExecuteNonQueryAsync("PRAGMA optimize") + .ExecuteNonQueryAsync("PRAGMA optimize", default) .ConfigureAwait(false); try @@ -3112,8 +3176,9 @@ namespace Duplicati.Library.Main.Database /// Writes the results of a basic operation to the log. /// /// The results to write. + /// Cancellation token to monitor for cancellation requests. /// A task that completes when the results have been written. - public async Task WriteResults(IBasicResults result) + public async Task WriteResults(IBasicResults result, CancellationToken token) { if (IsDisposed) return; @@ -3122,7 +3187,7 @@ namespace Duplicati.Library.Main.Database { if (result is BasicResults basicResults) { - await basicResults.FlushLog(this).ConfigureAwait(false); + await basicResults.FlushLog(this, token).ConfigureAwait(false); if (basicResults.EndTime.Ticks == 0) basicResults.EndTime = DateTime.UtcNow; } @@ -3130,7 +3195,8 @@ namespace Duplicati.Library.Main.Database var serializer = new JsonFormatSerializer(); await LogMessage("Result", serializer.SerializeResults(result), - null + null, + token ) .ConfigureAwait(false); } @@ -3149,8 +3215,9 @@ namespace Duplicati.Library.Main.Database /// Gets the path prefix ID, optionally creating it in the process.. /// /// The path to get the prefix for. + /// Cancellation token to monitor for cancellation requests. /// A task that when awaited returns the path prefix ID. - public async Task GetOrCreatePathPrefix(string prefix) + public async Task GetOrCreatePathPrefix(string prefix, CancellationToken token) { // Ring-buffer style lookup for (var i = 0; i < m_pathPrefixLookup.Length; i++) @@ -3163,7 +3230,7 @@ namespace Duplicati.Library.Main.Database var id = await m_findpathprefixCommand .SetTransaction(m_rtr) .SetParameterValue("@Prefix", prefix) - .ExecuteScalarInt64Async() + .ExecuteScalarInt64Async(token) .ConfigureAwait(false); if (id < 0) @@ -3171,7 +3238,7 @@ namespace Duplicati.Library.Main.Database id = await m_insertpathprefixCommand .SetTransaction(m_rtr) .SetParameterValue("@Prefix", prefix) - .ExecuteScalarInt64Async() + .ExecuteScalarInt64Async(token) .ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 30c66eb8f..afc234d3c 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -68,6 +69,11 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran /// public SqliteTransaction Transaction => m_disposed ? throw new InvalidOperationException("Transaction is disposed") : m_transaction; + public async Task CommitAsync(CancellationToken token) + { + await CommitAsync(token: token).ConfigureAwait(false); + } + /// /// Async version of Commit: /// @@ -75,7 +81,7 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran /// True if the transaction should be restarted. /// A task that completes when the commit is done and (potentially) a new transaction has been started. /// If the transaction is already Disposed. - public async Task CommitAsync(string? message = null, bool restart = true) + public async Task CommitAsync(string? message = null, bool restart = true, CancellationToken token = default) { if (m_disposed) throw new InvalidOperationException("Transaction is already disposed"); diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 24c817ee6..b91925154 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -260,20 +260,26 @@ namespace Duplicati.Library.Main } } - public async Task FlushLog(LocalDatabase db) + /// + /// Flushes the log messages to the database. + /// + /// The database to flush the log messages to. + /// The cancellation token to use. + /// A task that completes when the log messages have been flushed. + public async Task FlushLog(LocalDatabase db, CancellationToken token) { if (m_parent != null) - await m_parent.FlushLog(db).ConfigureAwait(false); + await m_parent.FlushLog(db, token).ConfigureAwait(false); else { - await m_lock.WaitAsync().ConfigureAwait(false); + await m_lock.WaitAsync(token).ConfigureAwait(false); try { while (m_dbqueue.Count > 0) { var el = m_dbqueue.Dequeue(); await db - .LogMessage(el.Type, el.Message, el.Exception) + .LogMessage(el.Type, el.Message, el.Exception, token) .ConfigureAwait(false); } } From 8039cbb0f6de5737c920304955aa30ccbf5400db Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 16:00:20 +0200 Subject: [PATCH 324/362] Made the LocalBackupDatabase methods accept a cancellation token --- .../Main/Database/LocalBackupDatabase.cs | 443 ++++++++++-------- 1 file changed, 239 insertions(+), 204 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index c594960c2..582b06a55 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -24,6 +24,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -158,15 +160,16 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The Duplicati options used by the current operation. /// An optional existing instance of to reuse. Used when derived classes need to call the base constructor. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, Options options, LocalBackupDatabase? dbnew = null) + public static async Task CreateAsync(string path, Options options, CancellationToken token, LocalBackupDatabase? dbnew = null) { dbnew ??= new LocalBackupDatabase(); dbnew = (LocalBackupDatabase) - await CreateLocalDatabaseAsync(path, "Backup", false, options.SqlitePageCache, dbnew) + await CreateLocalDatabaseAsync(path, "Backup", false, options.SqlitePageCache, dbnew, token) .ConfigureAwait(false); - dbnew = await CreateAsync(dbnew, options).ConfigureAwait(false); + dbnew = await CreateAsync(dbnew, options, null, token).ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; @@ -179,12 +182,13 @@ namespace Duplicati.Library.Main.Database /// The parent local database from which to create the new backup database. /// The Duplicati options used by the current operation. /// An optional existing instance of to reuse. Used when derived classes need to call the base constructor. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(LocalDatabase dbparent, Options options, LocalBackupDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase dbparent, Options options, LocalBackupDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalBackupDatabase(); - dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew) + dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew, token) .ConfigureAwait(false); dbnew.m_logQueries = options.ProfileAllDatabaseQueries; @@ -195,7 +199,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - ") + ", token) .ConfigureAwait(false); dbnew.m_findblocksetCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -204,7 +208,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Fullhash"" = @Fullhash AND ""Length"" = @Length - ") + ", token) .ConfigureAwait(false); dbnew.m_findmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -218,7 +222,7 @@ namespace Duplicati.Library.Main.Database AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = @Hash AND ""C"".""Size"" = @Size - ") + ", token) .ConfigureAwait(false); dbnew.m_findfilesetCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -229,7 +233,7 @@ namespace Duplicati.Library.Main.Database AND ""MetadataID"" = @MetadataId AND ""Path"" = @Path AND ""PrefixID"" = @PrefixId - ") + ", token) .ConfigureAwait(false); dbnew.m_insertblockCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -244,7 +248,7 @@ namespace Duplicati.Library.Main.Database @Size ); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); dbnew.m_insertfileOperationCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -258,7 +262,7 @@ namespace Duplicati.Library.Main.Database @FileId, @LastModified ) - ") + ", token) .ConfigureAwait(false); dbnew.m_insertfileCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -275,7 +279,7 @@ namespace Duplicati.Library.Main.Database @MetadataId ); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); dbnew.m_insertblocksetCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -287,7 +291,8 @@ namespace Duplicati.Library.Main.Database @Length, @Fullhash ); - SELECT last_insert_rowid();") + SELECT last_insert_rowid(); + ", token) .ConfigureAwait(false); dbnew.m_insertblocksetentryCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -303,7 +308,7 @@ namespace Duplicati.Library.Main.Database FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size - ") + ", token) .ConfigureAwait(false); dbnew.m_insertblocklistHashesCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -317,14 +322,14 @@ namespace Duplicati.Library.Main.Database @Index, @Hash ) - ") + ", token) .ConfigureAwait(false); dbnew.m_insertmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); dbnew.m_selectfilelastmodifiedCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -341,7 +346,7 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId - ") + ", token) .ConfigureAwait(false); dbnew.m_selectfilelastmodifiedWithSizeCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -371,7 +376,7 @@ namespace Duplicati.Library.Main.Database ) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" - ") + ", token) .ConfigureAwait(false); dbnew.m_selectfilemetadatahashandsizeCommand = await dbnew.Connection.CreateCommandAsync(@" @@ -386,7 +391,7 @@ namespace Duplicati.Library.Main.Database ""File"".""ID"" = @FileId AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" - ") + ", token) .ConfigureAwait(false); // Experimental toggling of the deleted block cache @@ -404,38 +409,38 @@ namespace Duplicati.Library.Main.Database await using var cmd = dbnew.Connection.CreateCommand(); dbnew.m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); await cmd.SetCommandAndParameters($@" - CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS - SELECT - MAX(""ID"") AS ""ID"", - ""Hash"", - ""Size"" - FROM ""DeletedBlock"" - WHERE ""VolumeID"" IN ( - SELECT ""ID"" - FROM ""RemoteVolume"" - WHERE ""State"" NOT IN (@States) - ) - GROUP BY - ""Hash"", - ""Size"" - ") + CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS + SELECT + MAX(""ID"") AS ""ID"", + ""Hash"", + ""Size"" + FROM ""DeletedBlock"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""State"" NOT IN (@States) + ) + GROUP BY + ""Hash"", + ""Size"" + ") .ExpandInClauseParameterMssqlite("@States", [ RemoteVolumeState.Deleted, RemoteVolumeState.Deleting ]) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$" SELECT COUNT(*) FROM ""{dbnew.m_tempDeletedBlockTable}"" - ", 0) + ", 0, token) .ConfigureAwait(false); // There are no deleted blocks, so we can drop the table if (deletedBlocks == 0) { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""", token) .ConfigureAwait(false); dbnew.m_tempDeletedBlockTable = null; @@ -453,8 +458,8 @@ namespace Duplicati.Library.Main.Database ") .ConfigureAwait(false); - await using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await reader.ReadAsync().ConfigureAwait(false)) + await using (var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await reader.ReadAsync(token).ConfigureAwait(false)) { var id = reader.ConvertValueToInt64(0); var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null"); @@ -465,7 +470,7 @@ namespace Duplicati.Library.Main.Database sizes[size] = id; } - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""", token) .ConfigureAwait(false); dbnew.m_tempDeletedBlockTable = null; } @@ -473,44 +478,44 @@ namespace Duplicati.Library.Main.Database else { await cmd.ExecuteNonQueryAsync($@" - CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" - ON ""{dbnew.m_tempDeletedBlockTable}"" ( - ""Hash"", - ""Size"" - ) - ") + CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" + ON ""{dbnew.m_tempDeletedBlockTable}"" ( + ""Hash"", + ""Size"" + ) + ", token) .ConfigureAwait(false); dbnew.m_findindeletedCommand = await dbnew.Connection.CreateCommandAsync($@" - SELECT ""ID"" - FROM ""{dbnew.m_tempDeletedBlockTable}"" - WHERE - ""Hash"" = @Hash - AND ""Size"" = @Size - ") + SELECT ""ID"" + FROM ""{dbnew.m_tempDeletedBlockTable}"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ", token) .ConfigureAwait(false); dbnew.m_moveblockfromdeletedCommand = await dbnew.Connection.CreateCommandAsync(@$" - INSERT INTO ""Block"" ( - ""Hash"", - ""Size"", - ""VolumeID"" - ) - SELECT - ""Hash"", - ""Size"", - ""VolumeID"" - FROM ""DeletedBlock"" - WHERE ""ID"" = @DeletedBlockId LIMIT 1; + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId LIMIT 1; - DELETE FROM ""DeletedBlock"" - WHERE ""ID"" = @DeletedBlockId; + DELETE FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId; - DELETE FROM ""{dbnew.m_tempDeletedBlockTable}"" - WHERE ""ID"" = @DeletedBlockId; + DELETE FROM ""{dbnew.m_tempDeletedBlockTable}"" + WHERE ""ID"" = @DeletedBlockId; - SELECT last_insert_rowid() - ") + SELECT last_insert_rowid() + ", token) .ConfigureAwait(false); } @@ -518,23 +523,23 @@ namespace Duplicati.Library.Main.Database if (deletedBlocks > 0) { dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(@$" - INSERT INTO ""Block"" ( - ""Hash"", - ""Size"", - ""VolumeID"" - ) - SELECT - ""Hash"", - ""Size"", - ""VolumeID"" - FROM ""DeletedBlock"" - WHERE ""ID"" = @DeletedBlockId LIMIT 1; + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId LIMIT 1; - DELETE FROM ""DeletedBlock"" - WHERE ""ID"" = @DeletedBlockId; + DELETE FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId; - SELECT last_insert_rowid() - ") + SELECT last_insert_rowid() + ", token) .ConfigureAwait(false); } } @@ -729,14 +734,15 @@ namespace Duplicati.Library.Main.Database /// /// The block key. /// The size of the block. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains true if the block should be added to the current output. - public async Task FindBlockID(string key, long size) + public async Task FindBlockID(string key, long size, CancellationToken token) { return await m_findblockCommand .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(m_logQueries, -1) + .ExecuteScalarInt64Async(m_logQueries, -1, token) .ConfigureAwait(false); } @@ -746,10 +752,11 @@ namespace Duplicati.Library.Main.Database /// The block key. /// The size of the block. /// The ID of the volume to which the block belongs. + /// The cancellation token to monitor for cancellation requests. /// A taskt that when awaited contains true if the block should be added to the current output. - public async Task AddBlock(string key, long size, long volumeid) + public async Task AddBlock(string key, long size, long volumeid, CancellationToken token) { - var r = await FindBlockID(key, size).ConfigureAwait(false); + var r = await FindBlockID(key, size, token).ConfigureAwait(false); if (r == -1L) { if (m_moveblockfromdeletedCommand != null) @@ -762,7 +769,7 @@ namespace Duplicati.Library.Main.Database await m_moveblockfromdeletedCommand .SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) - .ExecuteNonQueryAsync(m_logQueries) + .ExecuteNonQueryAsync(m_logQueries, token) .ConfigureAwait(false); sizes.Remove(size); @@ -778,7 +785,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(m_logQueries, -1) + .ExecuteScalarInt64Async(m_logQueries, -1, token) .ConfigureAwait(false); if (id != -1) @@ -786,7 +793,7 @@ namespace Duplicati.Library.Main.Database var c = await m_moveblockfromdeletedCommand .SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) - .ExecuteNonQueryAsync(m_logQueries) + .ExecuteNonQueryAsync(m_logQueries, token) .ConfigureAwait(false); if (c != 2) @@ -803,7 +810,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", key) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync(m_logQueries) + .ExecuteNonQueryAsync(m_logQueries, token) .ConfigureAwait(false); if (ins != 1) @@ -827,14 +834,15 @@ namespace Duplicati.Library.Main.Database /// The size of the blocks in the blockset. /// The list of hashes. /// The list of hashes for the blocklist, or null if no blocklist is used. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains a tuple with the first value indicating whether the blockset was created, and the second value being the blockset ID. - public async Task<(bool, long)> AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes) + public async Task<(bool, long)> AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, CancellationToken token) { long blocksetid = await m_findblocksetCommand .SetTransaction(m_rtr) .SetParameterValue("@Fullhash", filehash) .SetParameterValue("@Length", size) - .ExecuteScalarInt64Async(m_logQueries, -1) + .ExecuteScalarInt64Async(m_logQueries, -1, token) .ConfigureAwait(false); if (blocksetid != -1) @@ -844,7 +852,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@Fullhash", filehash) - .ExecuteScalarInt64Async(m_logQueries) + .ExecuteScalarInt64Async(m_logQueries, token) .ConfigureAwait(false); long ix = 0; @@ -859,7 +867,7 @@ namespace Duplicati.Library.Main.Database var c = await m_insertblocklistHashesCommand .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", bh) - .ExecuteNonQueryAsync(m_logQueries) + .ExecuteNonQueryAsync(m_logQueries, token) .ConfigureAwait(false); if (c != 1) @@ -882,7 +890,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", h) .SetParameterValue("@Size", exsize) - .ExecuteNonQueryAsync(m_logQueries) + .ExecuteNonQueryAsync(m_logQueries, token) .ConfigureAwait(false); if (c != 1) @@ -896,7 +904,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash ") .SetParameterValue("@Hash", h) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (bid == -1) @@ -909,7 +917,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Hash", h); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); } @@ -920,7 +928,7 @@ namespace Duplicati.Library.Main.Database remainsize -= blocksize; } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); return (true, blocksetid); } @@ -930,8 +938,9 @@ namespace Duplicati.Library.Main.Database /// /// The metadata hash. /// The size of the metadata. + /// The cancellation token to cancel the operation. /// A task that when awaited contains a tuple with the first value indicating if the metadataset was found, and the second value being the metadataset ID. - public async Task<(bool, long)> GetMetadatasetID(string filehash, long size) + public async Task<(bool, long)> GetMetadatasetID(string filehash, long size, CancellationToken token) { long metadataid; @@ -941,7 +950,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Hash", filehash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(m_logQueries, -1) + .ExecuteScalarInt64Async(m_logQueries, -1, token) .ConfigureAwait(false); return (metadataid != -1, metadataid); @@ -957,10 +966,11 @@ namespace Duplicati.Library.Main.Database /// The metadata hash. /// The size of the metadata. /// The id of the blockset to add. + /// The cancellation token to cancel the operation. /// A task that when awaited contains a tuple with the first value indicating if the metadata set was added, and the second value being the metadata ID. - public async Task<(bool, long)> AddMetadataset(string filehash, long size, long blocksetid) + public async Task<(bool, long)> AddMetadataset(string filehash, long size, long blocksetid, CancellationToken token) { - var (metadatafound, metadataid) = await GetMetadatasetID(filehash, size) + var (metadatafound, metadataid) = await GetMetadatasetID(filehash, size, token) .ConfigureAwait(false); if (metadatafound) return (false, metadataid); @@ -968,10 +978,10 @@ namespace Duplicati.Library.Main.Database metadataid = await m_insertmetadatasetCommand .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64Async(m_logQueries) + .ExecuteScalarInt64Async(m_logQueries, token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); return (true, metadataid); } @@ -984,8 +994,9 @@ namespace Duplicati.Library.Main.Database /// The time the file was modified. /// The ID of the hashkey for the file. /// The ID for the metadata. + /// The cancellation token to cancel the operation. /// A task that completes when the file is added. - public async Task AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID) + public async Task AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, CancellationToken token) { var fileidobj = await m_findfilesetCommand .SetTransaction(m_rtr) @@ -993,7 +1004,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@MetadataId", metadataID) .SetParameterValue("@Path", filename) .SetParameterValue("@PrefixId", pathprefixid) - .ExecuteScalarInt64Async(m_logQueries) + .ExecuteScalarInt64Async(m_logQueries, token) .ConfigureAwait(false); if (fileidobj == -1) @@ -1003,13 +1014,13 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", filename) .SetParameterValue("@BlocksetId", blocksetID) .SetParameterValue("@MetadataId", metadataID) - .ExecuteScalarInt64Async(m_logQueries) + .ExecuteScalarInt64Async(m_logQueries, token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token).ConfigureAwait(false); } - await AddKnownFile(fileidobj, lastmodified).ConfigureAwait(false); + await AddKnownFile(fileidobj, lastmodified, token).ConfigureAwait(false); } /// @@ -1019,17 +1030,19 @@ namespace Duplicati.Library.Main.Database /// The time the file was modified. /// The ID of the hashkey for the file. /// The ID for the metadata. + /// The cancellation token to cancel the operation. /// A task that completes when the file is added. - public async Task AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID) + public async Task AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, CancellationToken token) { var split = SplitIntoPrefixAndName(filename); await AddFile( - await GetOrCreatePathPrefix(split.Key).ConfigureAwait(false), + await GetOrCreatePathPrefix(split.Key, token).ConfigureAwait(false), split.Value, lastmodified, blocksetID, - metadataID + metadataID, + token ) .ConfigureAwait(false); } @@ -1039,15 +1052,16 @@ namespace Duplicati.Library.Main.Database /// /// Id of the file. /// The time the file was modified. + /// The cancellation token to cancel the operation. /// A task that completes when the file is added. - public async Task AddKnownFile(long fileid, DateTime lastmodified) + public async Task AddKnownFile(long fileid, DateTime lastmodified, CancellationToken token) { await m_insertfileOperationCommand .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", m_filesetId) .SetParameterValue("@FileId", fileid) .SetParameterValue("@LastModified", lastmodified.ToUniversalTime().Ticks) - .ExecuteNonQueryAsync(m_logQueries) + .ExecuteNonQueryAsync(m_logQueries, token) .ConfigureAwait(false); } @@ -1057,10 +1071,11 @@ namespace Duplicati.Library.Main.Database /// The path to the directory. /// The ID for the metadata. /// The time the directory was modified. + /// The cancellation token to cancel the operation. /// A task that completes when the directory entry is added. - public async Task AddDirectoryEntry(string path, long metadataID, DateTime lastmodified) + public async Task AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, CancellationToken token) { - await AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID) + await AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID, token) .ConfigureAwait(false); } @@ -1070,10 +1085,11 @@ namespace Duplicati.Library.Main.Database /// The path to the symlink. /// The ID for the metadata. /// The time the symlink was modified. + /// The cancellation token to cancel the operation. /// A task that completes when the symlink entry is added. - public async Task AddSymlinkEntry(string path, long metadataID, DateTime lastmodified) + public async Task AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, CancellationToken token) { - await AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID) + await AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, token) .ConfigureAwait(false); } @@ -1084,8 +1100,9 @@ namespace Duplicati.Library.Main.Database /// The path to the file. /// The ID of the fileset. /// Whether to include the file length in the result. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains a tuple with the file ID, last modified time, and file length. - public async Task<(long, DateTime, long)> GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength) + public async Task<(long, DateTime, long)> GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, CancellationToken token) { DateTime oldModified; long length; @@ -1097,8 +1114,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - await using var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null).ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) { oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); length = rd.ConvertValueToInt64(2); @@ -1112,8 +1129,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - await using var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries).ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries, token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) { length = -1; oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); @@ -1198,13 +1215,14 @@ namespace Duplicati.Library.Main.Database /// Gets the hash of a file. /// /// The ID of the file. + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the hash of the file, or null if the file does not exist. - public async Task GetFileHash(long fileid) + public async Task GetFileHash(long fileid, CancellationToken token) { var r = await m_selectfileHashCommand .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid) - .ExecuteScalarAsync(m_logQueries, null) + .ExecuteScalarAsync(m_logQueries, token) .ConfigureAwait(false); if (r == null || r == DBNull.Value) @@ -1224,7 +1242,7 @@ namespace Duplicati.Library.Main.Database try { await using (var cmd = m_connection.CreateCommand(m_rtr)) - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}""", default) .ConfigureAwait(false); await m_rtr.CommitAsync().ConfigureAwait(false); } @@ -1239,8 +1257,9 @@ namespace Duplicati.Library.Main.Database /// /// Gets the size of the last written DBlock volume. /// + /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the size of the last written DBlock volume, or -1 if no such volume exists. - public async Task GetLastWrittenDBlockVolumeSize() + public async Task GetLastWrittenDBlockVolumeSize(CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); return await cmd.SetCommandAndParameters(@" @@ -1254,7 +1273,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); } @@ -1262,10 +1281,11 @@ namespace Duplicati.Library.Main.Database /// Gets the ID of the previous fileset based on the operation timestamp and current fileset ID. /// /// The command to use for the query. + /// The cancellation token to cancel the operation. /// A task that when awaited contains the ID of the previous fileset, or -1 if no such fileset exists. - private async Task GetPreviousFilesetID(SqliteCommand cmd) + private async Task GetPreviousFilesetID(SqliteCommand cmd, CancellationToken token) { - return await GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId) + return await GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId, token) .ConfigureAwait(false); } @@ -1275,8 +1295,9 @@ namespace Duplicati.Library.Main.Database /// The command to use for the query. /// The timestamp to use for the query. /// The current fileset ID. + /// The cancellation token to cancel the operation. /// A task that when awaited contains the ID of the previous fileset, or -1 if no such fileset exists. - private async Task GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid) + private async Task GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid, CancellationToken token) { return await cmd .SetTransaction(m_rtr) @@ -1290,41 +1311,42 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) .SetParameterValue("@FilesetId", filesetid) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); } /// /// Gets the count and size of files in the last backup fileset. /// + /// The cancellation token to cancel the operation. /// A task that when awaited contains a tuple with the count of files and the total size of files in the last backup fileset. - internal async Task> GetLastBackupFileCountAndSize() + internal async Task> GetLastBackupFileCountAndSize(CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); var lastFilesetId = await cmd.ExecuteScalarInt64Async(@" - SELECT ""ID"" - FROM ""Fileset"" - ORDER BY ""Timestamp"" DESC - LIMIT 1 - ") + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + LIMIT 1 + ", token) .ConfigureAwait(false); var count = await cmd.SetCommandAndParameters(@" - SELECT COUNT(*) - FROM ""FileLookup"" - INNER JOIN ""FilesetEntry"" - ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" - WHERE - ""FilesetEntry"".""FilesetID"" = @FilesetId - AND ""FileLookup"".""BlocksetID"" NOT IN ( - @FolderBlocksetId, - @SymlinkBlocksetId - ) - ") + SELECT COUNT(*) + FROM ""FileLookup"" + INNER JOIN ""FilesetEntry"" + ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + WHERE + ""FilesetEntry"".""FilesetID"" = @FilesetId + AND ""FileLookup"".""BlocksetID"" NOT IN ( + @FolderBlocksetId, + @SymlinkBlocksetId + ) + ") .SetParameterValue("@FilesetId", lastFilesetId) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); var size = await cmd.SetCommandAndParameters(@" @@ -1345,7 +1367,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", lastFilesetId) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); return new Tuple(count, size); @@ -1355,13 +1377,14 @@ namespace Duplicati.Library.Main.Database /// Updates the change statistics for the current fileset based on the results of a backup operation. /// /// The results of the backup operation. + /// The cancellation token to cancel the operation. /// A task that completes when the change statistics are updated. - internal async Task UpdateChangeStatistics(BackupResults results) + internal async Task UpdateChangeStatistics(BackupResults results, CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); - var prevFileSetId = await GetPreviousFilesetID(cmd) + var prevFileSetId = await GetPreviousFilesetID(cmd, token) .ConfigureAwait(false); - await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId) + await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId, token) .ConfigureAwait(false); } @@ -1369,11 +1392,12 @@ namespace Duplicati.Library.Main.Database /// Populates FilesetEntry table with files from previous fileset, which aren't /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. /// - /// List of deleted paths, or null. + /// List of deleted paths, or null." + /// The cancellation token to cancel the operation. /// A task that completes when the files are appended. - public async Task AppendFilesFromPreviousSet(IEnumerable? deleted = null) + public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, CancellationToken token) { - await AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp) + await AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp, token) .ConfigureAwait(false); } @@ -1385,13 +1409,14 @@ namespace Duplicati.Library.Main.Database /// Current file-set ID. /// Source file-set ID. /// If filesetid == -1, used to locate previous file-set. + /// The cancellation token to cancel the operation. /// A task that completes when the files are appended. - public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) + public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp, CancellationToken token) { await using var cmd = m_connection.CreateCommand(); await using var cmdDelete = m_connection.CreateCommand(); long lastFilesetId = prevId < 0 ? - await GetPreviousFilesetID(cmd, timestamp, filesetid) + await GetPreviousFilesetID(cmd, timestamp, filesetid, token) .ConfigureAwait(false) : prevId; @@ -1424,13 +1449,13 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@CurrentFilesetId", filesetid) .SetParameterValue("@PreviousFilesetId", lastFilesetId) - .ExecuteNonQueryAsync(m_logQueries) + .ExecuteNonQueryAsync(m_logQueries, token) .ConfigureAwait(false); if (deleted != null) { await using var tmplist = await TemporaryDbValueList - .CreateAsync(this, deleted) + .CreateAsync(this, deleted, token) .ConfigureAwait(false); await ( @@ -1446,14 +1471,14 @@ namespace Duplicati.Library.Main.Database ) ") .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameterMssqliteAsync("@Paths", tmplist) + .ExpandInClauseParameterMssqliteAsync("@Paths", tmplist, token) .ConfigureAwait(false) ) - .ExecuteNonQueryAsync(m_logQueries) + .ExecuteNonQueryAsync(m_logQueries, token) .ConfigureAwait(false); } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token).ConfigureAwait(false); } /// @@ -1462,10 +1487,11 @@ namespace Duplicati.Library.Main.Database /// predicate. /// /// Optional exclusion predicate (true = exclude file). + /// The cancellation token to cancel the operation. /// A task that completes when the files are appended. - public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate) + public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, CancellationToken token) { - await AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp) + await AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp, token) .ConfigureAwait(false); } @@ -1478,12 +1504,13 @@ namespace Duplicati.Library.Main.Database /// Current fileset ID /// Source fileset ID /// If prevFileSetId == -1, used to locate previous fileset + /// The cancellation token to cancel the operation. /// A task that completes when the files are appended. - public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) + public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp, CancellationToken token) { if (exclusionPredicate == null) { - await AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp) + await AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp, token) .ConfigureAwait(false); return; } @@ -1491,7 +1518,7 @@ namespace Duplicati.Library.Main.Database await using var cmd = m_connection.CreateCommand(); await using var cmdDelete = m_connection.CreateCommand(); long lastFilesetId = prevFileSetId < 0 ? - await GetPreviousFilesetID(cmd, timestamp, fileSetId) + await GetPreviousFilesetID(cmd, timestamp, fileSetId, token) .ConfigureAwait(false) : prevFileSetId; @@ -1522,7 +1549,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@PreviousFilesetId", lastFilesetId) .SetParameterValue("@CurrentFilesetId", fileSetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); // now we need to remove, from the above, any entries that were enumerated by the @@ -1552,14 +1579,14 @@ namespace Duplicati.Library.Main.Database ON ""f"".""BlocksetID"" = ""bs"".""ID""; "); - await foreach (var row in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var row in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path"); var size = row.ConvertValueToInt64(3); if (exclusionPredicate(path, size)) await cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -1577,10 +1604,10 @@ namespace Duplicati.Library.Main.Database FROM ""{tempFileSetTable}"" ") .SetParameterValue("@FilesetId", fileSetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token).ConfigureAwait(false); } /// @@ -1588,10 +1615,11 @@ namespace Duplicati.Library.Main.Database /// /// The ID of the fileset volume to update. /// The timestamp of the operation to create. + /// The cancellation token to cancel the operation. /// A task that when awaited contains the ID of the created fileset. - public override async Task CreateFileset(long volumeid, DateTime timestamp) + public override async Task CreateFileset(long volumeid, DateTime timestamp, CancellationToken token) { - return m_filesetId = await base.CreateFileset(volumeid, timestamp) + return m_filesetId = await base.CreateFileset(volumeid, timestamp, token) .ConfigureAwait(false); } @@ -1599,24 +1627,25 @@ namespace Duplicati.Library.Main.Database /// Retrieves the names of temporary fileset volumes that are incomplete. /// /// If true, only the latest incomplete fileset volume will be returned. + /// The cancellation token to cancel the operation. /// A task that when awaited contains a list of volume names. - public async Task> GetTemporaryFilelistVolumeNames(bool latestOnly) + public async Task> GetTemporaryFilelistVolumeNames(bool latestOnly, CancellationToken token) { - var incompleteFilesetIDs = GetIncompleteFilesets().OrderBy(x => x.Value).Select(x => x.Key); + var incompleteFilesetIDs = GetIncompleteFilesets(token).OrderBy(x => x.Value).Select(x => x.Key); - if (!await incompleteFilesetIDs.AnyAsync().ConfigureAwait(false)) + if (!await incompleteFilesetIDs.AnyAsync(token).ConfigureAwait(false)) return []; if (latestOnly) incompleteFilesetIDs = new long[] { - await incompleteFilesetIDs.LastAsync().ConfigureAwait(false) + await incompleteFilesetIDs.LastAsync(token).ConfigureAwait(false) } .ToAsyncEnumerable(); var volumeNames = new List(); await foreach (var filesetID in incompleteFilesetIDs.ConfigureAwait(false)) volumeNames.Add(( - await GetRemoteVolumeFromFilesetID(filesetID) + await GetRemoteVolumeFromFilesetID(filesetID, token) .ConfigureAwait(false) ).Name); @@ -1687,11 +1716,12 @@ namespace Duplicati.Library.Main.Database /// If it does, an exception is thrown; otherwise, the volume is removed. /// /// The name of the remote volume to delete. + /// The cancellation token to cancel the operation. /// A task that completes when the remote volume is safely deleted. /// Thrown if the volume has associated blocks. - public async Task SafeDeleteRemoteVolume(string name) + public async Task SafeDeleteRemoteVolume(string name, CancellationToken token) { - var volumeid = await GetRemoteVolumeID(name).ConfigureAwait(false); + var volumeid = await GetRemoteVolumeID(name, token).ConfigureAwait(false); await using var cmd = m_connection.CreateCommand(m_rtr); var c = await cmd.SetCommandAndParameters(@" @@ -1700,23 +1730,24 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", volumeid) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (c != 0) throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); - await RemoveRemoteVolume(name).ConfigureAwait(false); + await RemoveRemoteVolume(name, token).ConfigureAwait(false); } /// /// Retrieves the hashes of blocks that are on the blocklist for a given volume. /// /// The name of the volume to check. + /// The cancellation token to cancel the operation. /// A task that when awaited contains an array of blocklist hashes. - public async Task GetBlocklistHashes(string name) + public async Task GetBlocklistHashes(string name, CancellationToken token) { - var volumeid = GetRemoteVolumeID(name); + var volumeid = GetRemoteVolumeID(name, token); await using var cmd = m_connection.CreateCommand(m_rtr); // Grab the strings and return as array to avoid concurrent access to the IEnumerable cmd.SetCommandAndParameters(@" @@ -1731,17 +1762,18 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@VolumeId", volumeid); - return await cmd.ExecuteReaderEnumerableAsync() + return await cmd.ExecuteReaderEnumerableAsync(token) .Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash")) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: token) .ConfigureAwait(false); } /// /// Retrieves the first path in the database, ordered by length in descending order. /// + /// The cancellation token to cancel the operation. /// A task that when awaited contains the first path, or null if no paths exist. - public async Task GetFirstPath() + public async Task GetFirstPath(CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); var v0 = await cmd.ExecuteScalarAsync(@" @@ -1749,7 +1781,7 @@ namespace Duplicati.Library.Main.Database FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1 - ") + ", token) .ConfigureAwait(false); if (v0 == null || v0 == DBNull.Value) @@ -1763,7 +1795,7 @@ namespace Duplicati.Library.Main.Database /// /// The Fileset-ID. /// An asynchronous enumerable of USN journal data entries. - public async IAsyncEnumerable GetChangeJournalData(long fileSetId) + public async IAsyncEnumerable GetChangeJournalData(long fileSetId, [EnumeratorCancellation] CancellationToken token) { var data = new List(); @@ -1779,8 +1811,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { yield return new Interface.USNJournalDataEntry { @@ -1796,9 +1828,10 @@ namespace Duplicati.Library.Main.Database /// Adds NTFS change journal data for file set and volume. /// /// Data to add. + /// The cancellation token to cancel the operation. /// A task that completes when the data is added. /// Thrown if unable to add change journal entry. - public async Task CreateChangeJournalData(IEnumerable data) + public async Task CreateChangeJournalData(IEnumerable data, CancellationToken token) { foreach (var entry in data) { @@ -1824,14 +1857,14 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@JournalId", entry.JournalId) .SetParameterValue("@NextUsn", entry.NextUsn) .SetParameterValue("@ConfigHash", entry.ConfigHash) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) throw new Exception("Unable to add change journal entry"); } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// @@ -1839,8 +1872,9 @@ namespace Duplicati.Library.Main.Database /// /// Data to add. /// Existing file set to update. + /// The cancellation token to cancel the operation. /// A task that completes when the data is added. - public async Task UpdateChangeJournalData(IEnumerable data, long fileSetId) + public async Task UpdateChangeJournalData(IEnumerable data, long fileSetId, CancellationToken token) { foreach (var entry in data) { @@ -1858,24 +1892,25 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", fileSetId) .SetParameterValue("@VolumeName", entry.Volume) .SetParameterValue("@JournalId", entry.JournalId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// /// Checks if a blocklist hash is known. /// /// The hash to check. + /// The cancellation token to cancel the operation. /// A task that when awaited returns true if the hash is known, false otherwise. - public async Task IsBlocklistHashKnown(string hash) + public async Task IsBlocklistHashKnown(string hash, CancellationToken token) { var res = await m_getfirstfilesetwithblockinblockset .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) - .ExecuteScalarInt64Async() + .ExecuteScalarInt64Async(token) .ConfigureAwait(false); if (res != -1 && res != m_filesetId) From ccce18bb00f4df02ab1c459bb32a73774cc3aa8b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 16:02:05 +0200 Subject: [PATCH 325/362] Made the LocalBugReportDatabase methods accept a cancellation token --- .../Main/Database/LocalBugReportDatabase.cs | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index d1610487e..7cf791c08 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -22,6 +22,7 @@ #nullable enable using System; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Common.IO; @@ -44,13 +45,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalBugReportDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalBugReportDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalBugReportDatabase(); dbnew = (LocalBugReportDatabase) - await CreateLocalDatabaseAsync(path, "BugReportCreate", false, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, "BugReportCreate", false, pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -60,8 +62,9 @@ namespace Duplicati.Library.Main.Database /// /// Obfuscates sensitive data in the database, readying it for a bug report. /// + /// A cancellation token to cancel the operation. /// A task that completes when the obfuscation is finished. - public async Task Fix() + public async Task Fix(CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -76,7 +79,7 @@ namespace Duplicati.Library.Main.Database ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL ) - ") + ", token) .ConfigureAwait(false); await upcmd.ExecuteNonQueryAsync($@" @@ -84,7 +87,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" - ") + ", token) .ConfigureAwait(false); await upcmd.SetCommandAndParameters($@" @@ -105,7 +108,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\") .SetParameterValue("@DirSep", Util.DirectorySeparatorString) .SetParameterValue("@FileExt", ".bin") - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); /*long id = 1; @@ -127,7 +130,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" @@ -136,14 +139,14 @@ namespace Duplicati.Library.Main.Database WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -157,24 +160,24 @@ namespace Duplicati.Library.Main.Database ""{tablename}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" - ") + ", token) .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" ") + await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" ", token) .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" ") + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" ", token) .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" ") + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" ", token) .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ", token) .ConfigureAwait(false); using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); cmd.SetTransaction(m_rtr); - await cmd.ExecuteNonQueryAsync("VACUUM").ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync("VACUUM", token).ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } } From c496d0bf8c506851ef70322d086e5d5ee388d0ce Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 16:07:46 +0200 Subject: [PATCH 326/362] Made the LocalDeleteDatabase methods accept a cancellation token --- .../Main/Database/LocalDeleteDatabase.cs | 131 ++++++++++-------- 1 file changed, 72 insertions(+), 59 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 83badd292..5109398fd 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -24,6 +24,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; using Microsoft.Data.Sqlite; @@ -74,17 +76,18 @@ namespace Duplicati.Library.Main.Database /// The operation name. /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, string operation, long pagecachesize, LocalDeleteDatabase? dbnew = null) + public static async Task CreateAsync(string path, string operation, long pagecachesize, LocalDeleteDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalDeleteDatabase(); dbnew = (LocalDeleteDatabase) - await CreateLocalDatabaseAsync(path, operation, true, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, operation, true, pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.m_registerDuplicateBlockCommand = - await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND) + await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND, token) .ConfigureAwait(false); return dbnew; @@ -95,17 +98,18 @@ namespace Duplicati.Library.Main.Database /// /// The parent database to use. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(LocalDatabase dbparent, LocalDeleteDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase dbparent, LocalDeleteDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalDeleteDatabase(); dbnew = (LocalDeleteDatabase) - await CreateLocalDatabaseAsync(dbparent, dbnew) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) .ConfigureAwait(false); dbnew.m_registerDuplicateBlockCommand = - await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND) + await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND, token) .ConfigureAwait(false); return dbnew; @@ -115,22 +119,23 @@ namespace Duplicati.Library.Main.Database /// Drops all entries related to operations listed in the table. /// /// The fileset entries to delete. + /// A cancellation token to cancel the operation. /// An async enumerable of key-value pairs, where the key is the fileset name and the value is the size of the fileset. - public async IAsyncEnumerable> DropFilesetsFromTable(DateTime[] toDelete) + public async IAsyncEnumerable> DropFilesetsFromTable(DateTime[] toDelete, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); var deleted = 0; - await using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds)).ConfigureAwait(false)) + await using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds), token).ConfigureAwait(false)) deleted += await ( await cmd.SetCommandAndParameters(@" DELETE FROM ""Fileset"" WHERE ""Timestamp"" IN (@Timestamps) ") - .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable) + .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable, token) .ConfigureAwait(false) ) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (deleted != toDelete.Length) @@ -143,7 +148,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""Fileset"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" @@ -152,7 +157,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""Fileset"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" @@ -161,7 +166,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" @@ -170,7 +175,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""MetadataID"" FROM ""FileLookup"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" @@ -182,7 +187,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" @@ -191,7 +196,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""Blockset"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" @@ -200,7 +205,7 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""ID"" FROM ""Blockset"" ) - ") + ", token) .ConfigureAwait(false); //We save the block info for the remote files, before we delete it @@ -225,7 +230,7 @@ namespace Duplicati.Library.Main.Database ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync(@" @@ -240,7 +245,7 @@ namespace Duplicati.Library.Main.Database ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ) - ") + ", token) .ConfigureAwait(false); //Find all remote filesets that are no longer required, and mark them as deleting @@ -263,7 +268,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Deleting.ToString() ]) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (deleted != updated) @@ -281,8 +286,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) .SetParameterValue("@State", RemoteVolumeState.Deleting.ToString()); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1) @@ -294,8 +299,9 @@ namespace Duplicati.Library.Main.Database /// exposed to the user. This is in contrast to other cases where the Version is the ID in the /// Fileset table. /// + /// A cancellation token to cancel the operation. /// An async enumerable of IListResultFileset. - internal async IAsyncEnumerable FilesetsWithBackupVersion() + internal async IAsyncEnumerable FilesetsWithBackupVersion([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); // TODO check if this is still the case? (shouldn't be with new sqlite driver): @@ -310,9 +316,9 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC "); - await using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + await using var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); int version = 0; - while (await reader.ReadAsync().ConfigureAwait(false)) + while (await reader.ReadAsync(token).ConfigureAwait(false)) { yield return new ListResultFileset( version++, @@ -367,12 +373,13 @@ namespace Duplicati.Library.Main.Database /// and the number of bytes no longer needed in each volume. /// The sizes are the uncompressed values. /// + /// A cancellation token to cancel the operation. /// /// An asynchronous enumerable of , holding the name of the volume, /// the size of the data stored in the volume, the size of the data that is no longer needed in the volume, /// and the size of the data that is compressed in the volume. /// - private async IAsyncEnumerable GetWastedSpaceReport() + private async IAsyncEnumerable GetWastedSpaceReport([EnumeratorCancellation] CancellationToken token) { var tmptablename = "UsageReport-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -507,7 +514,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString() ]) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); cmd.SetCommandAndParameters($@" @@ -523,8 +530,8 @@ namespace Duplicati.Library.Main.Database ORDER BY ""B"".""Sorttime"" ASC "); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new VolumeUsage( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), @@ -537,7 +544,7 @@ namespace Duplicati.Library.Main.Database try { await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"" ") + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"" ", token) .ConfigureAwait(false); } catch { } @@ -729,16 +736,17 @@ namespace Duplicati.Library.Main.Database /// The threshold for wasted space percentage to trigger compaction. /// The size of small files to trigger compaction. /// The maximum number of small files to trigger compaction. + /// A cancellation token to cancel the operation. /// A task that when awaited contains an instance of . - public async Task GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount) + public async Task GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, CancellationToken token) { return new CompactReport( volsize, wastethreshold, smallfilesize, maxsmallfilecount, - await GetWastedSpaceReport() - .ToListAsync() + await GetWastedSpaceReport(token) + .ToListAsync(cancellationToken: token) .ConfigureAwait(false) ); } @@ -755,9 +763,9 @@ namespace Duplicati.Library.Main.Database /// The hash of the block. /// The size of the block. /// The volume ID to check, or -1 to check all volumes. - /// The transaction to execute the command in. + /// A cancellation token to cancel the operation. /// A task that when awaited returns true if the block is in use, false otherwise. - Task UseBlock(string hash, long size, long volumeId); + Task UseBlock(string hash, long size, long volumeId, CancellationToken token); } /// @@ -791,8 +799,9 @@ namespace Duplicati.Library.Main.Database /// Creates a new instance of the class. /// /// The local database to use for the queries. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a new instance of . - public static async Task CreateAsync(LocalDatabase db) + public static async Task CreateAsync(LocalDatabase db, CancellationToken token) { return new BlockQuery { @@ -803,19 +812,19 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - ") + ", token) .ConfigureAwait(false) }; } /// - public async Task UseBlock(string hash, long size, long volumeId) + public async Task UseBlock(string hash, long size, long volumeId, CancellationToken token) { var r = await m_command .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (r == -1) @@ -844,9 +853,10 @@ namespace Duplicati.Library.Main.Database /// /// Builds a lookup table to enable faster response to block queries. /// - public async Task CreateBlockQueryHelper() + /// A cancellation token to cancel the operation. + public async Task CreateBlockQueryHelper(CancellationToken token) { - return await BlockQuery.CreateAsync(this).ConfigureAwait(false); + return await BlockQuery.CreateAsync(this, token).ConfigureAwait(false); } /// @@ -855,8 +865,9 @@ namespace Duplicati.Library.Main.Database /// The hash of the block. /// The size of the block. /// The new volume ID. + /// A cancellation token to cancel the operation. /// A task that when completed indicates the block has been registered. - public async Task RegisterDuplicatedBlock(string hash, long size, long volumeID) + public async Task RegisterDuplicatedBlock(string hash, long size, long volumeID, CancellationToken token) { // Using INSERT OR IGNORE to avoid duplicate entries, result may be 1 or 0 await m_registerDuplicateBlockCommand @@ -864,7 +875,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@VolumeId", volumeID) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -873,10 +884,11 @@ namespace Duplicati.Library.Main.Database /// /// The file to remove. /// The volume IDs that will be removed. + /// A cancellation token to cancel the operation. /// A task that when completed indicates the operation has finished. - public async Task PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved) + public async Task PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved, CancellationToken token) { - var deletedVolume = await GetRemoteVolume(filename) + var deletedVolume = await GetRemoteVolume(filename, token) .ConfigureAwait(false); if (deletedVolume.Type != RemoteVolumeType.Blocks) return; @@ -893,10 +905,10 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); - await using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved).ConfigureAwait(false)) + await using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved, token).ConfigureAwait(false)) await ( await cmd.SetCommandAndParameters($@" CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS @@ -912,16 +924,16 @@ namespace Duplicati.Library.Main.Database ) GROUP BY ""BlockID"" ") - .ExpandInClauseParameterMssqliteAsync("@VolumeIds", tempTable) + .ExpandInClauseParameterMssqliteAsync("@VolumeIds", tempTable, token) .ConfigureAwait(false) ) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); var targetCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{updatedBlocks}"" - ") + ", token) .ConfigureAwait(false); if (targetCount == 0) @@ -930,7 +942,7 @@ namespace Duplicati.Library.Main.Database var replacementCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{replacementBlocks}"" - ") + ", token) .ConfigureAwait(false); var updateCount = await cmd.SetCommandAndParameters(@$" @@ -945,7 +957,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Block"".""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); var deleteCount = await cmd.ExecuteNonQueryAsync(@$" @@ -962,7 +974,7 @@ namespace Duplicati.Library.Main.Database || ""RB"".""VolumeID"" FROM ""{replacementBlocks}"" ""RB"" ) - ") + ", token) .ConfigureAwait(false); if (targetCount != updateCount @@ -978,20 +990,20 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" = @VolumeId ") .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } finally { try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ", token) .ConfigureAwait(false); } catch { } try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ", token) .ConfigureAwait(false); } catch { } @@ -1002,8 +1014,9 @@ namespace Duplicati.Library.Main.Database /// Calculates the sequence in which files should be deleted based on their relations. /// /// Block volumes slated for deletion. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of that represents the order in which volumes should be deleted. - public async IAsyncEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes) + public async IAsyncEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); // Although the generated index volumes are always in pairs, @@ -1029,8 +1042,8 @@ namespace Duplicati.Library.Main.Database AND ""B"".""Size"" IS NOT NULL "); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var name = rd.ConvertValueToString(0) ?? ""; if (!lookupBlock.TryGetValue(name, out var indexfileList)) From 936742b58626393e1f37b5cb48f8371474ffcb39 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 16:11:10 +0200 Subject: [PATCH 327/362] Made the LocalListAffectedDatabase methods accept a cancellation token --- .../Database/LocalListAffectedDatabase.cs | 68 +++++++++++++------ 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 97f5d149c..18f70f1ce 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -25,6 +25,8 @@ using System; using System.Linq; using System.Collections.Generic; using System.Threading.Tasks; +using System.Threading; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Main.Database { @@ -40,13 +42,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The size of the page cache in bytes. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalListAffectedDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalListAffectedDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListAffectedDatabase(); dbnew = (LocalListAffectedDatabase) - await CreateLocalDatabaseAsync(path, "ListAffected", false, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, "ListAffected", false, pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -126,11 +129,12 @@ namespace Duplicati.Library.Main.Database /// Retrieves the fileset times from the database. /// /// The items to filter the filesets by. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of fileset times. - public async IAsyncEnumerable GetFilesets(IEnumerable items) + public async IAsyncEnumerable GetFilesets(IEnumerable items, [EnumeratorCancellation] CancellationToken token) { - var filesets = await FilesetTimes() - .ToArrayAsync() + var filesets = await FilesetTimes(token) + .ToArrayAsync(cancellationToken: token) .ConfigureAwait(false); var dict = new Dictionary(); @@ -170,10 +174,18 @@ namespace Duplicati.Library.Main.Database ) "; - await using var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false); - await using var cmd = await m_connection.CreateCommand(sql).SetTransaction(m_rtr).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var tmptable = await TemporaryDbValueList + .CreateAsync(this, items, token) + .ConfigureAwait(false); + await using var cmd = await m_connection + .CreateCommand(sql) + .SetTransaction(m_rtr) + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token) + .ConfigureAwait(false); + await using var rd = await cmd + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var v = dict[rd.ConvertValueToInt64(0)]; yield return new ListResultFileset() @@ -188,8 +200,9 @@ namespace Duplicati.Library.Main.Database /// Retrieves the list of files from the database. /// /// The items to filter the files by. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of file results. - public async IAsyncEnumerable GetFiles(IEnumerable items) + public async IAsyncEnumerable GetFiles(IEnumerable items, [EnumeratorCancellation] CancellationToken token) { var sql = $@" SELECT DISTINCT ""Path"" @@ -249,10 +262,17 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Path"" "; - await using var tmptable = await TemporaryDbValueList.CreateAsync(this, items).ConfigureAwait(false); - await using var cmd = await m_connection.CreateCommand(sql).ExpandInClauseParameterMssqliteAsync("@Names", tmptable).ConfigureAwait(false); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var tmptable = await TemporaryDbValueList + .CreateAsync(this, items, token) + .ConfigureAwait(false); + await using var cmd = await m_connection + .CreateCommand(sql) + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token) + .ConfigureAwait(false); + await using var rd = await cmd + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new ListResultFile() { Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"), @@ -265,8 +285,9 @@ namespace Duplicati.Library.Main.Database /// The log lines are retrieved from both the LogData and RemoteOperation tables. /// /// The items to filter the log lines by. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of log line results. - public async IAsyncEnumerable GetLogLines(IEnumerable items) + public async IAsyncEnumerable GetLogLines(IEnumerable items, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(); @@ -299,8 +320,8 @@ namespace Duplicati.Library.Main.Database foreach ((var x, var i) in items.Select((x, i) => (x, i))) cmd.SetParameterValue($"@Message{i}", "%" + x + "%"); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new ListResultRemoteLog() { Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)), @@ -314,8 +335,9 @@ namespace Duplicati.Library.Main.Database /// This method queries the database for remote volumes associated with filesets and metadata datasets. /// /// The names of the remote volumes to filter by. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of remote volume results. - public async IAsyncEnumerable GetVolumes(IEnumerable items) + public async IAsyncEnumerable GetVolumes(IEnumerable items, [EnumeratorCancellation] CancellationToken token) { var sql = $@" SELECT DISTINCT ""Name"" @@ -382,9 +404,13 @@ namespace Duplicati.Library.Main.Database ) "; - await using var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameterMssqlite("@Names", items.ToArray()); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var cmd = m_connection + .CreateCommand(sql) + .ExpandInClauseParameterMssqlite("@Names", items.ToArray()); + await using var rd = await cmd + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new ListResultRemoteVolume() { Name = rd.ConvertValueToString(0) ?? "" From 42877ebf968ac2bfcc3b0160df0abc6163786b06 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 16:23:10 +0200 Subject: [PATCH 328/362] Made the LocalListBrokenFilesDatabase methods accept a cancellation token --- .../Database/LocalListBrokenFilesDatabase.cs | 76 +++++++++++-------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index d3e59fae9..ad7f494b3 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -24,6 +24,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Main.Database @@ -165,13 +167,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalListBrokenFilesDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalListBrokenFilesDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListBrokenFilesDatabase(); dbnew = (LocalListBrokenFilesDatabase) - await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -183,13 +186,14 @@ namespace Duplicati.Library.Main.Database /// /// The parent database to use for the new database. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(LocalDatabase dbparent, LocalListBrokenFilesDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase dbparent, LocalListBrokenFilesDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListBrokenFilesDatabase(); dbnew = (LocalListBrokenFilesDatabase) - await CreateLocalDatabaseAsync(dbparent, dbnew) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = false; @@ -201,11 +205,12 @@ namespace Duplicati.Library.Main.Database /// /// The time to filter filesets by. /// Optional array of versions to filter filesets by. If null, all versions are considered. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of broken file IDs. - public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[]? versions) + public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[]? versions, [EnumeratorCancellation] CancellationToken token) { var query = BROKEN_FILE_SETS; - var clause = await GetFilelistWhereClause(time, versions) + var clause = await GetFilelistWhereClause(time, versions, null, false, token) .ConfigureAwait(false); if (!string.IsNullOrWhiteSpace(clause.Query)) @@ -223,7 +228,7 @@ namespace Duplicati.Library.Main.Database .SetCommandAndParameters(query) .SetParameterValues(clause.Values); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) if (!rd.IsDBNull(0)) yield return ( ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), @@ -236,14 +241,15 @@ namespace Duplicati.Library.Main.Database /// Returns all broken file IDs, i.e., files that reference blocksets or blocks that are not available. /// /// The fileset ID to filter by. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of broken file IDs. - public async IAsyncEnumerable> GetBrokenFilenames(long filesetid) + public async IAsyncEnumerable> GetBrokenFilenames(long filesetid, [EnumeratorCancellation] CancellationToken token) { await using var cmd = Connection.CreateCommand(m_rtr) .SetCommandAndParameters(BROKEN_FILE_NAMES) .SetParameterValue("@FilesetId", filesetid); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) if (!rd.IsDBNull(0)) yield return new Tuple( rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), @@ -254,8 +260,9 @@ namespace Duplicati.Library.Main.Database /// /// Returns all index files that are orphaned, i.e., not referenced by any block files. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of representing the orphaned index files. - public async IAsyncEnumerable GetOrphanedIndexFiles() + public async IAsyncEnumerable GetOrphanedIndexFiles([EnumeratorCancellation] CancellationToken token) { await using var cmd = Connection.CreateCommand($@" SELECT @@ -272,7 +279,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), @@ -286,14 +293,15 @@ namespace Duplicati.Library.Main.Database /// The filset id for the current operation. /// The name of the table to insert into. /// The name of the ID field in the table. + /// A cancellation token to cancel the operation. /// A task that completes when the insertion is finished. - public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname) + public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, CancellationToken token) { await using var cmd = Connection.CreateCommand(m_rtr) .SetCommandAndParameters(INSERT_BROKEN_IDS(tablename, IDfieldname)) .SetParameterValue("@FilesetId", filesetid); - await cmd.ExecuteNonQueryAsync() + await cmd.ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -304,8 +312,9 @@ namespace Duplicati.Library.Main.Database /// The volume ids to ignore when searching for a suitable metadata block. /// The hash of the empty blockset. /// The size of the empty blockset. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the ID of the empty metadata blockset, or -1 if no suitable blockset is found - public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize) + public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, CancellationToken token) { await using var cmd = Connection.CreateCommand(@" SELECT ""ID"" @@ -328,7 +337,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@EmptyHash", emptyHash) .SetParameterValue("@EmptyHashSize", emptyHashSize); - var res = await cmd.ExecuteScalarInt64Async(-1) + var res = await cmd.ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); // No empty block found, try to find a zero-length block instead @@ -350,7 +359,7 @@ namespace Duplicati.Library.Main.Database ") .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) .SetParameterValue("@EmptyHashSize", 0) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); // No empty block found, pick the smallest one @@ -371,7 +380,7 @@ namespace Duplicati.Library.Main.Database LIMIT 1 ") .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); return res; @@ -383,8 +392,9 @@ namespace Duplicati.Library.Main.Database /// /// The filesetId to target. /// The empty blockset ID to replace with. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the number of rows affected - public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId) + public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId, CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" UPDATE ""Metadataset"" @@ -411,15 +421,16 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@EmptyBlocksetID", emptyBlocksetId) .SetParameterValue("@FilesetID", filesetId); - return await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); + return await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); } /// /// Removes all blocks, blocksets, and index links that are missing from the specified volumes. /// /// The names of the volumes to check for missing blocks. + /// A cancellation token to cancel the operation. /// A task that completes when the removal is finished. - public async Task RemoveMissingBlocks(IEnumerable names) + public async Task RemoveMissingBlocks(IEnumerable names, CancellationToken token) { if (names == null || !names.Any()) return; @@ -432,10 +443,10 @@ namespace Duplicati.Library.Main.Database CREATE TEMP TABLE ""{volidstable}"" ( ""ID"" INTEGER PRIMARY KEY ) - ") + ", token) .ConfigureAwait(false); - await using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names).ConfigureAwait(false)) + await using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names, token).ConfigureAwait(false)) await ( await deletecmd.SetCommandAndParameters($@" INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") @@ -443,10 +454,10 @@ namespace Duplicati.Library.Main.Database FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names) ") - .ExpandInClauseParameterMssqliteAsync("@Names", tmptable) + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token) .ConfigureAwait(false) ) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); var volIdsSubQuery = $@" @@ -459,39 +470,39 @@ namespace Duplicati.Library.Main.Database WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); await deletecmd.ExecuteNonQueryAsync($@" DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery}) - ") + ", token) .ConfigureAwait(false); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // SQLite.SQLiteException (0x80004005): database table is locked await deletecmd - .ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ") + .ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ", token) .ConfigureAwait(false); try { deletecmd.CommandTimeout = 2; await deletecmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ") + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ", token) .ConfigureAwait(false); } catch { /* Ignore, will be deleted on close anyway. */ } @@ -501,8 +512,9 @@ namespace Duplicati.Library.Main.Database /// Gets the count of files in a specific fileset. /// /// The ID of the fileset to count files in. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the count of files in the specified fileset. - public async Task GetFilesetFileCount(long filesetid) + public async Task GetFilesetFileCount(long filesetid, CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT COUNT(*) @@ -513,7 +525,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", filesetid); return await cmd - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } } From 28ca30f6170c09c4c7a18d5a335b2fb88b866202 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 18 Jun 2025 16:28:23 +0200 Subject: [PATCH 329/362] Made the LocalListChangesDatabase methods accept a cancellation token --- .../Main/Database/LocalListChangesDatabase.cs | 110 ++++++++++-------- 1 file changed, 63 insertions(+), 47 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index bfb12d220..404db019d 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -23,6 +23,8 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -41,13 +43,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalListChangesDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalListChangesDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListChangesDatabase(); dbnew = (LocalListChangesDatabase) - await CreateLocalDatabaseAsync(path, "ListChanges", false, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, "ListChanges", false, pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -68,8 +71,9 @@ namespace Duplicati.Library.Main.Database /// The size of the file or folder. /// The type of the element (file, folder, symlink). /// If true, adds to the current table; otherwise, adds to the previous table. + /// A cancellation token to cancel the operation. /// A task that completes when the element is added. - Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew); + Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew, CancellationToken token); /// /// Adds elements from the database to the temporary storage. @@ -77,26 +81,30 @@ namespace Duplicati.Library.Main.Database /// The ID of the fileset to add. /// If true, adds to the current table; otherwise, adds to the previous table. /// An optional filter to apply when adding elements. + /// A cancellation token to cancel the operation. /// A task that completes when the elements are added. - Task AddFromDb(long filesetId, bool asNew, IFilter filter); + Task AddFromDb(long filesetId, bool asNew, IFilter filter, CancellationToken token); /// /// Creates a report containing the count of added, deleted, and modified elements. /// + /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an with the change counts. - Task CreateChangeCountReport(); + Task CreateChangeCountReport(CancellationToken token); /// /// Creates a report containing the size information for added, deleted, previous, and current elements. /// + /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an with the size details. - Task CreateChangeSizeReport(); + Task CreateChangeSizeReport(CancellationToken token); /// /// Asynchronously generates a report of changed files, yielding tuples that describe the change type, element type, and file path. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of tuples containing the change type, element type, and file path. - IAsyncEnumerable> CreateChangedFileReport(); + IAsyncEnumerable> CreateChangedFileReport(CancellationToken token); } /// @@ -245,8 +253,9 @@ namespace Duplicati.Library.Main.Database /// Asynchronously creates a new instance of the class. /// /// The local database instance to use. + /// A cancellation token to cancel the operation. /// A task that, when awaited, returns a new instance of . - public static async Task CreateAsync(LocalDatabase db) + public static async Task CreateAsync(LocalDatabase db, CancellationToken token) { var sh = new StorageHelper { @@ -265,7 +274,7 @@ namespace Duplicati.Library.Main.Database ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -276,7 +285,7 @@ namespace Duplicati.Library.Main.Database ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); } @@ -296,7 +305,7 @@ namespace Duplicati.Library.Main.Database @Size, @Type ) - ") + ", token) .ConfigureAwait(false); sh.m_insertCurrentElementCommand = await sh.m_db.Connection @@ -315,13 +324,14 @@ namespace Duplicati.Library.Main.Database @Size, @Type ) - ") + ", token) .ConfigureAwait(false); return sh; } - public async Task AddFromDb(long filesetId, bool asNew, IFilter filter) + /// + public async Task AddFromDb(long filesetId, bool asNew, IFilter filter, CancellationToken token) { var tablename = asNew ? m_currentTable : m_previousTable; @@ -410,7 +420,7 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) @@ -424,20 +434,20 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{filenamestable}"" ( ""Path"" TEXT NOT NULL ) - ") + ", token) .ConfigureAwait(false); await cmd.SetCommandAndParameters($@" INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path) ") - .PrepareAsync() + .PrepareAsync(token) .ConfigureAwait(false); foreach (var s in p) await cmd .SetParameterValue("@Path", s) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); string whereClause; @@ -481,11 +491,11 @@ namespace Duplicati.Library.Main.Database WHERE {whereClause} ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ") + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ", token) .ConfigureAwait(false); } else @@ -503,7 +513,7 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) - .PrepareAsync() + .PrepareAsync(token) .ConfigureAwait(false); await using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) @@ -523,13 +533,13 @@ namespace Duplicati.Library.Main.Database @Type ) "); - await cmd2.PrepareAsync().ConfigureAwait(false); + await cmd2.PrepareAsync(token).ConfigureAwait(false); await using var rd = await cmd - .ExecuteReaderAsync() + .ExecuteReaderAsync(token) .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { rd.GetValues(values); var path = values[0] as string; @@ -541,14 +551,15 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@MetaHash", values[2]) .SetParameterValue("@Size", values[3]) .SetParameterValue("@Type", values[4]) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } } } - public async Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew) + /// + public async Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew, CancellationToken token) { var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; await cmd @@ -557,7 +568,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@MetaHash", metahash) .SetParameterValue("@Size", size) .SetParameterValue("@Type", (int)type) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -565,11 +576,12 @@ namespace Duplicati.Library.Main.Database /// Converts a SqliteDataReader to an asynchronous enumerable of strings. /// /// The SqliteDataReader to read from. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of strings, where each string is a value from the first column of the reader. - private static async IAsyncEnumerable ReaderToStringList(SqliteDataReader rd) + private static async IAsyncEnumerable ReaderToStringList(SqliteDataReader rd, [EnumeratorCancellation] CancellationToken token) { await using (rd) - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var v = rd.GetValue(0); if (v == null || v == DBNull.Value) @@ -627,8 +639,9 @@ namespace Duplicati.Library.Main.Database /// /// Creates a report of changes in file sizes, including added, deleted, previous, and current sizes. /// + /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an with the size details. - public async Task CreateChangeSizeReport() + public async Task CreateChangeSizeReport(CancellationToken token) { var (Added, Deleted, Modified) = GetSqls(true); @@ -638,27 +651,27 @@ namespace Duplicati.Library.Main.Database PreviousSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_previousTable}"" - ", 0) + ", 0, token) .ConfigureAwait(false), CurrentSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_currentTable}"" - ", 0) + ", 0, token) .ConfigureAwait(false), AddedSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({Added}) - ", 0) + ", 0, token) .ConfigureAwait(false), DeletedSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({Deleted}) - ", 0) + ", 0, token) .ConfigureAwait(false) }; @@ -668,8 +681,9 @@ namespace Duplicati.Library.Main.Database /// /// Creates a report containing the count of added, deleted, and modified elements. /// + /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an with the change counts. - public async Task CreateChangeCountReport() + public async Task CreateChangeCountReport(CancellationToken token) { var (Added, Deleted, Modified) = GetSqls(false); @@ -695,47 +709,47 @@ namespace Duplicati.Library.Main.Database AddedFolders = await cmd .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), AddedSymlinks = await cmd .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), AddedFiles = await cmd .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), DeletedFolders = await cmd .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), DeletedSymlinks = await cmd .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), DeletedFiles = await cmd .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), ModifiedFolders = await cmd .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), ModifiedSymlinks = await cmd .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), ModifiedFiles = await cmd .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false) }; @@ -745,8 +759,9 @@ namespace Duplicati.Library.Main.Database /// /// Asynchronously creates a report of changed files, yielding tuples that describe the change type, element type, and file path. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of tuples containing the change type, element type, and file path. - public async IAsyncEnumerable> CreateChangedFileReport() + public async IAsyncEnumerable> CreateChangedFileReport([EnumeratorCancellation] CancellationToken token) { var (Added, Deleted, Modified) = GetSqls(false); @@ -762,7 +777,7 @@ namespace Duplicati.Library.Main.Database { cmd.SetCommandAndParameters(sql); foreach (var type in elTypes) - await foreach (var s in ReaderToStringList(await cmd.SetParameterValue("@Type", (int)type).ExecuteReaderAsync().ConfigureAwait(false)).ConfigureAwait(false)) + await foreach (var s in ReaderToStringList(await cmd.SetParameterValue("@Type", (int)type).ExecuteReaderAsync(token).ConfigureAwait(false), token).ConfigureAwait(false)) yield return new Tuple(changeType, type, s ?? ""); } @@ -835,10 +850,11 @@ namespace Duplicati.Library.Main.Database /// /// Creates a new instance of the for managing temporary storage of file changes. /// + /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an instance of . - public async Task CreateStorageHelper() + public async Task CreateStorageHelper(CancellationToken token) { - return await StorageHelper.CreateAsync(this).ConfigureAwait(false); + return await StorageHelper.CreateAsync(this, token).ConfigureAwait(false); } } } From 3ea8733a4d8b29197f48ba00e0a4a37d76b34527 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 06:54:24 +0200 Subject: [PATCH 330/362] Made the LocalListDatabase methods accept a cancellation token --- .../Main/Database/LocalListDatabase.cs | 212 +++++++++++------- 1 file changed, 127 insertions(+), 85 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index d9fd7fa30..ade62473d 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -29,7 +29,8 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Utility; using Duplicati.Library.Interface; using Microsoft.Data.Sqlite; -using Google.Protobuf.WellKnownTypes; +using System.Threading; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Main.Database { @@ -44,13 +45,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of LocalListDatabase. - public static async Task CreateAsync(string path, long pagecachesize, LocalListDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalListDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListDatabase(); dbnew = (LocalListDatabase) - await CreateLocalDatabaseAsync(path, "List", false, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, "List", false, pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -107,24 +109,42 @@ namespace Duplicati.Library.Main.Database /// /// Gets all of the filesets in the database as an asynchronous enumerable, including file count and total size. /// - IAsyncEnumerable Sets(); + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of filesets. + IAsyncEnumerable Sets(CancellationToken token); /// /// Gets all of the filesets in the database as an asynchronous enumerable, omitting file count and total size. /// - IAsyncEnumerable QuickSets(); + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of filesets. + IAsyncEnumerable QuickSets(CancellationToken token); /// /// Gets the files in the filesets that match the provided filter as an asynchronous enumerable. /// - IAsyncEnumerable SelectFiles(IFilter filter); + /// The filter to apply to the files. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file versions representing the selected files. + IAsyncEnumerable SelectFiles(IFilter filter, CancellationToken token); /// /// Gets the largest prefix of files in the filesets that match the provided filter as an asynchronous enumerable. /// - IAsyncEnumerable GetLargestPrefix(IFilter filter); + /// The filter to apply to the files. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file versions representing the largest prefix. + IAsyncEnumerable GetLargestPrefix(IFilter filter, CancellationToken token); /// /// Selects the contents of a folder in the filesets that match the provided filter as an asynchronous enumerable. /// - IAsyncEnumerable SelectFolderContents(IFilter filter); - Task TakeFirst(); + /// The filter to apply to the folder contents. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file versions representing the folder contents. + IAsyncEnumerable SelectFolderContents(IFilter filter, CancellationToken token); + /// + /// Deletes all filesets except the most recent one. + /// + /// A cancellation token to cancel the operation. + /// A task that completes when the filesets have been deleted. + Task TakeFirst(CancellationToken token); } /// @@ -165,19 +185,20 @@ namespace Duplicati.Library.Main.Database /// The owner database instance. /// The time to filter filesets by. /// An array of versions to filter filesets by. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(LocalListDatabase owner, DateTime time, long[] versions) + public static async Task CreateAsync(LocalListDatabase owner, DateTime time, long[] versions, CancellationToken token) { var fs = new FileSets { m_db = owner, m_filesets = await owner - .FilesetTimes() - .ToArrayAsync() + .FilesetTimes(token) + .ToArrayAsync(cancellationToken: token) .ConfigureAwait(false), m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) }; - var tmp = await owner.GetFilelistWhereClause(time, versions, fs.m_filesets) + var tmp = await owner.GetFilelistWhereClause(time, versions, fs.m_filesets, false, token) .ConfigureAwait(false); var query = tmp.Item1; var args = tmp.Item2; @@ -191,7 +212,7 @@ namespace Duplicati.Library.Main.Database ""IsFullBackup"" AS ""IsFullBackup"", ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query} - ", args) + ", args, token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -200,7 +221,7 @@ namespace Duplicati.Library.Main.Database ""FilesetID"", ""Timestamp"" DESC ) - ") + ", token) .ConfigureAwait(false); } @@ -299,9 +320,10 @@ namespace Duplicati.Library.Main.Database public IAsyncEnumerable Sizes() => AsyncEnumerable.Empty(); } - public IAsyncEnumerable GetLargestPrefix(IFilter filter) + /// + public IAsyncEnumerable GetLargestPrefix(IFilter filter, CancellationToken token) { - return GetLargestPrefix(filter, null); + return GetLargestPrefix(filter, null, token); } /// @@ -309,10 +331,13 @@ namespace Duplicati.Library.Main.Database /// /// The filter to apply to the files. /// The prefix rule to apply, or null if no prefix rule is specified. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of file versions representing the largest prefix. - private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) + private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule, [EnumeratorCancellation] CancellationToken token) { - await using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false); + await using var tmpnames = await FilteredFilenameTable + .CreateFilteredFilenameTableAsync(m_db, filter, token) + .ConfigureAwait(false); await using var cmd = m_db.Connection.CreateCommand(); //First we trim the filelist to exclude filenames not found in any of the filesets await cmd.ExecuteNonQueryAsync($@" @@ -329,7 +354,7 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tablename}"" ) ) - ") + ", token) .ConfigureAwait(false); //If we have a prefix rule, apply it @@ -339,7 +364,7 @@ namespace Duplicati.Library.Main.Database WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule ") .SetParameterValue("@Rule", prefixrule) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); // Then we recursively find the largest prefix @@ -348,7 +373,7 @@ namespace Duplicati.Library.Main.Database FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1 - ") + ", token) .ConfigureAwait(false); var maxpath = ""; @@ -360,7 +385,7 @@ namespace Duplicati.Library.Main.Database var filecount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" - ", 0) + ", 0, token) .ConfigureAwait(false); var foundfiles = -1L; @@ -371,7 +396,7 @@ namespace Duplicati.Library.Main.Database FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix ") - .PrepareAsync() + .PrepareAsync(token) .ConfigureAwait(false); while (filecount != foundfiles && maxpath.Length > 0) @@ -380,7 +405,7 @@ namespace Duplicati.Library.Main.Database foundfiles = await cmd .SetParameterValue("@PrefixLength", mp.Length) .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); if (filecount != foundfiles) @@ -400,7 +425,7 @@ namespace Duplicati.Library.Main.Database var paths = cmd.ExecuteReaderEnumerableAsync($@" SELECT ""Path"" FROM ""{tmpnames.Tablename}"" - ") + ", token) .Select(x => x.ConvertValueToString(0) ?? ""); var roots = paths @@ -418,7 +443,7 @@ namespace Duplicati.Library.Main.Database var result = roots .Concat(rootsUNC) - .Select(x => GetLargestPrefix(filter, x) + .Select(x => GetLargestPrefix(filter, x, token) .FirstAsync()) .Distinct(); @@ -438,8 +463,9 @@ namespace Duplicati.Library.Main.Database /// The SqliteCommand to execute. /// The prefix to filter folder entries by. /// The name of the table to query. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of folder entries. - private async IAsyncEnumerable SelectFolderEntries(SqliteCommand cmd, string prefix, string table) + private static async IAsyncEnumerable SelectFolderEntries(SqliteCommand cmd, string prefix, string table, [EnumeratorCancellation] CancellationToken token) { if (!string.IsNullOrEmpty(prefix)) prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); @@ -449,8 +475,8 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""Path"" FROM ""{table}"" "); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var s = rd.ConvertValueToString(0) ?? ""; if (!s.StartsWith(prefix, StringComparison.Ordinal)) @@ -467,7 +493,8 @@ namespace Duplicati.Library.Main.Database } } - public async IAsyncEnumerable SelectFolderContents(IFilter filter) + /// + public async IAsyncEnumerable SelectFolderContents(IFilter filter, [EnumeratorCancellation] CancellationToken token) { var tbname = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try @@ -487,7 +514,9 @@ namespace Duplicati.Library.Main.Database if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); - await using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true)).ConfigureAwait(false); + await using var tmpnames = await FilteredFilenameTable + .CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true), token) + .ConfigureAwait(false); await using var cmd = m_db.Connection.CreateCommand(); //First we trim the filelist to exclude filenames not found in any of the filesets await cmd.ExecuteNonQueryAsync($@" @@ -504,7 +533,7 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tablename}"" ) ) - ") + ", token) .ConfigureAwait(false); // If we had instr support this would work: @@ -520,7 +549,7 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{tbname}"" ( ""Path"" TEXT NOT NULL ) - ") + ", token) .ConfigureAwait(false); await using (var c2 = m_db.Connection.CreateCommand()) @@ -530,16 +559,16 @@ namespace Duplicati.Library.Main.Database VALUES (@Path) "); - await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct().ConfigureAwait(false)) + await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename, token).Distinct().ConfigureAwait(false)) await c2 .SetParameterValue("@Path", n) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); await c2.ExecuteNonQueryAsync($@" CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"") - ") + ", token) .ConfigureAwait(false); } @@ -594,10 +623,10 @@ namespace Duplicati.Library.Main.Database "; await using var rd = await cmd - .ExecuteReaderAsync(query) + .ExecuteReaderAsync(query, token) .ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) { bool more; do @@ -610,7 +639,7 @@ namespace Duplicati.Library.Main.Database } else { - more = await rd.ReadAsync() + more = await rd.ReadAsync(token) .ConfigureAwait(false); } @@ -622,7 +651,7 @@ namespace Duplicati.Library.Main.Database try { await using var c = m_db.Connection.CreateCommand(); - await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}""") + await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}""", token) .ConfigureAwait(false); } catch @@ -631,9 +660,12 @@ namespace Duplicati.Library.Main.Database } } - public async IAsyncEnumerable SelectFiles(IFilter filter) + /// + public async IAsyncEnumerable SelectFiles(IFilter filter, [EnumeratorCancellation] CancellationToken token) { - await using var tmpnames = await FilteredFilenameTable.CreateFilteredFilenameTableAsync(m_db, filter).ConfigureAwait(false); + await using var tmpnames = await FilteredFilenameTable + .CreateFilteredFilenameTableAsync(m_db, filter, token) + .ConfigureAwait(false); await using var cmd = m_db.Connection.CreateCommand(); //First we trim the filelist to exclude filenames not found in any of the filesets await cmd.ExecuteNonQueryAsync($@" @@ -650,7 +682,7 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tablename}"" ) ) - ") + ", token) .ConfigureAwait(false); //Then we select the matching results @@ -703,9 +735,9 @@ namespace Duplicati.Library.Main.Database AND ""C"".""Path"" = ""D"".""Path"" "; - await using var rd = await cmd.ExecuteReaderAsync(query) + await using var rd = await cmd.ExecuteReaderAsync(query, token) .ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) { bool more; do @@ -717,10 +749,8 @@ namespace Duplicati.Library.Main.Database } } - /// - /// Deletes all filesets except the most recent one. - /// - public async Task TakeFirst() + /// + public async Task TakeFirst(CancellationToken token) { await using var cmd = m_db.Connection.CreateCommand(); await cmd.ExecuteNonQueryAsync($@" @@ -731,11 +761,12 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Timestamp"" DESC LIMIT 1 ) - ") + ", token) .ConfigureAwait(false); } - public async IAsyncEnumerable QuickSets() + /// + public async IAsyncEnumerable QuickSets([EnumeratorCancellation] CancellationToken token) { var dict = new Dictionary(); for (var i = 0; i < m_filesets.Length; i++) @@ -748,8 +779,9 @@ namespace Duplicati.Library.Main.Database ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC - ").ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + ", token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var id = rd.ConvertValueToInt64(0); var backupType = rd.GetInt32(1); @@ -765,7 +797,8 @@ namespace Duplicati.Library.Main.Database } } - public async IAsyncEnumerable Sets() + /// + public async IAsyncEnumerable Sets([EnumeratorCancellation] CancellationToken token) { var dict = new Dictionary(); for (var i = 0; i < m_filesets.Length; i++) @@ -800,10 +833,10 @@ namespace Duplicati.Library.Main.Database LEFT OUTER JOIN ( {summation} ) ""B"" ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC - ") + ", token) .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var id = rd.ConvertValueToInt64(0); var isFullBackup = rd.GetInt32(1); @@ -833,7 +866,8 @@ namespace Duplicati.Library.Main.Database try { await using var cmd = m_db.Connection.CreateCommand(); - await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ") + await cmd + .ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ", default) .ConfigureAwait(false); } catch { } @@ -848,9 +882,9 @@ namespace Duplicati.Library.Main.Database /// The time to filter filesets by. /// An array of versions to filter filesets by. /// A task that when awaited returns a new instance of . - public async Task SelectFileSets(DateTime time, long[] versions) + public async Task SelectFileSets(DateTime time, long[] versions, CancellationToken token) { - return await FileSets.CreateAsync(this, time, versions) + return await FileSets.CreateAsync(this, time, versions, token) .ConfigureAwait(false); } @@ -867,8 +901,9 @@ namespace Duplicati.Library.Main.Database /// /// Lists all filesets with summary data. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of fileset entries. - public async IAsyncEnumerable ListFilesetsExtended() + public async IAsyncEnumerable ListFilesetsExtended([EnumeratorCancellation] CancellationToken token) { const string query = @" SELECT @@ -889,7 +924,7 @@ namespace Duplicati.Library.Main.Database "; await using var cmd = m_connection.CreateCommand(query); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var id = rd.ConvertValueToInt64(0); var time = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); @@ -924,8 +959,9 @@ namespace Duplicati.Library.Main.Database /// The fileset ID to filter the folder entries. /// The offset for pagination. /// >The limit for pagination. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a paginated result set of folder entries. - public async Task> ListFolder(IEnumerable prefixIds, long filesetid, long offset, long limit) + public async Task> ListFolder(IEnumerable prefixIds, long filesetid, long offset, long limit, CancellationToken token) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); @@ -977,7 +1013,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@offset", offset); var results = new List(); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); @@ -1019,7 +1055,7 @@ namespace Duplicati.Library.Main.Database ") .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) .SetParameterValue("@filesetid", filesetid) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } @@ -1035,8 +1071,9 @@ namespace Duplicati.Library.Main.Database /// The fileset ID to filter the folder entries. /// The offset for pagination. /// The limit for pagination. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a paginated result set of folder entries. - public async Task> ListFilesetEntries(IEnumerable prefixIds, long filesetid, long offset, long limit) + public async Task> ListFilesetEntries(IEnumerable prefixIds, long filesetid, long offset, long limit, CancellationToken token) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); @@ -1088,7 +1125,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@offset", offset); var results = new List(); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); @@ -1123,7 +1160,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", filesetid) .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } @@ -1135,8 +1172,9 @@ namespace Duplicati.Library.Main.Database /// Gets the prefix IDs for a given set of folder-prefixes. /// /// The prefixes to get the IDs for. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of prefix IDs. - public async IAsyncEnumerable GetPrefixIds(IEnumerable prefixes) + public async IAsyncEnumerable GetPrefixIds(IEnumerable prefixes, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(); @@ -1148,7 +1186,7 @@ namespace Duplicati.Library.Main.Database ") .ExpandInClauseParameter("@Prefixes", prefixes); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { yield return rd.ConvertValueToInt64(0); } @@ -1158,8 +1196,9 @@ namespace Duplicati.Library.Main.Database /// Gathers root prefixes from the fileset entries. /// /// The filesetId to get the prefixes for. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of root prefixes. - public async IAsyncEnumerable GetRootPrefixes(long filesetid) + public async IAsyncEnumerable GetRootPrefixes(long filesetid, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(); @@ -1189,7 +1228,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@filesetid", filesetid); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var prefixId = rd.ConvertValueToInt64(0, -1); if (prefixId != -1) @@ -1203,8 +1242,9 @@ namespace Duplicati.Library.Main.Database /// Gathers all prefixes and their parent prefix IDs from the fileset entries. /// /// The filesetId to get the prefixes for + /// A cancellation token to cancel the operation. /// A task that when awaited returns an asynchronous enumerable of (PrefixID, ParentPrefixID) pairs. - public async Task> GetRootPrefixesWithParents(long filesetid) + public async Task> GetRootPrefixesWithParents(long filesetid, CancellationToken token) { await using var cmd = m_connection.CreateCommand(); @@ -1214,7 +1254,7 @@ namespace Duplicati.Library.Main.Database FROM ""PathPrefix"" LIMIT 1 ") - .ExecuteScalarAsync() + .ExecuteScalarAsync(token) .ConfigureAwait(false) )?.ToString()); @@ -1256,7 +1296,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@dirSeparator", dirSeparator) .SetParameterValue("@filesetid", filesetid); - return cmd.ExecuteReaderEnumerableAsync() + return cmd.ExecuteReaderEnumerableAsync(token) .Select(x => ( PrefixId: x.ConvertValueToInt64(0, -1), ParentPrefixId: x.ConvertValueToInt64(1, -1) @@ -1270,8 +1310,9 @@ namespace Duplicati.Library.Main.Database /// Optional fileset IDs to restrict the search to. /// Pagination offset. /// Pagination limit. + /// A cancellation token to cancel the operation. /// A task that when awaited returns all matching file versions ordered by path and then version time. - public async Task> ListFileVersions(IEnumerable paths, long[]? filesetIds, long offset, long limit) + public async Task> ListFileVersions(IEnumerable paths, long[]? filesetIds, long offset, long limit, CancellationToken token) { if (paths == null || !paths.Any()) return new PaginatedResults(0, (int)(limit > 0 ? limit : long.MaxValue), 0, 0, Enumerable.Empty()); @@ -1294,9 +1335,9 @@ namespace Duplicati.Library.Main.Database "); var versionMap = await cmd - .ExecuteReaderEnumerableAsync() + .ExecuteReaderEnumerableAsync(token) .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) - .ToDictionaryAsync(x => x.FilesetId, x => x.Version) + .ToDictionaryAsync(x => x.FilesetId, x => x.Version, cancellationToken: token) .ConfigureAwait(false); // Then fetch the actual data @@ -1343,7 +1384,7 @@ namespace Duplicati.Library.Main.Database cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); var results = new List(); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); @@ -1391,7 +1432,7 @@ namespace Duplicati.Library.Main.Database if (filesetIds != null && filesetIds.Length > 0) cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); - totalCount = await cmd.ExecuteScalarInt64Async(0) + totalCount = await cmd.ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } @@ -1415,8 +1456,9 @@ namespace Duplicati.Library.Main.Database /// Optional fileset IDs to restrict the search to. /// The offset for pagination. /// The limit for pagination. + /// A cancellation token to cancel the operation. /// A task that when awaited returns all matching file versions ordered by path and then version time. - public async Task> SearchEntries(IEnumerable? pathprefixes, IFilter filter, long[]? filesetIds, long offset, long limit) + public async Task> SearchEntries(IEnumerable? pathprefixes, IFilter filter, long[]? filesetIds, long offset, long limit, CancellationToken token) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); @@ -1550,9 +1592,9 @@ namespace Duplicati.Library.Main.Database "); var versionMap = await cmd - .ExecuteReaderEnumerableAsync() + .ExecuteReaderEnumerableAsync(token) .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) - .ToDictionaryAsync(x => x.FilesetId, x => x.Version) + .ToDictionaryAsync(x => x.FilesetId, x => x.Version, cancellationToken: token) .ConfigureAwait(false); // Fetch results @@ -1606,7 +1648,7 @@ namespace Duplicati.Library.Main.Database cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); var results = new List(); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); @@ -1655,7 +1697,7 @@ namespace Duplicati.Library.Main.Database if (filesetIds != null && filesetIds.Length > 0) cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); - totalCount = await cmd.ExecuteScalarInt64Async(0) + totalCount = await cmd.ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } From 736b52c2fbef0054462cace3d397f4bcbbdc65d3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 07:00:14 +0200 Subject: [PATCH 331/362] Made the LocalPurgeDatabase methods accept a cancellation token --- .../Main/Database/LocalPurgeDatabase.cs | 110 +++++++++++------- 1 file changed, 66 insertions(+), 44 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 50cdb25e3..8434c8842 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -23,6 +23,8 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -40,13 +42,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The size of the page cache. /// The optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalPurgeDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalPurgeDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalPurgeDatabase(); dbnew = (LocalPurgeDatabase) - await LocalDeleteDatabase.CreateAsync(path, "Purge", pagecachesize, dbnew) + await LocalDeleteDatabase.CreateAsync(path, "Purge", pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -59,13 +62,14 @@ namespace Duplicati.Library.Main.Database /// /// The parent database to use. /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(LocalDatabase dbparent, LocalPurgeDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase dbparent, LocalPurgeDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalPurgeDatabase(); dbnew = (LocalPurgeDatabase) - await LocalDeleteDatabase.CreateAsync(dbparent, dbnew) + await LocalDeleteDatabase.CreateAsync(dbparent, dbnew, token) .ConfigureAwait(false); return dbnew; @@ -75,10 +79,11 @@ namespace Duplicati.Library.Main.Database /// Creates a new instance of the class. /// /// The ID of the parent fileset. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public async Task CreateTemporaryFileset(long parentid) + public async Task CreateTemporaryFileset(long parentid, CancellationToken token) { - return await TemporaryFileset.CreateAsync(parentid, this) + return await TemporaryFileset.CreateAsync(parentid, this, token) .ConfigureAwait(false); } @@ -86,8 +91,9 @@ namespace Duplicati.Library.Main.Database /// Gets the remote volume name for a fileset with the specified ID. /// /// The ID of the fileset. + /// A cancellation token to cancel the operation. /// A task that when awaited contains the name of the remote volume. - public async Task GetRemoteVolumeNameForFileset(long id) + public async Task GetRemoteVolumeNameForFileset(long id, CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT @@ -101,8 +107,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", id); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - if (!await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (!await rd.ReadAsync(token).ConfigureAwait(false)) throw new Exception($"No remote volume found for fileset with id {id}"); else return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); @@ -111,8 +117,9 @@ namespace Duplicati.Library.Main.Database /// /// Counts the number of orphan files in the database. /// + /// A cancellation token to cancel the operation. /// A task that when awaited contains the count of orphan files. - internal async Task CountOrphanFiles() + internal async Task CountOrphanFiles(CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); cmd.SetCommandAndParameters(@" @@ -123,8 +130,8 @@ namespace Duplicati.Library.Main.Database FROM ""FilesetEntry"" ) "); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) return rd.ConvertValueToInt64(0, 0); else return 0; @@ -156,27 +163,31 @@ namespace Duplicati.Library.Main.Database /// Applies a filter to the temporary fileset. /// /// The filter to apply. + /// A cancellation token to cancel the operation. /// A task that completes when the filter has been applied. - Task ApplyFilter(Library.Utility.IFilter filter); + Task ApplyFilter(Library.Utility.IFilter filter, CancellationToken token); /// /// Applies a filter using a custom command. /// /// The command to execute for filtering. + /// A cancellation token to cancel the operation. /// A task that completes when the filter has been applied. - Task ApplyFilter(Func> filtercommand); + Task ApplyFilter(Func> filtercommand, CancellationToken token); /// /// Converts the temporary fileset to a permanent fileset. /// /// The name of the new fileset. /// The timestamp for the new fileset. /// Indicates if this is a full backup. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a tuple with the remote volume ID and the new fileset ID. - Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); + Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup, CancellationToken token); /// /// Lists all deleted files in the temporary fileset. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of key-value pairs where the key is the file path and the value is the file size. - IAsyncEnumerable> ListAllDeletedFiles(); + IAsyncEnumerable> ListAllDeletedFiles(CancellationToken token); } /// @@ -217,8 +228,9 @@ namespace Duplicati.Library.Main.Database /// /// The ID of the parent fileset. /// The database instance to use. + /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(long parentid, LocalDatabase db) + public static async Task CreateAsync(long parentid, LocalDatabase db, CancellationToken token) { var tempf = new TemporaryFileset() { @@ -232,7 +244,7 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{tempf.m_tablename}"" ( ""FileID"" INTEGER PRIMARY KEY ) - ") + ", token) .ConfigureAwait(false); return tempf; @@ -242,23 +254,25 @@ namespace Duplicati.Library.Main.Database /// Applies a filter to the temporary fileset using a custom command. /// /// The command to execute for filtering. + /// A cancellation token to cancel the operation. /// A task that completes when the filter has been applied. - public async Task ApplyFilter(Func> filtercommand) + public async Task ApplyFilter(Func> filtercommand, CancellationToken token) { int updated; await using (var cmd = m_db.Connection.CreateCommand()) updated = await filtercommand(cmd, ParentID, m_tablename) .ConfigureAwait(false); - await PostFilterChecks(updated).ConfigureAwait(false); + await PostFilterChecks(updated, token).ConfigureAwait(false); } /// /// Applies a filter to the temporary fileset. /// /// The filter to apply. + /// A cancellation token to cancel the operation. /// A task that completes when the filter has been applied. - public async Task ApplyFilter(Library.Utility.IFilter filter) + public async Task ApplyFilter(Library.Utility.IFilter filter, CancellationToken token) { if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { @@ -272,19 +286,19 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{filenamestable}"" ( ""Path"" TEXT NOT NULL ) - ") + ", token) .ConfigureAwait(false); await cmd.SetCommandAndParameters($@" INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path) ") - .PrepareAsync() + .PrepareAsync(token) .ConfigureAwait(false); foreach (var s in p) await cmd.SetParameterValue("@Path", s) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); await cmd.SetCommandAndParameters($@" @@ -299,11 +313,11 @@ namespace Duplicati.Library.Main.Database AND ""B"".""Path"" IN ""{filenamestable}"" ") .SetParameterValue("@FilesetId", ParentID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ") + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ", token) .ConfigureAwait(false); } else @@ -316,7 +330,7 @@ namespace Duplicati.Library.Main.Database INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId) ") - .PrepareAsync() + .PrepareAsync(token) .ConfigureAwait(false); cmd.SetCommandAndParameters(@" @@ -332,8 +346,8 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@FilesetId", ParentID); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { rd.GetValues(values); var path = values[0] as string; @@ -341,21 +355,22 @@ namespace Duplicati.Library.Main.Database { await cmd2 .SetParameterValue("@FileId", values[1]) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } } - await PostFilterChecks(0).ConfigureAwait(false); + await PostFilterChecks(0, token).ConfigureAwait(false); } /// /// Applies the filter checks after filtering. /// /// The number of files updated by the filter. + /// A cancellation token to cancel the operation. /// A task that completes when the checks have been applied. - private async Task PostFilterChecks(int updated) + private async Task PostFilterChecks(int updated, CancellationToken token) { await using var cmd = m_db.Connection.CreateCommand(); UpdatedFileCount = updated; @@ -363,7 +378,7 @@ namespace Duplicati.Library.Main.Database RemovedFileCount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{m_tablename}"" - ", 0) + ", 0, token) .ConfigureAwait(false); RemovedFileSize = await cmd.ExecuteScalarInt64Async($@" @@ -382,7 +397,7 @@ namespace Duplicati.Library.Main.Database AND ""D"".""BlocksetID"" = ""C"".""ID"" ) ) - ", 0) + ", 0, token) .ConfigureAwait(false); var filesetcount = await cmd.SetCommandAndParameters($@" @@ -391,23 +406,27 @@ namespace Duplicati.Library.Main.Database WHERE ""FilesetID"" = @ParentID ") .SetParameterValue("@ParentID", ParentID) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); if (filesetcount == RemovedFileCount) throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } - public async Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup) + /// + public async Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup, CancellationToken token) { var remotevolid = - await m_db.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary) + await m_db + .RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, token) .ConfigureAwait(false); - var filesetid = await m_db.CreateFileset(remotevolid, timestamp) + var filesetid = await m_db + .CreateFileset(remotevolid, timestamp, token) .ConfigureAwait(false); - await m_db.UpdateFullBackupStateInFileset(filesetid, isFullBackup) + await m_db + .UpdateFullBackupStateInFileset(filesetid, isFullBackup, token) .ConfigureAwait(false); await using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) @@ -428,13 +447,14 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@TargetFilesetId", filesetid) .SetParameterValue("@SourceFilesetId", ParentID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); return new Tuple(remotevolid, filesetid); } - public async IAsyncEnumerable> ListAllDeletedFiles() + /// + public async IAsyncEnumerable> ListAllDeletedFiles([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_db.Connection.CreateCommand(); await using var rd = await cmd.ExecuteReaderAsync($@" @@ -448,8 +468,10 @@ namespace Duplicati.Library.Main.Database WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" - ").ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + ", token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1) @@ -467,7 +489,7 @@ namespace Duplicati.Library.Main.Database { await using var cmd = m_db.Connection.CreateCommand(); await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""", default) .ConfigureAwait(false); } catch { } From 1c103c18b8432ea50e7e475059e5da12856b4c4f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 07:15:28 +0200 Subject: [PATCH 332/362] Made the LocalRecreateDatabase methods accept a cancellation token --- .../Main/Database/LocalRecreateDatabase.cs | 178 ++++++++++-------- 1 file changed, 98 insertions(+), 80 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 56416a8cd..807ff4e4e 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -23,6 +23,8 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -189,13 +191,14 @@ namespace Duplicati.Library.Main.Database /// The parent database from which to restore. /// The options to use for the database creation. /// An optional existing instance of to use; if null, a new instance will be created. + /// A cancellation token to observe while waiting for the task to complete. /// A task that represents the asynchronous operation, containing the newly created instance. - public static async Task CreateAsync(LocalDatabase parentdb, Options options, LocalRecreateDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase parentdb, Options options, LocalRecreateDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalRecreateDatabase(); dbnew = (LocalRecreateDatabase) - await LocalRestoreDatabase.CreateAsync(parentdb, dbnew) + await LocalRestoreDatabase.CreateAsync(parentdb, dbnew, token) .ConfigureAwait(false); dbnew.m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -211,13 +214,13 @@ namespace Duplicati.Library.Main.Database ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" ON ""{dbnew.m_tempblocklist}"" (""BlockListHash""); - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -226,7 +229,7 @@ namespace Duplicati.Library.Main.Database ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -235,7 +238,7 @@ namespace Duplicati.Library.Main.Database ""FileHash"", ""BlockSize"" ); - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -244,7 +247,7 @@ namespace Duplicati.Library.Main.Database ""BlockHash"", ""BlockSize"" ); - ") + ", token) .ConfigureAwait(false); dbnew.m_insertFileCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -261,7 +264,7 @@ namespace Duplicati.Library.Main.Database @MetadataId ); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); dbnew.m_insertFilesetEntryCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -275,14 +278,14 @@ namespace Duplicati.Library.Main.Database @FileId, @LastModified ) - ") + ", token) .ConfigureAwait(false); dbnew.m_insertMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); dbnew.m_insertBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -295,7 +298,7 @@ namespace Duplicati.Library.Main.Database @FullHash ); SELECT last_insert_rowid(); - ") + ", token) .ConfigureAwait(false); dbnew.m_insertBlocklistHashCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -309,7 +312,7 @@ namespace Duplicati.Library.Main.Database @Index, @Hash ) - ") + ", token) .ConfigureAwait(false); dbnew.m_updateBlockVolumeCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -318,7 +321,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - ") + ", token) .ConfigureAwait(false); dbnew.m_insertTempBlockListHash = await dbnew.m_connection.CreateCommandAsync($@" @@ -332,7 +335,7 @@ namespace Duplicati.Library.Main.Database @BlockHash, @Index ) - ") + ", token) .ConfigureAwait(false); dbnew.m_insertSmallBlockset = await dbnew.m_connection.CreateCommandAsync($@" @@ -346,7 +349,7 @@ namespace Duplicati.Library.Main.Database @BlockHash, @BlockSize ) - ") + ", token) .ConfigureAwait(false); dbnew.m_findBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -355,7 +358,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash - ") + ", token) .ConfigureAwait(false); dbnew.m_findMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -367,7 +370,7 @@ namespace Duplicati.Library.Main.Database ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length - ") + ", token) .ConfigureAwait(false); dbnew.m_findFilesetCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -378,14 +381,14 @@ namespace Duplicati.Library.Main.Database AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId - ") + ", token) .ConfigureAwait(false); dbnew.m_findTempBlockListHashCommand = await dbnew.m_connection.CreateCommandAsync($@" SELECT DISTINCT ""BlockListHash"" FROM ""{dbnew.m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash - ") + ", token) .ConfigureAwait(false); dbnew.m_findHashBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -394,7 +397,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - ") + ", token) .ConfigureAwait(false); dbnew.m_insertBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -408,7 +411,7 @@ namespace Duplicati.Library.Main.Database @Size, @VolumeId ) - ") + ", token) .ConfigureAwait(false); dbnew.m_insertDuplicateBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" @@ -426,7 +429,7 @@ namespace Duplicati.Library.Main.Database ), @VolumeId ) - ") + ", token) .ConfigureAwait(false); return dbnew; @@ -437,8 +440,9 @@ namespace Duplicati.Library.Main.Database /// /// The size of the hashes in the blocklist. /// The size of the blocks in the blocklist. + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the operation is finished. - public async Task FindMissingBlocklistHashes(long hashsize, long blocksize) + public async Task FindMissingBlocklistHashes(long hashsize, long blocksize, CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); //Update all small blocklists and matching blocks @@ -508,7 +512,7 @@ namespace Duplicati.Library.Main.Database "; // Insert all known blocks into block table with volumeid = -1 - await cmd.ExecuteNonQueryAsync(insertBlocksCommand) + await cmd.ExecuteNonQueryAsync(insertBlocksCommand, token) .ConfigureAwait(false); // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati @@ -587,7 +591,7 @@ namespace Duplicati.Library.Main.Database try { - await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand) + await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand, token) .ConfigureAwait(false); } catch (Exception ex) @@ -600,21 +604,21 @@ namespace Duplicati.Library.Main.Database CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" - ") + ", token) .ConfigureAwait(false); await fixcmd.ExecuteNonQueryAsync($@" CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" - ") + ", token) .ConfigureAwait(false); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// @@ -636,8 +640,9 @@ namespace Duplicati.Library.Main.Database /// The size of the hashes in the blocklist. /// The size of the blocks in the blocklist. /// If true, only hash entries are processed, ignoring small blocks. + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the operation is finished. - public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, bool hashOnly = false) + public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, bool hashOnly, CancellationToken token) { // TODO should values be parameters, rather than hardcoded into the SQL queries? await using var cmd = m_connection.CreateCommand(m_rtr); @@ -738,11 +743,11 @@ namespace Duplicati.Library.Main.Database try { // Insert discovered new blocks into block table with volumeid = -1 - await cmd.ExecuteNonQueryAsync(insertBlocksCommand) + await cmd.ExecuteNonQueryAsync(insertBlocksCommand, token) .ConfigureAwait(false); // Insert corresponding entries into blockset - await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand) + await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand, token) .ConfigureAwait(false); } catch (Exception ex) @@ -755,14 +760,14 @@ namespace Duplicati.Library.Main.Database CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" - ") + ", token) .ConfigureAwait(false); await fixcmd.ExecuteNonQueryAsync($@" CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" - ") + ", token) .ConfigureAwait(false); } @@ -778,10 +783,11 @@ namespace Duplicati.Library.Main.Database /// The path of the directory entry. /// The last modified time of the directory entry. /// The ID of the metadata associated with the directory entry. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when completed, indicates that the directory entry has been added. - public async Task AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid) + public async Task AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, CancellationToken token) { - await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid) + await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, token) .ConfigureAwait(false); } @@ -793,10 +799,11 @@ namespace Duplicati.Library.Main.Database /// The path of the symlink entry. /// The last modified time of the symlink entry. /// The ID of the metadata associated with the symlink entry. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when completed, indicates that the symlink entry has been added. - public async Task AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid) + public async Task AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, CancellationToken token) { - await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid) + await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, token) .ConfigureAwait(false); } @@ -809,10 +816,11 @@ namespace Duplicati.Library.Main.Database /// The last modified time of the file entry. /// The ID of the blockset associated with the file entry. /// The ID of the metadata associated with the file entry. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when completed, indicates that the file entry has been added. - public async Task AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid) + public async Task AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, CancellationToken token) { - await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid) + await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, token) .ConfigureAwait(false); } @@ -827,8 +835,9 @@ namespace Duplicati.Library.Main.Database /// The last modified time of the file entry. /// The ID of the blockset associated with the file entry. /// The ID of the metadata associated with the file entry. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when completed, indicates that the file and fileset entry has been added. - private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid) + private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, CancellationToken token) { var fileid = await m_findFilesetCommand .SetTransaction(m_rtr) @@ -836,7 +845,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@MetadataId", metadataid) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (fileid < 0) @@ -847,7 +856,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@MetadataId", metadataid) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); } @@ -856,7 +865,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@FilesetId", filesetid) .SetParameterValue("@FileId", fileid) .SetParameterValue("@LastModified", time.ToUniversalTime().Ticks) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -868,8 +877,9 @@ namespace Duplicati.Library.Main.Database /// The size of the metadataset in bytes. /// A collection of blocklist hashes associated with the metadataset. /// The expected number of blocklist hashes for the metadataset. + /// A cancellation token to observe while waiting for the task to complete. /// A task that returns the ID of the added or existing metadataset. - public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes) + public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, CancellationToken token) { var metadataid = -1L; if (metahash == null) @@ -879,20 +889,20 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FullHash", metahash) .SetParameterValue("@Length", metahashsize) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (metadataid != -1) return metadataid; var blocksetid = - await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes) + await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, token) .ConfigureAwait(false); metadataid = await m_insertMetadatasetCommand .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); return metadataid; @@ -906,14 +916,15 @@ namespace Duplicati.Library.Main.Database /// The size of the blockset in bytes. /// A collection of blocklist hashes associated with the blockset. /// The expected number of blocklist hashes for the blockset. + /// A cancellation token to observe while waiting for the task to complete. /// A task that returns the ID of the added or existing blockset. - public async Task AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes) + public async Task AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, CancellationToken token) { var blocksetid = await m_findBlocksetCommand .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@FullHash", fullhash) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (blocksetid != -1) @@ -923,7 +934,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@FullHash", fullhash) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); long c = 0; @@ -944,7 +955,7 @@ namespace Duplicati.Library.Main.Database await m_insertBlocklistHashCommand .SetParameterValue("@Index", index++) .SetParameterValue("@Hash", hash) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } @@ -964,15 +975,16 @@ namespace Duplicati.Library.Main.Database /// The hash of the block to update. /// The size of the block in bytes. /// The ID of the volume to which the block belongs. + /// A cancellation token to observe while waiting for the task to complete. /// A task that returns a tuple indicating whether any changes were made and whether the block was newly inserted. - public async Task<(bool, bool)> UpdateBlock(string hash, long size, long volumeID) + public async Task<(bool, bool)> UpdateBlock(string hash, long size, long volumeID, CancellationToken token) { var anyChange = false; var currentVolumeId = await m_findHashBlockCommand .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64Async(-2) + .ExecuteScalarInt64Async(-2, token) .ConfigureAwait(false); if (currentVolumeId == volumeID) @@ -988,7 +1000,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@VolumeId", volumeID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); return (anyChange, true); @@ -1001,7 +1013,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@VolumeId", volumeID) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) @@ -1016,7 +1028,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@VolumeId", volumeID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); return (anyChange, false); @@ -1030,15 +1042,16 @@ namespace Duplicati.Library.Main.Database /// The hash of the file that contains the small block. /// The hash of the small block. /// The size of the small block in bytes. + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the small blockset link has been added. - public async Task AddSmallBlocksetLink(string filehash, string blockhash, long blocksize) + public async Task AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, CancellationToken token) { await m_insertSmallBlockset .SetTransaction(m_rtr) .SetParameterValue("@FileHash", filehash) .SetParameterValue("@BlockHash", blockhash) .SetParameterValue("@BlockSize", blocksize) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -1048,13 +1061,14 @@ namespace Duplicati.Library.Main.Database /// /// The hash of the temporary blocklist. /// A collection of block hashes associated with the temporary blocklist. + /// A cancellation token to observe while waiting for the task to complete. /// A task that returns a boolean indicating whether the temporary blocklist hash was successfully added. - public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes) + public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes, CancellationToken token) { var r = await m_findTempBlockListHashCommand .SetTransaction(m_rtr) .SetParameterValue("@BlocklistHash", hash) - .ExecuteScalarAsync() + .ExecuteScalarAsync(token) .ConfigureAwait(false); if (r != null && r != DBNull.Value) @@ -1071,7 +1085,7 @@ namespace Duplicati.Library.Main.Database await m_insertTempBlockListHash .SetParameterValue("@BlockHash", s) .SetParameterValue("@Index", index++) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -1083,8 +1097,9 @@ namespace Duplicati.Library.Main.Database /// This method is used to obtain the block hashes for a given volume, allowing for efficient retrieval of block data. /// /// The ID of the volume for which to retrieve block hashes. + /// A cancellation token to observe while waiting for the task to complete. /// An asynchronous enumerable collection of block hashes associated with the specified volume ID. - public async IAsyncEnumerable GetBlockLists(long volumeid) + public async IAsyncEnumerable GetBlockLists(long volumeid, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT DISTINCT ""BlocklistHash"".""Hash"" @@ -1098,8 +1113,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? ""; } @@ -1111,8 +1126,9 @@ namespace Duplicati.Library.Main.Database /// The size of the blocks in the blocklist. /// The size of the hashes in the blocklist. /// A boolean indicating whether to force the use of blocks, even if they are not missing. + /// A cancellation token to observe while waiting for the task to complete. /// An asynchronous enumerable collection of remote volumes that are missing blocks. - public async IAsyncEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) + public async IAsyncEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); var selectCommand = @" @@ -1177,7 +1193,7 @@ namespace Duplicati.Library.Main.Database //On anything but the first pass, we check if we are done var r = await cmd .SetCommandAndParameters(countMissingInformation) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); if (r == 0 && !forceBlockUse) @@ -1217,10 +1233,10 @@ namespace Duplicati.Library.Main.Database } await using var rd = await cmd - .ExecuteReaderAsync() + .ExecuteReaderAsync(token) .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var volumeID = rd.ConvertValueToInt64(3); @@ -1240,8 +1256,9 @@ namespace Duplicati.Library.Main.Database /// Cleans up missing volumes by replacing blocks for non-present remote files with existing ones, /// removing references to non-present remote files, and marking index files for later removal. /// + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the cleanup operation is finished. - public async Task CleanupMissingVolumes() + public async Task CleanupMissingVolumes(CancellationToken token) { var tablename = "SwapBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -1366,17 +1383,17 @@ namespace Duplicati.Library.Main.Database "; await using var cmd = m_connection.CreateCommand(m_rtr); - var cnt = await cmd.ExecuteScalarInt64Async(countsql) + var cnt = await cmd.ExecuteScalarInt64Async(countsql, token) .ConfigureAwait(false); if (cnt > 0) { try { - await cmd.ExecuteNonQueryAsync(sql) + await cmd.ExecuteNonQueryAsync(sql, token) .ConfigureAwait(false); - var cnt2 = await cmd.ExecuteScalarInt64Async(countsql) + var cnt2 = await cmd.ExecuteScalarInt64Async(countsql, token) .ConfigureAwait(false); Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); @@ -1393,8 +1410,9 @@ namespace Duplicati.Library.Main.Database /// Move blocks that are not referenced by any files to DeletedBlock table. /// Needs to be called after the last FindMissingBlocklistHashes, otherwise the tables are not up to date. /// + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the cleanup operation is finished. - public async Task CleanupDeletedBlocks() + public async Task CleanupDeletedBlocks(CancellationToken token) { // Find out which blocks are deleted and move them into DeletedBlock, so that compact notices these blocks are empty // Deleted blocks do not appear in the BlocksetEntry and not in the BlocklistHash table @@ -1420,7 +1438,7 @@ namespace Duplicati.Library.Main.Database SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" ) - ") + ", token) .ConfigureAwait(false); // 2. Insert blocks into DeletedBlock table @@ -1435,7 +1453,7 @@ namespace Duplicati.Library.Main.Database ""Size"", ""VolumeID"" FROM ""{tmptablename}"" - ") + ", token) .ConfigureAwait(false); // 3. Remove blocks from Block table @@ -1445,14 +1463,14 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""{tmptablename}"" ) - ") + ", token) .ConfigureAwait(false); await cmd - .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}""") + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}""", token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } public override void Dispose() @@ -1468,7 +1486,7 @@ namespace Duplicati.Library.Main.Database if (m_tempblocklist != null) try { - await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""") + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""", default) .ConfigureAwait(false); } catch (Exception ex) @@ -1483,7 +1501,7 @@ namespace Duplicati.Library.Main.Database if (m_tempsmalllist != null) try { - await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""") + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""", default) .ConfigureAwait(false); } catch (Exception ex) From 04c52010fe8ddc4f440db9f495a1ac2013ef039c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 07:26:54 +0200 Subject: [PATCH 333/362] Made the LocalRepairDatabase methods accept a cancellation token --- .../Main/Database/LocalRepairDatabase.cs | 265 ++++++++++-------- 1 file changed, 147 insertions(+), 118 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 035f116c6..fbf4f8969 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -23,6 +23,8 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; @@ -52,13 +54,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database. /// The page cache size. /// An optional existing database instance to use. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a new instance of . - public static async Task CreateRepairDatabase(string path, long pagecachesize, LocalRepairDatabase? dbnew = null) + public static async Task CreateRepairDatabase(string path, long pagecachesize, LocalRepairDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalRepairDatabase(); dbnew = (LocalRepairDatabase) - await CreateLocalDatabaseAsync(path, "Repair", true, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, "Repair", true, pagecachesize, dbnew, token) .ConfigureAwait(false); return dbnew; @@ -69,13 +72,14 @@ namespace Duplicati.Library.Main.Database /// /// The parent local database to use. /// An optional existing database instance to use. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a new instance of . - public static async Task CreateAsync(LocalDatabase dbparent, LocalRepairDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase dbparent, LocalRepairDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalRepairDatabase(); return (LocalRepairDatabase) - await CreateLocalDatabaseAsync(dbparent, dbnew) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) .ConfigureAwait(false); } @@ -83,9 +87,10 @@ namespace Duplicati.Library.Main.Database /// Gets the fileset ID from the remote name. /// /// The remote name of the fileset. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a tuple containing the fileset ID, timestamp, and whether it is a full backup. /// Thrown if the remote file does not exist. - public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist) + public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist, CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT @@ -102,9 +107,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Name", filelist); - var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); - if (!await rd.ReadAsync().ConfigureAwait(false)) + if (!await rd.ReadAsync(token).ConfigureAwait(false)) throw new Exception($"No such remote file: {filelist}"); return ( @@ -119,9 +124,10 @@ namespace Duplicati.Library.Main.Database /// /// Current fileset ID. /// Source fileset ID. + /// A cancellation token to cancel the operation. /// A task that represents the asynchronous operation. /// Thrown if either fileset ID is less than or equal to zero. - public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId) + public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId, CancellationToken token) { if (filesetid <= 0) throw new ArgumentException("filesetid must be > 0"); @@ -138,15 +144,16 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@CurrentFilesetId", filesetid) .SetParameterValue("@PreviousFilesetId", prevFilesetId); - await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); } /// /// Gets the list of index files that reference a given block file. /// /// The block file ID. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of index file names that reference the specified block file. - public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid) + public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT ""RemoteVolume"".""Name"" @@ -162,15 +169,16 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@BlockFileId", blockfileid) .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); } /// /// Gets a list of filesets that are missing files. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of key-value pairs where the key is the fileset ID and the value is the timestamp of the fileset. - public async IAsyncEnumerable> GetFilesetsWithMissingFiles() + public async IAsyncEnumerable> GetFilesetsWithMissingFiles([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT @@ -188,8 +196,8 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { yield return new KeyValuePair( rd.ConvertValueToInt64(0), @@ -202,8 +210,9 @@ namespace Duplicati.Library.Main.Database /// Deletes all fileset entries for a given fileset. /// /// The fileset ID. + /// A cancellation token to cancel the operation. /// A task that when awaited returns the number of deleted entries. - public async Task DeleteFilesetEntries(long filesetid) + public async Task DeleteFilesetEntries(long filesetid, CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" DELETE FROM ""FilesetEntry"" @@ -212,7 +221,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetid); - return await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); + return await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); } /// @@ -251,8 +260,9 @@ namespace Duplicati.Library.Main.Database /// Gets a list of block volumes that are associated with a given index name. /// /// The name of the index volume. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of representing the block volumes. - public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName) + public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT @@ -273,7 +283,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Name", indexName); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", @@ -326,40 +336,47 @@ namespace Duplicati.Library.Main.Database /// The block hash. /// The block size. /// The volume ID of the new target volume. + /// A cancellation token to cancel the operation. /// A task that when awaited returns true if the block was successfully marked as restored, false otherwise. - Task SetBlockRestored(string hash, long size, long volumeId); + Task SetBlockRestored(string hash, long size, long volumeId, CancellationToken token); /// /// Gets the list of files that contains missing blocks. /// /// The blocksize setting. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of representing the files with missing blocks. - IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize); + IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize, CancellationToken token); /// /// Gets a list for filesystem entries that contain missing blocks in metadata. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of representing the metadata blocks. - IAsyncEnumerable GetSourceItemsWithMetadataBlocks(); + IAsyncEnumerable GetSourceItemsWithMetadataBlocks(CancellationToken token); /// /// Gets missing blocklist hashes. /// /// The number of hashes for each block. + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of representing the blocklist hashes. - IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock); + IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock, CancellationToken token); /// /// Gets the number of missing blocks. /// + /// A cancellation token to cancel the operation. /// A task that when awaited returns the count of missing blocks. - Task GetMissingBlockCount(); + Task GetMissingBlockCount(CancellationToken token); /// /// Gets all the filesets that are affected by missing blocks. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of representing the filesets that contain missing blocks. - IAsyncEnumerable GetFilesetsUsingMissingBlocks(); + IAsyncEnumerable GetFilesetsUsingMissingBlocks(CancellationToken token); /// /// Gets a list of remote files that may contain missing blocks. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of representing the remote volumes that may contain missing blocks. - IAsyncEnumerable GetMissingBlockSources(); + IAsyncEnumerable GetMissingBlockSources(CancellationToken token); } /// @@ -431,8 +448,9 @@ namespace Duplicati.Library.Main.Database /// The name of the volume with missing blocks. /// The connection to the database. /// The transaction to use. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a new instance of . - public static async Task CreateMissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction) + public static async Task CreateMissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction, CancellationToken token) { var blocklist = new MissingBlockList(volumename, connection, transaction); await using (var cmd = connection.CreateCommand(transaction.Transaction)) @@ -443,7 +461,7 @@ namespace Duplicati.Library.Main.Database ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); cmd.SetCommandAndParameters($@" @@ -463,7 +481,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@Name", volumename); - var blockCount = await cmd.ExecuteNonQueryAsync() + var blockCount = await cmd.ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (blockCount == 0) @@ -476,7 +494,7 @@ namespace Duplicati.Library.Main.Database ""Size"", ""Restored"" ) - ") + ", token) .ConfigureAwait(false); } @@ -487,7 +505,7 @@ namespace Duplicati.Library.Main.Database ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue - ") + ", token) .ConfigureAwait(false); blocklist.m_copyIntoDuplicatedBlocks = await connection.CreateCommandAsync(@" @@ -502,7 +520,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Block"".""Hash"" = @Hash AND ""Block"".""Size"" = @Size - ") + ", token) .ConfigureAwait(false); blocklist.m_assignBlocksToNewVolume = await connection.CreateCommandAsync(@" @@ -511,7 +529,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Hash"" = @Hash AND ""Size"" = @Size - ") + ", token) .ConfigureAwait(false); var m_missingBlocksQuery = $@" @@ -522,21 +540,21 @@ namespace Duplicati.Library.Main.Database WHERE ""{blocklist.m_tablename}"".""Restored"" = @Restored "; blocklist.m_missingBlocksCommand = - await connection.CreateCommandAsync(m_missingBlocksQuery) + await connection.CreateCommandAsync(m_missingBlocksQuery, token) .ConfigureAwait(false); blocklist.m_missingBlocksCountCommand = await connection.CreateCommandAsync($@" SELECT COUNT(*) FROM ({m_missingBlocksQuery}) - ") + ", token) .ConfigureAwait(false); return blocklist; } /// - public async Task SetBlockRestored(string hash, long size, long targetVolumeId) + public async Task SetBlockRestored(string hash, long size, long targetVolumeId, CancellationToken token) { var restored = await m_insertCommand .SetTransaction(m_rtr) @@ -544,7 +562,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@PreviousRestoredValue", 0) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false) == 1; if (restored) @@ -552,14 +570,14 @@ namespace Duplicati.Library.Main.Database await m_copyIntoDuplicatedBlocks.SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); var c = await m_assignBlocksToNewVolume.SetTransaction(m_rtr) .SetParameterValue("@TargetVolumeId", targetVolumeId) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) @@ -570,7 +588,7 @@ namespace Duplicati.Library.Main.Database } /// - public async IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize) + public async IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" SELECT DISTINCT @@ -598,7 +616,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); var size = rd.ConvertValueToInt64(1); @@ -610,7 +628,7 @@ namespace Duplicati.Library.Main.Database } /// - public async IAsyncEnumerable GetSourceItemsWithMetadataBlocks() + public async IAsyncEnumerable GetSourceItemsWithMetadataBlocks([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" SELECT DISTINCT @@ -638,7 +656,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); var size = rd.ConvertValueToInt64(1); @@ -649,7 +667,7 @@ namespace Duplicati.Library.Main.Database } /// - public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock) + public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); @@ -691,7 +709,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@HashesPerBlock", hashesPerBlock) .SetParameterValue("@Restored", 0) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); cmd.SetCommandAndParameters($@" @@ -709,7 +727,7 @@ namespace Duplicati.Library.Main.Database ""BlocksetEntryIndex"" "); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); var blocksetId = rd.ConvertValueToInt64(1); @@ -734,7 +752,7 @@ namespace Duplicati.Library.Main.Database { await cmd.ExecuteNonQueryAsync($@" DROP TABLE IF EXISTS ""{blocklistTableName}"" - ") + ", token) .ConfigureAwait(false); } catch { } @@ -742,26 +760,27 @@ namespace Duplicati.Library.Main.Database } /// - public async Task GetMissingBlockCount() + public async Task GetMissingBlockCount(CancellationToken token) { return await m_missingBlocksCountCommand .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } /// /// Gets the list of missing blocks. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of tuples containing the block hash and size. - public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks() + public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks([EnumeratorCancellation] CancellationToken token) { m_missingBlocksCommand .SetTransaction(m_rtr) .SetParameterValue("@Restored", 0); - await foreach (var rd in m_missingBlocksCommand.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in m_missingBlocksCommand.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return ( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1) @@ -769,7 +788,7 @@ namespace Duplicati.Library.Main.Database } /// - public async Task MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId) + public async Task MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, CancellationToken token) { if (targetVolumeId <= 0) throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); @@ -797,7 +816,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@SourceVolumeId", sourceVolumeId) .SetParameterValue("@Restored", 1); - var moved = await cmd.ExecuteNonQueryAsync() + var moved = await cmd.ExecuteNonQueryAsync(token) .ConfigureAwait(false); // Then update the blocks table to point to the new volume @@ -817,7 +836,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TargetVolumeId", targetVolumeId) .SetParameterValue("@SourceVolumeId", sourceVolumeId) .SetParameterValue("@Restored", 1) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (updated != moved) @@ -827,7 +846,7 @@ namespace Duplicati.Library.Main.Database } /// - public async IAsyncEnumerable GetFilesetsUsingMissingBlocks() + public async IAsyncEnumerable GetFilesetsUsingMissingBlocks([EnumeratorCancellation] CancellationToken token) { var blocks = $@" SELECT DISTINCT ""FileLookup"".""ID"" AS ID @@ -884,7 +903,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", @@ -893,7 +912,7 @@ namespace Duplicati.Library.Main.Database } /// - public async IAsyncEnumerable GetMissingBlockSources() + public async IAsyncEnumerable GetMissingBlockSources([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" SELECT DISTINCT @@ -915,7 +934,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Restored", 0) .SetParameterValue("@Name", m_volumename); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", @@ -942,7 +961,7 @@ namespace Duplicati.Library.Main.Database if (m_tablename != null) { - await using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + await using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""", default) .ConfigureAwait(false); await cmd.SetTransaction(m_rtr) @@ -961,18 +980,20 @@ namespace Duplicati.Library.Main.Database /// Creates a new missing block list for the specified volume. /// /// The name of the volume with missing blocks. + /// A cancellation token to cancel the operation. /// A task that when awaited returns an instance of . - public async Task CreateBlockList(string volumename) + public async Task CreateBlockList(string volumename, CancellationToken token) { - return await MissingBlockList.CreateMissingBlockList(volumename, m_connection, m_rtr) + return await MissingBlockList.CreateMissingBlockList(volumename, m_connection, m_rtr, token) .ConfigureAwait(false); } /// /// Fixes duplicate metadata hashes in the database. /// + /// A cancellation token to cancel the operation. /// A task that when completed indicates that the repair has been attempted. - public async Task FixDuplicateMetahash() + public async Task FixDuplicateMetahash(CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -992,7 +1013,7 @@ namespace Duplicati.Library.Main.Database ) "; - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) .ConfigureAwait(false); if (x > 1) @@ -1005,7 +1026,7 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File"" - ") + ", token) .ConfigureAwait(false); var sql = @" @@ -1041,15 +1062,15 @@ namespace Duplicati.Library.Main.Database AND ""ID"" != @MetadataId "); - await using var rd = await cmd.ExecuteReaderAsync(sql) + await using var rd = await cmd.ExecuteReaderAsync(sql, token) .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { await c2 .SetParameterValue("@MetadataId", rd.GetValue(0)) .SetParameterValue("@BlocksetId", rd.GetValue(1)) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } @@ -1099,14 +1120,14 @@ namespace Duplicati.Library.Main.Database AND ""ID"" != @FileId "); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql).ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql, token).ConfigureAwait(false)) { await c2 .SetParameterValue("@FileId", rd.GetValue(0)) .SetParameterValue("@Path", rd.GetValue(1)) .SetParameterValue("@BlocksetId", rd.GetValue(2)) .SetParameterValue("@MetadataId", rd.GetValue(3)) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } @@ -1117,7 +1138,7 @@ namespace Duplicati.Library.Main.Database SELECT ""ID"" FROM ""{tablename}"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -1126,7 +1147,7 @@ namespace Duplicati.Library.Main.Database ""ID"", ""MetadataID"" ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -1136,13 +1157,13 @@ namespace Duplicati.Library.Main.Database FROM ""{tablename}"" ""A"" WHERE ""A"".""ID"" = ""FileLookup"".""ID"" ) - ") + ", token) .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{tablename}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{tablename}"" ", token) .ConfigureAwait(false); - x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) .ConfigureAwait(false); if (x > 1) @@ -1150,15 +1171,16 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } /// /// Fixes duplicate file entries in the database. /// + /// A cancellation token to cancel the operation. /// A task that when completed indicates that the repair has been attempted. - public async Task FixDuplicateFileentries() + public async Task FixDuplicateFileentries(CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -1181,7 +1203,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Duplicates"" > 1 "; - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) .ConfigureAwait(false); if (x > 0) @@ -1240,7 +1262,7 @@ namespace Duplicati.Library.Main.Database cmd.SetCommandAndParameters(sql); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { await c2 .SetParameterValue("@FileId", rd.GetValue(0)) @@ -1248,12 +1270,12 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", rd.GetValue(2)) .SetParameterValue("@BlocksetId", rd.GetValue(3)) .SetParameterValue("@MetadataId", rd.GetValue(4)) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } - x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) .ConfigureAwait(false); if (x > 1) @@ -1261,7 +1283,7 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } @@ -1271,8 +1293,9 @@ namespace Duplicati.Library.Main.Database /// /// The hash algorithm used for the blocklist hashes. /// The size of each block in bytes. + /// A cancellation token to cancel the operation. /// A task that when completed indicates that the repair has been attempted. - public async Task FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize) + public async Task FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize, CancellationToken token) { var blocklistbuffer = new byte[blocksize]; @@ -1318,7 +1341,7 @@ namespace Duplicati.Library.Main.Database "; var itemswithnoblocklisthash = await cmd - .ExecuteScalarInt64Async(countsql, 0) + .ExecuteScalarInt64Async(countsql, 0, token) .ConfigureAwait(false); if (itemswithnoblocklisthash != 0) @@ -1388,7 +1411,7 @@ namespace Duplicati.Library.Main.Database WHERE ""ID"" = @DeletedBlockId; "); - await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql).ConfigureAwait(false)) + await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql, token).ConfigureAwait(false)) { var blocksetid = e.ConvertValueToInt64(0); var ix = 0L; @@ -1399,7 +1422,7 @@ namespace Duplicati.Library.Main.Database WHERE ""BlocksetID"" = @BlocksetId ") .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); c2.SetCommandAndParameters(@" @@ -1414,7 +1437,7 @@ namespace Duplicati.Library.Main.Database ") .SetParameterValue("@BlocksetId", blocksetid); - await foreach (var h in c2.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var h in c2.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); if (blocklistbuffer.Length - blocklistoffset < tmp.Length) @@ -1425,7 +1448,7 @@ namespace Duplicati.Library.Main.Database var existingBlockId = await c4 .SetParameterValue("@Hash", blkey) .SetParameterValue("@Size", blocklistoffset) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (existingBlockId <= 0) @@ -1436,7 +1459,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (deletedBlockId <= 0) @@ -1445,7 +1468,7 @@ namespace Duplicati.Library.Main.Database { var rc = await c6 .SetParameterValue("@DeletedBlockId", deletedBlockId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (rc != 2) @@ -1457,7 +1480,7 @@ namespace Duplicati.Library.Main.Database await c3.SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", blkey) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); ix++; @@ -1477,7 +1500,7 @@ namespace Duplicati.Library.Main.Database var existingBlockId = await c4 .SetParameterValue("@Hash", blkeyfinal) .SetParameterValue("@Size", blocklistoffset) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (existingBlockId <= 0) @@ -1488,7 +1511,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (deletedBlockId == 0) @@ -1497,7 +1520,7 @@ namespace Duplicati.Library.Main.Database { var rc = await c6 .SetParameterValue("@DeletedBlockId", deletedBlockId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (rc != 2) throw new Exception($"Unexpected update count: {rc}"); @@ -1509,14 +1532,14 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@Index", ix) .SetParameterValue("@Hash", blkeyfinal) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } } itemswithnoblocklisthash = await cmd - .ExecuteScalarInt64Async(countsql, 0) + .ExecuteScalarInt64Async(countsql, 0, token) .ConfigureAwait(false); if (itemswithnoblocklisthash != 0) @@ -1524,7 +1547,7 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } @@ -1534,8 +1557,9 @@ namespace Duplicati.Library.Main.Database /// /// The size of each block in bytes. /// The size of each hash in bytes. + /// A cancellation token to cancel the operation. /// A task that when completed indicates that the repair has been attempted. - public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize) + public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize, CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -1559,7 +1583,7 @@ namespace Duplicati.Library.Main.Database FROM ({dup_sql}) "; - var x = await cmd.ExecuteScalarInt64Async(sql_count, 0) + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) .ConfigureAwait(false); if (x > 0) @@ -1574,7 +1598,7 @@ namespace Duplicati.Library.Main.Database ""Index"" FROM ""BlocklistHash"" ) - ", 0) + ", 0, token) .ConfigureAwait(false); await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) @@ -1591,14 +1615,14 @@ namespace Duplicati.Library.Main.Database ) "); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql).ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql, token).ConfigureAwait(false)) { var expected = rd.GetInt32(2) - 1; var actual = await c2 .SetParameterValue("@BlocksetId", rd.GetValue(0)) .SetParameterValue("@Index", rd.GetValue(1)) .SetParameterValue("@Limit", expected) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (actual != expected) @@ -1606,7 +1630,7 @@ namespace Duplicati.Library.Main.Database } } - x = await cmd.ExecuteScalarInt64Async(sql_count) + x = await cmd.ExecuteScalarInt64Async(sql_count, token) .ConfigureAwait(false); if (x > 1) @@ -1615,7 +1639,7 @@ namespace Duplicati.Library.Main.Database var real_count = await cmd.ExecuteScalarInt64Async(@" SELECT Count(*) FROM ""BlocklistHash"" - ", 0) + ", 0, token) .ConfigureAwait(false); if (real_count != unique_count) @@ -1623,7 +1647,7 @@ namespace Duplicati.Library.Main.Database try { - await VerifyConsistency(blocksize, hashsize, true) + await VerifyConsistency(blocksize, hashsize, true, token) .ConfigureAwait(false); } catch (Exception ex) @@ -1633,7 +1657,7 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } @@ -1642,9 +1666,10 @@ namespace Duplicati.Library.Main.Database /// /// The name of the volume to check. /// A collection of blocks to check, represented as key-value pairs where the key is the block hash and the value is the block size. + /// A cancellation token to cancel the operation. /// Thrown if not all blocks are found in the specified volume. /// A task that when awaited indicates the completion of the check. - public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks) + public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -1656,7 +1681,7 @@ namespace Duplicati.Library.Main.Database ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); cmd.SetCommandAndParameters($@" @@ -1675,7 +1700,7 @@ namespace Duplicati.Library.Main.Database await cmd .SetParameterValue("@Hash", kp.Key) .SetParameterValue("@Size", kp.Value) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -1685,7 +1710,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name ") .SetParameterValue("@Name", filename) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); var aliens = await cmd.SetCommandAndParameters($@" @@ -1700,7 +1725,7 @@ namespace Duplicati.Library.Main.Database WHERE ""VolumeID"" != @VolumeId ") .SetParameterValue("@VolumeId", id) - .ExecuteScalarInt64Async(0) + .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); if (aliens != 0) @@ -1708,7 +1733,7 @@ namespace Duplicati.Library.Main.Database } finally { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ", token) .ConfigureAwait(false); } } @@ -1721,9 +1746,10 @@ namespace Duplicati.Library.Main.Database /// The expected blocklist entries to compare against. /// The size of each block in bytes. /// The length of each block hash in bytes. + /// A cancellation token to cancel the operation. /// A task that when awaited indicates the completion of the check. /// Thrown if the blocklist does not match the expected entries. - public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength) + public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); @@ -1764,7 +1790,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", length); - await foreach (var r in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var r in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { if (!en.MoveNext()) throw new Exception($"Too few entries in source blocklist with hash {hash}"); @@ -1779,8 +1805,9 @@ namespace Duplicati.Library.Main.Database /// /// Checks if there are any missing local filesets in the database. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of missing local fileset names. - public async IAsyncEnumerable MissingLocalFilesets() + public async IAsyncEnumerable MissingLocalFilesets([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT ""Name"" @@ -1800,15 +1827,16 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Deleted.ToString() ]); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? ""; } /// /// Checks if there are any missing remote filesets in the database. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of tuples containing the fileset ID, timestamp, and whether it is a full backup. - public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets() + public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT @@ -1831,7 +1859,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Deleted.ToString() ]); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return ( rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), @@ -1842,8 +1870,9 @@ namespace Duplicati.Library.Main.Database /// /// Checks if there are any empty index files in the database. /// + /// A cancellation token to cancel the operation. /// An asynchronous enumerable of remote volumes that are empty index files. - public async IAsyncEnumerable EmptyIndexFiles() + public async IAsyncEnumerable EmptyIndexFiles([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT @@ -1866,7 +1895,7 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Verified.ToString() ]); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) yield return new RemoteVolume( rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", From 488d531bd5313e28b764d6cd8c04a6467ffa284c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 07:54:53 +0200 Subject: [PATCH 334/362] Made the LocalRestoreDatabase methods accept a cancellation token --- .../Main/Database/LocalRestoreDatabase.cs | 477 ++++++++++-------- 1 file changed, 278 insertions(+), 199 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index c7c1d9083..58d6126a2 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -25,7 +25,9 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Main.Operation.Restore; @@ -99,13 +101,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The size of the page cache for the database. /// An optional existing instance of to use, or null to create a new one. Used to mimic constructor chaining. + /// A cancellation token to monitor for cancellation requests. /// A task that when awaited on, returns a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalRestoreDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalRestoreDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalRestoreDatabase(); dbnew = (LocalRestoreDatabase) - await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -117,13 +120,14 @@ namespace Duplicati.Library.Main.Database /// /// The parent database from which to create the restore database. /// An optional existing instance of to use, or null to create a new one. Used to mimic constructor chaining. + /// A cancellation token to monitor for cancellation requests. /// A task that when awaited on, returns a new instance of . - public static async Task CreateAsync(LocalDatabase dbparent, LocalRestoreDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase dbparent, LocalRestoreDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalRestoreDatabase(); return (LocalRestoreDatabase) - await CreateLocalDatabaseAsync(dbparent, dbnew) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) .ConfigureAwait(false); } @@ -134,6 +138,7 @@ namespace Duplicati.Library.Main.Database /// /// This allows to create another table that keeps track /// of all files that are done (all data blocks restored). + /// A cancellation token to monitor for cancellation requests. /// /// The method is prepared to create a table that keeps track of all files being done completely. /// That means, it fires for files where the number of restored blocks equals the number of all blocks. @@ -143,7 +148,7 @@ namespace Duplicati.Library.Main.Database /// But if it reaches a restored state later, it will be re-added (trigger will fire) /// /// A task that completes when the progress tracker is created. - public async Task CreateProgressTracker(bool createFilesNewlyDoneTracker) + public async Task CreateProgressTracker(bool createFilesNewlyDoneTracker, CancellationToken token) { m_fileprogtable = "FileProgress-" + m_temptabsetguid; m_totalprogtable = "TotalProgress-" + m_temptabsetguid; @@ -153,7 +158,7 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr); // How to handle METADATA? - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -164,10 +169,10 @@ namespace Duplicati.Library.Main.Database ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -180,19 +185,19 @@ namespace Duplicati.Library.Main.Database ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); if (createFilesNewlyDoneTracker) { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( ""ID"" INTEGER PRIMARY KEY ) - ") + ", token) .ConfigureAwait(false); } @@ -242,7 +247,7 @@ namespace Duplicati.Library.Main.Database ON ""B"".""FileID"" = ""F"".""ID"" WHERE ""B"".""Metadata"" IS NOT 1 GROUP BY ""F"".""ID"" - ") + ", token) .ConfigureAwait(false); // Will result in a single line (no support to also track metadata) @@ -285,7 +290,7 @@ namespace Duplicati.Library.Main.Database IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) FROM ""{m_fileprogtable}"" ""P"" - ") + ", token) .ConfigureAwait(false); // Finally we create TRIGGERs to keep all our statistics up to date. @@ -311,7 +316,7 @@ namespace Duplicati.Library.Main.Database + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" ; END - ") + ", token) .ConfigureAwait(false); // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) @@ -357,7 +362,7 @@ namespace Duplicati.Library.Main.Database + NEW.""SizeRestored"" - OLD.""SizeRestored"" ; END - ") + ", token) .ConfigureAwait(false); if (createFilesNewlyDoneTracker) @@ -374,7 +379,7 @@ namespace Duplicati.Library.Main.Database INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") VALUES (NEW.""FileId""); END - ") + ", token) .ConfigureAwait(false); } @@ -388,7 +393,7 @@ namespace Duplicati.Library.Main.Database } finally { - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } @@ -398,8 +403,9 @@ namespace Duplicati.Library.Main.Database /// The time at which the restore operation is being performed. /// An array of version identifiers to filter the filesets. /// An optional filter to apply to the files being restored. + /// A cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a tuple containing the count of files to restore and the total size of those files. - public async Task> PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter) + public async Task> PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter, CancellationToken token) { m_tempfiletable = "Fileset-" + m_temptabsetguid; m_tempblocktable = "Blocks-" + m_temptabsetguid; @@ -408,8 +414,8 @@ namespace Duplicati.Library.Main.Database { cmd.SetTransaction(m_rtr); var filesetIds = - await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions) - .ToListAsync() + await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions, false, token) + .ToListAsync(cancellationToken: token) .ConfigureAwait(false); while (filesetIds.Count > 0) @@ -423,21 +429,24 @@ namespace Duplicati.Library.Main.Database WHERE ""ID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId); - m_restoreTime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0).ConfigureAwait(false)); + m_restoreTime = ParseFromEpochSeconds( + await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false) + ); - var ix = await FilesetTimes() + var ix = await FilesetTimes(token) .Select((value, index) => new { value.Key, index }) .Where(n => n.Key == filesetId) .Select(pair => pair.index + 1) - .FirstOrDefaultAsync() + .FirstOrDefaultAsync(cancellationToken: token) .ConfigureAwait(false) - 1; Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ", token) .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -451,7 +460,7 @@ namespace Duplicati.Library.Main.Database ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN ) - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -466,7 +475,7 @@ namespace Duplicati.Library.Main.Database ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); // TODO: Optimize to use the path prefix @@ -496,13 +505,13 @@ namespace Duplicati.Library.Main.Database AND ""FilesetEntry"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple) { using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset")) - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); cmd.SetTransaction(m_rtr); // If we get a list of filenames, the lookup table is faster @@ -514,7 +523,7 @@ namespace Duplicati.Library.Main.Database CREATE TEMPORARY TABLE ""{m_filenamestable}"" ( ""Path"" TEXT NOT NULL ) - ") + ", token) .ConfigureAwait(false); cmd.SetCommandAndParameters($@" @@ -527,7 +536,7 @@ namespace Duplicati.Library.Main.Database { await cmd .SetParameterValue("@Path", s) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -557,7 +566,7 @@ namespace Duplicati.Library.Main.Database ) ") .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != p.Length && c != 0) @@ -573,8 +582,8 @@ namespace Duplicati.Library.Main.Database FROM ""{m_tempfiletable}"" ) "); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) sb.AppendLine(rd.ConvertValueToString(0)); cmd.SetCommandAndParameters(@" @@ -583,16 +592,19 @@ namespace Duplicati.Library.Main.Database WHERE ""ID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId); - var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0).ConfigureAwait(false)); + var actualrestoretime = ParseFromEpochSeconds( + await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false) + ); Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); } - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ", token) .ConfigureAwait(false); using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset")) - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); cmd.SetTransaction(m_rtr); } else @@ -631,9 +643,9 @@ namespace Duplicati.Library.Main.Database 0 ) "); - await using var rd = await cmd.ExecuteReaderAsync() + await using var rd = await cmd.ExecuteReaderAsync(token) .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { rd.GetValues(values); var path = values[1] as string; @@ -644,7 +656,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", values[1]) .SetParameterValue("@BlocksetID", values[2]) .SetParameterValue("@MetadataID", values[3]) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } @@ -654,19 +666,19 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_ID"" ON ""{m_tempfiletable}"" (""ID"") - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_TargetPath"" ON ""{m_tempfiletable}"" (""TargetPath"") - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_Path"" ON ""{m_tempfiletable}"" (""Path"") - ") + ", token) .ConfigureAwait(false); cmd.SetCommandAndParameters($@" @@ -678,12 +690,12 @@ namespace Duplicati.Library.Main.Database ""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" "); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) { var filecount = 0L; var filesize = 0L; - if (await rd.ReadAsync().ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) { filecount = rd.ConvertValueToInt64(0, 0); filesize = rd.ConvertValueToInt64(1, 0); @@ -697,7 +709,7 @@ namespace Duplicati.Library.Main.Database } } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } return new Tuple(0, 0); @@ -706,8 +718,9 @@ namespace Duplicati.Library.Main.Database /// /// Retrieves the first path from the temporary file table. /// + /// A cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns the first path found in the temporary file table, or null if no paths are found. - public async Task GetFirstPath() + public async Task GetFirstPath(CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" SELECT ""Path"" @@ -717,7 +730,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - var v0 = await cmd.ExecuteScalarAsync().ConfigureAwait(false); + var v0 = await cmd.ExecuteScalarAsync(token).ConfigureAwait(false); if (v0 == null || v0 == DBNull.Value) return null; @@ -727,8 +740,9 @@ namespace Duplicati.Library.Main.Database /// /// Retrieves the largest prefix path from the temporary file table. /// + /// A cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns the largest prefix path found in the temporary file table. - public async Task GetLargestPrefix() + public async Task GetLargestPrefix(CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" SELECT ""Path"" @@ -738,7 +752,7 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - var v0 = await cmd.ExecuteScalarAsync().ConfigureAwait(false); + var v0 = await cmd.ExecuteScalarAsync(token).ConfigureAwait(false); var maxpath = ""; if (v0 != null && v0 != DBNull.Value) maxpath = v0.ToString()!; @@ -748,7 +762,7 @@ namespace Duplicati.Library.Main.Database var filecount = await cmd.ExecuteScalarInt64Async($@" SELECT COUNT(*) FROM ""{m_tempfiletable}"" - ", -1) + ", -1, token) .ConfigureAwait(false); var foundfiles = -1L; @@ -766,7 +780,7 @@ namespace Duplicati.Library.Main.Database foundfiles = await cmd .SetParameterValue("@PrefixLength", mp.Length) .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, token) .ConfigureAwait(false); if (filecount != foundfiles) @@ -789,11 +803,12 @@ namespace Duplicati.Library.Main.Database /// /// The largest prefix path to use for adjusting target paths. /// The destination path to prepend to the target paths. + /// A cancellation token to monitor for cancellation requests. /// A task that completes when the target paths have been set. - public async Task SetTargetPaths(string largest_prefix, string destination) + public async Task SetTargetPaths(string largest_prefix, string destination, CancellationToken token) { var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? - await GetFirstPath().ConfigureAwait(false) + await GetFirstPath(token).ConfigureAwait(false) : largest_prefix); @@ -816,7 +831,7 @@ namespace Duplicati.Library.Main.Database THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -827,7 +842,7 @@ namespace Duplicati.Library.Main.Database THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END - ") + ", token) .ConfigureAwait(false); } @@ -844,7 +859,7 @@ namespace Duplicati.Library.Main.Database END ") .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } else @@ -853,7 +868,7 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = ""Path"" - ") + ", token) .ConfigureAwait(false); } } @@ -872,7 +887,7 @@ namespace Duplicati.Library.Main.Database THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END - ") + ", token) .ConfigureAwait(false); // For UNC paths, we use \\server\folder -> / / @@ -884,7 +899,7 @@ namespace Duplicati.Library.Main.Database THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END - ") + ", token) .ConfigureAwait(false); } else @@ -895,7 +910,7 @@ namespace Duplicati.Library.Main.Database SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength) ") .SetParameterValue("@PrefixLength", largest_prefix.Length + 1) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } @@ -906,14 +921,14 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/') - ") + ", token) .ConfigureAwait(false); else if (OperatingSystem.IsWindows() && dirsep == "/") // For Linux paths on Windows await cmd.ExecuteNonQueryAsync($@" UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\') - ") + ", token) .ConfigureAwait(false); if (!string.IsNullOrEmpty(destination)) @@ -925,19 +940,20 @@ namespace Duplicati.Library.Main.Database SET ""TargetPath"" = @Destination || ""TargetPath"" ") .SetParameterValue("@Destination", Util.AppendDirSeparator(destination)) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// /// Finds missing blocks in the temporary block table. /// /// If true, skips metadata blocks when searching for missing blocks. + /// A cancellation token to monitor for cancellation requests. /// A task that completes when the missing blocks have been found and inserted into the temporary block table. - public async Task FindMissingBlocks(bool skipMetadata) + public async Task FindMissingBlocks(bool skipMetadata, CancellationToken token) { await using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); @@ -968,7 +984,7 @@ namespace Duplicati.Library.Main.Database WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - ") + ", token) .ConfigureAwait(false); var p2 = 0; @@ -1002,32 +1018,32 @@ namespace Duplicati.Library.Main.Database ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - ") + ", token) .ConfigureAwait(false); // Creating indexes after insertion is much faster await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" ON ""{m_tempblocktable}"" (""Hash"", ""Size"") - ") + ", token) .ConfigureAwait(false); // Better suited to speed up commit on UpdateBlocks await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" ON ""{m_tempblocktable}"" (""FileId"", ""Index"") - ") + ", token) .ConfigureAwait(false); var size = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_tempblocktable}"" - ", 0) + ", 0, token) .ConfigureAwait(false); Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// @@ -1035,8 +1051,9 @@ namespace Duplicati.Library.Main.Database /// /// The ID of the file in the temporary file table. /// The new target path to set for the file. + /// A cancellation token to monitor for cancellation requests. /// A task that completes when the target path has been updated. - public async Task UpdateTargetPath(long ID, string newname) + public async Task UpdateTargetPath(long ID, string newname, CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" UPDATE ""{m_tempfiletable}"" @@ -1047,10 +1064,10 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@TargetPath", newname) .SetParameterValue("@ID", ID) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// @@ -1096,7 +1113,8 @@ namespace Duplicati.Library.Main.Database /// /// An asynchronous enumerable of the blocks in the file. /// - IAsyncEnumerable Blocks(); + /// A cancellation token to monitor for cancellation requests. + IAsyncEnumerable Blocks(CancellationToken token); } /// @@ -1146,7 +1164,8 @@ namespace Duplicati.Library.Main.Database /// /// An asynchronous enumerable of block sources for this block descriptor. /// - IAsyncEnumerable BlockSources(); + /// A cancellation token to monitor for cancellation requests. + IAsyncEnumerable BlockSources(CancellationToken token); } /// @@ -1165,7 +1184,8 @@ namespace Duplicati.Library.Main.Database /// /// An asynchronous enumerable of block descriptors for the file. /// - IAsyncEnumerable Blocks(); + /// A cancellation token to monitor for cancellation requests. + IAsyncEnumerable Blocks(CancellationToken token); } /// @@ -1222,7 +1242,8 @@ namespace Duplicati.Library.Main.Database /// /// Gets the size of the remote volume in bytes. /// - IAsyncEnumerable Blocks(); + /// A cancellation token to monitor for cancellation requests. + IAsyncEnumerable Blocks(CancellationToken token); } /// @@ -1281,14 +1302,15 @@ namespace Duplicati.Library.Main.Database /// /// Gets the blocks in the existing file. /// + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of IExistingFileBlock objects representing the blocks in the file. - public async IAsyncEnumerable Blocks() + public async IAsyncEnumerable Blocks([EnumeratorCancellation] CancellationToken token) { string p = TargetPath; while (HasMore && p == TargetPath) { yield return new ExistingFileBlock(m_reader); - HasMore = await m_reader.ReadAsync().ConfigureAwait(false); + HasMore = await m_reader.ReadAsync(token).ConfigureAwait(false); } } @@ -1297,8 +1319,9 @@ namespace Duplicati.Library.Main.Database /// /// The LocalDatabase instance to query. /// The name of the table containing existing files. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of IExistingFile objects representing the existing files with their blocks. - public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename) + public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename, [EnumeratorCancellation] CancellationToken token) { await using var cmd = db.Connection.CreateCommand($@" SELECT @@ -1323,8 +1346,8 @@ namespace Duplicati.Library.Main.Database ""BlocksetEntry"".""Index"" ") .SetTransaction(db.Transaction); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) { var more = true; while (more) @@ -1335,7 +1358,7 @@ namespace Duplicati.Library.Main.Database more = f.HasMore; while (more && current == f.TargetPath) - more = await rd.ReadAsync().ConfigureAwait(false); + more = await rd.ReadAsync(token).ConfigureAwait(false); } } } @@ -1344,12 +1367,13 @@ namespace Duplicati.Library.Main.Database /// /// Retrieves existing files with their blocks from the temporary file table. /// + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of IExistingFile objects representing the existing files with their blocks. - public IAsyncEnumerable GetExistingFilesWithBlocks() + public IAsyncEnumerable GetExistingFilesWithBlocks(CancellationToken token) { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); - return ExistingFile.GetExistingFilesWithBlocks(this, m_tempfiletable); + return ExistingFile.GetExistingFilesWithBlocks(this, m_tempfiletable, token); } /// @@ -1417,7 +1441,8 @@ namespace Duplicati.Library.Main.Database /// public bool HasMore { get; private set; } - public async IAsyncEnumerable BlockSources() + /// + public async IAsyncEnumerable BlockSources([EnumeratorCancellation] CancellationToken token) { var p = TargetPath; var h = Hash; @@ -1426,7 +1451,7 @@ namespace Duplicati.Library.Main.Database while (HasMore && p == TargetPath && h == Hash && s == Size) { yield return new BlockSource(m_reader); - HasMore = await m_reader.ReadAsync().ConfigureAwait(false); + HasMore = await m_reader.ReadAsync(token).ConfigureAwait(false); } } @@ -1455,7 +1480,8 @@ namespace Duplicati.Library.Main.Database /// public bool HasMore { get; private set; } - public async IAsyncEnumerable Blocks() + /// + public async IAsyncEnumerable Blocks([EnumeratorCancellation] CancellationToken token) { var p = TargetPath; while (HasMore && p == TargetPath) @@ -1468,7 +1494,7 @@ namespace Duplicati.Library.Main.Database HasMore = c.HasMore; while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) - HasMore = await m_reader.ReadAsync().ConfigureAwait(false); + HasMore = await m_reader.ReadAsync(token).ConfigureAwait(false); } } @@ -1480,8 +1506,9 @@ namespace Duplicati.Library.Main.Database /// The name of the block table to query. /// The size of each block in bytes. /// If true, skips metadata blocks when retrieving files and source blocks. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of ILocalBlockSource objects representing the files and their source blocks. - public static async IAsyncEnumerable GetFilesAndSourceBlocks(LocalDatabase db, string filetablename, string blocktablename, long blocksize, bool skipMetadata) + public static async IAsyncEnumerable GetFilesAndSourceBlocks(LocalDatabase db, string filetablename, string blocktablename, long blocksize, bool skipMetadata, [EnumeratorCancellation] CancellationToken token) { // TODO: Skip metadata as required // Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows @@ -1515,8 +1542,8 @@ namespace Duplicati.Library.Main.Database ""B"".""Index"" ") .SetTransaction(db.Transaction); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - var more = await rd.ReadAsync().ConfigureAwait(false); + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + var more = await rd.ReadAsync(token).ConfigureAwait(false); while (more) { @@ -1526,7 +1553,7 @@ namespace Duplicati.Library.Main.Database more = f.HasMore; while (more && current == f.TargetPath) - more = await rd.ReadAsync().ConfigureAwait(false); + more = await rd.ReadAsync(token).ConfigureAwait(false); } } } @@ -1536,19 +1563,21 @@ namespace Duplicati.Library.Main.Database /// /// If true, skips metadata blocks when retrieving files and source blocks. /// The size of each block in bytes. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of ILocalBlockSource objects representing the files and their source blocks. - public IAsyncEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize) + public IAsyncEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize, CancellationToken token) { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); - return LocalBlockSource.GetFilesAndSourceBlocks(this, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata); + return LocalBlockSource.GetFilesAndSourceBlocks(this, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata, token); } /// /// Retrieves the missing remote volumes that need to be restored. /// + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of IRemoteVolume objects representing the missing remote volumes. - public async IAsyncEnumerable GetMissingVolumes() + public async IAsyncEnumerable GetMissingVolumes([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" SELECT @@ -1580,8 +1609,8 @@ namespace Duplicati.Library.Main.Database // Now it is likely to restore all files from front to back. Large files will always be done last. // One could also use like the average block number in a volume, that needs to be measured. - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { yield return new RemoteVolume( rd.ConvertValueToString(0), @@ -1598,8 +1627,16 @@ namespace Duplicati.Library.Main.Database { /// /// Retrieves files with missing blocks. - IAsyncEnumerable FilesWithMissingBlocks(); - IAsyncEnumerable MetadataWithMissingBlocks(); + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IVolumePatch objects representing the files with missing blocks. + IAsyncEnumerable FilesWithMissingBlocks(CancellationToken token); + /// + /// Retrieves metadata with missing blocks. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IVolumePatch objects representing the metadata with missing blocks. + IAsyncEnumerable MetadataWithMissingBlocks(CancellationToken token); } private class FilesAndMetadata : IFilesAndMetadata @@ -1619,7 +1656,17 @@ namespace Duplicati.Library.Main.Database private FilesAndMetadata() { } - public static async Task CreateAsync(LocalDatabase db, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + /// + /// Creates a new instance of FilesAndMetadata, initializing the temporary table and inserting blocks from the current volume. + /// + /// The LocalDatabase instance to use for database operations. + /// The name of the file table in the database. + /// The name of the block table in the database. + /// The size of each block in bytes. + /// The BlockVolumeReader containing the current volume's blocks. + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited returns an instance of . + public static async Task CreateAsync(LocalDatabase db, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume, CancellationToken token) { var fam = new FilesAndMetadata() { @@ -1637,7 +1684,7 @@ namespace Duplicati.Library.Main.Database ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL ) - ") + ", token) .ConfigureAwait(false); c.SetCommandAndParameters($@" @@ -1654,13 +1701,13 @@ namespace Duplicati.Library.Main.Database { await c.SetParameterValue("@Hash", s.Key) .SetParameterValue("@Size", s.Value) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks - await db.Transaction.CommitAsync().ConfigureAwait(false); + await db.Transaction.CommitAsync(token: token).ConfigureAwait(false); return fam; } @@ -1725,13 +1772,14 @@ namespace Duplicati.Library.Main.Database public long FileID { get { return m_reader.ConvertValueToInt64(1); } } public bool HasMore { get; private set; } - public async IAsyncEnumerable Blocks() + /// + public async IAsyncEnumerable Blocks([EnumeratorCancellation] CancellationToken token) { string p = Path; while (HasMore && p == Path) { yield return new PatchBlock(m_reader); - HasMore = await m_reader.ReadAsync().ConfigureAwait(false); + HasMore = await m_reader.ReadAsync(token).ConfigureAwait(false); } } } @@ -1739,8 +1787,9 @@ namespace Duplicati.Library.Main.Database /// /// Retrieves the files with missing blocks. /// + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of IVolumePatch objects representing the files with missing blocks. - public async IAsyncEnumerable FilesWithMissingBlocks() + public async IAsyncEnumerable FilesWithMissingBlocks([EnumeratorCancellation] CancellationToken token) { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! await using var cmd = m_db.Connection.CreateCommand($@" @@ -1772,8 +1821,8 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_db.Transaction); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) { var more = true; while (more) @@ -1784,7 +1833,7 @@ namespace Duplicati.Library.Main.Database more = f.HasMore; while (more && current == f.Path) - more = await rd.ReadAsync().ConfigureAwait(false); + more = await rd.ReadAsync(token).ConfigureAwait(false); } } } @@ -1792,8 +1841,9 @@ namespace Duplicati.Library.Main.Database /// /// Retrieves the metadata with missing blocks. /// + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of IVolumePatch objects representing the metadata with missing blocks. - public async IAsyncEnumerable MetadataWithMissingBlocks() + public async IAsyncEnumerable MetadataWithMissingBlocks([EnumeratorCancellation] CancellationToken token) { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! await using var cmd = m_db.Connection.CreateCommand($@" @@ -1825,8 +1875,8 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_db.Transaction); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) { var more = true; while (more) @@ -1837,7 +1887,7 @@ namespace Duplicati.Library.Main.Database more = f.HasMore; while (more && current == f.Path) - more = await rd.ReadAsync().ConfigureAwait(false); + more = await rd.ReadAsync(token).ConfigureAwait(false); } } } @@ -1848,20 +1898,22 @@ namespace Duplicati.Library.Main.Database /// /// The size of each block in bytes. /// The current volume reader containing the blocks to restore. + /// A cancellation token to monitor for cancellation requests. /// A task that when awaited returns an IFilesAndMetadata instance containing the files and metadata with missing blocks. - public async Task GetMissingBlockData(BlockVolumeReader curvolume, long blocksize) + public async Task GetMissingBlockData(BlockVolumeReader curvolume, long blocksize, CancellationToken token) { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); - return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume) + return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume, token) .ConfigureAwait(false); } /// /// Returns a connection and transaction from the connection pool. /// + /// A cancellation token to monitor for cancellation requests. /// A task that when awaited returns a tuple containing the SqliteConnection and ReusableTransaction. - public async Task<(SqliteConnection, ReusableTransaction)> GetConnectionFromPool() + public async Task<(SqliteConnection, ReusableTransaction)> GetConnectionFromPool(CancellationToken token) { if (!m_connection_pool.TryTake(out var entry)) { @@ -1869,7 +1921,7 @@ namespace Duplicati.Library.Main.Database .ConfigureAwait(false); connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared"; - await connection.OpenAsync().ConfigureAwait(false); + await connection.OpenAsync(token).ConfigureAwait(false); await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize) .ConfigureAwait(false); @@ -1909,8 +1961,9 @@ namespace Duplicati.Library.Main.Database /// Returns a list of files to restore, including their target paths, hashes, and lengths. /// /// If true, only returns files that have not been verified. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of IFileToRestore objects representing the files to restore. - public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified) + public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" SELECT @@ -1928,8 +1981,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@Verified", !onlyNonVerified); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new FileToRestore( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); } @@ -1937,8 +1990,9 @@ namespace Duplicati.Library.Main.Database /// /// Returns a list of files and symlinks to restore. /// + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of FileRequest objects representing the files and symlinks to restore. - public async IAsyncEnumerable GetFilesAndSymlinksToRestore() + public async IAsyncEnumerable GetFilesAndSymlinksToRestore([EnumeratorCancellation] CancellationToken token) { // Order by length descending, so that larger files are restored first. await using var cmd = m_connection.CreateCommand($@" @@ -1956,8 +2010,8 @@ namespace Duplicati.Library.Main.Database ORDER BY ""Length"" DESC ") .SetTransaction(m_rtr); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new FileRequest( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), @@ -1971,8 +2025,9 @@ namespace Duplicati.Library.Main.Database /// /// Returns a list of folders to restore. Used to restore folder metadata. /// + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of FileRequest objects representing the folders to restore. - public async IAsyncEnumerable GetFolderMetadataToRestore() + public async IAsyncEnumerable GetFolderMetadataToRestore([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(); cmd.SetTransaction(m_rtr); @@ -1989,10 +2044,10 @@ namespace Duplicati.Library.Main.Database ""F"".""BlocksetID"" = {FOLDER_BLOCKSET_ID} AND ""F"".""MetadataID"" IS NOT NULL AND ""F"".""MetadataID"" >= 0 - ") + ", token) .ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new FileRequest( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), @@ -2007,8 +2062,9 @@ namespace Duplicati.Library.Main.Database /// Returns a list of blocks and their volume IDs. Used by the to keep track of blocks and volumes to automatically evict them from the respective caches. /// /// Flag indicating whether the returned blocks should exclude the metadata blocks. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of tuples containing the block ID and volume ID. - public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) + public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata, [EnumeratorCancellation] CancellationToken token) { var metadata_query = skipMetadata ? "" : $@" UNION ALL @@ -2037,8 +2093,8 @@ namespace Duplicati.Library.Main.Database ") .SetTransaction(m_rtr); - await using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await reader.ReadAsync().ConfigureAwait(false)) + await using var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await reader.ReadAsync(token).ConfigureAwait(false)) yield return ( reader.ConvertValueToInt64(0), reader.ConvertValueToInt64(1) @@ -2049,10 +2105,11 @@ namespace Duplicati.Library.Main.Database /// Returns a list of for the given blockset ID. It is used by the to restore the blocks of a file. /// /// The BlocksetID of the file. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of representing the blocks of the file. - public async IAsyncEnumerable GetBlocksFromFile(long blocksetID) + public async IAsyncEnumerable GetBlocksFromFile(long blocksetID, [EnumeratorCancellation] CancellationToken token) { - var (connection, transaction) = await GetConnectionFromPool() + var (connection, transaction) = await GetConnectionFromPool(token) .ConfigureAwait(false); try @@ -2073,9 +2130,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(transaction) .SetParameterValue("@BlocksetID", blocksetID); - await using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync(token) .ConfigureAwait(false); - for (long i = 0; await reader.ReadAsync().ConfigureAwait(false); i++) + for (long i = 0; await reader.ReadAsync(token).ConfigureAwait(false); i++) yield return new BlockRequest( reader.ConvertValueToInt64(0), i, @@ -2096,10 +2153,11 @@ namespace Duplicati.Library.Main.Database /// Returns a list of for the metadata blocks of the given file. It is used by the to restore the metadata of a file. /// /// The ID of the file. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of representing the metadata blocks of the file. - public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID) + public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID, [EnumeratorCancellation] CancellationToken token) { - var (connection, transaction) = await GetConnectionFromPool() + var (connection, transaction) = await GetConnectionFromPool(token) .ConfigureAwait(false); try { @@ -2121,9 +2179,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(transaction) .SetParameterValue("@FileID", fileID); - await using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync(token) .ConfigureAwait(false); - for (long i = 0; await reader.ReadAsync().ConfigureAwait(false); i++) + for (long i = 0; await reader.ReadAsync(token).ConfigureAwait(false); i++) { yield return new BlockRequest( reader.ConvertValueToInt64(0), @@ -2146,8 +2204,9 @@ namespace Duplicati.Library.Main.Database /// Returns the volume information for the given volume ID. It is used by the to get the volume information for the given volume ID. /// /// The ID of the volume. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of tuples containing the volume name, size, and hash. - public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) + public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID, [EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(@" SELECT @@ -2160,9 +2219,9 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@VolumeID", VolumeID); - await using var reader = await cmd.ExecuteReaderAsync() + await using var reader = await cmd.ExecuteReaderAsync(token) .ConfigureAwait(false); - while (await reader.ReadAsync().ConfigureAwait(false)) + while (await reader.ReadAsync(token).ConfigureAwait(false)) yield return ( reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), @@ -2173,8 +2232,9 @@ namespace Duplicati.Library.Main.Database /// /// Drops all temporary tables used for the restore process and commits the transaction. /// + /// A cancellation token to monitor for cancellation requests. /// A task that completes when the temporary tables are dropped and the transaction is committed. - public async Task DropRestoreTable() + public async Task DropRestoreTable(CancellationToken token) { await using var cmd = m_connection.CreateCommand() .SetTransaction(m_rtr); @@ -2182,7 +2242,7 @@ namespace Duplicati.Library.Main.Database if (m_tempfiletable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""", token) .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -2191,7 +2251,7 @@ namespace Duplicati.Library.Main.Database if (m_tempblocktable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""", token) .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -2200,7 +2260,7 @@ namespace Duplicati.Library.Main.Database if (m_latestblocktable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""", token) .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -2209,7 +2269,7 @@ namespace Duplicati.Library.Main.Database if (m_fileprogtable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""", token) .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -2218,7 +2278,7 @@ namespace Duplicati.Library.Main.Database if (m_totalprogtable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""", token) .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -2227,13 +2287,13 @@ namespace Duplicati.Library.Main.Database if (m_filesnewlydonetable != null) try { - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""", token) .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_filesnewlydonetable = null; } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// @@ -2249,43 +2309,49 @@ namespace Duplicati.Library.Main.Database /// The hash of the block. /// The size of the block. /// Indicates whether the block is metadata. + /// A cancellation token to monitor for cancellation requests. /// A task that completes when the block is marked as restored. - Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata); + Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata, CancellationToken token); /// /// Marks all blocks of a specific file as missing. /// /// The ID of the target file. + /// A cancellation token to monitor for cancellation requests. /// A task that completes when all blocks of the file are marked as missing. - Task SetAllBlocksMissing(long targetfileid); + Task SetAllBlocksMissing(long targetfileid, CancellationToken token); /// /// Marks all blocks of a specific file as restored. /// /// The ID of the target file. /// Indicates whether to include metadata blocks. + /// A cancellation token to monitor for cancellation requests. /// A task that completes when all blocks of the file are marked as restored. - Task SetAllBlocksRestored(long targetfileid, bool includeMetadata); + Task SetAllBlocksRestored(long targetfileid, bool includeMetadata, CancellationToken token); /// /// Marks the data of a specific file as verified. /// /// The ID of the target file. + /// A cancellation token to monitor for cancellation requests. /// A task that completes when the file data is marked as verified. - Task SetFileDataVerified(long targetfileid); + Task SetFileDataVerified(long targetfileid, CancellationToken token); /// /// Commits the changes made by the block marker. /// + /// A cancellation token to monitor for cancellation requests. /// A task that completes when the changes are committed. - Task CommitAsync(); + Task CommitAsync(CancellationToken token); /// /// Updates the processed statistics for the blocks and files. /// /// An operation progress updater to report the processed statistics. + /// A cancellation token to monitor for cancellation requests. /// A task that completes when the processed statistics are updated. - Task UpdateProcessed(IOperationProgressUpdater writer); + Task UpdateProcessed(IOperationProgressUpdater writer, CancellationToken token); } /// @@ -2360,8 +2426,9 @@ namespace Duplicati.Library.Main.Database /// The name of the temporary block table. /// The name of the temporary file table. /// The name of the statistics table, or null if not available. + /// A cancellation token to monitor for cancellation requests. /// A task that when awaited returns a new instance of the class. - public static async Task CreateAsync(LocalDatabase db, string blocktablename, string filetablename, string statstablename) + public static async Task CreateAsync(LocalDatabase db, string blocktablename, string filetablename, string statstablename, CancellationToken token) { var dbm = new DirectBlockMarker() { @@ -2378,14 +2445,14 @@ namespace Duplicati.Library.Main.Database AND ""Size"" = @Size AND ""Metadata"" = @Metadata AND ""Restored"" = 0 - ") + ", token) .ConfigureAwait(false), m_resetfileCommand = await db.Connection.CreateCommandAsync($@" UPDATE ""{blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = @TargetFileId - ") + ", token) .ConfigureAwait(false), m_updateAsRestoredCommand = await db.Connection.CreateCommandAsync($@" @@ -2394,14 +2461,14 @@ namespace Duplicati.Library.Main.Database WHERE ""FileID"" = @TargetFileId AND ""Metadata"" <= @Metadata - ") + ", token) .ConfigureAwait(false), m_updateFileAsDataVerifiedCommand = await db.Connection.CreateCommandAsync($@" UPDATE ""{filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = @TargetFileId - ") + ", token) .ConfigureAwait(false), m_statUpdateCommand = statstablename == null ? @@ -2412,7 +2479,7 @@ namespace Duplicati.Library.Main.Database SUM(""Size"") FROM ""{blocktablename}"" WHERE ""Restored"" = 1 - ") + ", token) .ConfigureAwait(false) : // Fields in Stats: TotalFiles, TotalBlocks, TotalSize @@ -2422,14 +2489,15 @@ namespace Duplicati.Library.Main.Database SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" - ") + ", token) .ConfigureAwait(false) }; return dbm; } - public async Task UpdateProcessed(IOperationProgressUpdater updater) + /// + public async Task UpdateProcessed(IOperationProgressUpdater updater, CancellationToken token) { if (!m_hasUpdates) return; @@ -2437,7 +2505,7 @@ namespace Duplicati.Library.Main.Database m_hasUpdates = false; await using var rd = await m_statUpdateCommand .SetTransaction(m_db.Transaction) - .ExecuteReaderAsync() + .ExecuteReaderAsync(token) .ConfigureAwait(false); var filesprocessed = 0L; @@ -2452,20 +2520,22 @@ namespace Duplicati.Library.Main.Database updater.UpdatefilesProcessed(filesprocessed, processedsize); } - public async Task SetAllBlocksMissing(long targetfileid) + // + public async Task SetAllBlocksMissing(long targetfileid, CancellationToken token) { m_hasUpdates = true; m_resetfileCommand .SetTransaction(m_db.Transaction) .SetParameterValue("@TargetFileId", targetfileid); - var r = await m_resetfileCommand.ExecuteNonQueryAsync() + var r = await m_resetfileCommand.ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (r <= 0) throw new Exception("Unexpected reset result"); } - public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata) + /// + public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata, CancellationToken token) { m_hasUpdates = true; m_updateAsRestoredCommand @@ -2474,13 +2544,14 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Metadata", includeMetadata ? 1 : 0); var r = await m_updateAsRestoredCommand - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (r <= 0) throw new Exception("Unexpected reset result"); } - public async Task SetFileDataVerified(long targetfileid) + /// + public async Task SetFileDataVerified(long targetfileid, CancellationToken token) { m_hasUpdates = true; m_updateFileAsDataVerifiedCommand @@ -2488,13 +2559,14 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TargetFileId", targetfileid); var r = await m_updateFileAsDataVerifiedCommand - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (r != 1) throw new Exception("Unexpected result when marking file as verified."); } - public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata) + /// + public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata, CancellationToken token) { m_hasUpdates = true; m_insertblockCommand @@ -2506,17 +2578,18 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Metadata", metadata); var r = await m_insertblockCommand - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (r != 1) throw new Exception("Unexpected result when marking block."); } - public async Task CommitAsync() + /// + public async Task CommitAsync(CancellationToken token) { using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) - await m_db.Transaction.CommitAsync().ConfigureAwait(false); + await m_db.Transaction.CommitAsync(token: token).ConfigureAwait(false); } public void Dispose() @@ -2537,14 +2610,15 @@ namespace Duplicati.Library.Main.Database /// /// Creates a new block marker for marking blocks as restored, missing, or verified. /// + /// A cancellation token to monitor for cancellation requests. /// A task that when awaited returns an instance of . - public async Task CreateBlockMarkerAsync() + public async Task CreateBlockMarkerAsync(CancellationToken token) { if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) throw new InvalidOperationException("No temporary file table set up for this restore."); if (string.IsNullOrWhiteSpace(m_totalprogtable)) throw new InvalidOperationException("No progress table set up for this restore."); - return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable) + return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable, token) .ConfigureAwait(false); } @@ -2556,7 +2630,7 @@ namespace Duplicati.Library.Main.Database public override async ValueTask DisposeAsync() { await DisposePoolAsync().ConfigureAwait(false); - await DropRestoreTable().ConfigureAwait(false); + await DropRestoreTable(default).ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } @@ -2578,8 +2652,9 @@ namespace Duplicati.Library.Main.Database /// /// Returns a list of target folders for the folder blockset. /// + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of strings representing the target folders. - public async IAsyncEnumerable GetTargetFolders() + public async IAsyncEnumerable GetTargetFolders([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand($@" SELECT ""TargetPath"" @@ -2589,8 +2664,8 @@ namespace Duplicati.Library.Main.Database .SetTransaction(m_rtr) .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? ""; } @@ -2614,7 +2689,9 @@ namespace Duplicati.Library.Main.Database /// /// Gets an asynchronous enumerable of block entries for the file. /// - IAsyncEnumerable Blocks(); + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of representing the blocks of the file. + IAsyncEnumerable Blocks(CancellationToken token); } /// @@ -2706,14 +2783,15 @@ namespace Duplicati.Library.Main.Database public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } public string SourcePath { get { return m_rd.ConvertValueToString(1) ?? ""; } } - public async IAsyncEnumerable Blocks() + /// + public async IAsyncEnumerable Blocks([EnumeratorCancellation] CancellationToken token) { var tid = TargetFileID; do { yield return new BlockEntry(m_rd, m_blocksize); - } while ((MoreData = await m_rd.ReadAsync().ConfigureAwait(false)) && tid == TargetFileID); + } while ((MoreData = await m_rd.ReadAsync(token).ConfigureAwait(false)) && tid == TargetFileID); } } @@ -2721,8 +2799,9 @@ namespace Duplicati.Library.Main.Database /// Returns a list of files and their source blocks for fast restoration. /// /// The size of the blocks to be used for restoration. + /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of representing the files and their source blocks. - public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize) + public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize, [EnumeratorCancellation] CancellationToken token) { await using (var cmdReader = m_connection.CreateCommand()) await using (var cmd = m_connection.CreateCommand()) @@ -2738,16 +2817,16 @@ namespace Duplicati.Library.Main.Database SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"" "); - await using (var rd = await cmdReader.ExecuteReaderAsync().ConfigureAwait(false)) + await using (var rd = await cmdReader.ExecuteReaderAsync(token).ConfigureAwait(false)) { - while (await rd.ReadAsync().ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var sourcepath = rd.ConvertValueToString(0); if (SystemIO.IO_OS.FileExists(sourcepath)) { await cmd .SetParameterValue("@Path", sourcepath) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } else @@ -2761,10 +2840,10 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"") - ") + ", token) .ConfigureAwait(false); - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; @@ -2803,16 +2882,16 @@ namespace Duplicati.Library.Main.Database await using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ", token) .ConfigureAwait(false); - await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}") + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_latestblocktable}_path"" ON ""{m_latestblocktable}"" (""Path"") - ") + ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" @@ -2822,7 +2901,7 @@ namespace Duplicati.Library.Main.Database FROM ""{m_latestblocktable}"" WHERE ""Path"" = ""{m_tempfiletable}"".""Path"" ) - ") + ", token) .ConfigureAwait(false); } @@ -2852,9 +2931,9 @@ namespace Duplicati.Library.Main.Database await using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); - await using var rd = await cmd.ExecuteReaderAsync(sources) + await using var rd = await cmd.ExecuteReaderAsync(sources, token) .ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) { bool more; do @@ -2865,13 +2944,13 @@ namespace Duplicati.Library.Main.Database more = n.MoreData; while (more && n.TargetFileID == tid) - more = await rd.ReadAsync().ConfigureAwait(false); + more = await rd.ReadAsync(token).ConfigureAwait(false); } while (more); } } - await m_rtr.CommitAsync().ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } From 4f7760f09227d5820e35864ab1cee4aa2e1e6d96 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 08:04:20 +0200 Subject: [PATCH 335/362] Made the LocalTestDatabase methods accep a cancellation token --- .../Main/Database/LocalTestDatabase.cs | 180 +++++++++++------- 1 file changed, 107 insertions(+), 73 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 4eddb114e..95b7d9ba2 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -27,6 +27,8 @@ using System.Threading.Tasks; using System.Collections.Generic; using Microsoft.Data.Sqlite; using Duplicati.Library.Utility; +using System.Threading; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Main.Database { @@ -43,13 +45,14 @@ namespace Duplicati.Library.Main.Database /// The path to the database file. /// The size of the page cache in bytes. /// An optional existing instance to use. Used to mimic constructor chaining. + /// A cancellation token to observe while waiting for the task to complete. /// A task that represents the asynchronous operation. The task result contains the created instance. - public static async Task CreateAsync(string path, long pagecachesize, LocalTestDatabase? dbnew = null) + public static async Task CreateAsync(string path, long pagecachesize, LocalTestDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalTestDatabase(); dbnew = (LocalTestDatabase) - await CreateLocalDatabaseAsync(path, "Test", true, pagecachesize, dbnew) + await CreateLocalDatabaseAsync(path, "Test", true, pagecachesize, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -62,13 +65,14 @@ namespace Duplicati.Library.Main.Database /// /// The parent database to use for creating the new test database. /// An optional existing instance to use. Used to mimic constructor chaining. + /// A cancellation token to observe while waiting for the task to complete. /// A task that represents the asynchronous operation. The task result contains the created instance. - public static async Task CreateAsync(LocalDatabase dbparent, LocalTestDatabase? dbnew = null) + public static async Task CreateAsync(LocalDatabase dbparent, LocalTestDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalTestDatabase(); return (LocalTestDatabase) - await CreateLocalDatabaseAsync(dbparent, dbnew) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) .ConfigureAwait(false); } @@ -77,8 +81,9 @@ namespace Duplicati.Library.Main.Database /// Increments the count by 1, or sets it to the maximum of 1 if it was previously 0 or negative. /// /// The name of the remote volume to update. + /// A cancellation token to observe while waiting for the task to complete. /// A task that represents the asynchronous operation. - public async Task UpdateVerificationCount(string name) + public async Task UpdateVerificationCount(string name, CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); await cmd.SetCommandAndParameters(@" @@ -96,7 +101,7 @@ namespace Duplicati.Library.Main.Database WHERE ""Name"" = @Name ") .SetParameterValue("@Name", name) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } @@ -197,10 +202,11 @@ namespace Duplicati.Library.Main.Database /// /// The number of remote volumes to select for testing. /// The options that define selection criteria, such as time, version, and verification strategy. + /// A cancellation token to observe while waiting for the task to complete. /// An asynchronous enumerable of representing the selected test targets. - public async IAsyncEnumerable SelectTestTargets(long samples, Options options) + public async IAsyncEnumerable SelectTestTargets(long samples, Options options, [EnumeratorCancellation] CancellationToken token) { - var tp = await GetFilelistWhereClause(options.Time, options.Version) + var tp = await GetFilelistWhereClause(options.Time, options.Version, null, false, token) .ConfigureAwait(false); samples = Math.Max(1, samples); @@ -237,13 +243,13 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Uploaded.ToString() ]); - await using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + await using (var rd = await cmd.ExecuteReaderAsync(token)) + while (await rd.ReadAsync(token)) yield return new RemoteVolume(rd); //First we select some filesets var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); - await using (var rd = cmd.SetCommandAndParameters(@$" + await using (var rd = await cmd.SetCommandAndParameters(@$" SELECT ""A"".""VolumeID"", ""A"".""Name"", @@ -274,8 +280,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) .SetParameterValues(tp.Item2) - .ExecuteReader()) - while (rd.Read()) + .ExecuteReaderAsync(token)) + while (await rd.ReadAsync(token)) files.Add(new RemoteVolume(rd)); if (files.Count == 0) @@ -307,8 +313,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) files.Add(new RemoteVolume(rd)); foreach (var f in FilterByVerificationCount(files, samples, max)) @@ -336,8 +342,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - while (await rd.ReadAsync().ConfigureAwait(false)) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) files.Add(new RemoteVolume(rd)); foreach (var f in FilterByVerificationCount(files, samples, max)) @@ -390,8 +396,9 @@ namespace Duplicati.Library.Main.Database /// The prefix for the temporary table name. /// The SQL format for creating the temporary table. /// The SQL command for inserting data into the temporary table. + /// A cancellation token to observe while waiting for the task to complete. /// A task that represents the asynchronous operation. The task result contains the initialized instance. - protected static async Task CreateAsync(Basiclist bl, LocalDatabase db, string volumename, string tablePrefix, string tableFormat, string insertCommand) + protected static async Task CreateAsync(Basiclist bl, LocalDatabase db, string volumename, string tablePrefix, string tableFormat, string insertCommand, CancellationToken token) { bl.m_db = db; bl.m_volumename = volumename; @@ -402,7 +409,7 @@ namespace Duplicati.Library.Main.Database await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat} - ") + ", token) .ConfigureAwait(false); bl.m_tablename = tablename; @@ -411,7 +418,7 @@ namespace Duplicati.Library.Main.Database bl.m_insertCommand = await bl.m_db.Connection.CreateCommandAsync($@" INSERT INTO ""{bl.m_tablename}"" {insertCommand} - ") + ", token) .ConfigureAwait(false); return bl; @@ -428,7 +435,7 @@ namespace Duplicati.Library.Main.Database try { await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction); - await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""", default) .ConfigureAwait(false); } catch { } @@ -455,14 +462,16 @@ namespace Duplicati.Library.Main.Database /// A collection of blocklist hashes associated with the file. /// The type of the file entry. /// The timestamp of the file entry. + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the file entry has been added. - Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time); + Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time, CancellationToken token); /// /// Asynchronously compares the file list with remote volumes and yields differences. /// + /// A cancellation token to observe while waiting for the task to complete. /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path. - IAsyncEnumerable> Compare(); + IAsyncEnumerable> Compare(CancellationToken token); } /// @@ -528,16 +537,18 @@ namespace Duplicati.Library.Main.Database /// /// The local database to use for the file list. /// The name of the volume associated with this file list. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when awaited returns a new instance of the class. - public static async Task CreateAsync(LocalDatabase db, string volumename) + public static async Task CreateAsync(LocalDatabase db, string volumename, CancellationToken token) { var bl = new Filelist(); return (Filelist) - await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND, token) .ConfigureAwait(false); } - public async Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time) + /// + public async Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time, CancellationToken token) { await m_insertCommand .SetTransaction(m_db.Transaction) @@ -546,11 +557,12 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", hash) .SetParameterValue("@Metasize", metasize) .SetParameterValue("@Metahash", metahash) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } - public async IAsyncEnumerable> Compare() + /// + public async IAsyncEnumerable> Compare([EnumeratorCancellation] CancellationToken token) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -635,7 +647,7 @@ namespace Duplicati.Library.Main.Database await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); cmd @@ -644,8 +656,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair( (Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "" @@ -657,7 +669,7 @@ namespace Duplicati.Library.Main.Database try { await cmd - .ExecuteNonQueryAsync(drop) + .ExecuteNonQueryAsync(drop, token) .ConfigureAwait(false); } catch { } @@ -677,14 +689,16 @@ namespace Duplicati.Library.Main.Database /// The name of the file associated with the block link. /// The hash of the block link. /// The length of the block link in bytes. + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the block link has been added. - Task AddBlockLink(string filename, string hash, long length); + Task AddBlockLink(string filename, string hash, long length, CancellationToken token); /// /// Asynchronously compares the index list with remote volumes and yields differences. /// + /// A cancellation token to observe while waiting for the task to complete. /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path. - IAsyncEnumerable> Compare(); + IAsyncEnumerable> Compare(CancellationToken token); } /// @@ -744,27 +758,30 @@ namespace Duplicati.Library.Main.Database /// /// The local database to use for the index list. /// The name of the volume associated with this index list. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when awaited returns a new instance of the class. - public static async Task CreateAsync(LocalDatabase db, string volumename) + public static async Task CreateAsync(LocalDatabase db, string volumename, CancellationToken token) { var bl = new Indexlist(); return (Indexlist) - await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND, token) .ConfigureAwait(false); } - public async Task AddBlockLink(string filename, string hash, long length) + /// + public async Task AddBlockLink(string filename, string hash, long length, CancellationToken token) { await m_insertCommand .SetTransaction(m_db.Transaction) .SetParameterValue("@Name", filename) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", length) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } - public async IAsyncEnumerable> Compare() + /// + public async IAsyncEnumerable> Compare([EnumeratorCancellation] CancellationToken token) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var create = $@" @@ -828,7 +845,7 @@ namespace Duplicati.Library.Main.Database await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); cmd @@ -837,8 +854,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } @@ -847,7 +864,7 @@ namespace Duplicati.Library.Main.Database try { await cmd - .ExecuteNonQueryAsync(drop) + .ExecuteNonQueryAsync(drop, token) .ConfigureAwait(false); } catch { } @@ -866,14 +883,16 @@ namespace Duplicati.Library.Main.Database /// /// The key (hash) of the block. /// The size of the block in bytes. + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the block has been added. - Task AddBlock(string key, long value); + Task AddBlock(string key, long value, CancellationToken token); /// /// Asynchronously compares the blocklist with remote volumes and yields differences. /// + /// A cancellation token to observe while waiting for the task to complete. /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash. - IAsyncEnumerable> Compare(); + IAsyncEnumerable> Compare(CancellationToken token); } public interface IBlocklistHashList : IDisposable, IAsyncDisposable @@ -883,8 +902,9 @@ namespace Duplicati.Library.Main.Database /// /// The hash of the block. /// The size of the block in bytes. + /// A cancellation token to observe while waiting for the task to complete. /// A task that completes when the block hash has been added. - Task AddBlockHash(string hash, long size); + Task AddBlockHash(string hash, long size, CancellationToken token); /// /// Asynchronously compares the blocklist hash list with remote volumes and yields differences. @@ -892,8 +912,9 @@ namespace Duplicati.Library.Main.Database /// The number of hashes per block. /// The size of each hash in bytes. /// The size of each block in bytes. + /// A cancellation token to observe while waiting for the task to complete. /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash. - IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize); + IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize, CancellationToken token); } /// @@ -943,26 +964,29 @@ namespace Duplicati.Library.Main.Database /// Asynchronously creates a new instance of the class. /// /// The local database to use for the blocklist. + /// A cancellation token to observe while waiting for the task to complete. /// The name of the volume associated with this blocklist. - public static async Task CreateAsync(LocalDatabase db, string volumename) + public static async Task CreateAsync(LocalDatabase db, string volumename, CancellationToken token) { var bl = new Blocklist(); return (Blocklist) - await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND, token) .ConfigureAwait(false); } - public async Task AddBlock(string hash, long size) + /// + public async Task AddBlock(string hash, long size, CancellationToken token) { await m_insertCommand .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } - public async IAsyncEnumerable> Compare() + /// + public async IAsyncEnumerable> Compare([EnumeratorCancellation] CancellationToken token) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -1060,7 +1084,7 @@ namespace Duplicati.Library.Main.Database await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); cmd @@ -1073,8 +1097,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } @@ -1083,7 +1107,7 @@ namespace Duplicati.Library.Main.Database try { await cmd - .ExecuteNonQueryAsync(drop) + .ExecuteNonQueryAsync(drop, token) .ConfigureAwait(false); } catch { } @@ -1141,26 +1165,29 @@ namespace Duplicati.Library.Main.Database /// /// The local database to use for the blocklist hash list. /// The name of the volume associated with this blocklist hash list. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when awaited returns a new instance of the class. - public static async Task CreateAsync(LocalDatabase db, string volumename) + public static async Task CreateAsync(LocalDatabase db, string volumename, CancellationToken token) { var bl = new BlocklistHashList(); return (BlocklistHashList) - await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND, token) .ConfigureAwait(false); } - public async Task AddBlockHash(string hash, long size) + /// + public async Task AddBlockHash(string hash, long size, CancellationToken token) { await m_insertCommand .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } - public async IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize) + /// + public async IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize, [EnumeratorCancellation] CancellationToken token) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -1273,7 +1300,7 @@ namespace Duplicati.Library.Main.Database await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); // Compare against actual values inserted into temp table @@ -1283,8 +1310,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return new KeyValuePair( (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); @@ -1294,7 +1321,7 @@ namespace Duplicati.Library.Main.Database try { await cmd - .ExecuteNonQueryAsync(drop) + .ExecuteNonQueryAsync(drop, token) .ConfigureAwait(false); } catch { } @@ -1306,40 +1333,47 @@ namespace Duplicati.Library.Main.Database /// Creates a new filelist in the local test database. /// /// The name of the filelist to create. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when awaited returns a new instance of the interface. - public async Task CreateFilelist(string name) + public async Task CreateFilelist(string name, CancellationToken token) { - return await Filelist.CreateAsync(this, name).ConfigureAwait(false); + return await Filelist.CreateAsync(this, name, token) + .ConfigureAwait(false); } /// /// Creates a new indexlist in the local test database. /// /// The name of the indexlist to create. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when awaited returns a new instance of the interface. - public async Task CreateIndexlist(string name) + public async Task CreateIndexlist(string name, CancellationToken token) { - return await Indexlist.CreateAsync(this, name).ConfigureAwait(false); + return await Indexlist.CreateAsync(this, name, token) + .ConfigureAwait(false); } /// /// Creates a new blocklist in the local test database. /// /// The name of the blocklist to create. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when awaited returns a new instance of the interface. - public async Task CreateBlocklist(string name) + public async Task CreateBlocklist(string name, CancellationToken token) { - return await Blocklist.CreateAsync(this, name).ConfigureAwait(false); + return await Blocklist.CreateAsync(this, name, token) + .ConfigureAwait(false); } /// /// Creates a new blocklist hash list in the local test database. /// /// The name of the blocklist hash list to create. + /// A cancellation token to observe while waiting for the task to complete. /// A task that when awaited returns a new instance of the interface. - public async Task CreateBlocklistHashList(string name) + public async Task CreateBlocklistHashList(string name, CancellationToken token) { - return await BlocklistHashList.CreateAsync(this, name) + return await BlocklistHashList.CreateAsync(this, name, token) .ConfigureAwait(false); } } From b9952281ed9857f4d423b8481544526964660976 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 08:07:48 +0200 Subject: [PATCH 336/362] ReusableTransaction.RollbackAsync now also accepts cancellation tokens --- .../Main/Database/ReusableTransaction.cs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index afc234d3c..7034b49da 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -69,9 +69,14 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran /// public SqliteTransaction Transaction => m_disposed ? throw new InvalidOperationException("Transaction is disposed") : m_transaction; + /// + /// Commits the transaction and restarts it. + /// + /// A cancellation token to cancel the operation. + /// A task that completes when the commit is done and a new transaction has been started. public async Task CommitAsync(CancellationToken token) { - await CommitAsync(token: token).ConfigureAwait(false); + await CommitAsync(null, true, token).ConfigureAwait(false); } /// @@ -126,14 +131,25 @@ internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? tran } } + /// + /// Rolls back the transaction and restarts it. + /// + /// A cancellation token to cancel the operation. + /// A task that completes when the rollback is done and a new transaction has been started. + public async Task RollBackAsync(CancellationToken token) + { + await RollBackAsync(null, true, token).ConfigureAwait(false); + } + /// /// Rolls back the transaction and optionally restarts it. /// /// Message to log. /// Whether to restart the transaction after rolling back. + /// A cancellation token to cancel the operation. /// A task that completes when the rollback is done and (potentially) a new transaction has been started. /// If the transaction has already been disposed. - public async Task RollBackAsync(string? message = null, bool restart = true) + public async Task RollBackAsync(string? message = null, bool restart = true, CancellationToken token = default) { if (m_disposed) throw new InvalidOperationException("Transaction is already disposed"); From 7c425c4649f806bc02b87710497bf9127f96a4cf Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 08:08:17 +0200 Subject: [PATCH 337/362] Added some extra overloads to the async database extension methods --- .../Library/Main/Database/ExtensionMethods.cs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 820d03f44..a13f8e413 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -199,6 +199,19 @@ public static partial class ExtensionMethods .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns a . + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the . + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken) + { + return await ExecuteReaderAsync(self, writeLog, null, null, cancellationToken) + .ConfigureAwait(false); + } + /// /// Executes the command asynchronously and returns a . /// @@ -317,6 +330,19 @@ public static partial class ExtensionMethods .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. + public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken) + { + return await self.ExecuteScalarAsync(writeLog, null, null, cancellationToken) + .ConfigureAwait(false); + } + /// /// Executes the command asynchronously and returns the first column of the first row in the result set. /// @@ -369,6 +395,7 @@ public static partial class ExtensionMethods /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. /// /// The instance to execute on. + /// The default value to return if no rows are returned or the value cannot be converted. /// Cancellation token to cancel the operation. /// A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted. public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue, CancellationToken cancellationToken) @@ -377,6 +404,19 @@ public static partial class ExtensionMethods .ConfigureAwait(false); } + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, writeLog, null, null, -1, cancellationToken) + .ConfigureAwait(false); + } + /// /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. /// From 32efa4b845f8a88988134d2d7b44c52b38444c4b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 09:47:03 +0200 Subject: [PATCH 338/362] Updated all of the handler calls to database methods to forward their cancellation token --- .../Main/Database/LocalListDatabase.cs | 13 +- .../Library/Main/Operation/BackupHandler.cs | 74 ++--- .../Library/Main/Operation/CompactHandler.cs | 67 ++--- .../Main/Operation/CreateBugReportHandler.cs | 6 +- .../Library/Main/Operation/DeleteHandler.cs | 21 +- .../Main/Operation/FilelistProcessor.cs | 72 +++-- .../Library/Main/Operation/ListAffected.cs | 18 +- .../Main/Operation/ListBrokenFilesHandler.cs | 36 +-- .../Main/Operation/ListChangesHandler.cs | 24 +- .../Main/Operation/ListControlFilesHandler.cs | 4 +- .../Main/Operation/ListFileVersionsHandler.cs | 8 +- .../Main/Operation/ListFilesHandler.cs | 28 +- .../Main/Operation/ListFilesetsHandler.cs | 7 +- .../Main/Operation/ListFolderHandler.cs | 16 +- .../Main/Operation/PurgeBrokenFilesHandler.cs | 45 +-- .../Main/Operation/PurgeFilesHandler.cs | 62 ++--- .../Main/Operation/RecreateDatabaseHandler.cs | 161 ++++++----- .../Library/Main/Operation/RepairHandler.cs | 258 +++++++++--------- .../Operation/RestoreControlFilesHandler.cs | 11 +- .../Library/Main/Operation/RestoreHandler.cs | 126 ++++----- .../Main/Operation/SearchEntriesHandler.cs | 8 +- .../Library/Main/Operation/TestHandler.cs | 131 ++++----- .../Library/Main/Operation/VacuumHandler.cs | 4 +- Duplicati/Library/Main/Utility.cs | 22 +- 24 files changed, 638 insertions(+), 584 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index ade62473d..96d311f64 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -71,7 +71,9 @@ namespace Duplicati.Library.Main.Database /// /// Asynchronously retrieves the sizes of the file versions. /// - IAsyncEnumerable Sizes(); + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file sizes. + IAsyncEnumerable Sizes(CancellationToken token); } /// @@ -301,12 +303,13 @@ namespace Duplicati.Library.Main.Database More = true; } - public async IAsyncEnumerable Sizes() + /// + public async IAsyncEnumerable Sizes([EnumeratorCancellation] CancellationToken token) { while (More && Path == m_reader.ConvertValueToString(0)) { yield return m_reader.ConvertValueToInt64(1, -1); - More = await m_reader.ReadAsync().ConfigureAwait(false); + More = await m_reader.ReadAsync(token).ConfigureAwait(false); } } } @@ -316,8 +319,10 @@ namespace Duplicati.Library.Main.Database /// private class FileversionFixed : IFileversion { + /// public string? Path { get; internal set; } - public IAsyncEnumerable Sizes() => AsyncEnumerable.Empty(); + /// + public IAsyncEnumerable Sizes(CancellationToken token) => AsyncEnumerable.Empty(); } /// diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index b00e3207a..ab5cbf7cf 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -214,7 +214,7 @@ namespace Duplicati.Library.Main.Operation // all of the local database classes handle this), as the constructor // is not async and shouldn't be executing "heavy" code, such as // database queries. - var journalData = m_database.GetChangeJournalData(lastfilesetid).ToEnumerable(); + var journalData = m_database.GetChangeJournalData(lastfilesetid, m_taskReader.ProgressToken).ToEnumerable(); var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, journalData, cancellationTokenSource.Token); @@ -300,9 +300,9 @@ namespace Duplicati.Library.Main.Operation { try { - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); - var lastFilestTime = await database.FilesetTimes() + var lastFilestTime = await database.FilesetTimes(result.TaskControl.ProgressToken) .Select(x => x.Value) .Append(DateTime.UnixEpoch) .Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds) @@ -319,40 +319,40 @@ namespace Duplicati.Library.Main.Operation Log.WriteVerboseMessage(LOGTAG, "DatabaseTimestampError", "The database has a timestamp of {0}, but the last fileset has a timestamp of {1}. Sleeping for {2} seconds to allow the clock to catch up.", database.OperationTimestamp.ToLocalTime(), Library.Utility.Utility.EPOCH.AddSeconds(lastFilestTime).ToLocalTime(), sleepSeconds); Thread.Sleep(TimeSpan.FromSeconds(sleepSeconds)); database.Dispose(); - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); } result.Dryrun = options.Dryrun; // Check the database integrity - await Utility.UpdateOptionsFromDb(database, options) + await Utility.UpdateOptionsFromDb(database, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(database, options) + await Utility.VerifyOptionsAndUpdateDatabase(database, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); var probe_path = await database - .GetFirstPath() + .GetFirstPath(result.TaskControl.ProgressToken) .ConfigureAwait(false); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); - if (await database.PartiallyRecreated().ConfigureAwait(false)) + if (await database.PartiallyRecreated(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (await database.RepairInProgress().ConfigureAwait(false)) + if (await database.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress"); // Make sure the database is sane await database - .VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks) + .VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks, result.TaskControl.ProgressToken) .ConfigureAwait(false); // Guard the last filelist if (!options.DisableSyntheticFilelist) lastTempFilelist = await database - .GetLastIncompleteFilesetVolume() + .GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken) .ConfigureAwait(false); try @@ -364,8 +364,8 @@ namespace Duplicati.Library.Main.Operation } else { - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); - var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean, result.TaskControl.ProgressToken).ConfigureAwait(false); + var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name, result.TaskControl.ProgressToken).ConfigureAwait(false); if (!string.IsNullOrWhiteSpace(updatedLastTemp.Name) && updatedLastTemp.State == RemoteVolumeState.Deleting) { // The last temporary filelist was emptied, and scheduled for deletion, so we need to delete it @@ -381,7 +381,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(database, result.TaskControl.ProgressToken).ConfigureAwait(false); // In case the backend delete failed, manually set the state to deleted await database - .UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null, result.TaskControl.ProgressToken) .ConfigureAwait(false); // The last temporary filelist is no more lastTempFilelist = default; @@ -402,18 +402,18 @@ namespace Duplicati.Library.Main.Operation .ConfigureAwait(false); // Re-open the database and backend manager - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, result.TaskControl.ProgressToken).ConfigureAwait(false); } else throw; } // Return the updated version of the fileset, in case it was actually complete - return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume().ConfigureAwait(false)); + return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken).ConfigureAwait(false)); } catch { @@ -540,7 +540,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); - await using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) .DoRunAsync(db, true, currentIsSmall, backendManager) .ConfigureAwait(false); @@ -550,7 +550,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); - await using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) .DoCompactAsync(db, true, backendManager) .ConfigureAwait(false); @@ -561,7 +561,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager .WaitForEmptyAsync(m_database, m_taskReader.ProgressToken) @@ -569,7 +569,7 @@ namespace Duplicati.Library.Main.Operation // Calculate the number of samples to test, using the largest number of file of a given type long remoteVolumeCount = await m_database - .GetRemoteVolumes() + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) .Where(x => x.State == RemoteVolumeState.Verified) .GroupBy(x => x.Type) .SelectAwait(async x => await x.LongCountAsync()) @@ -582,7 +582,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest); m_result.TestResults = new TestResults(m_result); - await using var testdb = await LocalTestDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var testdb = await LocalTestDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await new TestHandler(m_options, (TestResults)m_result.TestResults) .DoRunAsync(samplesToTest, testdb, backendManager) .ConfigureAwait(false); @@ -597,33 +597,33 @@ namespace Duplicati.Library.Main.Operation if (result.BackendWriter != null) { result.BackendWriter.KnownFileCount = await database - .GetRemoteVolumes() - .CountAsync() + .GetRemoteVolumes(cancelToken) + .CountAsync(cancelToken) .ConfigureAwait(false); result.BackendWriter.KnownFilesets = await database - .GetRemoteVolumes() + .GetRemoteVolumes(cancelToken) .CountAsync(x => x.Type == RemoteVolumeType.Files) .ConfigureAwait(false); result.BackendWriter.KnownFileSize = await database - .GetRemoteVolumes() + .GetRemoteVolumes(cancelToken) .Select(x => Math.Max(0, x.Size)) - .SumAsync() + .SumAsync(cancelToken) .ConfigureAwait(false); result.BackendWriter.UnknownFileCount = 0; result.BackendWriter.UnknownFileSize = 0; result.BackendWriter.BackupListCount = await database - .FilesetTimes() - .CountAsync() + .FilesetTimes(cancelToken) + .CountAsync(cancelToken) .ConfigureAwait(false); result.BackendWriter.LastBackupDate = ( await database - .FilesetTimes() - .FirstOrDefaultAsync() + .FilesetTimes(cancelToken) + .FirstOrDefaultAsync(cancelToken) .ConfigureAwait(false) ) .Value.ToLocalTime(); @@ -711,7 +711,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, there is a chance we leave behind partial remote files if (!m_options.Dryrun) await database - .TerminatedWithActiveUploads(true) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true) .ConfigureAwait(false); await using (m_database = database) @@ -731,7 +731,7 @@ namespace Duplicati.Library.Main.Operation // Grab the previous backup ID, if any var prevfileset = await m_database - .FilesetTimes() + .FilesetTimes(m_result.TaskControl.ProgressToken) .FirstOrDefaultAsync() .ConfigureAwait(false); @@ -768,7 +768,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.DisableFileScanner) { var d = await m_database - .GetLastBackupFileCountAndSize() + .GetLastBackupFileCountAndSize(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true); @@ -815,7 +815,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await database - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) .ConfigureAwait(false); // Make sure we have the database up-to-date @@ -826,7 +826,7 @@ namespace Duplicati.Library.Main.Operation if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { var lastVolumeSize = await m_database - .GetLastWrittenDBlockVolumeSize() + .GetLastWrittenDBlockVolumeSize(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await CompactIfRequired(backendManager, lastVolumeSize) @@ -836,7 +836,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database) + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 781ca99f2..5374947fc 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -52,17 +52,17 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var changed = await DoCompactAsync(db, false, backendManager).ConfigureAwait(false); if (changed && m_options.UploadVerificationFile) - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, m_result.TaskControl.ProgressToken); if (!m_options.Dryrun) { @@ -72,7 +72,8 @@ namespace Duplicati.Library.Main.Operation if (changed) { - await db.WriteResults(m_result).ConfigureAwait(false); + await db.WriteResults(m_result, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (m_options.AutoVacuum) { m_result.VacuumResults = new VacuumResults(m_result); @@ -86,7 +87,7 @@ namespace Duplicati.Library.Main.Operation internal async Task DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, IBackendManager backendManager) { var report = await db - .GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount) + .GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); report.ReportCompactData(); @@ -95,11 +96,11 @@ namespace Duplicati.Library.Main.Operation { // Workaround where we allow a running backendmanager to be used if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var newvol = new BlockVolumeWriter(m_options); newvol.VolumeID = await db - .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); IndexVolumeWriter newvolindex = null; @@ -107,20 +108,20 @@ namespace Duplicati.Library.Main.Operation { newvolindex = new IndexVolumeWriter(m_options); newvolindex.VolumeID = await db - .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db - .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID) + .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } var blocksInVolume = 0L; var buffer = new byte[m_options.Blocksize]; var remoteList = await db - .GetRemoteVolumes() + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) .Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified) - .ToArrayAsync() + .ToArrayAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); //These are for bookkeeping @@ -152,7 +153,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, we may leave partial files if (!m_options.Dryrun) await db - .TerminatedWithActiveUploads(true) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true) .ConfigureAwait(false); newvolindex?.StartVolume(newvol.RemoteFilename); @@ -167,7 +168,7 @@ namespace Duplicati.Library.Main.Operation .ToList(); } - using (var q = await db.CreateBlockQueryHelper().ConfigureAwait(false)) + using (var q = await db.CreateBlockQueryHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { @@ -182,7 +183,7 @@ namespace Duplicati.Library.Main.Operation downloadedVolumes.Add(new KeyValuePair(entry.Name, entry.Size)); var volumeid = await db - .GetRemoteVolumeID(entry.Name) + .GetRemoteVolumeID(entry.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var inst = VolumeBase.ParseFilename(entry.Name); @@ -190,7 +191,7 @@ namespace Duplicati.Library.Main.Operation { foreach (var e in f.Blocks) { - if (await q.UseBlock(e.Key, e.Value, volumeid).ConfigureAwait(false)) + if (await q.UseBlock(e.Key, e.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { //TODO: How do we get the compression hint? Reverse query for filename in db? var s = f.ReadBlock(e.Key, buffer); @@ -205,7 +206,7 @@ namespace Duplicati.Library.Main.Operation newvolindex.AddBlock(e.Key, e.Value); await db - .RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID) + .RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); blocksInVolume++; @@ -217,18 +218,18 @@ namespace Duplicati.Library.Main.Operation newvol = new BlockVolumeWriter(m_options); newvol.VolumeID = await db - .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); newvolindex.VolumeID = await db - .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db - .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID) + .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); newvolindex.StartVolume(newvol.RemoteFilename); @@ -242,7 +243,7 @@ namespace Duplicati.Library.Main.Operation // Commit as we have uploaded a volume if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitCompact") + .CommitAsync("CommitCompact", true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (deleteableVolumes.Any()) @@ -268,13 +269,13 @@ namespace Duplicati.Library.Main.Operation else { await db - .RemoveRemoteVolume(newvol.RemoteFilename) + .RemoveRemoteVolume(newvol.RemoteFilename, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (newvolindex != null) { await db - .RemoveRemoteVolume(newvolindex.RemoteFilename) + .RemoveRemoteVolume(newvolindex.RemoteFilename, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); newvolindex.FinishVolume(null, 0); @@ -285,7 +286,7 @@ namespace Duplicati.Library.Main.Operation // The remainder of the operation cannot leave partial files if (!m_options.Dryrun) await db - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) .ConfigureAwait(false); } else @@ -355,27 +356,27 @@ namespace Duplicati.Library.Main.Operation { // Find volumes that can be deleted var remoteFilesToRemove = await db - .ReOrderDeleteableVolumes(deleteableVolumes) - .ToListAsync() + .ReOrderDeleteableVolumes(deleteableVolumes, cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); // Make sure we do not re-assign blocks to any of the volumes we are about to delete var toRemoveVolumeIds = await db - .GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name)) + .GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), cancellationToken) .Select(x => x.Value) .Distinct() - .ToListAsync() + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); // Mark all volumes and relevant index files as disposable foreach (var f in remoteFilesToRemove) { await db - .PrepareForDelete(f.Name, toRemoveVolumeIds) + .PrepareForDelete(f.Name, toRemoveVolumeIds, cancellationToken) .ConfigureAwait(false); await db - .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, cancellationToken) .ConfigureAwait(false); } @@ -384,7 +385,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitDelete") + .CommitAsync("CommitDelete", token: cancellationToken) .ConfigureAwait(false); await foreach (var d in PerformDelete(backend, remoteFilesToRemove, cancellationToken).ConfigureAwait(false)) @@ -397,7 +398,7 @@ namespace Duplicati.Library.Main.Operation if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { - await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize).ConfigureAwait(false)) + await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) newvolindex.WriteBlocklist(blocklist.Hash, blocklist.Buffer, 0, blocklist.Size); }; @@ -409,7 +410,7 @@ namespace Duplicati.Library.Main.Operation // because the transaction is not thread-safe, and shared with the upload await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await db - .UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // TODO: The upload here does not flush the database messages, diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index 69492ae26..9ff8ea208 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -68,11 +68,11 @@ namespace Duplicati.Library.Main.Operation using (var tmp = new TempFile()) { File.Copy(m_options.Dbpath, tmp, true); - await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await db.Fix().ConfigureAwait(false); + await db.Fix(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (m_options.AutoVacuum) - await db.Vacuum().ConfigureAwait(false); + await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false); } // Apply zip64 option if not already set diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 19498295f..260c33891 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -51,11 +51,11 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache).ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); @@ -69,13 +69,13 @@ namespace Duplicati.Library.Main.Operation public async Task DoRunAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) { if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // We collapse the async enumerablo into a array to avoid multiple // enumerations (and thus multiple database queries) var filesets = await db - .FilesetsWithBackupVersion() - .ToArrayAsync() + .FilesetsWithBackupVersion(m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); List versionsToDelete = @@ -109,19 +109,20 @@ namespace Duplicati.Library.Main.Operation .DropFilesetsFromTable( versionsToDelete .Select(x => x.Time) - .ToArray() + .ToArray(), + m_result.TaskControl.ProgressToken ) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); foreach (var f in lst) await db - .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null) + .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitBeforeDelete") + .CommitAsync("CommitBeforeDelete", true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); foreach (var f in lst) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 39d0e8134..5bf01708b 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -71,7 +71,7 @@ namespace Duplicati.Library.Main.Operation /// A task that completes when the verification is done. public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) { - var locallist = database.GetRemoteVolumes(); + var locallist = database.GetRemoteVolumes(cancellationToken); await foreach (var i in locallist.ConfigureAwait(false)) { switch (i.State) @@ -115,20 +115,21 @@ namespace Duplicati.Library.Main.Operation /// The backend writer to use. /// True if only the latest volumes should be verified. /// The mode to use for verification. + /// The cancellation token to use. /// An awaitable task. - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, CancellationToken cancellationToken) { if (!options.NoBackendverification) { LocalBackupDatabase backupDatabase = - await LocalBackupDatabase.CreateAsync(database, options) + await LocalBackupDatabase.CreateAsync(database, options, null, cancellationToken) .ConfigureAwait(false); IEnumerable protectedFiles = await backupDatabase - .GetTemporaryFilelistVolumeNames(latestVolumesOnly) + .GetTemporaryFilelistVolumeNames(latestVolumesOnly, cancellationToken) .ConfigureAwait(false); - await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); + await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode, cancellationToken).ConfigureAwait(false); } } @@ -144,10 +145,11 @@ namespace Duplicati.Library.Main.Operation /// Filenames that should be exempted from strict verification. /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless. /// The mode to use for verification. + /// The cancellation token to use. /// An awaitable task. - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode, CancellationToken cancellationToken) { - var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); + var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode, cancellationToken).ConfigureAwait(false); long extraCount = 0; long missingCount = 0; @@ -213,10 +215,17 @@ namespace Duplicati.Library.Main.Operation /// The verification file. /// The database instance /// The stream to write to - public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream) + /// The cancellation token to use. + public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream, CancellationToken cancellationToken) { var s = new Newtonsoft.Json.JsonSerializer(); - s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).ToEnumerable().Cast().ToArray()); + s.Serialize(stream, + db.GetRemoteVolumes(cancellationToken) + .Where(x => x.State != RemoteVolumeState.Temporary) + .ToEnumerable() + .Cast() + .ToArray() + ); } /// @@ -225,14 +234,15 @@ namespace Duplicati.Library.Main.Operation /// The backend manager instance to use. /// The options to use. /// The attached database. + /// The cancellation token to use. /// A task that completes when the upload is done. - public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db) + public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, CancellationToken cancellationToken) { using (var tempfile = new Library.Utility.TempFile()) { var remotename = options.Prefix + "-verification.json"; await using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8)) - CreateVerificationFile(db, stream); + CreateVerificationFile(db, stream, cancellationToken); if (options.Dryrun) { @@ -240,8 +250,8 @@ namespace Duplicati.Library.Main.Operation } else { - await backendManager.PutVerificationFileAsync(remotename, tempfile, CancellationToken.None).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(db, CancellationToken.None).ConfigureAwait(false); + await backendManager.PutVerificationFileAsync(remotename, tempfile, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); } } } @@ -257,18 +267,19 @@ namespace Duplicati.Library.Main.Operation /// Filenames that should be exempted from deletion. /// Filenames that should be exempted from strict verification. /// The mode to use for verification. + /// The cancellation token to use. /// A task that when awaited contains the analysis result. - public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) + public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode, CancellationToken cancellationToken) { // If the last operation completed, no cleanup should be required - if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads().ConfigureAwait(false)) + if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false)) verifyMode = VerifyMode.VerifyStrict; // Force cleanup should set the mode to cleanup if (verifyMode == VerifyMode.VerifyAndCleanForced) verifyMode = VerifyMode.VerifyAndClean; - var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false); + var rawlist = await backendManager.ListAsync(cancellationToken).ConfigureAwait(false); var lookup = new Dictionary(); protectedFiles = protectedFiles ?? []; strictExcemptFiles = strictExcemptFiles ?? []; @@ -300,8 +311,8 @@ namespace Duplicati.Library.Main.Operation log.UnknownFileCount = unknownlist.Count; log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum(); log.BackupListCount = await database - .FilesetTimes() - .CountAsync() + .FilesetTimes(cancellationToken) + .CountAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); @@ -316,17 +327,17 @@ namespace Duplicati.Library.Main.Operation var missingUploadingVolumes = new HashSet(); var protectedVolumes = new HashSet(); - await foreach (var e in database.DuplicateRemoteVolumes().ConfigureAwait(false)) + await foreach (var e in database.DuplicateRemoteVolumes(cancellationToken).ConfigureAwait(false)) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) await database - .UnlinkRemoteVolume(e.Key, e.Value) + .UnlinkRemoteVolume(e.Key, e.Value, cancellationToken) .ConfigureAwait(false); else throw new RemoteListVerificationException(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()), "AmbiguousStateRemoteFiles"); } - var locallist = database.GetRemoteVolumes(); + var locallist = database.GetRemoteVolumes(cancellationToken); await foreach (var i in locallist.ConfigureAwait(false)) { var remoteFound = lookup.TryGetValue(i.Name, out var r); @@ -336,11 +347,11 @@ namespace Duplicati.Library.Main.Operation correctSize = true; if (archived && i.ArchiveTime <= DateTime.UnixEpoch) await database - .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true) + .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true, cancellationToken) .ConfigureAwait(false); else if (!archived && i.ArchiveTime > DateTime.UnixEpoch) await database - .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false) + .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false, cancellationToken) .ConfigureAwait(false); lookup.Remove(i.Name); @@ -397,7 +408,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); await database - .UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash) + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, cancellationToken) .ConfigureAwait(false); } else if (!remoteFound) @@ -407,7 +418,7 @@ namespace Duplicati.Library.Main.Operation protectedVolumes.Add(i.Name); Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); await database - .UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null) + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, cancellationToken) .ConfigureAwait(false); } else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) @@ -420,7 +431,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); missingUploadingVolumes.Add(i.Name); await database - .UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); } } @@ -460,7 +471,7 @@ namespace Duplicati.Library.Main.Operation missing.Add(i); else if (correctSize) await database - .UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash) + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, cancellationToken) .ConfigureAwait(false); else missingHash.Add(new Tuple(r.File.Size, i)); @@ -491,17 +502,18 @@ namespace Duplicati.Library.Main.Operation await database .RemoveRemoteVolumes( missingUploadingVolumes - .Concat(temporaryAndDeletedVolumes) + .Concat(temporaryAndDeletedVolumes), + cancellationToken ) .ConfigureAwait(false); // Clear the flag after we have cleaned up if (!options.Dryrun && protectedVolumes.Count == 0) await database - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(cancellationToken, false) .ConfigureAwait(false); } - else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads().ConfigureAwait(false)) + else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false)) { Logging.Log.WriteWarningMessage(LOGTAG, "ActiveUploadsDetected", null, "Active uploads detected, but no cleanup was performed. Run the \"repair\" command to clean up the incomplete files"); } diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 33b0fa1bc..97284398e 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -44,21 +44,21 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (callback == null) { m_result.SetResult( - await db.GetFilesets(args) + await db.GetFilesets(args, m_result.TaskControl.ProgressToken) .OrderByDescending(x => x.Time) .ToArrayAsync() .ConfigureAwait(false), - await db.GetFiles(args) + await db.GetFiles(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false), - await db.GetLogLines(args) + await db.GetLogLines(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false), - await db.GetVolumes(args) + await db.GetVolumes(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false) ); @@ -66,17 +66,17 @@ namespace Duplicati.Library.Main.Operation else { m_result.SetResult( - await db.GetFilesets(args) + await db.GetFilesets(args, m_result.TaskControl.ProgressToken) .OrderByDescending(x => x.Time) .ToArrayAsync() .ConfigureAwait(false), - await db.GetFiles(args) + await db.GetFiles(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false), - await db.GetLogLines(args) + await db.GetLogLines(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false), - await db.GetVolumes(args) + await db.GetVolumes(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false) ); diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 13dc5502c..8f37f4226 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -52,10 +52,10 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); await db.Transaction.RollBackAsync().ConfigureAwait(false); @@ -65,18 +65,18 @@ namespace Duplicati.Library.Main.Operation { List? missing = null; var brokensets = await db - .GetBrokenFilesets(options.Time, options.Version) - .ToArrayAsync() + .GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); if (brokensets.Length == 0) { - if (await db.RepairInProgress().ConfigureAwait(false)) + if (await db.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("Cannot continue because the database is marked as being under repair, but does not have broken files.", "CannotListOnDatabaseInRepair"); Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly, result.TaskControl.ProgressToken).ConfigureAwait(false); if (!remotestate.ParsedVolumes.Any()) throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); @@ -90,24 +90,24 @@ namespace Duplicati.Library.Main.Operation // Mark all volumes as disposable foreach (var f in missing) await db - .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken) .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count); // Drop all content from tables await db - .RemoveMissingBlocks(missing.Select(x => x.Name)) + .RemoveMissingBlocks(missing.Select(x => x.Name), result.TaskControl.ProgressToken) .ConfigureAwait(false); // Mark all orphaned index files as disposable after removing the missing block files - await foreach (var f in db.GetOrphanedIndexFiles().ConfigureAwait(false)) + await foreach (var f in db.GetOrphanedIndexFiles(result.TaskControl.ProgressToken).ConfigureAwait(false)) await db - .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken) .ConfigureAwait(false); brokensets = await db - .GetBrokenFilesets(options.Time, options.Version) + .GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false); } @@ -120,7 +120,7 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles"); - if (await db.PartiallyRecreated().ConfigureAwait(false)) + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase"); (var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); @@ -142,8 +142,8 @@ namespace Duplicati.Library.Main.Operation } var fstimes = await db - .FilesetTimes() - .ToListAsync() + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var brokenfilesets = @@ -165,8 +165,8 @@ namespace Duplicati.Library.Main.Operation x.Timestamp, callbackhandler == null && !m_options.ListSetsOnly ? await db - .GetBrokenFilenames(x.FilesetID) - .ToArrayAsync() + .GetBrokenFilenames(x.FilesetID, m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false) : new MockList>((int)x.BrokenCount) ))) @@ -175,7 +175,7 @@ namespace Duplicati.Library.Main.Operation if (callbackhandler != null) foreach (var bs in brokenfilesets) - await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID).ConfigureAwait(false)) + await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2)) break; } diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index 8ee6cc459..e1c82db45 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -80,13 +80,13 @@ namespace Duplicati.Library.Main.Operation DateTime compareVersionTime; using var tmpdb = useLocalDb ? null : new TempFile(); - await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache).ConfigureAwait(false); - await using var storageKeeper = await db.CreateStorageHelper().ConfigureAwait(false); + await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var storageKeeper = await db.CreateStorageHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (useLocalDb) { var dbtimes = await db - .FilesetTimes() - .ToListAsync() + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (dbtimes.Count < 2) @@ -101,10 +101,10 @@ namespace Duplicati.Library.Main.Operation SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId); await storageKeeper - .AddFromDb(baseVersionId, false, filter) + .AddFromDb(baseVersionId, false, filter, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await storageKeeper - .AddFromDb(compareVersionId, true, filter) + .AddFromDb(compareVersionId, true, filter, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } else @@ -151,7 +151,7 @@ namespace Duplicati.Library.Main.Operation foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) await storageKeeper - .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false) + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) @@ -162,20 +162,20 @@ namespace Duplicati.Library.Main.Operation foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) await storageKeeper - .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true) + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } var changes = await storageKeeper - .CreateChangeCountReport() + .CreateChangeCountReport(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var sizes = await storageKeeper - .CreateChangeSizeReport() + .CreateChangeSizeReport(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var lst = (m_options.FullResult || callback != null) ? - (from n in storageKeeper.CreateChangedFileReport() + (from n in storageKeeper.CreateChangedFileReport(m_result.TaskControl.ProgressToken) select n) : null; m_result.SetResult( @@ -184,7 +184,7 @@ namespace Duplicati.Library.Main.Operation changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles, changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles, sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize, - (lst == null || callback == null) ? null : await lst.ToArrayAsync().ConfigureAwait(false) + (lst == null || callback == null) ? null : await lst.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false) ); if (callback != null) diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index ee5d229a3..d2f6bf925 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.Operation { var cancellationToken = CancellationToken.None; using var tmpdb = new TempFile(); - await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); try @@ -62,7 +62,7 @@ namespace Duplicati.Library.Main.Operation var file = fileversion.Value.File; var entry = await db - .GetRemoteVolume(file.Name) + .GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var files = new List(); diff --git a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs index 90533677b..fb0cf4427 100644 --- a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs @@ -48,14 +48,14 @@ internal static class ListFileVersionsHandler throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); await using var db = - await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) { filesetIds = await db - .GetFilesetIDs(options.Time, options.Version) - .ToArrayAsync() + .GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); if (filesetIds.Length == 0) @@ -67,7 +67,7 @@ internal static class ListFileVersionsHandler paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); result.FileVersions = await db - .ListFileVersions(paths, filesetIds, offset, limit) + .ListFileVersions(paths, filesetIds, offset, limit, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 77479350c..6d3375ddd 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -55,15 +55,15 @@ namespace Duplicati.Library.Main.Operation //Use a speedy local query if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) - await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version).ConfigureAwait(false); + await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (!filter.Empty) { if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) { await filesets - .TakeFirst() + .TakeFirst(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -71,11 +71,11 @@ namespace Duplicati.Library.Main.Operation IAsyncEnumerable files; if (m_options.ListFolderContents) { - files = filesets.SelectFolderContents(filter); + files = filesets.SelectFolderContents(filter, m_result.TaskControl.ProgressToken); } else if (m_options.ListPrefixOnly) { - files = filesets.GetLargestPrefix(filter); + files = filesets.GetLargestPrefix(filter, m_result.TaskControl.ProgressToken); } else if (filter.Empty) { @@ -83,16 +83,16 @@ namespace Duplicati.Library.Main.Operation } else { - files = filesets.SelectFiles(filter); + files = filesets.SelectFiles(filter, m_result.TaskControl.ProgressToken); } if (m_options.ListSetsOnly) { m_result.SetResult( await filesets - .QuickSets() + .QuickSets(m_result.TaskControl.ProgressToken) .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false), null ); @@ -101,9 +101,9 @@ namespace Duplicati.Library.Main.Operation { m_result.SetResult( await filesets - .Sets() + .Sets(m_result.TaskControl.ProgressToken) .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false), files == null ? null @@ -112,14 +112,14 @@ namespace Duplicati.Library.Main.Operation new ListResultFile( n.Path, await n - .Sizes() - .ToArrayAsync() + .Sizes(m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false) ) ) .Select(x => x.Result) .Cast() - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false) ); } @@ -137,7 +137,7 @@ namespace Duplicati.Library.Main.Operation // Otherwise, grab info from remote location using (var tmpdb = new TempFile()) - await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) diff --git a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs index b1169b1a2..8629f33e8 100644 --- a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs @@ -22,6 +22,7 @@ #nullable enable using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; @@ -49,12 +50,12 @@ internal static class ListFilesetsHandler // Use a speedy local query if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb) { - await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); result.Filesets = await db - .ListFilesetsExtended() - .ToArrayAsync() + .ListFilesetsExtended(result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); return; diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index b71e0a81c..5a296f564 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -46,12 +46,12 @@ internal static class ListFolderHandler if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); var filesetIds = await db - .GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true) - .ToArrayAsync() + .GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); if (filesetIds.Length == 0) @@ -65,10 +65,11 @@ internal static class ListFolderHandler throw new UserInformationException("When no folder is specified, only one folder can be listed", "MultipleFoldersFound"); result.Entries = await db .ListFilesetEntries( - db.GetRootPrefixes(filesetIds[0]).ToEnumerable(), + db.GetRootPrefixes(filesetIds[0], result.TaskControl.ProgressToken).ToEnumerable(), filesetIds[0], offset, - limit + limit, + result.TaskControl.ProgressToken ) .ConfigureAwait(false); } @@ -76,10 +77,11 @@ internal static class ListFolderHandler { result.Entries = await db .ListFolder( - db.GetPrefixIds(folders).ToEnumerable(), + db.GetPrefixIds(folders, result.TaskControl.ProgressToken).ToEnumerable(), filesetIds[0], offset, - limit + limit, + result.TaskControl.ProgressToken ) .ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 47afcb0eb..348ebf383 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -54,13 +54,13 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); - if (await db.PartiallyRecreated().ConfigureAwait(false)) + await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); @@ -84,8 +84,8 @@ namespace Duplicati.Library.Main.Operation var pgspan = 0.95f / sets.Length; var filesets = await db - .FilesetTimes() - .ToListAsync() + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var compare_list = sets.Select(async x => new @@ -95,7 +95,7 @@ namespace Duplicati.Library.Main.Operation RemoveCount = x.RemoveCount, Version = filesets.FindIndex(y => y.Key == x.FilesetID), SetCount = await db - .GetFilesetFileCount(x.FilesetID) + .GetFilesetFileCount(x.FilesetID, m_result.TaskControl.ProgressToken) .ConfigureAwait(false) }) .Select(x => x.Result) @@ -109,7 +109,8 @@ namespace Duplicati.Library.Main.Operation .GetEmptyMetadataBlocksetId( (missing ?? []).Select(x => x.ID), emptymetadata.FileHash, - emptymetadata.Blob.Length + emptymetadata.Blob.Length, + m_result.TaskControl.ProgressToken ) .ConfigureAwait(false); if (replacementMetadataBlocksetId < 0) @@ -119,11 +120,11 @@ namespace Duplicati.Library.Main.Operation var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); - if (fully_emptied.Length == await db.FilesetTimes().CountAsync().ConfigureAwait(false)) + if (fully_emptied.Length == await db.FilesetTimes(m_result.TaskControl.ProgressToken).CountAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); if (!m_options.Dryrun) - await db.Transaction.CommitAsync().ConfigureAwait(false); + await db.Transaction.CommitAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (fully_emptied.Length != 0) @@ -134,7 +135,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); m_result.DeleteResults = new DeleteResults(m_result); - await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) + await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var opts = new Options(new Dictionary(m_options.RawOptions)); opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); @@ -146,7 +147,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await rmdb.Transaction - .CommitAsync("CommitDelete") + .CommitAsync("CommitDelete", true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -163,12 +164,12 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); var opts = new Options(new Dictionary(m_options.RawOptions)); - await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db).ConfigureAwait(false)) + await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { // Recompute the version number after we deleted the versions before filesets = await pgdb - .FilesetTimes() - .ToListAsync() + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); if (thisversion < 0) @@ -186,9 +187,9 @@ namespace Duplicati.Library.Main.Operation // Update entries that would be removed because of missing metadata var updatedEntries = 0; if (!m_options.DisableReplaceMissingMetadata) - updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId); + updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId, m_result.TaskControl.ProgressToken); - await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID"); + await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", m_result.TaskControl.ProgressToken); return updatedEntries; }).ConfigureAwait(false); } @@ -201,20 +202,20 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0.95f); - if (!m_options.Dryrun && await db.RepairInProgress().ConfigureAwait(false)) + if (!m_options.Dryrun && await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); - await db.RepairInProgress(false).ConfigureAwait(false); + await db.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false); } else { - await db.Transaction.RollBackAsync().ConfigureAwait(false); + await db.Transaction.RollBackAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index dd7aece5a..378439872 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -52,10 +52,10 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -69,44 +69,44 @@ namespace Duplicati.Library.Main.Operation var doCompactStep = !m_options.NoAutoCompact && filtercommand == null; - if (await db.PartiallyRecreated().ConfigureAwait(false)) + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase"); - if (await db.RepairInProgress().ConfigureAwait(false) && filtercommand == null) + if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) && filtercommand == null) throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase"); var versions = await db - .GetFilesetIDs(m_options.Time, m_options.Version) + .GetFilesetIDs(m_options.Time, m_options.Version, false, m_result.TaskControl.ProgressToken) .OrderByDescending(x => x) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (versions.Length <= 0) throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion"); - var orphans = await db.CountOrphanFiles().ConfigureAwait(false); + var orphans = await db.CountOrphanFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (orphans != 0) throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans"); - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (filtercommand == null) { - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false) + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (m_options.NoBackendverification) await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } var filesets = await db - .FilesetTimes() + .FilesetTimes(m_result.TaskControl.ProgressToken) .OrderByDescending(x => x.Value) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan; @@ -118,7 +118,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, it is possible that the remote storage contains partial files if (!m_options.Dryrun) - await db.TerminatedWithActiveUploads(true).ConfigureAwait(false); + await db.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); // Reverse makes sure we re-write the old versions first foreach (var versionid in versions.Reverse()) @@ -132,18 +132,18 @@ namespace Duplicati.Library.Main.Operation var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal) .ConfigureAwait(false); - var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key) + var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (ix != 0 && filesets[ix - 1].Value <= ts) throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); - await using (var tempset = await db.CreateTemporaryFileset(versionid).ConfigureAwait(false)) + await using (var tempset = await db.CreateTemporaryFileset(versionid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { if (filtercommand == null) - await tempset.ApplyFilter(filter).ConfigureAwait(false); + await tempset.ApplyFilter(filter, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - await tempset.ApplyFilter(filtercommand) + await tempset.ApplyFilter(filtercommand, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) @@ -161,10 +161,10 @@ namespace Duplicati.Library.Main.Operation using (var vol = new FilesetVolumeWriter(m_options, ts)) { var isOriginalFilesetFullBackup = await db - .IsFilesetFullBackup(tsOriginal) + .IsFilesetFullBackup(tsOriginal, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var newids = await tempset - .ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup) + .ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); vol.VolumeID = newids.Item1; vol.CreateFilesetFile(isOriginalFilesetFullBackup); @@ -172,7 +172,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); await db - .WriteFileset(vol, newids.Item2) + .WriteFileset(vol, newids.Item2, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.RemovedFileSize += tempset.RemovedFileSize; @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.Dryrun || m_options.FullResult) { - await foreach (var fe in tempset.ListAllDeletedFiles().ConfigureAwait(false)) + await foreach (var fe in tempset.ListAllDeletedFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); @@ -212,19 +212,19 @@ namespace Duplicati.Library.Main.Operation else { await db - .UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var lst = await db - .DropFilesetsFromTable(new[] { tsOriginal }) - .ToArrayAsync() + .DropFilesetsFromTable(new[] { tsOriginal }, m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); foreach (var f in lst) await db - .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null) + .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await backendManager.PutAsync(vol, null, null, true, async () => @@ -235,7 +235,7 @@ namespace Duplicati.Library.Main.Operation // passing down a transaction, but maybe is // not needed anymore. await db.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); @@ -251,7 +251,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await db - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) .ConfigureAwait(false); if (doCompactStep) @@ -265,12 +265,12 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); m_result.CompactResults = new CompactResults(m_result); - await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false); + await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) .DoCompactAsync(cdb, true, backendManager) .ConfigureAwait(false); - await cdb.Transaction.CommitAsync("PostCompact"); + await cdb.Transaction.CommitAsync("PostCompact", true, m_result.TaskControl.ProgressToken); } m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan); diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 3c00e6c08..c5a116c54 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -23,6 +23,7 @@ using System.Collections.Generic; //using System.Data; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; @@ -65,14 +66,14 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); await using var db = - await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache) + await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); // Ensure database is consistent after the recreate await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -89,9 +90,9 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); await using var db = - await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache) + await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version).ConfigureAwait(false)).Any()) + if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Any()) { if (m_options.IgnoreUpdateIfVersionExists) { @@ -103,17 +104,17 @@ namespace Duplicati.Library.Main.Operation } // Mark as incomplete - await db.PartiallyRecreated(true).ConfigureAwait(false); + await db.PartiallyRecreated(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); var preexistingOptionsInDatabase = - await Utility.ContainsOptionsForVerification(db) + await Utility.ContainsOptionsForVerification(db, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // Make sure the options have not changed between calls, unless there are no previous options if (preexistingOptionsInDatabase) - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); @@ -135,9 +136,9 @@ namespace Duplicati.Library.Main.Operation //We build a local database in steps. await using var restoredb = - await LocalRecreateDatabase.CreateAsync(dbparent, m_options) + await LocalRecreateDatabase.CreateAsync(dbparent, m_options, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await restoredb.RepairInProgress(true).ConfigureAwait(false); + await restoredb.RepairInProgress(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); var expRecreateDb = false; // experimental recreate db code flag var volumeIds = new Dictionary(); @@ -205,9 +206,9 @@ namespace Duplicati.Library.Main.Operation // If we are updating, all files should be accounted for foreach (var fl in remotefiles) volumeIds[fl.File.Name] = updating - ? await restoredb.GetRemoteVolumeID(fl.File.Name) + ? await restoredb.GetRemoteVolumeID(fl.File.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false) - : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded) + : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var hasUpdatedOptions = false; @@ -224,7 +225,7 @@ namespace Duplicati.Library.Main.Operation foreach (var n in filelists) if (volumeIds[n.File.Name] == -1) volumeIds[n.File.Name] = await restoredb - .RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0)) + .RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -240,7 +241,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); return; } @@ -262,7 +263,7 @@ namespace Duplicati.Library.Main.Operation if (!string.IsNullOrWhiteSpace(hash) && size > 0) await restoredb - .UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var parsed = VolumeBase.ParseFilename(entry.Name); @@ -280,10 +281,10 @@ namespace Duplicati.Library.Main.Operation // Create timestamped operations based on the file timestamp var filesetid = await restoredb - .CreateFileset(volumeIds[entry.Name], parsed.Time) + .CreateFileset(volumeIds[entry.Name], parsed.Time, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter) + await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -295,7 +296,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -305,7 +306,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -314,7 +315,7 @@ namespace Duplicati.Library.Main.Operation { // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -323,12 +324,12 @@ namespace Duplicati.Library.Main.Operation //Make sure we write the config if it has been read from a manifest if (hasUpdatedOptions) - await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) await restoredb.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // do we stop after just handling the dlist files ? @@ -361,7 +362,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); return; } @@ -374,7 +375,7 @@ namespace Duplicati.Library.Main.Operation { if (!string.IsNullOrWhiteSpace(hash) && size > 0) await restoredb - .UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) @@ -383,13 +384,13 @@ namespace Duplicati.Library.Main.Operation { var filename = a.Filename; var volumeID = await restoredb - .GetRemoteVolumeID(filename) + .GetRemoteVolumeID(filename, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // No such file if (volumeID < 0) (volumeID, filename) = - await ProbeForMatchingFilename(filename, restoredb) + await ProbeForMatchingFilename(filename, restoredb, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var missing = false; @@ -402,7 +403,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); missing = true; volumeID = await restoredb - .RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -410,16 +411,17 @@ namespace Duplicati.Library.Main.Operation //Add all block/volume mappings foreach (var b in a.Blocks) { - anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID).ConfigureAwait(false)).Item1; + anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1; } await restoredb - .UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash) + .UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await restoredb .AddIndexBlockLink( - await restoredb.GetRemoteVolumeID(name).ConfigureAwait(false), - volumeID + await restoredb.GetRemoteVolumeID(name, m_result.TaskControl.ProgressToken).ConfigureAwait(false), + volumeID, + m_result.TaskControl.ProgressToken ) .ConfigureAwait(false); } @@ -436,7 +438,7 @@ namespace Duplicati.Library.Main.Operation // done before we add it to the database, since we do not have nested transactions var list = b.Blocklist.ToList(); await restoredb - .AddTempBlockListHash(b.Hash, list) + .AddTempBlockListHash(b.Hash, list, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } catch (System.IO.InvalidDataException e) @@ -461,7 +463,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -470,7 +472,7 @@ namespace Duplicati.Library.Main.Operation { // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -479,25 +481,25 @@ namespace Duplicati.Library.Main.Operation using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) await restoredb.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // TODO: In some cases, we can avoid downloading all index files, // if we are lucky and pick the right ones } - await restoredb.CleanupMissingVolumes().ConfigureAwait(false); + await restoredb.CleanupMissingVolumes(m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Update the real tables from the temp tables if (expRecreateDb) // add missing blocks and blocksetentry data (at this point // we have not yet anything in the blocksetentry table) await restoredb - .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize) + .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); else await restoredb - .FindMissingBlocklistHashes(hashsize, m_options.Blocksize) + .FindMissingBlocklistHashes(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // We have now grabbed as much information as possible, @@ -507,7 +509,7 @@ namespace Duplicati.Library.Main.Operation { // Grab the list matching the pass type var lst = await restoredb - .GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse) + .GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse, m_result.TaskControl.ProgressToken) .ToListAsync() .ConfigureAwait(false); @@ -545,7 +547,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); return; } @@ -555,26 +557,26 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); var volumeid = await restoredb - .GetRemoteVolumeID(name) + .GetRemoteVolumeID(name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await restoredb - .UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash) + .UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); bool anyChange = false; // Update the block table so we know about the block/volume map foreach (var h in rd.Blocks) { - anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid).ConfigureAwait(false)).Item1; + anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1; } // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes // to the needed list blocks in the volume, so grab them from the database // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table - await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid).ConfigureAwait(false)) + await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize)).ConfigureAwait(false)) + if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { anyChange = true; } @@ -593,17 +595,17 @@ namespace Duplicati.Library.Main.Operation } if (expRecreateDb) await restoredb - .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false) + .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); else await restoredb - .FindMissingBlocklistHashes(hashsize, m_options.Blocksize) + .FindMissingBlocklistHashes(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) await restoredb.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); //At this point we can patch files with data from the block volume @@ -618,7 +620,7 @@ namespace Duplicati.Library.Main.Operation { // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -634,10 +636,10 @@ namespace Duplicati.Library.Main.Operation // All blocks are collected and added into the Block table // Find out which blocks are deleted and move them into DeletedBlock, // so that compact can calculate the unused space - await restoredb.CleanupDeletedBlocks().ConfigureAwait(false); + await restoredb.CleanupDeletedBlocks(m_result.TaskControl.ProgressToken).ConfigureAwait(false); } - await restoredb.CleanupMissingVolumes().ConfigureAwait(false); + await restoredb.CleanupMissingVolumes(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (m_options.RepairOnlyPaths) { @@ -652,11 +654,11 @@ namespace Duplicati.Library.Main.Operation // except to continue a backup m_result.EndTime = DateTime.UtcNow; - await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb).ConfigureAwait(false)) + await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var broken = await lbfdb - .GetBrokenFilesets(new DateTime(0), null) - .CountAsync() + .GetBrokenFilesets(new DateTime(0), null, m_result.TaskControl.ProgressToken) + .CountAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (broken != 0) @@ -664,18 +666,28 @@ namespace Duplicati.Library.Main.Operation } await restoredb - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); - await restoredb.RepairInProgress(false).ConfigureAwait(false); + await restoredb.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false); } m_result.EndTime = DateTime.UtcNow; } - public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter) + /// + /// Recreate a fileset from the remote filelist. + /// + /// The database to restore to. + /// The compression module to use. + /// The ID of the fileset to recreate. + /// The options to use for the operation. + /// The filter to apply to the files. + /// The cancellation token to monitor for cancellation requests. + /// A task that completes when the fileset has been recreated. + public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter, CancellationToken cancellationToken) { var blocksize = options.Blocksize; var hashes_pr_block = blocksize / options.BlockhashSize; @@ -685,11 +697,11 @@ namespace Duplicati.Library.Main.Operation // update fileset using filesetData await restoredb - .UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup) + .UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, cancellationToken) .ConfigureAwait(false); // clear any existing fileset entries - await restoredb.ClearFilesetEntries(filesetid).ConfigureAwait(false); + await restoredb.ClearFilesetEntries(filesetid, cancellationToken).ConfigureAwait(false); using (var filelistreader = new FilesetVolumeReader(compressor, options)) foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) @@ -703,17 +715,17 @@ namespace Duplicati.Library.Main.Operation var metadataid = long.MinValue; var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path); var prefixid = await restoredb - .GetOrCreatePathPrefix(split.Key) + .GetOrCreatePathPrefix(split.Key, cancellationToken) .ConfigureAwait(false); switch (fe.Type) { case FilelistEntryType.Folder: metadataid = await restoredb - .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) .ConfigureAwait(false); await restoredb - .AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid) + .AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken) .ConfigureAwait(false); break; @@ -723,24 +735,24 @@ namespace Duplicati.Library.Main.Operation if (expectedblocks <= 1) expectedblocklisthashes = 0; var blocksetid = await restoredb - .AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes) + .AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, cancellationToken) .ConfigureAwait(false); metadataid = await restoredb - .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) .ConfigureAwait(false); await restoredb - .AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid) + .AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, cancellationToken) .ConfigureAwait(false); if (fe.Size <= blocksize) { if (!string.IsNullOrWhiteSpace(fe.Blockhash)) await restoredb - .AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize) + .AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, cancellationToken) .ConfigureAwait(false); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) await restoredb - .AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size) + .AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, cancellationToken) .ConfigureAwait(false); else if (fe.Size > 0) Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); @@ -749,10 +761,10 @@ namespace Duplicati.Library.Main.Operation break; case FilelistEntryType.Symlink: metadataid = await restoredb - .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) .ConfigureAwait(false); await restoredb - .AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid) + .AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken) .ConfigureAwait(false); break; default: @@ -764,11 +776,11 @@ namespace Duplicati.Library.Main.Operation { if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) await restoredb - .AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize) + .AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, cancellationToken) .ConfigureAwait(false); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) await restoredb - .AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize) + .AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, cancellationToken) .ConfigureAwait(false); else Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path); @@ -780,7 +792,7 @@ namespace Duplicati.Library.Main.Operation } } - await restoredb.Transaction.CommitAsync().ConfigureAwait(false); + await restoredb.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } /// @@ -788,8 +800,9 @@ namespace Duplicati.Library.Main.Operation /// /// The filename read and written. /// The database to query. + /// A cancellation token to cancel the operation. /// The volume id of the item. - public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb) + public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb, CancellationToken cancellationToken) { var p = VolumeBase.ParseFilename(filename); if (p != null) @@ -799,7 +812,7 @@ namespace Duplicati.Library.Main.Operation { var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule); var tvid = await restoredb - .GetRemoteVolumeID(testfilename) + .GetRemoteVolumeID(testfilename, cancellationToken) .ConfigureAwait(false); if (tvid >= 0) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 1383e1fda..547abae58 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -66,11 +66,11 @@ namespace Duplicati.Library.Main.Operation try { await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); knownRemotes = await db - .GetRemoteVolumes() - .CountAsync() + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) + .CountAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } catch (Exception ex) @@ -134,50 +134,50 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, cancellationToken) .ConfigureAwait(false); try { - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, cancellationToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, cancellationToken) .ConfigureAwait(false); - if (await db.PartiallyRecreated().ConfigureAwait(false)) + if (await db.PartiallyRecreated(cancellationToken).ConfigureAwait(false)) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (await db.RepairInProgress().ConfigureAwait(false)) + if (await db.RepairInProgress(cancellationToken).ConfigureAwait(false)) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); // Ensure the database is consistent before we start fixing the remote await db - .VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, cancellationToken) .ConfigureAwait(false); // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist var lastTempFilelist = await db - .GetLastIncompleteFilesetVolume() + .GetLastIncompleteFilesetVolume(cancellationToken) .ConfigureAwait(false); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced, cancellationToken).ConfigureAwait(false); await db.Transaction - .CommitAsync("CommitRemoteListAnalysisTransaction") + .CommitAsync("CommitRemoteListAnalysisTransaction", token: cancellationToken) .ConfigureAwait(false); var buffer = new byte[m_options.Blocksize]; var hashsize = m_options.BlockhashSize; var missingRemoteFilesets = await db - .MissingRemoteFilesets() + .MissingRemoteFilesets(cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); var missingLocalFilesets = await db - .MissingLocalFilesets() + .MissingLocalFilesets(cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); var emptyIndexFiles = await db - .EmptyIndexFiles() + .EmptyIndexFiles(cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -192,12 +192,12 @@ namespace Duplicati.Library.Main.Operation // Find the most recent timestamp from either a fileset or a remote volume var mostRecentLocal = await db - .GetRemoteVolumes() + .GetRemoteVolumes(cancellationToken) .Where(x => x.Type == RemoteVolumeType.Files) .Select(x => VolumeBase.ParseFilename(x.Name).Time.ToLocalTime()) - .Concat(db.FilesetTimes().Select(x => x.Value.ToLocalTime())) + .Concat(db.FilesetTimes(cancellationToken).Select(x => x.Value.ToLocalTime())) .Append(DateTime.MinValue) - .MaxAsync() + .MaxAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); @@ -231,7 +231,7 @@ namespace Duplicati.Library.Main.Operation if (tp.VerificationRequiredVolumes.Any()) { await using var testdb = - await LocalTestDatabase.CreateAsync(db) + await LocalTestDatabase.CreateAsync(db, null, cancellationToken) .ConfigureAwait(false); foreach (var n in tp.VerificationRequiredVolumes) @@ -242,11 +242,11 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(testdb, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await testdb.Transaction - .CommitAsync("CommitEarlyExit", false) + .CommitAsync("CommitEarlyExit", false, cancellationToken) .ConfigureAwait(false); else await testdb.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); return; } @@ -258,7 +258,7 @@ namespace Duplicati.Library.Main.Operation var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); using (tf) res = - await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1) + await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1, cancellationToken) .ConfigureAwait(false); if (res.Value.Any()) @@ -268,7 +268,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); await db - .UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash, cancellationToken) .ConfigureAwait(false); } @@ -281,7 +281,7 @@ namespace Duplicati.Library.Main.Operation } await db.Transaction - .CommitAsync("CommitVerificationTransaction") + .CommitAsync("CommitVerificationTransaction", true, cancellationToken) .ConfigureAwait(false); } @@ -294,11 +294,11 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitEarlyExit", false) + .CommitAsync("CommitEarlyExit", false, cancellationToken) .ConfigureAwait(false); else await db.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); return; } @@ -319,7 +319,7 @@ namespace Duplicati.Library.Main.Operation foreach (var rv in ifr.Volumes) { var entry = await db - .GetRemoteVolume(rv.Filename) + .GetRemoteVolume(rv.Filename, cancellationToken) .ConfigureAwait(false); if (entry.ID < 0) throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); @@ -331,44 +331,44 @@ namespace Duplicati.Library.Main.Operation throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); await db - .CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks) + .CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, cancellationToken) .ConfigureAwait(false); } var blocksize = m_options.Blocksize; foreach (var ixb in ifr.BlockLists) await db - .CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize) + .CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, cancellationToken) .ConfigureAwait(false); // Register the new index file and link it to the block files var selfid = await db - .RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0)) + .RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), cancellationToken) .ConfigureAwait(false); foreach (var rv in ifr.Volumes) { // Guard against unknown block files long id = await db - .GetRemoteVolumeID(rv.Filename) + .GetRemoteVolumeID(rv.Filename, cancellationToken) .ConfigureAwait(false); if (id == -1) Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); else await db - .AddIndexBlockLink(selfid, id) + .AddIndexBlockLink(selfid, id, cancellationToken) .ConfigureAwait(false); } if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitIndexFileTransaction") + .CommitAsync("CommitIndexFileTransaction", true, cancellationToken) .ConfigureAwait(false); } // All checks fine, we accept the new index file Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); await db - .UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, cancellationToken) .ConfigureAwait(false); continue; @@ -382,7 +382,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { await db - .RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting) + .RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting, cancellationToken) .ConfigureAwait(false); await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); } @@ -408,11 +408,11 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitEarlyExit", false) + .CommitAsync("CommitEarlyExit", false, cancellationToken) .ConfigureAwait(false); else await db.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); return; } @@ -432,17 +432,17 @@ namespace Duplicati.Library.Main.Operation fsw.CreateFilesetFile(isfull); await db - .WriteFileset(fsw, filesetId) + .WriteFileset(fsw, filesetId, cancellationToken) .ConfigureAwait(false); fsw.Close(); fsw.VolumeID = await db - .RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary, cancellationToken) .ConfigureAwait(false); await db - .LinkFilesetToVolume(filesetId, fsw.VolumeID) + .LinkFilesetToVolume(filesetId, fsw.VolumeID, cancellationToken) .ConfigureAwait(false); if (m_options.Dryrun) @@ -460,39 +460,39 @@ namespace Duplicati.Library.Main.Operation await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitFilesetTransaction") + .CommitAsync("CommitFilesetTransaction", token: cancellationToken) .ConfigureAwait(false); } foreach (var volumename in missingLocalFilesets) { var remoteVolume = await db - .GetRemoteVolume(volumename) + .GetRemoteVolume(volumename, cancellationToken) .ConfigureAwait(false); using var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false); var parsed = VolumeBase.ParseFilename(remoteVolume.Name); using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); - await using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false); + await using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options, null, cancellationToken).ConfigureAwait(false); if (compressor == null) throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); var filesetid = await db - .CreateFileset(remoteVolume.ID, parsed.Time) + .CreateFileset(remoteVolume.ID, parsed.Time, cancellationToken) .ConfigureAwait(false); - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()) + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression(), cancellationToken) .ConfigureAwait(false); if (!m_options.Dryrun) await recreatedb.Transaction - .CommitAsync("CommitRecreateFilesetTransaction") + .CommitAsync("CommitRecreateFilesetTransaction", token: cancellationToken) .ConfigureAwait(false); } if (!m_options.Dryrun && tp.MissingVolumes.Any()) await db - .TerminatedWithActiveUploads(true) + .TerminatedWithActiveUploads(cancellationToken, true) .ConfigureAwait(false); if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) @@ -548,7 +548,7 @@ namespace Duplicati.Library.Main.Operation try { // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = (await db.GetRemoteVolume(n.Name).ConfigureAwait(false)).State; + var currentState = (await db.GetRemoteVolume(n.Name, cancellationToken).ConfigureAwait(false)).State; if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) continue; @@ -577,7 +577,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); // Implicit rollback await db.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); return; } @@ -604,7 +604,7 @@ namespace Duplicati.Library.Main.Operation // FlushPendingMessagesAsync, but maybe it // is not needed anymore. await db.Transaction - .CommitAsync() + .CommitAsync(cancellationToken) .ConfigureAwait(false); } } @@ -628,23 +628,23 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { await db - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(cancellationToken, false) .ConfigureAwait(false); await db.Transaction - .CommitAsync("CommitRepairTransaction") + .CommitAsync("CommitRepairTransaction", true, cancellationToken) .ConfigureAwait(false); } else { await db.Transaction - .RollBackAsync("RollbackRepairTransaction") + .RollBackAsync("RollbackRepairTransaction", true, cancellationToken) .ConfigureAwait(false); } } catch (Exception) { // Implicit rollback - await db.Transaction.RollBackAsync().ConfigureAwait(false); + await db.Transaction.RollBackAsync(cancellationToken).ConfigureAwait(false); throw; } } @@ -662,10 +662,10 @@ namespace Duplicati.Library.Main.Operation private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) { volumeWriter.VolumeID = await db - .RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) .ConfigureAwait(false); (var prevFilesetId, var _, var isPrevFull) = await db - .GetFilesetFromRemotename(originalVolume.Name) + .GetFilesetFromRemotename(originalVolume.Name, cancellationToken) .ConfigureAwait(false); if (!string.IsNullOrEmpty(m_options.ControlFiles)) @@ -674,28 +674,28 @@ namespace Duplicati.Library.Main.Operation volumeWriter.CreateFilesetFile(isPrevFull); var newFilesetID = await db - .CreateFileset(volumeWriter.VolumeID, filesetTime) + .CreateFileset(volumeWriter.VolumeID, filesetTime, cancellationToken) .ConfigureAwait(false); await db - .UpdateFullBackupStateInFileset(newFilesetID, isPrevFull) + .UpdateFullBackupStateInFileset(newFilesetID, isPrevFull, cancellationToken) .ConfigureAwait(false); await db - .LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID) + .LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, cancellationToken) .ConfigureAwait(false); await db - .MoveFilesFromFileset(newFilesetID, prevFilesetId) + .MoveFilesFromFileset(newFilesetID, prevFilesetId, cancellationToken) .ConfigureAwait(false); await db - .WriteFileset(volumeWriter, newFilesetID) + .WriteFileset(volumeWriter, newFilesetID, cancellationToken) .ConfigureAwait(false); volumeWriter.Close(); await db - .UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken) .ConfigureAwait(false); await db - .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); @@ -704,7 +704,7 @@ namespace Duplicati.Library.Main.Operation else { await db.Transaction - .CommitAsync("CommitPriorToFilesetUpload") + .CommitAsync("CommitPriorToFilesetUpload", true, cancellationToken) .ConfigureAwait(false); await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } @@ -712,7 +712,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitFilesetTransaction") + .CommitAsync("CommitFilesetTransaction", true, cancellationToken) .ConfigureAwait(false); } @@ -729,26 +729,26 @@ namespace Duplicati.Library.Main.Operation public static async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, IndexVolumeWriter indexWriter, IRemoteVolume originalVolume, Options options, CancellationToken cancellationToken) { indexWriter.VolumeID = await db - .RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero) + .RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, cancellationToken) .ConfigureAwait(false); var blockvolumeids = new List(); using var h = HashFactory.CreateHasher(options.BlockHashAlgorithm); - await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name).ConfigureAwait(false)) + await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name, cancellationToken).ConfigureAwait(false)) { indexWriter.StartVolume(blockvolume.Name); var volumeid = await db - .GetRemoteVolumeID(blockvolume.Name) + .GetRemoteVolumeID(blockvolume.Name, cancellationToken) .ConfigureAwait(false); - await foreach (var b in db.GetBlocks(volumeid).ConfigureAwait(false)) + await foreach (var b in db.GetBlocks(volumeid, cancellationToken).ConfigureAwait(false)) indexWriter.AddBlock(b.Hash, b.Size); indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size); blockvolumeids.Add(volumeid); if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) - await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize).ConfigureAwait(false)) + await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize, cancellationToken).ConfigureAwait(false)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Buffer, 0, b.Size)); if (bh != b.Hash) @@ -760,7 +760,7 @@ namespace Duplicati.Library.Main.Operation foreach (var blockvolumeid in blockvolumeids) await db - .AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid) + .AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, cancellationToken) .ConfigureAwait(false); indexWriter.Close(); @@ -770,7 +770,7 @@ namespace Duplicati.Library.Main.Operation else { await db - .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); @@ -779,7 +779,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!options.Dryrun) await db.Transaction - .CommitAsync("CommitRepairTransaction") + .CommitAsync("CommitRepairTransaction", true, cancellationToken) .ConfigureAwait(false); } @@ -835,12 +835,13 @@ namespace Duplicati.Library.Main.Operation /// /// The options for the current operation. /// The database to use for registering the initial volume. + /// Cancellation token to cancel the operation. /// A task that completes with the new instance of . - public static async Task CreateAsync(Options options, LocalDatabase db) + public static async Task CreateAsync(Options options, LocalDatabase db, CancellationToken cancellationToken) { var ipdv = new InProgressDblockVolumes(options, db); ipdv.m_activeWriter.VolumeID = await db - .RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) .ConfigureAwait(false); return ipdv; } @@ -879,21 +880,24 @@ namespace Duplicati.Library.Main.Operation /// The offset in the buffer where the block data starts. /// The size of the block data. /// The compression hint for the block. - public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) + /// Cancellation token to cancel the operation. + /// A task that completes when the block is added. + public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint, CancellationToken cancellationToken) { await m_activeWriter .AddBlock(hash, buffer, offset, size, hint) .ConfigureAwait(false); m_anyData = true; if (IsFull) - await StartNewVolume().ConfigureAwait(false); + await StartNewVolume(cancellationToken).ConfigureAwait(false); } /// /// Starts a new volume for the current operation. /// + /// Cancellation token to cancel the operation. /// A task that completes when the new volume is started. - public async Task StartNewVolume() + public async Task StartNewVolume(CancellationToken cancellationToken) { if (!m_anyData) return; @@ -902,7 +906,7 @@ namespace Duplicati.Library.Main.Operation m_activeWriter = new BlockVolumeWriter(m_options); m_activeWriter.VolumeID = await m_database - .RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) .ConfigureAwait(false); m_anyData = false; } @@ -920,7 +924,7 @@ namespace Duplicati.Library.Main.Operation private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) { var currentVolume = - await InProgressDblockVolumes.CreateAsync(m_options, db) + await InProgressDblockVolumes.CreateAsync(m_options, db, cancellationToken) .ConfigureAwait(false); foreach (var n in missingDblockFiles) @@ -932,7 +936,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitEarlyExit", false) + .CommitAsync("CommitEarlyExit", false, cancellationToken) .ConfigureAwait(false); return; } @@ -953,13 +957,13 @@ namespace Duplicati.Library.Main.Operation // If we have any data in the volume, complete it so it will be uploaded if (currentVolume.AnyData) - await currentVolume.StartNewVolume().ConfigureAwait(false); + await currentVolume.StartNewVolume(cancellationToken).ConfigureAwait(false); // Complete any pending uploads await UploadCompletedBlockVolumes(backendManager, currentVolume, db, cancellationToken).ConfigureAwait(false); // After reset, we need to remove the temporary volume from the database await db - .RemoveRemoteVolume(currentVolume.RemoteFilename) + .RemoveRemoteVolume(currentVolume.RemoteFilename, cancellationToken) .ConfigureAwait(false); } @@ -976,7 +980,7 @@ namespace Duplicati.Library.Main.Operation foreach (var completedVolume in activeVolume.CompletedWriters) { await db - .UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken) .ConfigureAwait(false); // Create a new index file that points to the new volume @@ -986,19 +990,19 @@ namespace Duplicati.Library.Main.Operation { newvolindex = new IndexVolumeWriter(m_options); newvolindex.VolumeID = await db - .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken) .ConfigureAwait(false); newvolindex.StartVolume(completedVolume.RemoteFilename); - await foreach (var b in db.GetBlocks(completedVolume.VolumeID).ConfigureAwait(false)) + await foreach (var b in db.GetBlocks(completedVolume.VolumeID, cancellationToken).ConfigureAwait(false)) newvolindex.AddBlock(b.Hash, b.Size); await db - .AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID) + .AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, cancellationToken) .ConfigureAwait(false); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { - await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize).ConfigureAwait(false)) + await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize, cancellationToken).ConfigureAwait(false)) newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); }; } @@ -1010,7 +1014,7 @@ namespace Duplicati.Library.Main.Operation else { await db.Transaction - .CommitAsync("PostRepairPreUploadBlockVolume") + .CommitAsync("PostRepairPreUploadBlockVolume", token: cancellationToken) .ConfigureAwait(false); await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); } @@ -1022,13 +1026,13 @@ namespace Duplicati.Library.Main.Operation // Prepare for deleting the old stuff foreach (var vol in activeVolume.VolumesToDelete) await db - .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); // Persist desired state prior to deleting the old files if (!m_options.Dryrun) await db.Transaction - .CommitAsync("PostRepairBlockVolume") + .CommitAsync("PostRepairBlockVolume", token: cancellationToken) .ConfigureAwait(false); // Delete the old files @@ -1042,7 +1046,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("PostRepairBlockVolume") + .CommitAsync("PostRepairBlockVolume", token: cancellationToken) .ConfigureAwait(false); activeVolume.CompletedWriters.Clear(); @@ -1064,18 +1068,18 @@ namespace Duplicati.Library.Main.Operation // as data can be either file contents, metadata or blocklist hashes // We attempt to recover all three source parts in the steps below await using var mbl = await db - .CreateBlockList(originalVolume.Name) + .CreateBlockList(originalVolume.Name, cancellationToken) .ConfigureAwait(false); var originalMissingBlockCount = await mbl - .GetMissingBlockCount() + .GetMissingBlockCount(cancellationToken) .ConfigureAwait(false); var recoveredSourceBlocks = 0L; // First we grab all known blocks from local files string lastRestoredHash = null; long lastRestoredSize = -1; - await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize).ConfigureAwait(false)) + await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize, cancellationToken).ConfigureAwait(false)) { if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) continue; @@ -1111,11 +1115,11 @@ namespace Duplicati.Library.Main.Operation // Found it, no need to look again lastRestoredHash = block.Hash; lastRestoredSize = block.Size; - if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID).ConfigureAwait(false)) + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) { recoveredSourceBlocks++; await pendingVolume - .AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default) + .AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default, cancellationToken) .ConfigureAwait(false); } } @@ -1126,7 +1130,7 @@ namespace Duplicati.Library.Main.Operation } // Then grab all blocks with metadata - await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks().ConfigureAwait(false)) + await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks(cancellationToken).ConfigureAwait(false)) { if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) continue; @@ -1162,11 +1166,11 @@ namespace Duplicati.Library.Main.Operation // Found it, no need to look again lastRestoredHash = block.Hash; lastRestoredSize = block.Size; - if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID).ConfigureAwait(false)) + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) { recoveredSourceBlocks++; await pendingVolume - .AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default) + .AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default, cancellationToken) .ConfigureAwait(false); } } @@ -1201,9 +1205,9 @@ namespace Duplicati.Library.Main.Operation if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) { - if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID).ConfigureAwait(false)) + if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) await pendingVolume - .AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default) + .AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default, cancellationToken) .ConfigureAwait(false); } else @@ -1214,7 +1218,7 @@ namespace Duplicati.Library.Main.Operation blocklistHash.SetLength(0); } - await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock).ConfigureAwait(false)) + await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock, cancellationToken).ConfigureAwait(false)) { lastBlocklist ??= blocklist; @@ -1231,7 +1235,7 @@ namespace Duplicati.Library.Main.Operation await EmitBlockListBlock().ConfigureAwait(false); //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources().ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false)) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false)) { try { @@ -1239,10 +1243,10 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) foreach (var b in f.Blocks) - if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID).ConfigureAwait(false)) + if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) if (f.ReadBlock(b.Key, buffer) == b.Value) await pendingVolume - .AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default) + .AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default, cancellationToken) .ConfigureAwait(false); } catch (Exception ex) @@ -1253,13 +1257,13 @@ namespace Duplicati.Library.Main.Operation } var missingBlocks = await mbl - .GetMissingBlockCount() + .GetMissingBlockCount(cancellationToken) .ConfigureAwait(false); var recoveredBlocks = originalMissingBlockCount - missingBlocks; if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) { Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); - await foreach (var f in mbl.GetFilesetsUsingMissingBlocks().ConfigureAwait(false)) + await foreach (var f in mbl.GetFilesetsUsingMissingBlocks(cancellationToken).ConfigureAwait(false)) Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); @@ -1280,7 +1284,7 @@ namespace Duplicati.Library.Main.Operation { pendingVolume.VolumesToDelete.Add(originalVolume); var oldIndexFiles = await db - .GetIndexFilesReferencingBlockFile(originalVolume.ID) + .GetIndexFilesReferencingBlockFile(originalVolume.ID, cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -1288,12 +1292,12 @@ namespace Duplicati.Library.Main.Operation foreach (var oldIndexFile in oldIndexFiles) { var oldVolume = await db - .GetRemoteVolume(oldIndexFile) + .GetRemoteVolume(oldIndexFile, cancellationToken) .ConfigureAwait(false); if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) { var blockVolumesReferenced = await db - .GetBlockVolumesFromIndexName(oldIndexFile) + .GetBlockVolumesFromIndexName(oldIndexFile, cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) @@ -1322,12 +1326,12 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var sets = await db - .GetFilesetsWithMissingFiles() - .ToListAsync() + .GetFilesetsWithMissingFiles(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (sets.Count == 0) @@ -1340,7 +1344,7 @@ namespace Duplicati.Library.Main.Operation ix++; Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); var volume = await db - .GetRemoteVolumeFromFilesetID(entry.Key) + .GetRemoteVolumeFromFilesetID(entry.Key, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var parsed = VolumeBase.ParseFilename(volume.Name); using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); @@ -1350,15 +1354,17 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); // Clear out the old fileset - await db.DeleteFilesetEntries(entry.Key).ConfigureAwait(false); - await using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false)) + await db + .DeleteFilesetEntries(entry.Key, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression()) + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression(), m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } await db.Transaction - .CommitAsync("PostRepairFileset") + .CommitAsync("PostRepairFileset", true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -1377,22 +1383,22 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - if (await db.RepairInProgress().ConfigureAwait(false) || await db.PartiallyRecreated().ConfigureAwait(false)) + if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) || await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); - await db.FixDuplicateMetahash().ConfigureAwait(false); - await db.FixDuplicateFileentries().ConfigureAwait(false); + await db.FixDuplicateMetahash(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await db.FixDuplicateFileentries(m_result.TaskControl.ProgressToken).ConfigureAwait(false); await db - .FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize) + .FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db - .FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize) + .FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index d3d18639e..80658a033 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -46,7 +46,14 @@ namespace Duplicati.Library.Main.Operation Directory.CreateDirectory(m_options.Restorepath); using var tmpdb = new TempFile(); - await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync( + File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", + true, + m_options.SqlitePageCache, + null, + m_result.TaskControl.ProgressToken + ).ConfigureAwait(false); + var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); try @@ -68,7 +75,7 @@ namespace Duplicati.Library.Main.Operation var file = fileversion.Value.File; var entry = await db - .GetRemoteVolume(file.Name) + .GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var res = new List(); diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index c6c9af53e..6d3b26311 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -115,7 +115,7 @@ namespace Duplicati.Library.Main.Operation { if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) { - db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache) + db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } else @@ -123,7 +123,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); tmpdb = new TempFile(); RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); - db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache) + db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) @@ -152,16 +152,16 @@ namespace Duplicati.Library.Main.Operation } } - private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage) + private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage, CancellationToken cancellationToken) { var blocksize = options.Blocksize; var updateCounter = 0L; var fullblockverification = options.FullBlockVerification; using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm); - await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); - await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize).ConfigureAwait(false); - await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks().ConfigureAwait(false)) + await using var blockmarker = await database.CreateBlockMarkerAsync(cancellationToken).ConfigureAwait(false); + await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize, cancellationToken).ConfigureAwait(false); + await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks(cancellationToken).ConfigureAwait(false)) { var targetpath = restorelist.Path; @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Operation // TODO: Much faster if we iterate the volume and checks what blocks are used, // because the compressors usually like sequential reading using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + await foreach (var targetblock in restorelist.Blocks(cancellationToken).ConfigureAwait(false)) { file.Position = targetblock.Offset; var size = blocks.ReadBlock(targetblock.Key, blockbuffer); @@ -205,7 +205,7 @@ namespace Duplicati.Library.Main.Operation { file.Write(blockbuffer, 0, size); await blockmarker - .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false) + .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false, cancellationToken) .ConfigureAwait(false); } } @@ -217,7 +217,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCounter) % 20 == 0) await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, cancellationToken) .ConfigureAwait(false); } catch (Exception ex) @@ -231,7 +231,7 @@ namespace Duplicati.Library.Main.Operation if (!options.SkipMetadata) { - await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks().ConfigureAwait(false)) + await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks(cancellationToken).ConfigureAwait(false)) { var targetpath = restoremetadata.Path; Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); @@ -241,7 +241,7 @@ namespace Duplicati.Library.Main.Operation // TODO: When we support multi-block metadata this needs to deal with it using (var ms = new System.IO.MemoryStream()) { - await foreach (var targetblock in restoremetadata.Blocks().ConfigureAwait(false)) + await foreach (var targetblock in restoremetadata.Blocks(cancellationToken).ConfigureAwait(false)) { ms.Position = targetblock.Offset; var size = blocks.ReadBlock(targetblock.Key, blockbuffer); @@ -249,7 +249,7 @@ namespace Duplicati.Library.Main.Operation { ms.Write(blockbuffer, 0, size); await blockmarker - .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true) + .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true, cancellationToken) .ConfigureAwait(false); } } @@ -268,9 +268,9 @@ namespace Duplicati.Library.Main.Operation } } await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, cancellationToken) .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); + await blockmarker.CommitAsync(cancellationToken).ConfigureAwait(false); } private static void ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage) @@ -316,16 +316,16 @@ namespace Duplicati.Library.Main.Operation private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { // Perform initial setup - await Utility.UpdateOptionsFromDb(database, m_options) + await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(database, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken) .ConfigureAwait(false); // Verify the backend if necessary if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false); } // Prepare the block and file list and create the directory structure @@ -339,7 +339,7 @@ namespace Duplicati.Library.Main.Operation // At this point, there should be no more writes to the database, so we have to unlock the database: await database.Transaction - .CommitAsync("CommitBeforeRestore") + .CommitAsync("CommitBeforeRestore", token: cancellationToken) .ConfigureAwait(false); using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); @@ -391,7 +391,7 @@ namespace Duplicati.Library.Main.Operation } await database.Transaction - .CommitAsync("CommitAfterRestore") + .CommitAsync("CommitAfterRestore", token: cancellationToken) .ConfigureAwait(false); await database.DisposePoolAsync().ConfigureAwait(false); @@ -422,7 +422,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - await database.DropRestoreTable().ConfigureAwait(false); + await database.DropRestoreTable(cancellationToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Report that the restore is complete @@ -443,16 +443,16 @@ namespace Duplicati.Library.Main.Operation //using (var database = new LocalRestoreDatabase(dbparent)) using (var metadatastorage = new RestoreHandlerMetadataStorage()) { - await Utility.UpdateOptionsFromDb(database, m_options) + await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(database, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken) .ConfigureAwait(false); m_blockbuffer = new byte[m_options.Blocksize]; if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false); } //Figure out what files are to be patched, and what blocks are needed @@ -510,8 +510,8 @@ namespace Duplicati.Library.Main.Operation List volumes; using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes")) volumes = await database - .GetMissingVolumes() - .ToListAsync() + .GetMissingVolumes(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); if (volumes.Count > 0) @@ -535,7 +535,7 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) - await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage) + await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage, cancellationToken) .ConfigureAwait(false); } catch (Exception ex) @@ -550,7 +550,7 @@ namespace Duplicati.Library.Main.Operation var fileErrors = 0L; // Restore empty files. They might not have any blocks so don't appear in any volume. - await foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0).ConfigureAwait(false)) + await foreach (var file in database.GetFilesToRestore(true, cancellationToken).Where(item => item.Length == 0).ConfigureAwait(false)) { Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path); @@ -589,7 +589,7 @@ namespace Duplicati.Library.Main.Operation // After all blocks in the files are restored, verify the file hash using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification")) - await foreach (var file in database.GetFilesToRestore(true).ConfigureAwait(false)) + await foreach (var file in database.GetFilesToRestore(true, cancellationToken).ConfigureAwait(false)) { try { @@ -632,7 +632,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - await database.DropRestoreTable().ConfigureAwait(false); + await database.DropRestoreTable(cancellationToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); } @@ -712,9 +712,9 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) { // Fill BLOCKS with data from known local source files - await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); var updateCount = 0L; - await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize).ConfigureAwait(false)) + await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false)) { var targetpath = entry.TargetPath; var targetfileid = entry.TargetFileID; @@ -737,7 +737,7 @@ namespace Duplicati.Library.Main.Operation try { using var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath); - await foreach (var block in entry.Blocks().ConfigureAwait(false)) + await foreach (var block in entry.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -761,7 +761,7 @@ namespace Duplicati.Library.Main.Operation targetstream.Write(blockbuffer, 0, size); } - await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false) + await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -779,7 +779,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) { await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -810,17 +810,17 @@ namespace Duplicati.Library.Main.Operation } await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); + await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); } private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) { // Fill BLOCKS with data from known local source files - await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); var updateCount = 0L; - await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize).ConfigureAwait(false)) + await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false)) { var targetpath = restorelist.TargetPath; var targetfileid = restorelist.TargetFileID; @@ -838,9 +838,9 @@ namespace Duplicati.Library.Main.Operation } using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await foreach (var source in targetblock.BlockSources().ConfigureAwait(false)) + await foreach (var source in targetblock.BlockSources(result.TaskControl.ProgressToken).ConfigureAwait(false)) { try { @@ -878,7 +878,7 @@ namespace Duplicati.Library.Main.Operation } await blockmarker - .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false) + .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false, result.TaskControl.ProgressToken) .ConfigureAwait(false); patched = true; break; @@ -898,7 +898,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); } catch (Exception ex) @@ -918,9 +918,9 @@ namespace Duplicati.Library.Main.Operation } await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); + await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); } private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) @@ -930,7 +930,7 @@ namespace Duplicati.Library.Main.Operation using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList")) { var c = await database - .PrepareRestoreFilelist(options.Time, options.Version, filter) + .PrepareRestoreFilelist(options.Time, options.Version, filter, result.TaskControl.ProgressToken) .ConfigureAwait(false); result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); } @@ -941,30 +941,30 @@ namespace Duplicati.Library.Main.Operation // Find the largest common prefix var largest_prefix = options.DontCompressRestorePaths ? "" : - await database.GetLargestPrefix().ConfigureAwait(false); + await database.GetLargestPrefix(result.TaskControl.ProgressToken).ConfigureAwait(false); Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath)); // Set the target paths, special care with C:\ and / await database - .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)) + .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath), result.TaskControl.ProgressToken) .ConfigureAwait(false); } else { - await database.SetTargetPaths("", "").ConfigureAwait(false); + await database.SetTargetPaths("", "", result.TaskControl.ProgressToken).ConfigureAwait(false); } // Create a temporary table BLOCKS that lists all blocks that needs to be recovered using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks")) await database - .FindMissingBlocks(options.SkipMetadata) + .FindMissingBlocks(options.SkipMetadata, result.TaskControl.ProgressToken) .ConfigureAwait(false); // Create temporary tables and triggers that automatically track progress using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker")) await database - .CreateProgressTracker(false) + .CreateProgressTracker(false, result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -983,7 +983,7 @@ namespace Duplicati.Library.Main.Operation SystemIO.IO_OS.DirectoryCreate(options.Restorepath); } - await foreach (var folder in database.GetTargetFolders().ConfigureAwait(false)) + await foreach (var folder in database.GetTargetFolders(result.TaskControl.ProgressToken).ConfigureAwait(false)) { try { @@ -1014,9 +1014,9 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) { // Scan existing files for existing BLOCKS - await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); var updateCount = 0L; - await foreach (var restorelist in database.GetExistingFilesWithBlocks().ConfigureAwait(false)) + await foreach (var restorelist in database.GetExistingFilesWithBlocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { var rename = !options.Overwrite; var targetpath = restorelist.TargetPath; @@ -1069,7 +1069,7 @@ namespace Duplicati.Library.Main.Operation using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) using (var block = new Blockprocessor(file, blockbuffer)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { var size = block.Readblock(); if (size <= 0) @@ -1095,7 +1095,7 @@ namespace Duplicati.Library.Main.Operation if (key == targetblock.Hash) { await blockmarker - .SetBlockRestored(targetfileid, targetblock.Index, key, size, false) + .SetBlockRestored(targetfileid, targetblock.Index, key, size, false, result.TaskControl.ProgressToken) .ConfigureAwait(false); blockhashmatch = true; } @@ -1132,7 +1132,7 @@ namespace Duplicati.Library.Main.Operation { //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. await blockmarker - .SetFileDataVerified(targetfileid) + .SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken) .ConfigureAwait(false); Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); rename = false; @@ -1142,7 +1142,7 @@ namespace Duplicati.Library.Main.Operation // The new file will have none of the correct blocks, // even if the scanned file had some await blockmarker - .SetAllBlocksMissing(targetfileid) + .SetAllBlocksMissing(targetfileid, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -1150,7 +1150,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) { await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -1199,10 +1199,10 @@ namespace Duplicati.Library.Main.Operation //TODO: Also needs metadata check to make correct decision. // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. await blockmarker - .SetAllBlocksRestored(targetfileid, false) + .SetAllBlocksRestored(targetfileid, false, result.TaskControl.ProgressToken) .ConfigureAwait(false); await blockmarker - .SetFileDataVerified(targetfileid) + .SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken) .ConfigureAwait(false); break; } @@ -1220,17 +1220,17 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); await database - .UpdateTargetPath(targetfileid, newname) + .UpdateTargetPath(targetfileid, newname, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); await blockmarker - .CommitAsync() + .CommitAsync(result.TaskControl.ProgressToken) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs index 6f09c175b..fe66207c2 100644 --- a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs +++ b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs @@ -50,14 +50,14 @@ internal static class SearchEntriesHandler throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); await using var db = - await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) { filesetIds = await db - .GetFilesetIDs(options.Time, options.Version) - .ToArrayAsync() + .GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); if (filesetIds.Length == 0) @@ -68,7 +68,7 @@ internal static class SearchEntriesHandler paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); result.FileVersions = await db - .SearchEntries(paths, filter, filesetIds, offset, limit) + .SearchEntries(paths, filter, filesetIds, offset, limit, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 58852c7a6..523398465 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -39,12 +39,12 @@ namespace Duplicati.Library.Main.Operation private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly Options m_options; - private readonly TestResults m_results; + private readonly TestResults m_result; public TestHandler(Options options, TestResults results) { m_options = options; - m_results = results; + m_result = results; } public async Task RunAsync(long samples, IBackendManager backendManager) @@ -52,17 +52,17 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); await db.Transaction .CommitAsync("TestHandlerComplete") @@ -72,35 +72,35 @@ namespace Duplicati.Library.Main.Operation public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend) { var files = await db - .SelectTestTargets(samples, m_options) - .ToListAsync() + .SelectTestTargets(samples, m_options, m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); - m_results.OperationProgressUpdater.UpdateProgress(0); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); + m_result.OperationProgressUpdater.UpdateProgress(0); var progress = 0L; if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False) { var faultyIndexFiles = new List(); - await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_results.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var vol = new RemoteVolume(name, hash, size); try { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backend.WaitForEmptyAsync(db, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - m_results.EndTime = DateTime.UtcNow; + await backend.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + m_result.EndTime = DateTime.UtcNow; return; } progress++; - m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); + m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); KeyValuePair>> res; using (tf) - res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2) + res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var parsedInfo = VolumeBase.ParseFilename(vol.Name); @@ -119,14 +119,14 @@ namespace Duplicati.Library.Main.Operation faultyIndexFiles.Add(vol); } - m_results.AddResult(res.Key, res.Value); + m_result.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0) { if (res.Value == null || !res.Value.Any()) { var rv = await db - .GetRemoteVolume(vol.Name) + .GetRemoteVolume(vol.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (rv.ID < 0) @@ -141,7 +141,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", "Successfully captured hash and size for {0}, updating database", vol.Name); await db - .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash) + .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -150,16 +150,16 @@ namespace Duplicati.Library.Main.Operation } await db - .UpdateVerificationCount(vol.Name) + .UpdateVerificationCount(vol.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } catch (Exception ex) { - m_results.AddResult(vol.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); + m_result.AddResult(vol.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileProcessingFailed", ex, "Failed to process file {0}", vol.Name); if (ex.IsAbortException()) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; throw; } } @@ -170,7 +170,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.ReplaceFaultyIndexFiles) { Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, "Found {0} faulty index files, repairing now", faultyIndexFiles.Count); - await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } else Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, "Found {0} faulty index files, use the option {1} to repair them", faultyIndexFiles.Count, "--replace-faulty-index-files"); @@ -183,26 +183,26 @@ namespace Duplicati.Library.Main.Operation { try { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; return; } progress++; - m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); + m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); if (f.Size <= 0 || string.IsNullOrWhiteSpace(f.Hash)) { Logging.Log.WriteInformationMessage(LOGTAG, "MissingRemoteHash", "No hash or size recorded for {0}, performing full verification", f.Name); KeyValuePair>> res; - (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false); using (tf) - res = await TestVolumeInternals(db, f, tf, m_options, 1) + res = await TestVolumeInternals(db, f, tf, m_options, 1, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - m_results.AddResult(res.Key, res.Value); + m_result.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(hash) && size > 0) { @@ -216,7 +216,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", "Successfully captured hash and size for {0}, updating database", f.Name); await db - .UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -224,39 +224,39 @@ namespace Duplicati.Library.Main.Operation } else { - using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { } } await db - .UpdateVerificationCount(f.Name) + .UpdateVerificationCount(f.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - m_results.AddResult(f.Name, []); + m_result.AddResult(f.Name, []); } catch (Exception ex) { - m_results.AddResult(f.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); + m_result.AddResult(f.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); Logging.Log.WriteErrorMessage(LOGTAG, "FailedToProcessFile", ex, "Failed to process file {0}", f.Name); if (ex.IsAbortOrCancelException()) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; throw; } } } } - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; // generate a backup error status when any test is failing - except for 'extra' status // because these problems don't block database rebuilding. - var filtered = from n in m_results.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n; + var filtered = from n in m_result.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n; if (!filtered.Any()) { - Logging.Log.WriteInformationMessage(LOGTAG, "Test results", "Successfully verified {0} remote files", m_results.VerificationsActualLength); + Logging.Log.WriteInformationMessage(LOGTAG, "Test results", "Successfully verified {0} remote files", m_result.VerificationsActualLength); } else { - Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, "Verified {0} remote files with {1} problem(s)", m_results.VerificationsActualLength, filtered.Count()); + Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, "Verified {0} remote files with {1} problem(s)", m_result.VerificationsActualLength, filtered.Count()); } } @@ -268,8 +268,9 @@ namespace Duplicati.Library.Main.Operation /// The path to the downloaded copy of the file. /// The options to use for the test. /// A value between 0 and 1 that indicates how many blocks are tested in a dblock file. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a key-value pair where the key is the volume name and the value is a list of test results. - public static async Task>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent) + public static async Task>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent, CancellationToken cancellationToken) { var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm); var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name); @@ -280,15 +281,15 @@ namespace Duplicati.Library.Main.Operation case RemoteVolumeType.Files: //Compare with db and see if all files are accounted for // with correct file hashes and blocklist hashes - await using (var fl = await db.CreateFilelist(vol.Name).ConfigureAwait(false)) + await using (var fl = await db.CreateFilelist(vol.Name, cancellationToken).ConfigureAwait(false)) { using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options)) foreach (var f in rd.Files) await fl - .Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time) + .Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time, cancellationToken) .ConfigureAwait(false); - return new KeyValuePair>>(vol.Name, await fl.Compare().ToListAsync().ConfigureAwait(false)); + return new KeyValuePair>>(vol.Name, await fl.Compare(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false)); } case RemoteVolumeType.Index: @@ -301,17 +302,17 @@ namespace Duplicati.Library.Main.Operation foreach (var v in rd.Volumes) { blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); - await using (var bl = await db.CreateBlocklist(v.Filename).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklist(v.Filename, cancellationToken).ConfigureAwait(false)) { foreach (var h in v.Blocks) await bl - .AddBlock(h.Key, h.Value) + .AddBlock(h.Key, h.Value, cancellationToken) .ConfigureAwait(false); combined.AddRange( await bl - .Compare() - .ToListAsync() + .Compare(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } @@ -320,17 +321,17 @@ namespace Duplicati.Library.Main.Operation if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) { var hashesPerBlock = options.Blocksize / options.BlockhashSize; - await using (var bl = await db.CreateBlocklistHashList(vol.Name).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklistHashList(vol.Name, cancellationToken).ConfigureAwait(false)) { foreach (var b in rd.BlockLists) await bl - .AddBlockHash(b.Hash, b.Length) + .AddBlockHash(b.Hash, b.Length, cancellationToken) .ConfigureAwait(false); combined.AddRange( await bl - .Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize) - .ToListAsync() + .Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize, cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } @@ -338,17 +339,17 @@ namespace Duplicati.Library.Main.Operation } // Compare with db and see that all blocklists are listed - await using (var il = await db.CreateIndexlist(vol.Name).ConfigureAwait(false)) + await using (var il = await db.CreateIndexlist(vol.Name, cancellationToken).ConfigureAwait(false)) { foreach (var t in blocklinks) await il - .AddBlockLink(t.Item1, t.Item2, t.Item3) + .AddBlockLink(t.Item1, t.Item2, t.Item3, cancellationToken) .ConfigureAwait(false); combined.AddRange( await il - .Compare() - .ToArrayAsync() + .Compare(cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } @@ -356,13 +357,13 @@ namespace Duplicati.Library.Main.Operation return new KeyValuePair>>(vol.Name, combined); case RemoteVolumeType.Blocks: using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - await using (var bl = await db.CreateBlocklist(vol.Name).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklist(vol.Name, cancellationToken).ConfigureAwait(false)) using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options)) { //Verify that all blocks are in the file foreach (var b in rd.Blocks) await bl - .AddBlock(b.Key, b.Value) + .AddBlock(b.Key, b.Value, cancellationToken) .ConfigureAwait(false); //Select random blocks and verify their hashes match the filename and size @@ -392,8 +393,8 @@ namespace Duplicati.Library.Main.Operation vol.Name, changes.Union( await bl - .Compare() - .ToListAsync() + .Compare(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ) ); @@ -407,14 +408,14 @@ namespace Duplicati.Library.Main.Operation private async Task ReplaceFaultyIndexFilesAsync(List faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, CancellationToken cancellationToken) { await using var repairdb = - await LocalRepairDatabase.CreateAsync(db) + await LocalRepairDatabase.CreateAsync(db, null, cancellationToken) .ConfigureAwait(false); foreach (var vol in faultyIndexFiles) { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; return; } @@ -429,8 +430,8 @@ namespace Duplicati.Library.Main.Operation } else { - await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(repairdb, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(repairdb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await repairdb.Transaction .CommitAsync("ReplaceFaultyIndexFileCommit") .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index dfc62247c..9556363fa 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -38,10 +38,10 @@ namespace Duplicati.Library.Main.Operation public virtual async Task RunAsync() { await using var db = - await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache) + await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); - await db.Vacuum().ConfigureAwait(false); + await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; } } diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index feaf14dbc..0d01d04b7 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -28,6 +28,7 @@ using System.Threading.Tasks; using Newtonsoft.Json; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; +using System.Threading; namespace Duplicati.Library.Main { @@ -103,11 +104,12 @@ namespace Duplicati.Library.Main /// /// The database to read from. /// The options to update. + /// A cancellation token to cancel the operation. /// A task that completes when the options have been updated. - internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options) + internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options, CancellationToken cancellationToken) { string n = null; - var opts = await db.GetDbOptions().ConfigureAwait(false); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); if (opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n))) options.RawOptions["blocksize"] = opts["blocksize"] + "b"; @@ -121,11 +123,12 @@ namespace Duplicati.Library.Main /// Checks if the database contains options that need to be verified, such as the blocksize. /// /// The database to check. + /// A cancellation token to cancel the operation. /// A task that when awaited returns true if the database contains options that need to be verified; false otherwise. - internal static async Task ContainsOptionsForVerification(LocalDatabase db) + internal static async Task ContainsOptionsForVerification(LocalDatabase db, CancellationToken cancellationToken) { - var opts = await db.GetDbOptions().ConfigureAwait(false); - await db.Transaction.CommitAsync().ConfigureAwait(false); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); + await db.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false); return new[] { "blocksize", "blockhash", @@ -140,8 +143,9 @@ namespace Duplicati.Library.Main /// /// The database to check. /// The options to verify. + /// A cancellation token to cancel the operation. /// A task that completes when the options have been verified and the database has been updated if needed. - internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options) + internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options, CancellationToken cancellationToken) { var newDict = new Dictionary { @@ -149,7 +153,7 @@ namespace Duplicati.Library.Main { "blockhash", options.BlockHashAlgorithm }, { "filehash", options.FileHashAlgorithm } }; - var opts = await db.GetDbOptions().ConfigureAwait(false); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); if (options.NoEncryption) { @@ -199,7 +203,7 @@ namespace Duplicati.Library.Main } //Extra sanity check - if (await db.GetBlocksLargerThan(options.Blocksize).ConfigureAwait(false) > 0) + if (await db.GetBlocksLargerThan(options.Blocksize, cancellationToken).ConfigureAwait(false) > 0) throw new Duplicati.Library.Interface.UserInformationException("You have attempted to change the block-size on an existing backup, which is not supported. Please configure a new clean backup if you want to change the block-size.", "BlockSizeChangeNotSupported"); if (needsUpdate) @@ -209,7 +213,7 @@ namespace Duplicati.Library.Main if (!newDict.ContainsKey(k.Key)) newDict[k.Key] = k.Value; - await db.SetDbOptions(newDict).ConfigureAwait(false); + await db.SetDbOptions(newDict, cancellationToken).ConfigureAwait(false); } } } From 0f0da82f2828b96b11719f41ace1d507c5396d64 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 10:59:22 +0200 Subject: [PATCH 339/362] Removed whitespace --- .../Operation/Backup/FileBlockProcessor.cs | 30 +++++++++---------- .../Operation/Backup/FilePreFilterProcess.cs | 30 +++++++++---------- .../Operation/Backup/StreamBlockSplitter.cs | 30 +++++++++---------- .../Operation/Backup/UploadRealFilelist.cs | 30 +++++++++---------- .../Backup/UploadSyntheticFilelist.cs | 30 +++++++++---------- .../Operation/Common/IndexVolumeCreator.cs | 30 +++++++++---------- 6 files changed, 90 insertions(+), 90 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index b404232f0..c9f8a6f12 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 744606614..510b877e9 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs index 6d63dfd91..dd88b8404 100644 --- a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs +++ b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs index aa8a85d9c..7a90cc498 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index 76b80f852..afe037f66 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Main.Database; diff --git a/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs b/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs index caa6349e1..32f69fe51 100644 --- a/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs +++ b/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 78d649f5a39718a960cce5fe0bd22c714da1d95a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 11:00:24 +0200 Subject: [PATCH 340/362] All of the non-internal-library methods calling the database functions forwards CancellationToken.None --- Duplicati/CommandLine/DatabaseTool/Helper.cs | 6 +++--- Duplicati/UnitTest/DirectListHandlerTests.cs | 17 +++++++++-------- Duplicati/UnitTest/DisruptionTests.cs | 2 +- Duplicati/UnitTest/RepairHandlerTests.cs | 10 +++++----- Duplicati/UnitTest/SQLiteTests.cs | 17 +++++++++-------- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Helper.cs b/Duplicati/CommandLine/DatabaseTool/Helper.cs index c3ed85113..e8ae324c8 100644 --- a/Duplicati/CommandLine/DatabaseTool/Helper.cs +++ b/Duplicati/CommandLine/DatabaseTool/Helper.cs @@ -66,7 +66,7 @@ public static class Helper SELECT ""DBPath"" FROM ""Backup"" "); - await foreach (var rd in cmd.ExecuteReaderEnumerableAsync().ConfigureAwait(false)) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(CancellationToken.None).ConfigureAwait(false)) dbpaths.Add(rd.ConvertValueToString(0) ?? ""); } catch @@ -104,13 +104,13 @@ public static class Helper type='table' AND name='Backup' OR name='Schedule' - ") + ", CancellationToken.None) .ConfigureAwait(false) == 2; var version = (int)await cmd.ExecuteScalarInt64Async(@" SELECT MAX(Version) FROM Version - ") + ", CancellationToken.None) .ConfigureAwait(false); return (version, isserverdb); diff --git a/Duplicati/UnitTest/DirectListHandlerTests.cs b/Duplicati/UnitTest/DirectListHandlerTests.cs index 3a82641f9..a15768b56 100644 --- a/Duplicati/UnitTest/DirectListHandlerTests.cs +++ b/Duplicati/UnitTest/DirectListHandlerTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; @@ -326,7 +327,7 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1) + using var db = await LocalListDatabase.CreateAsync(tempFile, 1, null, CancellationToken.None) .ConfigureAwait(false); SeedTestData(db, [ "/folder1/", // 1 @@ -338,7 +339,7 @@ namespace Duplicati.UnitTest // Act var result = await db - .GetRootPrefixes(1) + .GetRootPrefixes(1, CancellationToken.None) .ToListAsync() .ConfigureAwait(false); @@ -356,7 +357,7 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1) + using var db = await LocalListDatabase.CreateAsync(tempFile, 1, null, CancellationToken.None) .ConfigureAwait(false); SeedTestData(db, [ "C:\\folder1\\", @@ -367,7 +368,7 @@ namespace Duplicati.UnitTest // Act var result = await db - .GetRootPrefixes(1) + .GetRootPrefixes(1, CancellationToken.None) .ToListAsync() .ConfigureAwait(false); @@ -384,7 +385,7 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1) + using var db = await LocalListDatabase.CreateAsync(tempFile, 1, null, CancellationToken.None) .ConfigureAwait(false); SeedTestData(db, [ "\\\\server\\share\\folder\\", @@ -394,7 +395,7 @@ namespace Duplicati.UnitTest // Act var result = await db - .GetRootPrefixes(1) + .GetRootPrefixes(1, CancellationToken.None) .ToListAsync() .ConfigureAwait(false); @@ -410,7 +411,7 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1) + using var db = await LocalListDatabase.CreateAsync(tempFile, 1, null, CancellationToken.None) .ConfigureAwait(false); SeedTestData(db, [ "C:\\data\\", // 1 @@ -424,7 +425,7 @@ namespace Duplicati.UnitTest // Act var result = await db - .GetRootPrefixes(1) + .GetRootPrefixes(1, CancellationToken.None) .ToListAsync() .ConfigureAwait(false); diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 18982f1d9..4ea98c964 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -805,7 +805,7 @@ namespace Duplicati.UnitTest } catch (TestUtils.TestVerificationException e) { - using var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, 0).ConfigureAwait(false); + using var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, 0, null, CancellationToken.None).ConfigureAwait(false); using var cmd = db.Connection.CreateCommand(); var sb = new StringBuilder(); diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index f4ccbb995..7bfe4e6d1 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -426,13 +426,13 @@ namespace Duplicati.UnitTest } // Check database block link - using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, 0).ConfigureAwait(false)) + using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, 0, null, CancellationToken.None).ConfigureAwait(false)) { var indexVolumeId = await db - .GetRemoteVolumeID(Path.GetFileName(origFile)) + .GetRemoteVolumeID(Path.GetFileName(origFile), CancellationToken.None) .ConfigureAwait(false); var duplicateVolumeId = await db - .GetRemoteVolumeID(Path.GetFileName(dupFile)) + .GetRemoteVolumeID(Path.GetFileName(dupFile), CancellationToken.None) .ConfigureAwait(false); Assert.AreNotEqual(-1, indexVolumeId); Assert.AreNotEqual(-1, duplicateVolumeId); @@ -443,12 +443,12 @@ namespace Duplicati.UnitTest var linkedIndexId = await cmd .SetCommandAndParameters(sql) .SetParameterValue("@VolumeId", indexVolumeId) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, CancellationToken.None) .ConfigureAwait(false); var linkedDuplicateId = await cmd .SetCommandAndParameters(sql) .SetParameterValue("@VolumeId", duplicateVolumeId) - .ExecuteScalarInt64Async(-1) + .ExecuteScalarInt64Async(-1, CancellationToken.None) .ConfigureAwait(false); Assert.AreEqual(linkedIndexId, linkedDuplicateId); } diff --git a/Duplicati/UnitTest/SQLiteTests.cs b/Duplicati/UnitTest/SQLiteTests.cs index 92f881c64..f9ae9164d 100755 --- a/Duplicati/UnitTest/SQLiteTests.cs +++ b/Duplicati/UnitTest/SQLiteTests.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; @@ -75,25 +76,25 @@ namespace Duplicati.UnitTest using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1, 2, 3 }); - Assert.AreEqual(3, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); + Assert.AreEqual(3, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1, 2 }); - Assert.AreEqual(2, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); + Assert.AreEqual(2, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1 }); - Assert.AreEqual(1, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); + Assert.AreEqual(1, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[0]); - Assert.AreEqual(0, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); + Assert.AreEqual(0, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } var list = new List(); @@ -102,10 +103,10 @@ namespace Duplicati.UnitTest using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { - using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list) + using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list, CancellationToken.None) .ConfigureAwait(false); command.ExpandInClauseParameter("@List", list); - Assert.AreEqual(3, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); + Assert.AreEqual(3, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) @@ -113,10 +114,10 @@ namespace Duplicati.UnitTest list.Remove(1); list.Remove(2); - using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list) + using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list, CancellationToken.None) .ConfigureAwait(false); command.ExpandInClauseParameter("@List", list); - Assert.AreEqual(1, await command.ExecuteScalarInt64Async().ConfigureAwait(false)); + Assert.AreEqual(1, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } } } From 7e979a4e9b5858721a85cf0b81e42baedab6b510 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 11:01:12 +0200 Subject: [PATCH 341/362] Found some methods in LocalBackupDatabase that didn't accept a cancellation token --- .../Main/Database/LocalBackupDatabase.cs | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 582b06a55..aa5804e72 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -1143,8 +1143,16 @@ namespace Duplicati.Library.Main.Database return (-1, oldModified, length); } - - public async Task<(long, DateTime, long, string?, long)> GetFileEntry(long prefixid, string path, long filesetid) + /// + /// Gets the file entry for a given path in the fileset. + /// + /// The ID of the path prefix. + /// The path to the file. + /// The ID of the fileset. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains a tuple with the file ID, last modified time, file size, metadata hash, and metadata size. + /// + public async Task<(long, DateTime, long, string?, long)> GetFileEntry(long prefixid, string path, long filesetid, CancellationToken token) { DateTime oldModified; long lastFileSize; @@ -1157,8 +1165,8 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - await using var rd = await m_findfileCommand.ExecuteReaderAsync().ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await m_findfileCommand.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) { oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); lastFileSize = rd.ConvertValueToInt64(2); @@ -1194,15 +1202,16 @@ namespace Duplicati.Library.Main.Database /// Gets the metadata hash and size for a file. /// /// The ID of the file. + /// A cancellation token to monitor for cancellation requests. /// A task that when awaited contains a tuple with the metadata hash and size, or null if the file does not exist. - public async Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFile(long fileid) + public async Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFile(long fileid, CancellationToken token) { m_selectfilemetadatahashandsizeCommand .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid); - await using var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync().ConfigureAwait(false); - if (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) return ( rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null"), rd.ConvertValueToInt64(0) @@ -1655,8 +1664,9 @@ namespace Duplicati.Library.Main.Database /// /// Retrieves the names of remote volumes that are missing index files. /// + /// The cancellation token to cancel the operation. /// An asynchronous enumerable of volume names that are missing index files. - public async IAsyncEnumerable GetMissingIndexFiles() + public async IAsyncEnumerable GetMissingIndexFiles([EnumeratorCancellation] CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr) .SetCommandAndParameters(@" @@ -1676,8 +1686,8 @@ namespace Duplicati.Library.Main.Database RemoteVolumeState.Verified.ToString() ]); - await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (await rd.ReadAsync().ConfigureAwait(false)) + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } @@ -1688,8 +1698,9 @@ namespace Duplicati.Library.Main.Database /// The size of the block to move. /// The ID of the source volume. /// The ID of the target volume. + /// The cancellation token to cancel the operation. /// A task that completes when the block is moved. - public async Task MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid) + public async Task MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid, CancellationToken token) { await using var cmd = m_connection.CreateCommand(m_rtr); var c = await cmd.SetCommandAndParameters(@" @@ -1704,7 +1715,7 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@Hash", blockkey) .SetParameterValue("@Size", size) .SetParameterValue("@PreviousVolumeId", sourcevolumeid) - .ExecuteNonQueryAsync() + .ExecuteNonQueryAsync(token) .ConfigureAwait(false); if (c != 1) From 31c5441bdbfb2c76fbdcc343113274352108b952 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 11:04:09 +0200 Subject: [PATCH 342/362] Forwarded cancellation tokens, added some missing .ConfigureAwait() and added some missing docstrings --- .../BackendManager.DatabaseCollector.cs | 16 +- .../Library/Main/Backend/BackendManager.cs | 2 +- Duplicati/Library/Main/Controller.cs | 26 +-- .../Main/Operation/Backup/BackupDatabase.cs | 173 +++++++++--------- .../Operation/Backup/DataBlockProcessor.cs | 22 ++- .../Operation/Backup/FileBlockProcessor.cs | 24 +-- .../Operation/Backup/FilePreFilterProcess.cs | 8 +- .../Operation/Backup/MetadataPreProcess.cs | 69 ++++--- .../Backup/RecreateMissingIndexFiles.cs | 10 +- .../Operation/Backup/SpillCollectorProcess.cs | 14 +- .../Operation/Backup/StreamBlockSplitter.cs | 2 +- .../Operation/Backup/UploadRealFilelist.cs | 16 +- .../Backup/UploadSyntheticFilelist.cs | 28 +-- .../Library/Main/Operation/BackupHandler.cs | 29 +-- .../Main/Operation/Common/DatabaseCommon.cs | 60 +++--- .../Operation/Common/IndexVolumeCreator.cs | 20 +- .../Main/Operation/PurgeFilesHandler.cs | 2 +- .../Library/Main/Operation/RepairHandler.cs | 4 +- .../Main/Operation/Restore/BlockManager.cs | 12 +- .../Main/Operation/Restore/FileLister.cs | 4 +- .../Main/Operation/Restore/FileProcessor.cs | 12 +- .../Operation/Restore/VolumeDownloader.cs | 4 +- .../Library/Main/Operation/RestoreHandler.cs | 2 +- .../Main/Volumes/FilesetVolumeWriter.cs | 14 +- 24 files changed, 303 insertions(+), 270 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs index 8a64605e6..6f54e71fc 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Logging; using Duplicati.Library.Main.Database; @@ -115,8 +116,9 @@ partial class BackendManager /// Flushes all messages to the database. /// /// The database to write pending messages to. + /// A cancellation token to cancel the operation. /// A task that when awaited contains whether any messages were flushed. - public async Task FlushPendingMessages(LocalDatabase db) + public async Task FlushPendingMessages(LocalDatabase db, CancellationToken cancellationToken) { List entries; lock (m_dbqueuelock) @@ -134,19 +136,19 @@ partial class BackendManager // As we replace the list, we can now freely access the elements without locking foreach (var e in entries) if (e is RemoteOperationLogEntry operation) - await db.LogRemoteOperation(operation.Action, operation.File, operation.Result) + await db.LogRemoteOperation(operation.Action, operation.File, operation.Result, cancellationToken) .ConfigureAwait(false); else if (e is RemoteVolumeUpdate update && update.State == RemoteVolumeState.Deleted) { - await db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null) + await db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); volsRemoved.Add(update.Remotename); } else if (e is RemoteVolumeUpdate dbUpdate) - await db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash) + await db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, cancellationToken) .ConfigureAwait(false); else if (e is RenameRemoteVolume rename) - await db.RenameRemoteFile(rename.Oldname, rename.Newname) + await db.RenameRemoteFile(rename.Oldname, rename.Newname, cancellationToken) .ConfigureAwait(false); else if (e != null) Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e); @@ -154,11 +156,11 @@ partial class BackendManager // Finally remove volumes from DB. if (volsRemoved.Count > 0) { - await db.RemoveRemoteVolumes(volsRemoved).ConfigureAwait(false); + await db.RemoveRemoteVolumes(volsRemoved, cancellationToken).ConfigureAwait(false); // This commit is to mimic the pre Microsoft.Data.Sqlite backend // behavior, in which RemoveRemoteVolumes started a new // transaction, if none was passed down. - await db.Transaction.CommitAsync().ConfigureAwait(false); + await db.Transaction.CommitAsync(token: cancellationToken).ConfigureAwait(false); } return true; diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index 1e96ca2ab..45b859917 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -299,7 +299,7 @@ internal partial class BackendManager : IBackendManager /// A task that completes when the messages are flushed. public async Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken) { - await context.Database.FlushPendingMessages(database).ConfigureAwait(false); + await context.Database.FlushPendingMessages(database, cancellationToken).ConfigureAwait(false); } /// diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index f714fea60..4f4f24616 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -253,7 +253,7 @@ namespace Duplicati.Library.Main return RunAction(new ListRemoteResults(), async (result, backendManager) => { using var tf = File.Exists(m_options.Dbpath) ? null : new TempFile(); - await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache, null, result.TaskControl.ProgressToken).ConfigureAwait(false); result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); }); } @@ -277,11 +277,11 @@ namespace Duplicati.Library.Main // and uses the same prefix (see issues #2678, #3845, and #4244). if (File.Exists(m_options.Dbpath)) { - await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache).ConfigureAwait(false); - var dbRemoteVolumes = db.GetRemoteVolumes(); + await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + var dbRemoteVolumes = db.GetRemoteVolumes(result.TaskControl.ProgressToken); var dbRemoteFiles = await dbRemoteVolumes .Select(x => x.Name) - .ToHashSetAsync() + .ToHashSetAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); list = list.Where(x => dbRemoteFiles.Contains(x.File.Name) @@ -522,7 +522,7 @@ namespace Duplicati.Library.Main // instead of relying on the operations to correctly toggle the flag if (File.Exists(m_options.Dbpath) && !m_options.NoLocalDb) { - using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache).Await(); + using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache, null, CancellationToken.None).Await(); backend.StopRunnerAndFlushMessages(db).Await(); } else @@ -538,10 +538,10 @@ namespace Duplicati.Library.Main if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) { - using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache).Await(); - db.WriteResults(result).Await(); - db.PurgeLogData(m_options.LogRetention).Await(); - db.PurgeDeletedVolumes(DateTime.UtcNow).Await(); + using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache, null, CancellationToken.None).Await(); + db.WriteResults(result, CancellationToken.None).Await(); + db.PurgeLogData(m_options.LogRetention, CancellationToken.None).Await(); + db.PurgeDeletedVolumes(DateTime.UtcNow, CancellationToken.None).Await(); // Vacuum is done AFTER the results are written to the database // This means that the information about the vacuum is not stored in the database, @@ -580,8 +580,8 @@ namespace Duplicati.Library.Main { // No operation was started in database, so write logs to new operation if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) - using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache).Await()) - db.WriteResults(result).Await(); + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache, null, CancellationToken.None).Await()) + db.WriteResults(result, CancellationToken.None).Await(); } catch (Exception we) { @@ -605,8 +605,8 @@ namespace Duplicati.Library.Main resultSetter.Fatal = true; // Write logs to previous operation if database exists if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) - using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache).Await()) - db.WriteResults(result).Await(); + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache, null, CancellationToken.None).Await()) + db.WriteResults(result, CancellationToken.None).Await(); } catch (Exception we) { diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index 74d355c87..e4f5635f4 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -28,6 +28,7 @@ using System.Collections.Generic; using Duplicati.Library.Main.Volumes; using System.Linq; using Duplicati.Library.Interface; +using System.Threading; namespace Duplicati.Library.Main.Operation.Backup { @@ -54,93 +55,93 @@ namespace Duplicati.Library.Main.Operation.Backup m_database = database; } - public Task FindBlockIDAsync(string hash, long size) + public Task FindBlockIDAsync(string hash, long size, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .FindBlockID(hash, size) + .FindBlockID(hash, size, cancellationToken) .ConfigureAwait(false) ); } - public Task AddBlockAsync(string hash, long size, long volumeid) + public Task AddBlockAsync(string hash, long size, long volumeid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AddBlock(hash, size, volumeid) + .AddBlock(hash, size, volumeid, cancellationToken) .ConfigureAwait(false) ); } - public Task GetFileHashAsync(long fileid) + public Task GetFileHashAsync(long fileid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .GetFileHash(fileid) + .GetFileHash(fileid, cancellationToken) .ConfigureAwait(false) ); } - public Task<(bool, long)> AddMetadatasetAsync(string hash, long size, long blocksetid) + public Task<(bool, long)> AddMetadatasetAsync(string hash, long size, long blocksetid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AddMetadataset(hash, size, blocksetid) + .AddMetadataset(hash, size, blocksetid, cancellationToken) .ConfigureAwait(false) ); } - public Task<(bool, long)> GetMetadataIDAsync(string hash, long size) + public Task<(bool, long)> GetMetadataIDAsync(string hash, long size, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .GetMetadatasetID(hash, size) + .GetMetadatasetID(hash, size, cancellationToken) .ConfigureAwait(false) ); } - public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified) + public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AddDirectoryEntry(filename, metadataid, lastModified) + .AddDirectoryEntry(filename, metadataid, lastModified, cancellationToken) .ConfigureAwait(false) ); } - public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified) + public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AddSymlinkEntry(filename, metadataid, lastModified) + .AddSymlinkEntry(filename, metadataid, lastModified, cancellationToken) .ConfigureAwait(false) ); } - public Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFileAsync(long fileid) + public Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFileAsync(long fileid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .GetMetadataHashAndSizeForFile(fileid) + .GetMetadataHashAndSizeForFile(fileid, cancellationToken) .ConfigureAwait(false) ); } - public Task<(long, DateTime, long)> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength) + public Task<(long, DateTime, long)> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .GetFileLastModified(prefixid, path, lastfilesetid, includeLength) + .GetFileLastModified(prefixid, path, lastfilesetid, includeLength, cancellationToken) .ConfigureAwait(false) ); } - public Task GetFileEntryAsync(long prefixid, string path, long lastfilesetid) + public Task GetFileEntryAsync(long prefixid, string path, long lastfilesetid, CancellationToken cancellationToken) { return RunOnMain(async () => { var (id, oldModified, lastFileSize, oldMetahash, oldMetasize) = - await m_database.GetFileEntry(prefixid, path, lastfilesetid) + await m_database.GetFileEntry(prefixid, path, lastfilesetid, cancellationToken) .ConfigureAwait(false); return @@ -158,74 +159,74 @@ namespace Duplicati.Library.Main.Operation.Backup }); } - public Task AddBlocksetAsync(string filehash, long size, int blocksize, IEnumerable hashlist, IEnumerable blocklisthashes) + public Task AddBlocksetAsync(string filehash, long size, int blocksize, IEnumerable hashlist, IEnumerable blocklisthashes, CancellationToken cancellationToken) { return RunOnMain(async () => { var (_, blocksetid) = await m_database - .AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes) + .AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes, cancellationToken) .ConfigureAwait(false); return blocksetid; }); } - public Task GetOrCreatePathPrefix(string prefix) + public Task GetOrCreatePathPrefix(string prefix, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .GetOrCreatePathPrefix(prefix) + .GetOrCreatePathPrefix(prefix, cancellationToken) .ConfigureAwait(false) ); } - public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid) + public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AddFile(prefixid, filename, lastmodified, blocksetid, metadataid) + .AddFile(prefixid, filename, lastmodified, blocksetid, metadataid, cancellationToken) .ConfigureAwait(false) ); } - public Task AddUnmodifiedAsync(long fileid, DateTime lastModified) + public Task AddUnmodifiedAsync(long fileid, DateTime lastModified, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AddKnownFile(fileid, lastModified) + .AddKnownFile(fileid, lastModified, cancellationToken) .ConfigureAwait(false) ); } - public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid) + public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid) + .MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid, cancellationToken) .ConfigureAwait(false) ); } - public Task SafeDeleteRemoteVolumeAsync(string remotename) + public Task SafeDeleteRemoteVolumeAsync(string remotename, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .SafeDeleteRemoteVolume(remotename) + .SafeDeleteRemoteVolume(remotename, cancellationToken) .ConfigureAwait(false) ); } - public Task GetBlocklistHashesAsync(string remotename) + public Task GetBlocklistHashesAsync(string remotename, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .GetBlocklistHashes(remotename) + .GetBlocklistHashes(remotename, cancellationToken) .ConfigureAwait(false) ); } - public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume) + public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume, CancellationToken cancellationToken) { if (indexvolume != null) return Task.FromResult(true); @@ -233,43 +234,43 @@ namespace Duplicati.Library.Main.Operation.Backup return RunOnMain(async () => { await m_database - .AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID) + .AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID, cancellationToken) .ConfigureAwait(false); indexvolume.StartVolume(blockvolume.RemoteFilename); - await foreach (var b in m_database.GetBlocks(blockvolume.VolumeID).ConfigureAwait(false)) + await foreach (var b in m_database.GetBlocks(blockvolume.VolumeID, cancellationToken).ConfigureAwait(false)) indexvolume.AddBlock(b.Hash, b.Size); await m_database - .UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken) .ConfigureAwait(false); }); } - public Task AppendFilesFromPreviousSetAsync() + public Task AppendFilesFromPreviousSetAsync(CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AppendFilesFromPreviousSet() + .AppendFilesFromPreviousSet(null, cancellationToken) .ConfigureAwait(false) ); } - public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist) + public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AppendFilesFromPreviousSet(deletedFilelist) + .AppendFilesFromPreviousSet(deletedFilelist, cancellationToken) .ConfigureAwait(false) ); } - public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp) + public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AppendFilesFromPreviousSet(deletedFilelist, filesetid, prevId, timestamp) + .AppendFilesFromPreviousSet(deletedFilelist, filesetid, prevId, timestamp, cancellationToken) .ConfigureAwait(false) ); } @@ -280,11 +281,11 @@ namespace Duplicati.Library.Main.Operation.Backup /// predicate. /// /// Optional exclusion predicate (true = exclude file) - public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate) + public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AppendFilesFromPreviousSetWithPredicate(exclusionPredicate) + .AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, cancellationToken) .ConfigureAwait(false) ); } @@ -294,156 +295,158 @@ namespace Duplicati.Library.Main.Operation.Backup /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// - /// Optional exclusion predicate (true = exclude file) - /// Current fileset ID - /// Source fileset ID - /// If prevFileSetId == -1, used to locate previous fileset - public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) + /// Optional exclusion predicate (true = exclude file). + /// Current fileset ID. + /// Source fileset ID. + /// If prevFileSetId == -1, used to locate previous fileset. + /// The cancellation token to cancel the operation. + /// Task that completes when the files have been appended. + public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, fileSetId, prevFileSetId, timestamp) + .AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, fileSetId, prevFileSetId, timestamp, cancellationToken) .ConfigureAwait(false) ); } - public Task[]> GetIncompleteFilesetsAsync() + public Task[]> GetIncompleteFilesetsAsync(CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .GetIncompleteFilesets() + .GetIncompleteFilesets(cancellationToken) .OrderBy(x => x.Value) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } - public Task[]> GetFilesetTimesAsync() + public Task[]> GetFilesetTimesAsync(CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .FilesetTimes() - .ToArrayAsync() + .FilesetTimes(cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } - public Task CreateFilesetAsync(long volumeID, DateTime fileTime) + public Task CreateFilesetAsync(long volumeID, DateTime fileTime, CancellationToken cancellationToken) { return RunOnMain(async () => { var fs = await m_database - .CreateFileset(volumeID, fileTime) + .CreateFileset(volumeID, fileTime, cancellationToken) .ConfigureAwait(false); await m_database.Transaction - .CommitAsync() + .CommitAsync(cancellationToken) .ConfigureAwait(false); return fs; }); } - public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid) + public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .LinkFilesetToVolume(filesetid, volumeid) + .LinkFilesetToVolume(filesetid, volumeid, cancellationToken) .ConfigureAwait(false) ); } - public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid) + public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .WriteFileset(fsw, filesetid) + .WriteFileset(fsw, filesetid, cancellationToken) .ConfigureAwait(false) ); } - public Task PushTimestampChangesToPreviousVersionAsync(long filesetid) + public Task PushTimestampChangesToPreviousVersionAsync(long filesetid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .PushTimestampChangesToPreviousVersion(filesetid) + .PushTimestampChangesToPreviousVersion(filesetid, cancellationToken) .ConfigureAwait(false) ); } - public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid) + public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .UpdateFullBackupStateInFileset(filesetid, true) + .UpdateFullBackupStateInFileset(filesetid, true, cancellationToken) .ConfigureAwait(false) ); } - public Task> GetMissingIndexFilesAsync() + public Task> GetMissingIndexFilesAsync(CancellationToken cancellationToken) { - return RunOnMain(() => m_database.GetMissingIndexFiles()); + return RunOnMain(() => m_database.GetMissingIndexFiles(cancellationToken)); } - public Task UpdateChangeStatisticsAsync(BackupResults result) + public Task UpdateChangeStatisticsAsync(BackupResults result, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .UpdateChangeStatistics(result) + .UpdateChangeStatistics(result, cancellationToken) .ConfigureAwait(false) ); } - public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists) + public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .VerifyConsistency(blocksize, blockhashSize, verifyfilelists) + .VerifyConsistency(blocksize, blockhashSize, verifyfilelists, cancellationToken) .ConfigureAwait(false) ); } - public Task RemoveRemoteVolumeAsync(string remoteFilename) + public Task RemoveRemoteVolumeAsync(string remoteFilename, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .RemoveRemoteVolume(remoteFilename) + .RemoveRemoteVolume(remoteFilename, cancellationToken) .ConfigureAwait(false) ); } - public Task GetRemoteVolumeFromFilesetIDAsync(long filesetID) + public Task GetRemoteVolumeFromFilesetIDAsync(long filesetID, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .GetRemoteVolumeFromFilesetID(filesetID) + .GetRemoteVolumeFromFilesetID(filesetID, cancellationToken) .ConfigureAwait(false) ); } - public Task CreateChangeJournalDataAsync(IEnumerable journalData) + public Task CreateChangeJournalDataAsync(IEnumerable journalData, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .CreateChangeJournalData(journalData) + .CreateChangeJournalData(journalData, cancellationToken) .ConfigureAwait(false) ); } - public Task UpdateChangeJournalDataAsync(IEnumerable journalData, long lastfilesetid) + public Task UpdateChangeJournalDataAsync(IEnumerable journalData, long lastfilesetid, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .UpdateChangeJournalData(journalData, lastfilesetid) + .UpdateChangeJournalData(journalData, lastfilesetid, cancellationToken) .ConfigureAwait(false) ); } - public Task IsBlocklistHashKnownAsync(string hash) + public Task IsBlocklistHashKnownAsync(string hash, CancellationToken cancellationToken) { return RunOnMain(async () => await m_database - .IsBlocklistHashKnown(hash) + .IsBlocklistHashKnown(hash, cancellationToken) .ConfigureAwait(false) ); } diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 34e598b5b..bcee2e525 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -82,7 +82,7 @@ namespace Duplicati.Library.Main.Operation.Backup // There can be a race, such that two workers determine that // the block is missing, but this will be solved by the AddBlock call // which runs atomically - if (!isMandatoryBlocklistHash && await database.FindBlockIDAsync(b.HashKey, b.Size) >= 0) + if (!isMandatoryBlocklistHash && await database.FindBlockIDAsync(b.HashKey, b.Size, taskreader.ProgressToken) >= 0) { b.TaskCompletion.TrySetResult(false); sw_workload.Stop(); @@ -90,13 +90,13 @@ namespace Duplicati.Library.Main.Operation.Backup } blockvolume = new BlockVolumeWriter(options); - blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); + blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, taskreader.ProgressToken); indexvolume = noIndexFiles ? null : new TemporaryIndexVolume(options); blocklistHashesAdded = 0; } - var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID); + var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID, taskreader.ProgressToken); b.TaskCompletion.TrySetResult(newBlock); // If we are recording blocklist hashes, add them to the index file, @@ -105,7 +105,7 @@ namespace Duplicati.Library.Main.Operation.Backup { // This can cause a race between workers, // but the side-effect is that the index files are slightly larger - if (newBlock || !await database.IsBlocklistHashKnownAsync(b.HashKey)) + if (newBlock || !await database.IsBlocklistHashKnownAsync(b.HashKey, taskreader.ProgressToken)) { blocklistHashesAdded++; indexvolume.AddBlockListHash(b.HashKey, b.Size, b.Data); @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Operation.Backup { //When uploading a new volume, we register the volumes and then flush the transaction // this ensures that the local database and remote storage are as closely related as possible - await database.UpdateRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await database.UpdateRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken); blockvolume.Close(); @@ -133,17 +133,17 @@ namespace Duplicati.Library.Main.Operation.Backup { // TODO: It is much easier to let the BackendManager deal with index files, // but it adds a bit of strain to the database - indexVolumeCopy = await indexvolume.CreateVolume(blockvolume.RemoteFilename, options, database); + indexVolumeCopy = await indexvolume.CreateVolume(blockvolume.RemoteFilename, options, database, taskreader.ProgressToken).ConfigureAwait(false); // Create link before upload is started, it will be removed later if upload fails - await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, blockvolume.VolumeID).ConfigureAwait(false); + await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, blockvolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); } var blockVolumeCopy = blockvolume; blockvolume = null; indexvolume = null; - await database.CommitTransactionAsync("CommitAddBlockToOutputFlush"); - await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken); + await database.CommitTransactionAsync("CommitAddBlockToOutputFlush", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); } @@ -170,7 +170,9 @@ namespace Duplicati.Library.Main.Operation.Backup } else { - await database.RemoveRemoteVolumeAsync(blockvolume.RemoteFilename); + await database + .RemoveRemoteVolumeAsync(blockvolume.RemoteFilename, taskreader.ProgressToken) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index c9f8a6f12..d1ffd97d3 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -49,7 +49,7 @@ namespace Duplicati.Library.Main.Operation.Backup { while (true) { - var e = await self.Input.ReadAsync(); + var e = await self.Input.ReadAsync().ConfigureAwait(false); // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressRendevouz().ConfigureAwait(false); @@ -57,7 +57,7 @@ namespace Duplicati.Library.Main.Operation.Backup try { var hint = options.GetCompressionHintFromFilename(e.Entry.Path); - var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId); + var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId, taskreader.ProgressToken).ConfigureAwait(false); StreamProcessResult filestreamdata; @@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation.Backup // If we have determined that metadata has not changed, just grab the ID if (!e.MetadataChanged) { - var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length); + var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, taskreader.ProgressToken).ConfigureAwait(false); if (res.Item1) return res.Item2; @@ -76,15 +76,15 @@ namespace Duplicati.Library.Main.Operation.Backup e.MetadataChanged = true; } - return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2; + return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel, taskreader.ProgressToken).ConfigureAwait(false)).Item2; }); - using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken)) - filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint); + using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken).ConfigureAwait(false)) + filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint).ConfigureAwait(false); - await stats.AddOpenedFile(filestreamdata.Streamlength); + await stats.AddOpenedFile(filestreamdata.Streamlength).ConfigureAwait(false); - var metadataid = await metatask; + var metadataid = await metatask.ConfigureAwait(false); var filekey = filestreamdata.Streamhash; var filesize = filestreamdata.Streamlength; @@ -110,18 +110,18 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize)); } - await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); + await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid, taskreader.ProgressToken).ConfigureAwait(false); } else if (e.MetadataChanged) { Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Entry.Path); - await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); + await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid, taskreader.ProgressToken).ConfigureAwait(false); } else if (e.TimestampChanged) { await stats.AddTimestampChangedFile(); Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Entry.Path); - await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken).ConfigureAwait(false); } else /*if (e.OldId >= 0)*/ { @@ -130,7 +130,7 @@ namespace Duplicati.Library.Main.Operation.Backup try { - await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken).ConfigureAwait(false); } catch (Exception ex) { diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 510b877e9..7001cae05 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FilePreFilterProcess)) + ".FileEntry"; - public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database) + public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database, ITaskReader taskreader) { return AutomationExtensions.RunTask( new @@ -105,7 +105,7 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Entry.Path); try { - await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken); } catch (Exception ex) { @@ -121,7 +121,7 @@ namespace Duplicati.Library.Main.Operation.Backup // but we want to know if the metadata is potentially changed if (!isNewFile && DISABLEFILETIMECHECK) { - var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId); + var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId, taskreader.ProgressToken); if (tp != null) { e.OldMetaSize = tp.Value.Size; @@ -145,7 +145,7 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Entry.Path); try { - await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken); } catch (Exception ex) { diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index 97135856d..465e7c4c5 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -26,6 +26,7 @@ using System.IO; using System.Collections.Generic; using Duplicati.Library.Interface; using Duplicati.Library.Main.Operation.Common; +using System.Threading; namespace Duplicati.Library.Main.Operation.Backup { @@ -63,7 +64,7 @@ namespace Duplicati.Library.Main.Operation.Backup public bool TimestampChanged; } - public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader) + public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskreader) { return AutomationExtensions.RunTask(new { @@ -85,7 +86,7 @@ namespace Duplicati.Library.Main.Operation.Backup var entry = await self.Input.ReadAsync(); // We ignore the stop signal, but not the pause and terminate - await taskReader.ProgressRendevouz().ConfigureAwait(false); + await taskreader.ProgressRendevouz().ConfigureAwait(false); var lastwrite = new DateTime(0, DateTimeKind.Utc); var attributes = entry.IsFolder @@ -111,7 +112,7 @@ namespace Duplicati.Library.Main.Operation.Backup } // If we only have metadata, stop here - if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false)) + if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel, taskreader.ProgressToken).ConfigureAwait(false)) { try { @@ -122,13 +123,13 @@ namespace Duplicati.Library.Main.Operation.Backup prefixid = prevprefix.Value; else { - prefixid = await database.GetOrCreatePathPrefix(split.Key); + prefixid = await database.GetOrCreatePathPrefix(split.Key, taskreader.ProgressToken); prevprefix = new KeyValuePair(split.Key, prefixid); } if (CHECKFILETIMEONLY || DISABLEFILETIMECHECK) { - var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid, false); + var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid, false, taskreader.ProgressToken); await self.Output.WriteAsync(new FileEntry { OldId = tmp.Item1, @@ -145,7 +146,7 @@ namespace Duplicati.Library.Main.Operation.Backup } else { - var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid); + var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid, taskreader.ProgressToken); await self.Output.WriteAsync(new FileEntry { OldId = res == null ? -1 : res.id, @@ -178,8 +179,16 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// Processes the metadata for the given path. /// + /// The source provider entry to process. + /// The file attributes of the entry. + /// The last write timestamp of the entry. + /// The options for the backup operation. + /// An empty metadata object to use if no metadata is generated. + /// The backup database to use for storing metadata. + /// The channel to write stream blocks to. + /// The cancellation token to use for the operation. /// True if the path should be submitted to more analysis, false if there is nothing else to do - private static async Task ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) + private static async Task ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel, CancellationToken cancellationToken) { if (entry.IsSymlink) { @@ -213,7 +222,7 @@ namespace Duplicati.Library.Main.Operation.Backup metadata["CoreSymlinkTarget"] = symlinkTarget; var metahash = Utility.WrapMetadata(metadata, options); - await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false); + await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel, cancellationToken).ConfigureAwait(false); Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", entry.Path); // Don't process further @@ -241,7 +250,7 @@ namespace Duplicati.Library.Main.Operation.Backup } Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", entry.Path); - await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false); + await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel, cancellationToken).ConfigureAwait(false); return false; } @@ -256,25 +265,15 @@ namespace Duplicati.Library.Main.Operation.Backup /// The metadata entry. /// The database connection. /// The channel to write streams to. + /// The cancellation token to use. /// A task that when awaited returns a tuple with the first value indicating if the metadata set was added, and the second value being the metadata ID. - internal static async Task<(bool, long)> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) + internal static async Task<(bool, long)> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel, CancellationToken cancellationToken) { StreamProcessResult res; using (var ms = new MemoryStream(meta.Blob)) res = await StreamBlock.ProcessStream(streamblockchannel, path, ms, true, CompressionHint.Default); - return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid); - } - - /// - /// Adds a file to the output, - /// - /// The name of the file to record - /// The value of the lastModified timestamp - private static async Task AddFolderToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) - { - var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel).ConfigureAwait(false); - await database.AddDirectoryEntryAsync(filename, metadataid.Item2, lastModified); + return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid, cancellationToken).ConfigureAwait(false); } /// @@ -282,13 +281,31 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The name of the file to record /// The value of the lastModified timestamp + /// The metadata to record /// The database to use /// The channel to write blocks to - /// The metadata ti record - private static async Task AddSymlinkToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) + /// The cancellation token to use + /// A task that completes when the file has been added to the output. + private static async Task AddFolderToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel, CancellationToken cancellationToken) { - var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel).ConfigureAwait(false); - await database.AddSymlinkEntryAsync(filename, metadataid.Item2, lastModified); + var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel, cancellationToken).ConfigureAwait(false); + await database.AddDirectoryEntryAsync(filename, metadataid.Item2, lastModified, cancellationToken).ConfigureAwait(false); + } + + /// + /// Adds a file to the output, + /// + /// The name of the file to record + /// The value of the lastModified timestamp + /// The metadata ti record + /// The database to use + /// The channel to write blocks to + /// The cancellation token to use + /// A task that completes when the file has been added to the output. + private static async Task AddSymlinkToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel, CancellationToken cancellationToken) + { + var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel, cancellationToken).ConfigureAwait(false); + await database.AddSymlinkEntryAsync(filename, metadataid.Item2, lastModified, cancellationToken).ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs index f8a204f9a..0b1008d23 100644 --- a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs +++ b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs @@ -35,20 +35,20 @@ internal static class RecreateMissingIndexFiles { if (options.IndexfilePolicy != Options.IndexFileStrategy.None) { - await foreach (var blockfile in (await database.GetMissingIndexFilesAsync().ConfigureAwait(false)).ConfigureAwait(false)) + await foreach (var blockfile in (await database.GetMissingIndexFilesAsync(taskreader.ProgressToken).ConfigureAwait(false)).ConfigureAwait(false)) { if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile); - var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database).ConfigureAwait(false); + var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database, taskreader.ProgressToken).ConfigureAwait(false); if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; - await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false); - await database.CommitTransactionAsync("RecreateMissingIndexFile").ConfigureAwait(false); - await backendManager.PutAsync(w, null, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false); + await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); + await database.CommitTransactionAsync("RecreateMissingIndexFile", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(w, null, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs index ee4c38c0e..6a906da16 100644 --- a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs @@ -79,7 +79,7 @@ namespace Duplicati.Library.Main.Operation.Backup { // No more targets, make one target = new SpillVolumeRequest(new BlockVolumeWriter(options), source.IndexVolume == null ? null : new TemporaryIndexVolume(options)); - target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary).ConfigureAwait(false); + target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, taskreader.ProgressToken).ConfigureAwait(false); } else { @@ -125,7 +125,7 @@ namespace Duplicati.Library.Main.Operation.Backup var len = rd.ReadBlock(file.Key, buffer); await target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default).ConfigureAwait(false); - await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false); + await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); if (target.IndexVolume != null) target.IndexVolume.AddBlock(file.Key, len); @@ -141,7 +141,7 @@ namespace Duplicati.Library.Main.Operation.Backup // Make sure they are out of the database System.IO.File.Delete(source.BlockVolume.LocalFilename); - await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename).ConfigureAwait(false); + await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename, taskreader.ProgressToken).ConfigureAwait(false); // Re-inject the target if it has content if (target != null) @@ -167,13 +167,13 @@ namespace Duplicati.Library.Main.Operation.Backup { // TODO: It is much easier to let the BackendManager deal with index files, // but it adds a bit of strain to the database - indexVolumeCopy = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, options, database).ConfigureAwait(false); + indexVolumeCopy = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, options, database, taskreader.ProgressToken).ConfigureAwait(false); // Create link before upload is started, it will be removed later if upload fails - await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false); + await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); } - await database.CommitTransactionAsync("UploadSpillVolume").ConfigureAwait(false); - await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false); + await database.CommitTransactionAsync("UploadSpillVolume", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs index dd88b8404..d41d477fa 100644 --- a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs +++ b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs @@ -162,7 +162,7 @@ namespace Duplicati.Library.Main.Operation.Backup filehasher.TransformFinalBlock(new byte[0], 0, 0); var filehash = Convert.ToBase64String(filehasher.Hash); - var blocksetid = await database.AddBlocksetAsync(filehash, filesize, blocksize, hashcollector, blocklisthashes); + var blocksetid = await database.AddBlocksetAsync(filehash, filesize, blocksize, hashcollector, blocklisthashes, taskreader.ProgressToken); cur.SetResult(new StreamProcessResult() { Streamlength = filesize, Streamhash = filehash, Blocksetid = blocksetid }); cur = null; } diff --git a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs index 7a90cc498..f7ba8f087 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs @@ -39,7 +39,7 @@ internal static class UploadRealFilelist // Update the reported source and backend changes using (new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics")) - await db.UpdateChangeStatisticsAsync(result); + await db.UpdateChangeStatisticsAsync(result, taskreader.ProgressToken).ConfigureAwait(false); var changeCount = result.AddedFiles + result.ModifiedFiles + result.DeletedFiles + @@ -58,16 +58,16 @@ internal static class UploadRealFilelist // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressRendevouz().ConfigureAwait(false); - await db.WriteFilesetAsync(filesetvolume, filesetid).ConfigureAwait(false); + await db.WriteFilesetAsync(filesetvolume, filesetid, taskreader.ProgressToken).ConfigureAwait(false); filesetvolume.Close(); // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressRendevouz().ConfigureAwait(false); - await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false); - await db.CommitTransactionAsync("CommitUpdateRemoteVolume").ConfigureAwait(false); + await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); + await db.CommitTransactionAsync("CommitUpdateRemoteVolume", true, taskreader.ProgressToken).ConfigureAwait(false); - await backendManager.PutAsync(filesetvolume, null, null, false, () => db.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(filesetvolume, null, null, false, () => db.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); } } else @@ -75,14 +75,14 @@ internal static class UploadRealFilelist if (result.TimestampChangedFiles != 0) { Logging.Log.WriteInformationMessage(LOGTAG, "DetectedTimestampChanges", "Detected timestamp changes, but no data needs to be uploaded, pushing timestamp changes to the latest fileset"); - await db.PushTimestampChangesToPreviousVersionAsync(filesetid).ConfigureAwait(false); + await db.PushTimestampChangesToPreviousVersionAsync(filesetid, taskreader.ProgressToken).ConfigureAwait(false); } Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded"); - await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename); + await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename, taskreader.ProgressToken).ConfigureAwait(false); } - await db.CommitTransactionAsync("CommitUpdateRemoteVolume"); + await db.CommitTransactionAsync("CommitUpdateRemoteVolume", true, taskreader.ProgressToken).ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index afe037f66..e369779bc 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -62,8 +62,8 @@ namespace Duplicati.Library.Main.Operation.Backup } // Ready to build and upload the synthetic list - await database.CommitTransactionAsync("PreSyntheticFilelist"); - var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList(); + await database.CommitTransactionAsync("PreSyntheticFilelist", true, taskreader.ProgressToken).ConfigureAwait(false); + var incompleteFilesets = (await database.GetIncompleteFilesetsAsync(taskreader.ProgressToken)).OrderBy(x => x.Value).ToList(); if (!incompleteFilesets.Any()) return; @@ -77,7 +77,7 @@ namespace Duplicati.Library.Main.Operation.Backup var incompleteSet = incompleteFilesets.Last(); var badIds = incompleteFilesets.Select(n => n.Key); - var prevs = (await database.GetFilesetTimesAsync()) + var prevs = (await database.GetFilesetTimesAsync(taskreader.ProgressToken)) .Where(n => n.Key < incompleteSet.Key && !badIds.Contains(n.Key)) .OrderBy(n => n.Key) .Select(n => n.Key) @@ -88,9 +88,9 @@ namespace Duplicati.Library.Main.Operation.Backup FilesetVolumeWriter fsw = null; try { - var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(database, options, incompleteSet.Value).ConfigureAwait(false); + var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(database, options, incompleteSet.Value, taskreader.ProgressToken).ConfigureAwait(false); fsw = new FilesetVolumeWriter(options, fileTime); - fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); + fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, taskreader.ProgressToken).ConfigureAwait(false); if (!string.IsNullOrEmpty(options.ControlFiles)) foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) @@ -99,27 +99,27 @@ namespace Duplicati.Library.Main.Operation.Backup // We declare this to be a partial backup since the synthetic filelist is only created // when a backup is interrupted. fsw.CreateFilesetFile(false); - var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime); - await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID); - await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime); + var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime, taskreader.ProgressToken).ConfigureAwait(false); + await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); + await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime, taskreader.ProgressToken).ConfigureAwait(false); - await database.WriteFilesetAsync(fsw, newFilesetID); + await database.WriteFilesetAsync(fsw, newFilesetID, taskreader.ProgressToken).ConfigureAwait(false); fsw.Close(); if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; - await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); // If the previous filelist was not uploaded, we register it for deletion, as we have created a new synthetic one // Because it is registered as "Deleting", it will be removed from remote storage by the cleanup process if it exists if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name) && (lastTempFilelist.State == RemoteVolumeState.Uploading || lastTempFilelist.State == RemoteVolumeState.Temporary)) - await database.UpdateRemoteVolumeAsync(lastTempFilelist.Name, RemoteVolumeState.Deleting, -1, null); - await database.CommitTransactionAsync("CommitUpdateFilelistVolume"); - await backendManager.PutAsync(fsw, null, null, true, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken); + await database.UpdateRemoteVolumeAsync(lastTempFilelist.Name, RemoteVolumeState.Deleting, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); + await database.CommitTransactionAsync("CommitUpdateFilelistVolume", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(fsw, null, null, true, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken); } catch { - await database.RollbackTransactionAsync(); + await database.RollbackTransactionAsync(taskreader.ProgressToken).ConfigureAwait(false); fsw?.Dispose(); throw; } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index ab5cbf7cf..938a8dc30 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -445,7 +445,7 @@ namespace Duplicati.Library.Main.Operation options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, GetBlacklistedPaths(options), options.ChangedFilelist, taskreader, () => result.PartialBackup = true, CancellationToken.None), - Backup.FilePreFilterProcess.Run(channels, options, stats, database), + Backup.FilePreFilterProcess.Run(channels, options, stats, database, taskreader), Backup.MetadataPreProcess.Run(channels, options, database, lastfilesetid, taskreader), Backup.SpillCollectorProcess.Run(channels, options, database, backendManager, taskreader), Backup.ProgressHandler.Run(channels, result) @@ -473,7 +473,7 @@ namespace Duplicati.Library.Main.Operation if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1) { await database - .AppendFilesFromPreviousSetAsync(options.DeletedFilelist) + .AppendFilesFromPreviousSetAsync(options.DeletedFilelist, taskreader.ProgressToken) .ConfigureAwait(false); } else if (journalService != null) @@ -495,7 +495,7 @@ namespace Duplicati.Library.Main.Operation stats.AddExaminedFile(fileSize); return false; - }) + }, taskreader.ProgressToken) .ConfigureAwait(false); // store journal data in database, unless job is being canceled @@ -506,12 +506,12 @@ namespace Duplicati.Library.Main.Operation { // always record change journal data for current fileset (entry may be dropped later if nothing is uploaded) await database - .CreateChangeJournalDataAsync(data) + .CreateChangeJournalDataAsync(data, taskreader.ProgressToken) .ConfigureAwait(false); // update the previous fileset's change journal entry to resume at this point in case nothing was backed up await database - .UpdateChangeJournalDataAsync(data, lastfilesetid) + .UpdateChangeJournalDataAsync(data, lastfilesetid, taskreader.ProgressToken) .ConfigureAwait(false); } } @@ -524,7 +524,7 @@ namespace Duplicati.Library.Main.Operation else { await database - .UpdateFilesetAndMarkAsFullBackupAsync(filesetid) + .UpdateFilesetAndMarkAsFullBackupAsync(filesetid, taskreader.ProgressToken) .ConfigureAwait(false); } @@ -675,9 +675,9 @@ namespace Duplicati.Library.Main.Operation try { - await database.FlushBackendMessagesAndCommitAsync(backendManager).ConfigureAwait(false); + await database.FlushBackendMessagesAndCommitAsync(backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); - await database.FlushBackendMessagesAndCommitAsync(backendManager).ConfigureAwait(false); + await database.FlushBackendMessagesAndCommitAsync(backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); } catch (RetiredException) { @@ -748,18 +748,18 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles); var repcnt = 0; - while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename).ConfigureAwait(false) >= 0) + while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename, m_taskReader.ProgressToken).ConfigureAwait(false) >= 0) filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++)); - if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename).ConfigureAwait(false) >= 0) + if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename, m_taskReader.ProgressToken).ConfigureAwait(false) >= 0) throw new Exception("Unable to generate a unique fileset name"); var filesetvolumeid = await db - .RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary) + .RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_taskReader.ProgressToken) .ConfigureAwait(false); filesetid = await db - .CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time) + .CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time, m_taskReader.ProgressToken) .ConfigureAwait(false); var journalService = GetJournalService(source, filter, lastfilesetid); @@ -798,7 +798,7 @@ namespace Duplicati.Library.Main.Operation // Ensure the database is in a sane state after adding data using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency")) await db - .VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false) + .VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false, m_taskReader.ProgressToken) .ConfigureAwait(false); await FlushBackend(db, m_result, backendManager) @@ -819,7 +819,8 @@ namespace Duplicati.Library.Main.Operation .ConfigureAwait(false); // Make sure we have the database up-to-date - await db.CommitTransactionAsync("CommitAfterUpload") + await db + .CommitTransactionAsync("CommitAfterUpload", true, m_taskReader.ProgressToken) .ConfigureAwait(false); // If this throws, we should roll back the transaction diff --git a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs index fba1a951b..312a79260 100644 --- a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs +++ b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs @@ -49,127 +49,127 @@ namespace Duplicati.Library.Main.Operation.Common m_options = options; } - public Task RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state) + public Task RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state, CancellationToken cancellationToken) { return RunOnMain(async () => await m_db - .RegisterRemoteVolume(name, type, state) + .RegisterRemoteVolume(name, type, state, cancellationToken) .ConfigureAwait(false) ); } - public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan), bool? setArchived = null) + public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, CancellationToken cancellationToken) { return RunOnMain(async () => await m_db - .UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived) + .UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived, cancellationToken) .ConfigureAwait(false) ); } - public async Task FlushBackendMessagesAndCommitAsync(IBackendManager backendManager) + public async Task FlushBackendMessagesAndCommitAsync(IBackendManager backendManager, CancellationToken cancellationToken) { - await FlushPendingBackendMessagesAsync(backendManager).ConfigureAwait(false); - await CommitTransactionAsync("FlushBackendMessagesAndCommitAsync").ConfigureAwait(false); + await FlushPendingBackendMessagesAsync(backendManager, cancellationToken).ConfigureAwait(false); + await CommitTransactionAsync("FlushBackendMessagesAndCommitAsync", true, cancellationToken).ConfigureAwait(false); } - private Task FlushPendingBackendMessagesAsync(IBackendManager backendManager) - => RunOnMain(async () => await backendManager.FlushPendingMessagesAsync(m_db, CancellationToken.None).ConfigureAwait(false)); + private Task FlushPendingBackendMessagesAsync(IBackendManager backendManager, CancellationToken cancellationToken) + => RunOnMain(async () => await backendManager.FlushPendingMessagesAsync(m_db, cancellationToken).ConfigureAwait(false)); - public Task CommitTransactionAsync(string message, bool restart = true) + public Task CommitTransactionAsync(string message, bool restart, CancellationToken cancellationToken) { return RunOnMain(async () => { if (m_options.Dryrun) { await m_db.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); } else { using (new Logging.Timer(LOGTAG, "CommitTransactionAsync", message)) await m_db.Transaction - .CommitAsync(message, restart) + .CommitAsync(message, restart, cancellationToken) .ConfigureAwait(false); } }); } - public Task RollbackTransactionAsync() + public Task RollbackTransactionAsync(CancellationToken cancellationToken) { return RunOnMain(async () => await m_db.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false) ); } - public Task RenameRemoteFileAsync(string oldname, string newname) + public Task RenameRemoteFileAsync(string oldname, string newname, CancellationToken cancellationToken) { return RunOnMain(async () => await m_db - .RenameRemoteFile(oldname, newname) + .RenameRemoteFile(oldname, newname, cancellationToken) .ConfigureAwait(false) ); } - public Task LogRemoteOperationAsync(string operation, string path, string data) + public Task LogRemoteOperationAsync(string operation, string path, string data, CancellationToken cancellationToken) { return RunOnMain(async () => await m_db - .LogRemoteOperation(operation, path, data) + .LogRemoteOperation(operation, path, data, cancellationToken) .ConfigureAwait(false) ); } - public Task GetBlocksAsync(long volumeid) + public Task GetBlocksAsync(long volumeid, CancellationToken cancellationToken) { // TODO: Figure out how to return the enumerable, while keeping the lock // and not creating the entire result in memory return RunOnMain(async () => await m_db - .GetBlocks(volumeid) - .ToArrayAsync() + .GetBlocks(volumeid, cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } - public Task GetVolumeInfoAsync(string remotename) + public Task GetVolumeInfoAsync(string remotename, CancellationToken cancellationToken) { return RunOnMain(async () => await m_db - .GetRemoteVolume(remotename) + .GetRemoteVolume(remotename, cancellationToken) .ConfigureAwait(false) ); } - public Task<(string Hash, byte[] Buffer, int Size)[]> GetBlocklistsAsync(long volumeid, int blocksize, int hashsize) + public Task<(string Hash, byte[] Buffer, int Size)[]> GetBlocklistsAsync(long volumeid, int blocksize, int hashsize, CancellationToken cancellationToken) { // TODO: Figure out how to return the enumerable, while keeping the lock // and not creating the entire result in memory return RunOnMain(async () => await m_db - .GetBlocklists(volumeid, blocksize, hashsize) - .ToArrayAsync() + .GetBlocklists(volumeid, blocksize, hashsize, cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } - public Task GetRemoteVolumeIDAsync(string remotename) + public Task GetRemoteVolumeIDAsync(string remotename, CancellationToken cancellationToken) { return RunOnMain(async () => await m_db - .GetRemoteVolumeID(remotename) + .GetRemoteVolumeID(remotename, cancellationToken) .ConfigureAwait(false) ); } - public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID) + public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID, CancellationToken cancellationToken) { return RunOnMain(async () => await m_db - .AddIndexBlockLink(indexVolumeID, blockVolumeID) + .AddIndexBlockLink(indexVolumeID, blockVolumeID, cancellationToken) .ConfigureAwait(false) ); } diff --git a/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs b/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs index 32f69fe51..1360b0b58 100644 --- a/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs +++ b/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs @@ -21,6 +21,7 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; @@ -67,14 +68,15 @@ namespace Duplicati.Library.Main.Operation.Common /// /// Creates an index volume with the temporary contents /// - /// The index volume. /// The name of the block file. /// The options used in this run. /// The database to use. - public async Task CreateVolume(string blockfilename, Options options, DatabaseCommon database) + /// The cancellation token to use. + /// The index volume. + public async Task CreateVolume(string blockfilename, Options options, DatabaseCommon database, CancellationToken cancellationToken) { var w = new IndexVolumeWriter(options); - w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); + w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken).ConfigureAwait(false); var enumerator = blockListHashes.GetEnumerator(); while (enumerator.MoveNext()) @@ -158,23 +160,23 @@ namespace Duplicati.Library.Main.Operation.Common /// internal static class IndexVolumeCreator { - public static async Task CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database) + public static async Task CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database, CancellationToken cancellationToken) { using (var h = HashFactory.CreateHasher(options.BlockHashAlgorithm)) { var w = new IndexVolumeWriter(options); - w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); + w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken).ConfigureAwait(false); - var blockvolume = await database.GetVolumeInfoAsync(blockname); + var blockvolume = await database.GetVolumeInfoAsync(blockname, cancellationToken).ConfigureAwait(false); w.StartVolume(blockname); - foreach (var b in await database.GetBlocksAsync(blockvolume.ID)) + foreach (var b in await database.GetBlocksAsync(blockvolume.ID, cancellationToken).ConfigureAwait(false)) w.AddBlock(b.Hash, b.Size); w.FinishVolume(blockvolume.Hash, blockvolume.Size); if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) - foreach (var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize)) + foreach (var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize, cancellationToken).ConfigureAwait(false)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); if (bh != b.Item1) @@ -185,7 +187,7 @@ namespace Duplicati.Library.Main.Operation.Common w.Close(); // Register that the index file is tracking the block file - await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID); + await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID, cancellationToken).ConfigureAwait(false); return w; } diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 378439872..cbf512518 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -130,7 +130,7 @@ namespace Duplicati.Library.Main.Operation if (ix < 0 || tsOriginal.Ticks == 0) throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); - var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal) + var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 547abae58..ff9c5800d 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -420,7 +420,7 @@ namespace Duplicati.Library.Main.Operation progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); var fileTime = - await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp) + await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp, cancellationToken) .ConfigureAwait(false); var fsw = new FilesetVolumeWriter(m_options, fileTime); @@ -520,7 +520,7 @@ namespace Duplicati.Library.Main.Operation { var timestamp = VolumeBase.ParseFilename(n.Name).Time; var fileTime = - await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp) + await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp, cancellationToken) .ConfigureAwait(false); var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 5e77a0ef4..2e8a8fdd4 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -181,12 +181,13 @@ namespace Duplicati.Library.Main.Operation.Restore /// CoCoL channel for submitting block requests from a volume. /// The restore options. /// The number of readers accessing this dictionary. Used during shutdown / cleanup. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a new instance of the class. - public static async Task CreateAsync(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers) + public static async Task CreateAsync(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers, CancellationToken cancellationToken) { var sd = new SleepableDictionary(db, volume_request, options, readers); - await foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata).ConfigureAwait(false)) + await foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata, cancellationToken).ConfigureAwait(false)) { var bc = sd.m_blockcount.TryGetValue(block_id, out var c); sd.m_blockcount[block_id] = bc ? c + 1 : 1; @@ -437,10 +438,11 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// The named channels for the restore operation. /// The database holding information about how many of each block this restore requires. - /// The restore options. /// The channels for reading block requests from the `FileProcessor`. /// The channels for writing block responses back to the `FileProcessor`. - public static Task Run(Channels channels, LocalRestoreDatabase db, Options options, IChannel[] fp_requests, IChannel>[] fp_responses) + /// The restore options. + /// The results of the restore operation. + public static Task Run(Channels channels, LocalRestoreDatabase db, IChannel[] fp_requests, IChannel>[] fp_responses, Options options, RestoreResults results) { return AutomationExtensions.RunTask( new @@ -452,7 +454,7 @@ namespace Duplicati.Library.Main.Operation.Restore { // Create a cache for the blocks, using SleepableDictionary cache = - await SleepableDictionary.CreateAsync(db, self.Output, options, fp_requests.Length) + await SleepableDictionary.CreateAsync(db, self.Output, options, fp_requests.Length, results.TaskControl.ProgressToken) .ConfigureAwait(false); // The volume consumer will read blocks from the input channel (data blocks from the volumes) and store them in the cache. diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index 1a7e3fbd1..7abeffd78 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_get_files?.Start(); // The enumerables are cast to arrays to force the query to be executed and release the database lock. var files = await db - .GetFilesAndSymlinksToRestore() + .GetFilesAndSymlinksToRestore(result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false); @@ -85,7 +85,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_get_folders?.Start(); // The enumerables are cast to arrays to force the query to be executed and release the database lock. var folders = await db - .GetFolderMetadataToRestore() + .GetFolderMetadataToRestore(result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false); sw_get_folders?.Stop(); diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 79454c9ac..adecbb0ea 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -120,7 +120,7 @@ namespace Duplicati.Library.Main.Operation.Restore decremented = true; } - await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp) + await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken) .ConfigureAwait(false); continue; @@ -130,7 +130,7 @@ namespace Duplicati.Library.Main.Operation.Restore // TODO rather than keeping all of the blocks in memory, we could do a single pass over the blocks using a cursor, only keeping the relevant block requests in memory. Maybe even only a single block request at a time. sw_block?.Start(); var blocks = await db - .GetBlocksFromFile(file.BlocksetID) + .GetBlocksFromFile(file.BlocksetID, results.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false); sw_block?.Stop(); @@ -435,7 +435,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (!options.SkipMetadata) { - empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp).ConfigureAwait(false); + empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken).ConfigureAwait(false); sw_work_meta?.Stop(); } @@ -601,7 +601,9 @@ namespace Duplicati.Library.Main.Operation.Restore /// The stopwatch for internal profiling of the general processing. /// The stopwatch for internal profiling of the block requests. /// The stopwatch for internal profiling of the block responses. - private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp) + /// The cancellation token to cancel the operation. + /// An awaitable `Task`, which returns `true` if the metadata was restored successfully, `false` otherwise. + private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp, CancellationToken cancellationToken) { sw_meta?.Start(); // Since each FileProcessor should have its own connection, it's ok @@ -609,7 +611,7 @@ namespace Duplicati.Library.Main.Operation.Restore // time, as the other processes should still be able to read. // Therefore, we don't need to read the entire result set into // memory. - var blocks = db.GetMetadataBlocksFromFile(file.ID); + var blocks = db.GetMetadataBlocksFromFile(file.ID, cancellationToken); sw_meta?.Stop(); using var ms = new MemoryStream(); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index 251f2f48c..a2c4aed9c 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -76,12 +76,12 @@ namespace Duplicati.Library.Main.Operation.Restore sw_wait?.Start(); TempFile f; var (volume_name, size, hash) = await db - .GetVolumeInfo(volume_id) + .GetVolumeInfo(volume_id, results.TaskControl.ProgressToken) .FirstAsync() .ConfigureAwait(false); try { - f = await backend.GetDirectAsync(volume_name, hash, size, CancellationToken.None).ConfigureAwait(false); + f = await backend.GetDirectAsync(volume_name, hash, size, results.TaskControl.TransferToken).ConfigureAwait(false); } catch (Exception) { diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 6d3b26311..c08858ca9 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -353,7 +353,7 @@ namespace Duplicati.Library.Main.Operation var filelister = Restore.FileLister.Run(channels, database, m_options, m_result); Restore.FileProcessor.file_processors_restoring_files = m_options.RestoreFileProcessors; var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], m_options, m_result)).ToArray(); - var blockmanager = Restore.BlockManager.Run(channels, database, m_options, fileprocessor_requests, fileprocessor_responses); + var blockmanager = Restore.BlockManager.Run(channels, database, fileprocessor_requests, fileprocessor_responses, m_options, m_result); var volumecache = Restore.VolumeManager.Run(channels, m_options); var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, backendManager, m_options, m_result)).ToArray(); var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, backendManager, m_options)).ToArray(); diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index ebaf50ab3..ec7e4ade7 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -28,6 +28,7 @@ using Duplicati.Library.Main.Operation.Common; using System.Threading.Tasks; using Duplicati.Library.Utility; using System.Data; +using System.Threading; namespace Duplicati.Library.Main.Volumes { @@ -236,11 +237,12 @@ namespace Duplicati.Library.Main.Volumes /// The database to check for clashes /// The options to use for the filename /// The starting time to probe from + /// The cancellation token to use for the operation /// The time to increment by each probe /// The maximum number of tries to probe /// The first unused filename - internal static Task ProbeUnusedFilenameName(DatabaseCommon database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) - => ProbeUnusedFilenameName(name => database.GetRemoteVolumeIDAsync(name), options, start, increment, maxTries); + internal static Task ProbeUnusedFilenameName(DatabaseCommon database, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60) + => ProbeUnusedFilenameName((name, CancellationToken) => database.GetRemoteVolumeIDAsync(name, cancellationToken), options, start, cancellationToken, increment, maxTries); /// /// Probes for an unused filename, using the current time as a starting point. @@ -251,8 +253,8 @@ namespace Duplicati.Library.Main.Volumes /// The time to increment by each probe. /// The maximum number of tries to probe. /// A task that when awaited returns the first unused filename. - internal static async Task ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) - => await ProbeUnusedFilenameName(database.GetRemoteVolumeID, options, start, increment, maxTries) + internal static async Task ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60) + => await ProbeUnusedFilenameName(database.GetRemoteVolumeID, options, start, cancellationToken, increment, maxTries) .ConfigureAwait(false); /// @@ -264,7 +266,7 @@ namespace Duplicati.Library.Main.Volumes /// The time to increment by each probe /// The maximum number of tries to probe /// The first unused filename - private static async Task ProbeUnusedFilenameName(Func> ProbeForId, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) + private static async Task ProbeUnusedFilenameName(Func> ProbeForId, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60) { if (increment == default) increment = TimeSpan.FromSeconds(1); @@ -274,7 +276,7 @@ namespace Duplicati.Library.Main.Volumes while (s < maxTries) { - var id = await ProbeForId(GenerateFilename(RemoteVolumeType.Files, options, null, time)).ConfigureAwait(false); + var id = await ProbeForId(GenerateFilename(RemoteVolumeType.Files, options, null, time), cancellationToken).ConfigureAwait(false); if (id < 0) return time; From 9b0d89363baeda1d84db19388e18792d4dd4c92a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 11:40:23 +0200 Subject: [PATCH 343/362] Removed whitespace --- .../DatabaseTool/Commands/Execute.cs | 30 +++++++++---------- .../CommandLine/DatabaseTool/Commands/List.cs | 30 +++++++++---------- Duplicati/UnitTest/DatabaseToolTests.cs | 30 +++++++++---------- Duplicati/UnitTest/Issue4693.cs | 30 +++++++++---------- .../UnitTest/RepairWithMissingRemoteFiles.cs | 30 +++++++++---------- .../Endpoints/V1/Backup/BackupGet.cs | 30 +++++++++---------- 6 files changed, 90 insertions(+), 90 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs index d2f57df6a..4e535e7e9 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs @@ -1,23 +1,23 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.NamingConventionBinder; diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/List.cs b/Duplicati/CommandLine/DatabaseTool/Commands/List.cs index 5b9429251..00ad6954e 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/List.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/List.cs @@ -1,23 +1,23 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.NamingConventionBinder; diff --git a/Duplicati/UnitTest/DatabaseToolTests.cs b/Duplicati/UnitTest/DatabaseToolTests.cs index 11c069733..3167f09e2 100644 --- a/Duplicati/UnitTest/DatabaseToolTests.cs +++ b/Duplicati/UnitTest/DatabaseToolTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Threading.Tasks; using Duplicati.CommandLine.DatabaseTool; diff --git a/Duplicati/UnitTest/Issue4693.cs b/Duplicati/UnitTest/Issue4693.cs index ff881cc06..597582829 100644 --- a/Duplicati/UnitTest/Issue4693.cs +++ b/Duplicati/UnitTest/Issue4693.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs index 827f23fc3..fa9d16ed5 100644 --- a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs +++ b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index 413226cfe..487a71f38 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Text.Json; using System.Text.Json.Serialization; From 2412800c0c58df1a6a071f20e766f307418531d6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 11:41:39 +0200 Subject: [PATCH 344/362] Removed the sqlite page cache option. It should be set through the environment variable, or some other way of providing a custom database configurotion --- .../DatabaseTool/Commands/Downgrade.cs | 2 +- .../DatabaseTool/Commands/Execute.cs | 2 +- .../CommandLine/DatabaseTool/Commands/List.cs | 2 +- .../DatabaseTool/Commands/Upgrade.cs | 2 +- Duplicati/CommandLine/DatabaseTool/Helper.cs | 4 +-- Duplicati/Library/Main/Controller.cs | 12 ++++---- .../Main/Database/LocalBackupDatabase.cs | 2 +- .../Main/Database/LocalBugReportDatabase.cs | 5 ++-- .../Library/Main/Database/LocalDatabase.cs | 11 +++----- .../Main/Database/LocalDeleteDatabase.cs | 5 ++-- .../Database/LocalListAffectedDatabase.cs | 5 ++-- .../Database/LocalListBrokenFilesDatabase.cs | 5 ++-- .../Main/Database/LocalListChangesDatabase.cs | 5 ++-- .../Main/Database/LocalListDatabase.cs | 5 ++-- .../Main/Database/LocalPurgeDatabase.cs | 5 ++-- .../Main/Database/LocalRepairDatabase.cs | 5 ++-- .../Main/Database/LocalRestoreDatabase.cs | 7 ++--- .../Main/Database/LocalTestDatabase.cs | 5 ++-- .../Library/Main/Operation/CompactHandler.cs | 2 +- .../Main/Operation/CreateBugReportHandler.cs | 2 +- .../Library/Main/Operation/DeleteHandler.cs | 2 +- .../Library/Main/Operation/ListAffected.cs | 2 +- .../Main/Operation/ListBrokenFilesHandler.cs | 2 +- .../Main/Operation/ListChangesHandler.cs | 2 +- .../Main/Operation/ListControlFilesHandler.cs | 2 +- .../Main/Operation/ListFileVersionsHandler.cs | 2 +- .../Main/Operation/ListFilesHandler.cs | 4 +-- .../Main/Operation/ListFilesetsHandler.cs | 2 +- .../Main/Operation/ListFolderHandler.cs | 2 +- .../Main/Operation/PurgeBrokenFilesHandler.cs | 2 +- .../Main/Operation/PurgeFilesHandler.cs | 2 +- .../Main/Operation/RecreateDatabaseHandler.cs | 4 +-- .../Library/Main/Operation/RepairHandler.cs | 8 +++--- .../Operation/RestoreControlFilesHandler.cs | 1 - .../Library/Main/Operation/RestoreHandler.cs | 4 +-- .../Main/Operation/SearchEntriesHandler.cs | 2 +- .../Library/Main/Operation/TestHandler.cs | 2 +- .../Library/Main/Operation/VacuumHandler.cs | 2 +- Duplicati/Library/Main/Options.cs | 19 ------------- Duplicati/Library/Main/Strings.cs | 2 -- .../Library/SQLiteHelper/SQLiteLoader.cs | 18 ++++-------- Duplicati/UnitTest/DatabaseToolTests.cs | 28 +++++++++---------- Duplicati/UnitTest/DirectListHandlerTests.cs | 8 +++--- Duplicati/UnitTest/DisruptionTests.cs | 2 +- Duplicati/UnitTest/Issue4693.cs | 8 +++--- Duplicati/UnitTest/RepairHandlerTests.cs | 18 ++++++------ .../UnitTest/RepairWithMissingRemoteFiles.cs | 2 +- Duplicati/UnitTest/SQLiteTests.cs | 2 +- .../Endpoints/V1/Backup/BackupGet.cs | 4 +-- 49 files changed, 104 insertions(+), 147 deletions(-) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs index f2de7e23d..369812aeb 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs @@ -161,7 +161,7 @@ public static class Downgrade if (!nobackups) Helper.CreateFileBackup(db); - await using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) + await using var con = await SQLiteLoader.LoadConnectionAsync(db) .ConfigureAwait(false); await using var tr = con.BeginTransaction(); await using var cmd = con.CreateCommand(tr); diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs index 4e535e7e9..30372ea18 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs @@ -48,7 +48,7 @@ public static class Execute if (createbackup) Helper.CreateFileBackup(database); - using var connection = SQLiteLoader.LoadConnection(database, 0); + using var connection = SQLiteLoader.LoadConnection(database); try { if (command.IndexOfAny(Path.GetInvalidPathChars()) < 0 && File.Exists(command)) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/List.cs b/Duplicati/CommandLine/DatabaseTool/Commands/List.cs index 00ad6954e..c8b8f10a7 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/List.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/List.cs @@ -54,7 +54,7 @@ public static class List return; } - using var con = SQLiteLoader.LoadConnection(database, 0); + using var con = SQLiteLoader.LoadConnection(database); using var cmd = con.CreateCommand(); if (tables == null || tables.Length == 0) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs index 2e1cd7aeb..4bf77c1fb 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs @@ -173,7 +173,7 @@ public static class Upgrade if (!nobackups) Helper.CreateFileBackup(db); - await using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) + await using var con = await SQLiteLoader.LoadConnectionAsync(db) .ConfigureAwait(false); await using var tr = con.BeginTransaction(); await using var cmd = con.CreateCommand(tr); diff --git a/Duplicati/CommandLine/DatabaseTool/Helper.cs b/Duplicati/CommandLine/DatabaseTool/Helper.cs index e8ae324c8..022f1cdd6 100644 --- a/Duplicati/CommandLine/DatabaseTool/Helper.cs +++ b/Duplicati/CommandLine/DatabaseTool/Helper.cs @@ -60,7 +60,7 @@ public static class Helper { try { - await using var con = await SQLiteLoader.LoadConnectionAsync(serverdb, 0) + await using var con = await SQLiteLoader.LoadConnectionAsync(serverdb) .ConfigureAwait(false); await using var cmd = con.CreateCommand(@" SELECT ""DBPath"" @@ -93,7 +93,7 @@ public static class Helper /// A task that when awaited contains a tuple with the version and whether it is a server database public static async Task<(int Version, bool isserver)> ExamineDatabase(string db) { - await using var con = await SQLiteLoader.LoadConnectionAsync(db, 0) + await using var con = await SQLiteLoader.LoadConnectionAsync(db) .ConfigureAwait(false); await using var cmd = con.CreateCommand(); diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 4f4f24616..7ed4aca8a 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -253,7 +253,7 @@ namespace Duplicati.Library.Main return RunAction(new ListRemoteResults(), async (result, backendManager) => { using var tf = File.Exists(m_options.Dbpath) ? null : new TempFile(); - await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, null, result.TaskControl.ProgressToken).ConfigureAwait(false); result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); }); } @@ -277,7 +277,7 @@ namespace Duplicati.Library.Main // and uses the same prefix (see issues #2678, #3845, and #4244). if (File.Exists(m_options.Dbpath)) { - await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, null, result.TaskControl.ProgressToken).ConfigureAwait(false); var dbRemoteVolumes = db.GetRemoteVolumes(result.TaskControl.ProgressToken); var dbRemoteFiles = await dbRemoteVolumes .Select(x => x.Name) @@ -522,7 +522,7 @@ namespace Duplicati.Library.Main // instead of relying on the operations to correctly toggle the flag if (File.Exists(m_options.Dbpath) && !m_options.NoLocalDb) { - using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache, null, CancellationToken.None).Await(); + using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).Await(); backend.StopRunnerAndFlushMessages(db).Await(); } else @@ -538,7 +538,7 @@ namespace Duplicati.Library.Main if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) { - using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache, null, CancellationToken.None).Await(); + using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).Await(); db.WriteResults(result, CancellationToken.None).Await(); db.PurgeLogData(m_options.LogRetention, CancellationToken.None).Await(); db.PurgeDeletedVolumes(DateTime.UtcNow, CancellationToken.None).Await(); @@ -580,7 +580,7 @@ namespace Duplicati.Library.Main { // No operation was started in database, so write logs to new operation if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) - using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache, null, CancellationToken.None).Await()) + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).Await()) db.WriteResults(result, CancellationToken.None).Await(); } catch (Exception we) @@ -605,7 +605,7 @@ namespace Duplicati.Library.Main resultSetter.Fatal = true; // Write logs to previous operation if database exists if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) - using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, m_options.SqlitePageCache, null, CancellationToken.None).Await()) + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).Await()) db.WriteResults(result, CancellationToken.None).Await(); } catch (Exception we) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index aa5804e72..d28425ac5 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -167,7 +167,7 @@ namespace Duplicati.Library.Main.Database dbnew ??= new LocalBackupDatabase(); dbnew = (LocalBackupDatabase) - await CreateLocalDatabaseAsync(path, "Backup", false, options.SqlitePageCache, dbnew, token) + await CreateLocalDatabaseAsync(path, "Backup", false, dbnew, token) .ConfigureAwait(false); dbnew = await CreateAsync(dbnew, options, null, token).ConfigureAwait(false); dbnew.ShouldCloseConnection = true; diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 7cf791c08..aeb5415e2 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -43,16 +43,15 @@ namespace Duplicati.Library.Main.Database /// Creates a new instance of the class. /// /// The path to the database file. - /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalBugReportDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, LocalBugReportDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalBugReportDatabase(); dbnew = (LocalBugReportDatabase) - await CreateLocalDatabaseAsync(path, "BugReportCreate", false, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, "BugReportCreate", false, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 8571f3c4f..635087b26 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -183,15 +183,14 @@ namespace Duplicati.Library.Main.Database /// This method ensures that the directory for the database exists and upgrades the database schema if necessary. /// /// The path to the SQLite database file. - /// The size of the SQLite page cache in bytes. /// A task that, when awaited, returns a new to the specified database. - protected static async Task CreateConnectionAsync(string path, long pagecachesize) + protected static async Task CreateConnectionAsync(string path) { path = Path.GetFullPath(path); if (!Directory.Exists(Path.GetDirectoryName(path))) Directory.CreateDirectory(Path.GetDirectoryName(path) ?? throw new DirectoryNotFoundException("Path was a root folder.")); - var c = await SQLiteHelper.SQLiteLoader.LoadConnectionAsync(path, pagecachesize) + var c = await SQLiteHelper.SQLiteLoader.LoadConnectionAsync(path) .ConfigureAwait(false); try @@ -215,21 +214,19 @@ namespace Duplicati.Library.Main.Database /// The path to the SQLite database file. /// The description of the operation being performed. /// Indicates whether the connection should be closed when the database is disposed. - /// The size of the SQLite page cache in bytes. /// An optional existing instance to use. If not provided, a new instance will be created. /// Cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a new instance of . - public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize, LocalDatabase? db, CancellationToken token) + public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, LocalDatabase? db, CancellationToken token) { db ??= new LocalDatabase(); - var connection = await CreateConnectionAsync(path, pagecachesize) + var connection = await CreateConnectionAsync(path) .ConfigureAwait(false); db = await CreateLocalDatabaseAsync(connection, operation, db, token) .ConfigureAwait(false); db.ShouldCloseConnection = shouldclose; - db.m_pagecachesize = pagecachesize; return db; } diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 5109398fd..9737f71ab 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -74,16 +74,15 @@ namespace Duplicati.Library.Main.Database /// /// The path to the database file. /// The operation name. - /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, string operation, long pagecachesize, LocalDeleteDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, string operation, LocalDeleteDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalDeleteDatabase(); dbnew = (LocalDeleteDatabase) - await CreateLocalDatabaseAsync(path, operation, true, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, operation, true, dbnew, token) .ConfigureAwait(false); dbnew.m_registerDuplicateBlockCommand = diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 18f70f1ce..02f10ce58 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -40,16 +40,15 @@ namespace Duplicati.Library.Main.Database /// Creates a new instance of . /// /// The path to the database file. - /// The size of the page cache in bytes. /// An optional existing database instance to use. Used to mimic constructor chaining. /// A cancellation token to cancel the operation. /// A task that when awaited returns a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalListAffectedDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, LocalListAffectedDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListAffectedDatabase(); dbnew = (LocalListAffectedDatabase) - await CreateLocalDatabaseAsync(path, "ListAffected", false, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, "ListAffected", false, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index ad7f494b3..2cacd60ae 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -165,16 +165,15 @@ namespace Duplicati.Library.Main.Database /// Creates a new instance of the class. /// /// The path to the database file. - /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalListBrokenFilesDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, LocalListBrokenFilesDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListBrokenFilesDatabase(); dbnew = (LocalListBrokenFilesDatabase) - await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 404db019d..83bad5cf5 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -41,16 +41,15 @@ namespace Duplicati.Library.Main.Database /// Creates a new instance of the class. /// /// The path to the database file. - /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalListChangesDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, LocalListChangesDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListChangesDatabase(); dbnew = (LocalListChangesDatabase) - await CreateLocalDatabaseAsync(path, "ListChanges", false, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, "ListChanges", false, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 96d311f64..a54577843 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -43,16 +43,15 @@ namespace Duplicati.Library.Main.Database /// Creates a new instance of the LocalListDatabase. /// /// The path to the database file. - /// The size of the page cache. /// An optional existing database instance to use. Used to mimic constructor chaining. /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of LocalListDatabase. - public static async Task CreateAsync(string path, long pagecachesize, LocalListDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, LocalListDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListDatabase(); dbnew = (LocalListDatabase) - await CreateLocalDatabaseAsync(path, "List", false, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, "List", false, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 8434c8842..a5ed28302 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -40,16 +40,15 @@ namespace Duplicati.Library.Main.Database /// Creates a new instance of the class. /// /// The path to the database file. - /// The size of the page cache. /// The optional existing database instance to use. Used to mimic constructor chaining. /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalPurgeDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, LocalPurgeDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalPurgeDatabase(); dbnew = (LocalPurgeDatabase) - await LocalDeleteDatabase.CreateAsync(path, "Purge", pagecachesize, dbnew, token) + await LocalDeleteDatabase.CreateAsync(path, "Purge", dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index fbf4f8969..02e68eb4d 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -52,16 +52,15 @@ namespace Duplicati.Library.Main.Database /// Creates a new local repair database. /// /// The path to the database. - /// The page cache size. /// An optional existing database instance to use. /// A cancellation token to cancel the operation. /// A task that when awaited returns a new instance of . - public static async Task CreateRepairDatabase(string path, long pagecachesize, LocalRepairDatabase? dbnew, CancellationToken token) + public static async Task CreateRepairDatabase(string path, LocalRepairDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalRepairDatabase(); dbnew = (LocalRepairDatabase) - await CreateLocalDatabaseAsync(path, "Repair", true, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, "Repair", true, dbnew, token) .ConfigureAwait(false); return dbnew; diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 58d6126a2..3274e6a2d 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -99,16 +99,15 @@ namespace Duplicati.Library.Main.Database /// Initializes a new instance of the class. /// /// The path to the database file. - /// The size of the page cache for the database. /// An optional existing instance of to use, or null to create a new one. Used to mimic constructor chaining. /// A cancellation token to monitor for cancellation requests. /// A task that when awaited on, returns a new instance of . - public static async Task CreateAsync(string path, long pagecachesize, LocalRestoreDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, LocalRestoreDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalRestoreDatabase(); dbnew = (LocalRestoreDatabase) - await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, "Restore", false, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; @@ -1922,7 +1921,7 @@ namespace Duplicati.Library.Main.Database connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared"; await connection.OpenAsync(token).ConfigureAwait(false); - await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize) + await SQLiteLoader.ApplyCustomPragmasAsync(connection) .ConfigureAwait(false); var transaction = new ReusableTransaction(connection); diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 95b7d9ba2..89d891ad9 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -43,16 +43,15 @@ namespace Duplicati.Library.Main.Database /// Initializes a new instance of the class. /// /// The path to the database file. - /// The size of the page cache in bytes. /// An optional existing instance to use. Used to mimic constructor chaining. /// A cancellation token to observe while waiting for the task to complete. /// A task that represents the asynchronous operation. The task result contains the created instance. - public static async Task CreateAsync(string path, long pagecachesize, LocalTestDatabase? dbnew, CancellationToken token) + public static async Task CreateAsync(string path, LocalTestDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalTestDatabase(); dbnew = (LocalTestDatabase) - await CreateLocalDatabaseAsync(path, "Test", true, pagecachesize, dbnew, token) + await CreateLocalDatabaseAsync(path, "Test", true, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 5374947fc..3da67f257 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index 9ff8ea208..164c3ad4c 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation using (var tmp = new TempFile()) { File.Copy(m_options.Dbpath, tmp, true); - await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { await db.Fix(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (m_options.AutoVacuum) diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 260c33891..f45562038 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -51,7 +51,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 97284398e..571123e8a 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -44,7 +44,7 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (callback == null) { m_result.SetResult( diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 8f37f4226..b554ac130 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index e1c82db45..71bfe0fd2 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -80,7 +80,7 @@ namespace Duplicati.Library.Main.Operation DateTime compareVersionTime; using var tmpdb = useLocalDb ? null : new TempFile(); - await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await using var storageKeeper = await db.CreateStorageHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (useLocalDb) { diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index d2f6bf925..0d21ebf02 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.Operation { var cancellationToken = CancellationToken.None; using var tmpdb = new TempFile(); - await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); try diff --git a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs index fb0cf4427..8829f14cb 100644 --- a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs @@ -48,7 +48,7 @@ internal static class ListFileVersionsHandler throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); await using var db = - await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) + await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 6d3375ddd..8ad83b976 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -55,7 +55,7 @@ namespace Duplicati.Library.Main.Operation //Use a speedy local query if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) - await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (!filter.Empty) @@ -137,7 +137,7 @@ namespace Duplicati.Library.Main.Operation // Otherwise, grab info from remote location using (var tmpdb = new TempFile()) - await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) diff --git a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs index 8629f33e8..22ce6e35c 100644 --- a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs @@ -50,7 +50,7 @@ internal static class ListFilesetsHandler // Use a speedy local query if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb) { - await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); result.Filesets = await db diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index 5a296f564..2ceff3590 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -46,7 +46,7 @@ internal static class ListFolderHandler if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); var filesetIds = await db diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 348ebf383..fb13f13c3 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -54,7 +54,7 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index cbf512518..11650c4a1 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); await db .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index c5a116c54..f6f9945c1 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -66,7 +66,7 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); await using var db = - await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) + await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); @@ -90,7 +90,7 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); await using var db = - await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) + await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Any()) { diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index ff9c5800d..1494fc70b 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -66,7 +66,7 @@ namespace Duplicati.Library.Main.Operation try { await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); knownRemotes = await db .GetRemoteVolumes(m_result.TaskControl.ProgressToken) @@ -134,7 +134,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, cancellationToken) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, null, cancellationToken) .ConfigureAwait(false); try @@ -1326,7 +1326,7 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var sets = await db @@ -1383,7 +1383,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index 80658a033..b64382de9 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -49,7 +49,6 @@ namespace Duplicati.Library.Main.Operation await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync( File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, - m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken ).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index c08858ca9..9766d78af 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -115,7 +115,7 @@ namespace Duplicati.Library.Main.Operation { if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) { - db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) + db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } else @@ -123,7 +123,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); tmpdb = new TempFile(); RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); - db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) + db = await LocalRestoreDatabase.CreateAsync(tmpdb, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) diff --git a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs index fe66207c2..b8c4610d5 100644 --- a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs +++ b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs @@ -50,7 +50,7 @@ internal static class SearchEntriesHandler throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); await using var db = - await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) + await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 523398465..210a92924 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index 9556363fa..ef90425a7 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -38,7 +38,7 @@ namespace Duplicati.Library.Main.Operation public virtual async Task RunAsync() { await using var db = - await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) + await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 0fbdbc808..1730bda83 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -176,11 +176,6 @@ namespace Duplicati.Library.Main /// private static readonly int DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE = Environment.ProcessorCount; - /// - /// The default value for the size of the SQLite page cache - /// - private static readonly string DEFAULT_SQLITE_PAGE_CACHE_SIZE = MemoryInfo.GetTotalMemoryString(0.01, SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE); // 1% of the total memory - /// /// An enumeration that describes the supported strategies for an optimization /// @@ -536,7 +531,6 @@ namespace Duplicati.Library.Main new CommandLineArgument("restore-volume-downloaders", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDownloadersShort, Strings.Options.RestoreVolumeDownloadersLong, DEFAULT_RESTORE_VOLUME_DOWNLOADERS.ToString()), new CommandLineArgument("restore-channel-buffer-size", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreChannelBufferSizeShort, Strings.Options.RestoreChannelBufferSizeLong, DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE.ToString()), new CommandLineArgument("internal-profiling", CommandLineArgument.ArgumentType.Boolean, Strings.Options.InternalProfilingShort, Strings.Options.InternalProfilingLong, "false"), - new CommandLineArgument("sqlite-page-cache", CommandLineArgument.ArgumentType.Size, Strings.Options.SqlitePageCacheShort, Strings.Options.SqlitePageCacheLong(SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE), DEFAULT_SQLITE_PAGE_CACHE_SIZE), new CommandLineArgument("ignore-update-if-version-exists", CommandLineArgument.ArgumentType.Boolean, Strings.Options.IgnoreUpdateIfVersionExistsShort, Strings.Options.IgnoreUpdateIfVersionExistsLong, "false"), ]; @@ -1677,19 +1671,6 @@ namespace Duplicati.Library.Main /// public bool InternalProfiling => GetBool("internal-profiling"); - /// - /// Gets the size of file-blocks - /// - public long SqlitePageCache - { - get - { - var pagesize = GetSize("sqlite-page-cache", "kb", DEFAULT_SQLITE_PAGE_CACHE_SIZE); - return pagesize <= SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE - ? 0 - : pagesize; - } - } /// /// Ignores the update if the version already exists in the database. /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index d7d168a2b..2e58f71a7 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -363,8 +363,6 @@ namespace Duplicati.Library.Main.Strings public static string RestoreChannelBufferSizeLong { get { return LC.L("Use this option to set the size of the buffers of the channels used during restore. The buffers are used to allow for better asynchronous communication between the processes in the restore flow. Increasing the buffer size may improve restore performance."); } } public static string InternalProfilingShort { get { return LC.L("Enable internal profiling"); } } public static string InternalProfilingLong { get { return LC.L("Use this option to enable internal profiling. Profiling is used to measure the performance of the internal code. The profiling data is written to the log file and can be used to identify performance bottlenecks."); } } - public static string SqlitePageCacheShort { get { return LC.L("Size of the SQLite page cache"); } } - public static string SqlitePageCacheLong(long minimumvalue) { return LC.L("Use this option to set the size of the SQLite page cache. The page cache is used to store the pages of the database in memory. Increasing the page cache size may improve performance, but will also increase memory usage. If the supplied value is the same or less than {0} bytes, the default SQLite cache value is used.", minimumvalue); } public static string IgnoreUpdateIfVersionExistsShort { get { return LC.L("Ignore update if version exists"); } } public static string IgnoreUpdateIfVersionExistsLong { get { return LC.L("Use this option to ignore the update if the version already exists. This can be used to avoid errors if asking to update the database with a version that already exists."); } } } diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index af086b424..29ae638e7 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -35,11 +35,6 @@ namespace Duplicati.Library.SQLiteHelper /// public static class SQLiteLoader { - /// - /// The minimum value for the SQLite page cache size. - /// - public const long MINIMUM_SQLITE_PAGE_CACHE_SIZE = 2048000L; - /// /// The tag used for logging. /// @@ -129,9 +124,8 @@ namespace Duplicati.Library.SQLiteHelper /// Applies user-supplied custom pragmas to the SQLite connection. /// /// The connection to apply the pragmas to. - /// The page cache size to set. /// A task that when awaited returns the connection with the pragmas applied. - public static async Task ApplyCustomPragmasAsync(Microsoft.Data.Sqlite.SqliteConnection con, long pagecachesize) + public static async Task ApplyCustomPragmasAsync(Microsoft.Data.Sqlite.SqliteConnection con) { // TODO more default custom options //await cmd.ExecuteNonQueryAsync("PRAGMA synchronous = NORMAL;"); @@ -172,23 +166,21 @@ namespace Duplicati.Library.SQLiteHelper /// Loads an SQLite connection instance and opening the database with a specified page cache size. /// /// The optional path to the database. - /// The page cache size to set. /// The SQLite connection instance. /// /// This method is synchronous and should be used when you need to load the connection immediately. It calls the asynchronous version and waits for it to complete. /// - public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection(string targetpath, long pagecachesize) + public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection(string targetpath) { - return LoadConnectionAsync(targetpath, pagecachesize).Await(); + return LoadConnectionAsync(targetpath).Await(); } /// /// Loads an SQLite connection instance and opening the database. /// /// The optional path to the database. - /// The page cache size to set. /// A task that when waited returns the SQLite connection instance. - public static async Task LoadConnectionAsync(string targetpath, long pagecachesize) + public static async Task LoadConnectionAsync(string targetpath) { if (string.IsNullOrWhiteSpace(targetpath)) throw new ArgumentNullException(nameof(targetpath)); @@ -208,7 +200,7 @@ namespace Duplicati.Library.SQLiteHelper } // set custom Sqlite options - return await ApplyCustomPragmasAsync(con, pagecachesize) + return await ApplyCustomPragmasAsync(con) .ConfigureAwait(false); } diff --git a/Duplicati/UnitTest/DatabaseToolTests.cs b/Duplicati/UnitTest/DatabaseToolTests.cs index 3167f09e2..93c0f405e 100644 --- a/Duplicati/UnitTest/DatabaseToolTests.cs +++ b/Duplicati/UnitTest/DatabaseToolTests.cs @@ -34,7 +34,7 @@ namespace Duplicati.UnitTest public async Task TestLocalDbMethods() { using var dbfile = new TempFile(); - using var db = SQLiteLoader.LoadConnection(dbfile, 0); + using var db = SQLiteLoader.LoadConnection(dbfile); using var cmd = db.CreateCommand(); cmd.CommandText = LocalSchemaV12; cmd.ExecuteNonQuery(); @@ -54,7 +54,7 @@ namespace Duplicati.UnitTest public async Task TestServerDbMethods() { using var dbfile = new TempFile(); - using var db = SQLiteLoader.LoadConnection(dbfile, 0); + using var db = SQLiteLoader.LoadConnection(dbfile); using var cmd = db.CreateCommand(); cmd.CommandText = ServerSchemaV6; cmd.ExecuteNonQuery(); @@ -141,8 +141,8 @@ CREATE TABLE ""Notification"" ( ""ID"" INTEGER PRIMARY KEY, ""Type"" TEXT NOT NULL, ""Title"" TEXT NOT NULL, - ""Message"" TEXT NOT NULL, - ""Exception"" TEXT NOT NULL, + ""Message"" TEXT NOT NULL, + ""Exception"" TEXT NOT NULL, ""BackupID"" TEXT NULL, ""Action"" TEXT NOT NULL, ""Timestamp"" INTEGER NOT NULL, @@ -152,15 +152,15 @@ CREATE TABLE ""Notification"" ( ); CREATE TABLE ""UIStorage"" ( - ""Scheme"" TEXT NOT NULL, - ""Key"" TEXT NOT NULL, + ""Scheme"" TEXT NOT NULL, + ""Key"" TEXT NOT NULL, ""Value"" TEXT NOT NULL ); CREATE TABLE ""TempFile"" ( ""ID"" INTEGER PRIMARY KEY, - ""Origin"" TEXT NOT NULL, - ""Path"" TEXT NOT NULL, + ""Origin"" TEXT NOT NULL, + ""Path"" TEXT NOT NULL, ""Timestamp"" INTEGER NOT NULL, ""Expires"" INTEGER NOT NULL ); @@ -250,7 +250,7 @@ CREATE TABLE ""Blockset"" ( ); CREATE UNIQUE INDEX ""BlocksetFullHash"" ON ""Blockset"" (""FullHash"", ""Length""); - + CREATE TABLE ""BlocksetEntry"" ( ""BlocksetID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, @@ -325,11 +325,11 @@ CREATE TABLE ""Configuration"" ( CREATE TABLE ""ChangeJournalData"" ( ""ID"" INTEGER PRIMARY KEY, - ""FilesetID"" INTEGER NOT NULL, - ""VolumeName"" TEXT NOT NULL, - ""JournalID"" INTEGER NOT NULL, - ""NextUsn"" INTEGER NOT NULL, - ""ConfigHash"" TEXT NOT NULL + ""FilesetID"" INTEGER NOT NULL, + ""VolumeName"" TEXT NOT NULL, + ""JournalID"" INTEGER NOT NULL, + ""NextUsn"" INTEGER NOT NULL, + ""ConfigHash"" TEXT NOT NULL ); INSERT INTO ""Version"" (""Version"") VALUES (12); diff --git a/Duplicati/UnitTest/DirectListHandlerTests.cs b/Duplicati/UnitTest/DirectListHandlerTests.cs index a15768b56..a5137571b 100644 --- a/Duplicati/UnitTest/DirectListHandlerTests.cs +++ b/Duplicati/UnitTest/DirectListHandlerTests.cs @@ -327,7 +327,7 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1, null, CancellationToken.None) + using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) .ConfigureAwait(false); SeedTestData(db, [ "/folder1/", // 1 @@ -357,7 +357,7 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1, null, CancellationToken.None) + using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) .ConfigureAwait(false); SeedTestData(db, [ "C:\\folder1\\", @@ -385,7 +385,7 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1, null, CancellationToken.None) + using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) .ConfigureAwait(false); SeedTestData(db, [ "\\\\server\\share\\folder\\", @@ -411,7 +411,7 @@ namespace Duplicati.UnitTest { // Arrange using var tempFile = new TempFile(); - using var db = await LocalListDatabase.CreateAsync(tempFile, 1, null, CancellationToken.None) + using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) .ConfigureAwait(false); SeedTestData(db, [ "C:\\data\\", // 1 diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 4ea98c964..63caa2dbc 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -805,7 +805,7 @@ namespace Duplicati.UnitTest } catch (TestUtils.TestVerificationException e) { - using var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, 0, null, CancellationToken.None).ConfigureAwait(false); + using var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, null, CancellationToken.None).ConfigureAwait(false); using var cmd = db.Connection.CreateCommand(); var sb = new StringBuilder(); diff --git a/Duplicati/UnitTest/Issue4693.cs b/Duplicati/UnitTest/Issue4693.cs index 597582829..a3555bfec 100644 --- a/Duplicati/UnitTest/Issue4693.cs +++ b/Duplicati/UnitTest/Issue4693.cs @@ -156,7 +156,7 @@ namespace Duplicati.UnitTest using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) TestUtils.AssertResults(c.Backup([DATAFOLDER])); - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock"""); @@ -172,14 +172,14 @@ namespace Duplicati.UnitTest using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) TestUtils.AssertResults(c.Backup([DATAFOLDER])); - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock"""); Assert.That(deletedBlocks, Is.GreaterThan(0), "DeletedBlock table is empty"); } - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock"""); @@ -193,7 +193,7 @@ namespace Duplicati.UnitTest using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) TestUtils.AssertResults(c.Backup([DATAFOLDER])); - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { var alldeletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock"""); diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 7bfe4e6d1..f80c6b65c 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -85,7 +85,7 @@ namespace Duplicati.UnitTest List expectedBlocksetIDs = new List(); List expectedIndexes = new List(); List expectedHashes = new List(); - using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"])) { // Read the contents of the BlocklistHash table so that we can // compare them to the contents after the repair operation. @@ -122,7 +122,7 @@ namespace Duplicati.UnitTest List repairedBlocksetIDs = new List(); List repairedIndexes = new List(); List repairedHashes = new List(); - using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"])) { using (IDbCommand command = connection.CreateCommand()) { @@ -426,7 +426,7 @@ namespace Duplicati.UnitTest } // Check database block link - using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, 0, null, CancellationToken.None).ConfigureAwait(false)) + using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, null, CancellationToken.None).ConfigureAwait(false)) { var indexVolumeId = await db .GetRemoteVolumeID(Path.GetFileName(origFile), CancellationToken.None) @@ -505,7 +505,7 @@ namespace Duplicati.UnitTest dlistFiles = dlistFiles.Reverse(); var dlistFile = dlistFiles.First(); - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand("DELETE FROM RemoteVolume WHERE Name = @Name")) Assert.AreEqual(1, cmd.SetParameterValue("@Name", Path.GetFileName(dlistFile)).ExecuteNonQuery()); @@ -525,7 +525,7 @@ namespace Duplicati.UnitTest } // Check that the entry was recreated - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand("SELECT COUNT(*) FROM RemoteVolume WHERE Name LIKE '%.dlist.%' AND State != @State")) Assert.AreEqual(2, cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()).ExecuteScalarInt64(-1)); @@ -562,7 +562,7 @@ namespace Duplicati.UnitTest TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); // Remove a fileset - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { var filesetId = cmd.ExecuteScalarInt64("SELECT Id FROM Fileset ORDER BY Id DESC LIMIT 1"); @@ -592,7 +592,7 @@ namespace Duplicati.UnitTest } // Check that entry was recreated - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) Assert.AreEqual(2, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM Fileset")); } @@ -616,7 +616,7 @@ namespace Duplicati.UnitTest // Remove a fileset long filesetEntries; long fileLookupEntries; - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { filesetEntries = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM FilesetEntry"); @@ -639,7 +639,7 @@ namespace Duplicati.UnitTest } // Check that entry was recreated - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { Assert.AreEqual(filesetEntries, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM FilesetEntry")); diff --git a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs index fa9d16ed5..ba67acb2e 100644 --- a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs +++ b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs @@ -324,7 +324,7 @@ namespace Duplicati.UnitTest (long remoteVolumes, long duplicatedBlocks, List blockVolumeIds, List<(long, long)> indexVolumeLinks) GetDbSignature() { - using var db = SQLiteLoader.LoadConnection(DBFILE, 0); + using var db = SQLiteLoader.LoadConnection(DBFILE); using var cmd = db.CreateCommand(); var remoteVolumes = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM RemoteVolume"); diff --git a/Duplicati/UnitTest/SQLiteTests.cs b/Duplicati/UnitTest/SQLiteTests.cs index f9ae9164d..8e2d1c09b 100755 --- a/Duplicati/UnitTest/SQLiteTests.cs +++ b/Duplicati/UnitTest/SQLiteTests.cs @@ -35,7 +35,7 @@ namespace Duplicati.UnitTest { private static async Task CreateDummyDatabase(string path) { - var connection = SQLiteLoader.LoadConnection(path, 0); + var connection = SQLiteLoader.LoadConnection(path); using (var command = connection.CreateCommand()) { diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index 487a71f38..11ba2c83f 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -179,7 +179,7 @@ public class BackupGet : IEndpointV1 if (!File.Exists(bk.DBPath)) return new List>(); - using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath, 0)) + using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath)) using (var cmd = con.CreateCommand()) return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize); } @@ -189,7 +189,7 @@ public class BackupGet : IEndpointV1 if (!File.Exists(bk.DBPath)) return new List>(); - using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath, 0)) + using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath)) using (var cmd = con.CreateCommand()) { var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize); From 322e4f15e4b39128821279a662fc1ee0d3d59170 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 11:48:05 +0200 Subject: [PATCH 345/362] Implemented a lightweight parser to the CUSTOMSQLITEOPTIONS environment variable along with new default options. --- .../Library/SQLiteHelper/SQLiteLoader.cs | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 29ae638e7..73c50db51 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -22,7 +22,9 @@ #nullable enable using System; +using System.Collections.Generic; using System.Globalization; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; @@ -127,26 +129,33 @@ namespace Duplicati.Library.SQLiteHelper /// A task that when awaited returns the connection with the pragmas applied. public static async Task ApplyCustomPragmasAsync(Microsoft.Data.Sqlite.SqliteConnection con) { - // TODO more default custom options - //await cmd.ExecuteNonQueryAsync("PRAGMA synchronous = NORMAL;"); - //await cmd.ExecuteNonQueryAsync("PRAGMA temp_store = MEMORY;"); - //await cmd.ExecuteNonQueryAsync("PRAGMA journal_mode = WAL;"); - //await cmd.ExecuteNonQueryAsync("PRAGMA cache_size = -512000;"); - //await cmd.ExecuteNonQueryAsync("PRAGMA threads = 8;"); - //await cmd.ExecuteNonQueryAsync("PRAGMA read_uncommitted = 1;"); - //await cmd.ExecuteNonQueryAsync("PRAGMA mmap_size = 41943040;"); - //await cmd.ExecuteNonQueryAsync("PRAGMA shared_cache = true;"); - var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI") ?? ""; - if (pagecachesize > MINIMUM_SQLITE_PAGE_CACHE_SIZE) - opts = string.Format(CultureInfo.InvariantCulture, "cache_size=-{0};{1}", pagecachesize / 1024L, opts); - - if (string.IsNullOrWhiteSpace(opts)) - return con; + Dictionary customOptions = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "synchronous", "NORMAL" }, // NORMAL is more performant than FULL (the default), but less safe, as it no longer blocks until the disk sync syscall to the OS has completed. + { "temp_store", "MEMORY" }, // Use memory for temporary storage to improve performance. + { "journal_mode", "WAL" }, // Use Write-Ahead Logging for better concurrency and performance. + { "cache_size", "-65536" }, // Set cache size to 64 MB (negative value means in KB, so -64000 = 64 MB). Default is 2000 pages, which is 2 MB (2000 * 1024 bytes). + { "mmap_size", "67108864" }, // 64 MB. + { "threads", "8" }, // Use 8 threads for parallel processing where applicable. + { "shared_cache", "true" } // + }; + // Override the default options with any custom options set in the environment variable. + var customOptionsEnv = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS"); + if (!string.IsNullOrWhiteSpace(customOptionsEnv)) + { + foreach (var opt in customOptionsEnv.Split([';'], StringSplitOptions.RemoveEmptyEntries)) + { + var parts = opt.Split(new[] { '=' }, 2); + if (parts.Length == 2) + customOptions[parts[0].Trim()] = parts[1].Trim(); + } + } using (var cmd = con.CreateCommand()) { - foreach (var opt in opts.Split([';'], StringSplitOptions.RemoveEmptyEntries)) + foreach (var (key, value) in customOptions) { + var opt = $"{key}={value}"; Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt); try { From 26c48c7e62125c655224dcf94bcbe03dc6dba0cc Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 12:24:22 +0200 Subject: [PATCH 346/362] Post merge with master fixes --- Duplicati/UnitTest/EmptyMetadataTests.cs | 22 ++++++++++++---------- Duplicati/UnitTest/RepairHandlerTests.cs | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Duplicati/UnitTest/EmptyMetadataTests.cs b/Duplicati/UnitTest/EmptyMetadataTests.cs index f4e9c2a0e..ad76d9938 100644 --- a/Duplicati/UnitTest/EmptyMetadataTests.cs +++ b/Duplicati/UnitTest/EmptyMetadataTests.cs @@ -28,6 +28,8 @@ using Duplicati.Library.Main.Database; using Duplicati.Library.Interface; using Duplicati.Library.SQLiteHelper; using NUnit.Framework; +using System.Threading; +using System.Threading.Tasks; namespace Duplicati.UnitTest; @@ -35,7 +37,7 @@ public class EmptyMetadataTests : BasicSetupHelper { [Test] [Category("Targeted")] - public void ReplaceMissingMetadataRestoresConsistency() + public async Task ReplaceMissingMetadataRestoresConsistency() { var testopts = TestOptions.Expand(new { no_encryption = true }); @@ -49,7 +51,7 @@ public class EmptyMetadataTests : BasicSetupHelper long metaBlocksetId; long filesetId; - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { // Find metadata blockset ID and fileset ID for the backed up folder @@ -81,24 +83,24 @@ public class EmptyMetadataTests : BasicSetupHelper var opts = new Options(testopts); var emptyMeta = Utility.WrapMetadata(new Dictionary(), opts); - Assert.Throws(() => + Assert.Throws(async () => { - using var db = new LocalDatabase(DBFILE, "verify", true, 0); - db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, null); + using var db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None); + await db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None); }); - using (var db = new LocalListBrokenFilesDatabase(DBFILE, 0)) + await using (var db = await LocalListBrokenFilesDatabase.CreateAsync(DBFILE, null, CancellationToken.None).ConfigureAwait(false)) { var blockVolumeIds = Array.Empty(); - var emptyId = db.GetEmptyMetadataBlocksetId(blockVolumeIds, emptyMeta.FileHash, emptyMeta.Blob.Length, null); + var emptyId = await db.GetEmptyMetadataBlocksetId(blockVolumeIds, emptyMeta.FileHash, emptyMeta.Blob.Length, CancellationToken.None); Assert.That(emptyId, Is.GreaterThanOrEqualTo(0), "Empty metadata blockset not found"); - var replaced = db.ReplaceMetadata(filesetId, emptyId, null); + var replaced = await db.ReplaceMetadata(filesetId, emptyId, CancellationToken.None); Assert.That(replaced, Is.GreaterThan(0), "No metadata rows replaced"); } - using (var db = new LocalDatabase(DBFILE, "verify", true, 0)) - db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, null); + await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None)) + await db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None); } } diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index c5974bfcb..d6fe983bf 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -659,7 +659,7 @@ namespace Duplicati.UnitTest using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { var lookupId = cmd.ExecuteScalarInt64("SELECT ID FROM FileLookup LIMIT 1"); @@ -677,7 +677,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, res.Errors.Count()); Assert.AreEqual(0, res.Warnings.Count()); } - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { var remaining = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM Metadataset JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID WHERE Blockset.Length = 0"); From 089dc8c53dc9fea530140e1c43777b6d4e6f1227 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 13:19:36 +0200 Subject: [PATCH 347/362] Assert.Throws did not like async lambda functions --- Duplicati/UnitTest/EmptyMetadataTests.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Duplicati/UnitTest/EmptyMetadataTests.cs b/Duplicati/UnitTest/EmptyMetadataTests.cs index ad76d9938..0451fda3c 100644 --- a/Duplicati/UnitTest/EmptyMetadataTests.cs +++ b/Duplicati/UnitTest/EmptyMetadataTests.cs @@ -30,6 +30,7 @@ using Duplicati.Library.SQLiteHelper; using NUnit.Framework; using System.Threading; using System.Threading.Tasks; +using Duplicati.Library.Utility; namespace Duplicati.UnitTest; @@ -81,12 +82,12 @@ public class EmptyMetadataTests : BasicSetupHelper } var opts = new Options(testopts); - var emptyMeta = Utility.WrapMetadata(new Dictionary(), opts); + var emptyMeta = Duplicati.Library.Main.Utility.WrapMetadata(new Dictionary(), opts); - Assert.Throws(async () => + Assert.Throws(() => { - using var db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None); - await db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None); + using var db = LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None).Await(); + db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None).Await(); }); await using (var db = await LocalListBrokenFilesDatabase.CreateAsync(DBFILE, null, CancellationToken.None).ConfigureAwait(false)) From b9fa8bd0bb5984ba1e0e0b74eaa8d616596651bc Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 13:20:01 +0200 Subject: [PATCH 348/362] Previous version was using 'null' as a transaction, which implies a commit after each query. --- Duplicati/UnitTest/EmptyMetadataTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Duplicati/UnitTest/EmptyMetadataTests.cs b/Duplicati/UnitTest/EmptyMetadataTests.cs index 0451fda3c..aca37e554 100644 --- a/Duplicati/UnitTest/EmptyMetadataTests.cs +++ b/Duplicati/UnitTest/EmptyMetadataTests.cs @@ -99,6 +99,8 @@ public class EmptyMetadataTests : BasicSetupHelper var replaced = await db.ReplaceMetadata(filesetId, emptyId, CancellationToken.None); Assert.That(replaced, Is.GreaterThan(0), "No metadata rows replaced"); + + await db.Transaction.CommitAsync(CancellationToken.None).ConfigureAwait(false); } await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None)) From 0a047a37e31e0e20e213b5852ea52bb7a29e7862 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 13:20:44 +0200 Subject: [PATCH 349/362] SQL query inside GetEmptyMetadataBlocksetId had not only moved files, but had also changed --- Duplicati/Library/Main/Database/LocalDatabase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index b4f4c8a32..70d47b4b1 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -3148,6 +3148,7 @@ namespace Duplicati.Library.Main.Database AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) + AND ""Blockset"".""Length"" > 0 ORDER BY ""Blockset"".""Length"" ASC LIMIT 1 ") From 6460e337871ab5507944604ea730493d2e40f22a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 19 Jun 2025 13:23:39 +0200 Subject: [PATCH 350/362] Added missing double quotes to the SQL queries in LocalRepairDatabase.FixEmptyMetadatasets() --- .../Main/Database/LocalRepairDatabase.cs | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 96ce1a92d..74ec8b2a1 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1935,17 +1935,18 @@ namespace Duplicati.Library.Main.Database try { + // TODO quotes await cmd.SetCommandAndParameters(@$" CREATE TEMP TABLE ""{tablename}"" AS SELECT - m.ID AS MetadataID, - m.BlocksetID - FROM Metadataset m - JOIN Blockset b - ON m.BlocksetID = b.ID + ""m"".""ID"" AS ""MetadataID"", + ""m"".""BlocksetID"" + FROM Metadataset ""m"" + JOIN Blockset ""b"" + ON ""m"".""BlocksetID"" = ""b"".""ID"" WHERE - b.Length = 0 - AND m.BlocksetID != @KeepBlockset + ""b"".""Length"" = 0 + AND ""m"".""BlocksetID"" != @KeepBlockset ") .SetParameterValue("@KeepBlockset", emptyBlocksetId) .ExecuteNonQueryAsync(token) @@ -1953,15 +1954,15 @@ namespace Duplicati.Library.Main.Database // Step 2: Update FileLookup to use a valid metadata ID await cmd.SetCommandAndParameters(@$" - UPDATE FileLookup - SET MetadataID = ( - SELECT ID - FROM Metadataset - WHERE BlocksetID = @KeepBlockset + UPDATE ""FileLookup"" + SET ""MetadataID"" = ( + SELECT ""ID"" + FROM ""Metadataset"" + WHERE ""BlocksetID"" = @KeepBlockset LIMIT 1 ) - WHERE MetadataID IN ( - SELECT MetadataID + WHERE ""MetadataID"" IN ( + SELECT ""MetadataID"" FROM ""{tablename}"" ) ") @@ -1971,9 +1972,9 @@ namespace Duplicati.Library.Main.Database // Step 3: Delete obsolete Metadataset entries await cmd.SetCommandAndParameters(@$" - DELETE FROM Metadataset + DELETE FROM ""Metadataset"" WHERE ID IN ( - SELECT MetadataID + SELECT ""MetadataID"" FROM ""{tablename}"" ) ") @@ -1982,21 +1983,21 @@ namespace Duplicati.Library.Main.Database // Step 4: Delete orphaned blocksets (affected only) await cmd.SetCommandAndParameters(@$" - DELETE FROM Blockset + DELETE FROM ""Blockset"" WHERE ID IN ( - SELECT BlocksetID + SELECT ""BlocksetID"" FROM ""{tablename}"" ) AND NOT EXISTS ( SELECT 1 - FROM Metadataset - WHERE BlocksetID = Blockset.ID + FROM ""Metadataset"" + WHERE ""BlocksetID"" = ""Blockset"".""ID"" ) AND NOT EXISTS ( SELECT 1 - FROM File - WHERE BlocksetID = Blockset.ID + FROM ""File"" + WHERE ""BlocksetID"" = ""Blockset"".""ID"" ) ") .ExecuteNonQueryAsync(token) @@ -2005,10 +2006,10 @@ namespace Duplicati.Library.Main.Database // Step 5: Confirm all broken metadata entries are resolved cmd.SetCommandAndParameters(@" SELECT COUNT(*) - FROM Metadataset - JOIN Blockset - ON Metadataset.BlocksetID = Blockset.ID - WHERE Blockset.Length = 0 + FROM ""Metadataset"" + JOIN ""Blockset"" + ON ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + WHERE ""Blockset"".""Length"" = 0 "); var remaining = await cmd.ExecuteScalarInt64Async(0, token); From 6ee76a29d31b922b5b536da3dc626f8f6bfc444a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 23 Jun 2025 10:27:09 +0200 Subject: [PATCH 351/362] Made the callback on a put command to the backend be a function returning an awaitable Task rather than an Action. --- .../Library/Main/Backend/BackendManager.PutOperation.cs | 5 +++-- Duplicati/Library/Main/Backend/BackendManager.cs | 2 +- Duplicati/Library/Main/IBackendManager.cs | 2 +- Duplicati/Library/Main/Operation/CompactHandler.cs | 2 +- Duplicati/Library/Main/Operation/RepairHandler.cs | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs index 4cffff12f..8046da063 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs @@ -98,7 +98,7 @@ partial class BackendManager /// /// A callback that is invoked when the index volume is finished /// - public required Action? IndexVolumeFinishedCallback { get; init; } + public required Func? IndexVolumeFinishedCallback { get; init; } /// /// The default callback for database updates (no-op) /// @@ -234,7 +234,8 @@ partial class BackendManager OriginalIndexFile.FinishVolume(hash, size); - IndexVolumeFinishedCallback?.Invoke(); + if (IndexVolumeFinishedCallback != null) + await IndexVolumeFinishedCallback(); OriginalIndexFile.Close(); indexOperation = (OriginalIndexFile, req2); diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index 45b859917..f235d7669 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -246,7 +246,7 @@ internal partial class BackendManager : IBackendManager /// The callback to call when the database is updated /// The cancellation token /// An awaitable task - public async Task PutAsync(VolumeWriterBase volume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, Func? onDbUpdate, CancellationToken cancelToken) + public async Task PutAsync(VolumeWriterBase volume, IndexVolumeWriter? indexVolume, Func? indexVolumeFinished, bool waitForComplete, Func? onDbUpdate, CancellationToken cancelToken) { volume.Close(); diff --git a/Duplicati/Library/Main/IBackendManager.cs b/Duplicati/Library/Main/IBackendManager.cs index 8e6d8c86a..18df634b5 100644 --- a/Duplicati/Library/Main/IBackendManager.cs +++ b/Duplicati/Library/Main/IBackendManager.cs @@ -28,7 +28,7 @@ internal interface IBackendManager : IDisposable /// The action to call when the database should be updated /// The cancellation token /// An awaitable task - Task PutAsync(VolumeWriterBase blockVolume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, Func? onDbUpdate, CancellationToken cancelToken); + Task PutAsync(VolumeWriterBase blockVolume, IndexVolumeWriter? indexVolume, Func? indexVolumeFinished, bool waitForComplete, Func? onDbUpdate, CancellationToken cancelToken); /// /// Uploads a file to the backend without encryption diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 3da67f257..e3b8112f2 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -394,7 +394,7 @@ namespace Duplicati.Library.Main.Operation private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes) { - Action indexVolumeFinished = null; + Func indexVolumeFinished = null; if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 75a372f3d..803436c0e 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -985,7 +985,7 @@ namespace Duplicati.Library.Main.Operation // Create a new index file that points to the new volume IndexVolumeWriter newvolindex = null; - Action indexVolumeFinished = null; + Func indexVolumeFinished = null; if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); From a3d2ffbcd034b9287e4930ff999926e0e3d43e9b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 23 Jun 2025 10:27:35 +0200 Subject: [PATCH 352/362] Removed whitespace --- .../Backend/BackendManager.PutOperation.cs | 4 +-- .../Build/Command.Compile.Verify.cs | 34 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs index 8046da063..160622b3c 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs @@ -105,7 +105,7 @@ partial class BackendManager public static Func OnDbUpdateDefault = () => Task.CompletedTask; /// - /// Creates a new put operation + /// Creates a new put operation /// /// The execution context /// True if the operation should wait for completion @@ -345,7 +345,7 @@ partial class BackendManager Context.Database.LogRemoteVolumeRenamed(oldname, newname); remoteFilename = newname; - // If there is an index file attached to the block file, + // If there is an index file attached to the block file, // it references the block filename, so we create a new index file // which is a copy of the current, but with the new name if (indexOperation != null) diff --git a/ReleaseBuilder/Build/Command.Compile.Verify.cs b/ReleaseBuilder/Build/Command.Compile.Verify.cs index ed33a0928..f4cdb90d3 100644 --- a/ReleaseBuilder/Build/Command.Compile.Verify.cs +++ b/ReleaseBuilder/Build/Command.Compile.Verify.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Reflection; using System.Text; @@ -248,7 +248,7 @@ public static partial class Command { // Using v3.3 for assembly, but 3.7 in nuget { "AWSSDK.Core", new Version(3, 3, 0, 0) }, - + // Using the Framework version, not the package version { "Microsoft.CSharp", new Version(8, 0, 0, 0) }, { "System.Memory", new Version(8, 0, 0, 0) }, @@ -256,7 +256,7 @@ public static partial class Command { "System.Security.Principal.Windows", new Version(8, 0, 0, 0) }, { "System.Security.Cryptography.Algorithms", new Version(8, 0, 0, 0) }, { "System.Security.Cryptography.Cng", new Version(8, 0, 0, 0) }, - + // Using v9.0 for assembly, but 9.0.2 in nuget { "System.IO.Pipelines", new Version(9, 0, 0, 0) } }; From 46499033b818addb11fc9c57ce8c64aa582d37c5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 23 Jun 2025 10:28:25 +0200 Subject: [PATCH 353/362] Exchanged one of the known packages in the ReleaseBuilder to expect Microsoft.Data.Sqlite instead of the old SQLite package. --- ReleaseBuilder/Build/Command.Compile.Verify.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReleaseBuilder/Build/Command.Compile.Verify.cs b/ReleaseBuilder/Build/Command.Compile.Verify.cs index f4cdb90d3..18dab5df8 100644 --- a/ReleaseBuilder/Build/Command.Compile.Verify.cs +++ b/ReleaseBuilder/Build/Command.Compile.Verify.cs @@ -54,7 +54,7 @@ public static partial class Command "Google.Apis.Auth.dll", "Google.Apis.Core.dll", "SQLiteHelper.dll", - "SQLite.Interop.dll", + "Microsoft.Data.Sqlite.dll", "Microsoft.IdentityModel.Abstractions.dll", "System.Reactive.dll" }; From e71adc8ad9dfaa4448db92106b79754d6658ebbb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 23 Jun 2025 13:28:37 +0200 Subject: [PATCH 354/362] Added specific package versions to the Storj backend, as the default would pull in old sqlite packages that could conflict with the newer sqlite packages pulled in by Microsoft.Data.Sqlite --- .../Backend/Storj/Duplicati.Library.Backend.Storj.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj b/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj index 38fd940be..2564e783d 100644 --- a/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj +++ b/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj @@ -10,6 +10,8 @@ + + From f0b56c058ff407f37e371e837b801d9ffb3f2b7e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 23 Jun 2025 13:29:02 +0200 Subject: [PATCH 355/362] Removed duplicate project references in the SQLiteHelper project --- .../SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj index a15c5f442..7019eecdf 100644 --- a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj +++ b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj @@ -9,14 +9,6 @@ Copyright © 2025 Team Duplicati, MIT license - - - - - - - - From e536c73be2e35deb2e25c2788bf963f7fab61820 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 23 Jun 2025 13:30:28 +0200 Subject: [PATCH 356/362] Removed whitespace --- ReleaseBuilder/Build/Command.Compile.Post.cs | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ReleaseBuilder/Build/Command.Compile.Post.cs b/ReleaseBuilder/Build/Command.Compile.Post.cs index 0383b4b75..b8ab3e86d 100644 --- a/ReleaseBuilder/Build/Command.Compile.Post.cs +++ b/ReleaseBuilder/Build/Command.Compile.Post.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Text.RegularExpressions; From b729a1929ab73b71cdd00c213cf3a320ccdf48c7 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 23 Jun 2025 13:30:46 +0200 Subject: [PATCH 357/362] SQLite no longer has to be manually patched --- ReleaseBuilder/Build/Command.Compile.Post.cs | 23 -------------------- 1 file changed, 23 deletions(-) diff --git a/ReleaseBuilder/Build/Command.Compile.Post.cs b/ReleaseBuilder/Build/Command.Compile.Post.cs index b8ab3e86d..f7301369e 100644 --- a/ReleaseBuilder/Build/Command.Compile.Post.cs +++ b/ReleaseBuilder/Build/Command.Compile.Post.cs @@ -56,7 +56,6 @@ public static partial class Command case OSType.Linux: await ReplaceLibMonoUnix(baseDir, buildDir, target.Arch); - await ReplaceSQLiteInterop(baseDir, buildDir, target.Arch); break; default: @@ -280,28 +279,6 @@ public static partial class Command return Task.CompletedTask; } - /// - /// Replaces the library SQLiteInterop.dll with a version that is built against GLIBC_2.33 for ARM7 - /// - /// The base directory - /// The build directory - /// The architecture to build for - /// An awaitable task - static Task ReplaceSQLiteInterop(string baseDir, string buildDir, ArchType arch) - { - if (arch != ArchType.Arm7) - return Task.CompletedTask; - - var sourceFile = Path.Combine(baseDir, "ReleaseBuilder", "Resources", "linux-arm-binary", "SQLite.Interop.dll"); - var targetFile = Path.Combine(buildDir, "SQLite.Interop.dll"); - if (!File.Exists(targetFile)) - throw new Exception($"Expected file \"{targetFile}\" not found, has build changed?"); - - File.Copy(sourceFile, targetFile, overwrite: true); - - return Task.CompletedTask; - } - /// /// Signs all .exe and .dll files with Authenticode /// From 09d79512bc0efb75a79704e5321a628792d7b9e8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 23 Jun 2025 13:31:05 +0200 Subject: [PATCH 358/362] Handled package / assembly version mismatch due to revision numbers --- ReleaseBuilder/Build/Command.Compile.Verify.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ReleaseBuilder/Build/Command.Compile.Verify.cs b/ReleaseBuilder/Build/Command.Compile.Verify.cs index 18dab5df8..0784d41f1 100644 --- a/ReleaseBuilder/Build/Command.Compile.Verify.cs +++ b/ReleaseBuilder/Build/Command.Compile.Verify.cs @@ -257,6 +257,9 @@ public static partial class Command { "System.Security.Cryptography.Algorithms", new Version(8, 0, 0, 0) }, { "System.Security.Cryptography.Cng", new Version(8, 0, 0, 0) }, + // The assembly version also has a revision number, but the nuget version does not. + { "SQLitePCLRaw.core", new Version(2, 1, 10, 2445) }, + // Using v9.0 for assembly, but 9.0.2 in nuget { "System.IO.Pipelines", new Version(9, 0, 0, 0) } }; From 79b63ee4b2a0b46501d08b118b8c114cc5991a66 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 24 Jun 2025 12:04:31 +0200 Subject: [PATCH 359/362] Fixed race condition on creating and disposing SqliteCommands from multiple threads sharing the same SqliteConnection --- .../Main/Database/LocalRestoreDatabase.cs | 273 ++++++++++-------- 1 file changed, 159 insertions(+), 114 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 3274e6a2d..6c108f483 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -85,6 +85,11 @@ namespace Duplicati.Library.Main.Database /// protected string? m_filesnewlydonetable; + /// + /// A semaphore used to ensure that only one thread can access the database at a time. + /// + private SemaphoreSlim m_dbLock = new(1, 1); + /// /// The time at which the restore operation begins. /// @@ -1989,48 +1994,63 @@ namespace Duplicati.Library.Main.Database /// /// Returns a list of files and symlinks to restore. /// + /// At its current state, this method is designed to be called by Duplicati.Library.Main.Operation.Restore.FileLister. It locks the database to ensure that calls from Duplicati.Library.Main.Operation.Restore.BlockManager do not interfere with each other. /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of FileRequest objects representing the files and symlinks to restore. public async IAsyncEnumerable GetFilesAndSymlinksToRestore([EnumeratorCancellation] CancellationToken token) { - // Order by length descending, so that larger files are restored first. - await using var cmd = m_connection.CreateCommand($@" - SELECT - ""F"".""ID"", - ""F"".""Path"", - ""F"".""TargetPath"", - IFNULL(""B"".""FullHash"", ''), - IFNULL(""B"".""Length"", 0) AS ""Length"", - ""F"".""BlocksetID"" - FROM ""{m_tempfiletable}"" ""F"" - LEFT JOIN ""Blockset"" ""B"" - ON ""F"".""BlocksetID"" = ""B"".""ID"" - WHERE ""F"".""BlocksetID"" != {FOLDER_BLOCKSET_ID} - ORDER BY ""Length"" DESC - ") - .SetTransaction(m_rtr); - await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); - while (await rd.ReadAsync(token).ConfigureAwait(false)) - yield return new FileRequest( - rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1), - rd.ConvertValueToString(2), - rd.ConvertValueToString(3), - rd.ConvertValueToInt64(4), - rd.ConvertValueToInt64(5) - ); + await m_dbLock.WaitAsync(token).ConfigureAwait(false); + try + { + // Order by length descending, so that larger files are restored first. + await using var cmd = m_connection.CreateCommand($@" + SELECT + ""F"".""ID"", + ""F"".""Path"", + ""F"".""TargetPath"", + IFNULL(""B"".""FullHash"", ''), + IFNULL(""B"".""Length"", 0) AS ""Length"", + ""F"".""BlocksetID"" + FROM ""{m_tempfiletable}"" ""F"" + LEFT JOIN ""Blockset"" ""B"" + ON ""F"".""BlocksetID"" = ""B"".""ID"" + WHERE ""F"".""BlocksetID"" != {FOLDER_BLOCKSET_ID} + ORDER BY ""Length"" DESC + ") + .SetTransaction(m_rtr); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new FileRequest( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToString(3), + rd.ConvertValueToInt64(4), + rd.ConvertValueToInt64(5) + ); + } + finally + { + m_dbLock.Release(); + } } /// /// Returns a list of folders to restore. Used to restore folder metadata. /// + /// At its current state, this method is designed to be called by Duplicati.Library.Main.Operation.Restore.FileLister. It locks the database to ensure that calls from Duplicati.Library.Main.Operation.Restore.BlockManager do not interfere with each other. /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of FileRequest objects representing the folders to restore. public async IAsyncEnumerable GetFolderMetadataToRestore([EnumeratorCancellation] CancellationToken token) { - await using var cmd = m_connection.CreateCommand(); - cmd.SetTransaction(m_rtr); - await using var rd = await cmd.ExecuteReaderAsync($@" + await m_dbLock.WaitAsync(token).ConfigureAwait(false); + + try + { + await using var cmd = m_connection.CreateCommand(); + cmd.SetTransaction(m_rtr); + await using var rd = await cmd.ExecuteReaderAsync($@" SELECT ""F"".""ID"", '', @@ -2044,60 +2064,75 @@ namespace Duplicati.Library.Main.Database AND ""F"".""MetadataID"" IS NOT NULL AND ""F"".""MetadataID"" >= 0 ", token) - .ConfigureAwait(false); + .ConfigureAwait(false); - while (await rd.ReadAsync(token).ConfigureAwait(false)) - yield return new FileRequest( - rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1), - rd.ConvertValueToString(2), - rd.ConvertValueToString(3), - rd.ConvertValueToInt64(4), - rd.ConvertValueToInt64(5) - ); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new FileRequest( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToString(3), + rd.ConvertValueToInt64(4), + rd.ConvertValueToInt64(5) + ); + } + finally + { + m_dbLock.Release(); + } } /// /// Returns a list of blocks and their volume IDs. Used by the to keep track of blocks and volumes to automatically evict them from the respective caches. /// + /// At its current state, this method is designed to be called by Duplicati.Library.Main.Operation.Restore.BlockManager. It locks the database to ensure that calls from Duplicati.Library.Main.Operation.Restore.FileLister do not interfere with each other. /// Flag indicating whether the returned blocks should exclude the metadata blocks. /// A cancellation token to monitor for cancellation requests. /// An asynchronous enumerable of tuples containing the block ID and volume ID. public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata, [EnumeratorCancellation] CancellationToken token) { - var metadata_query = skipMetadata ? "" : $@" - UNION ALL - SELECT - ""Block"".""ID"", - ""Block"".""VolumeID"" - FROM ""{m_tempfiletable}"" - INNER JOIN ""Metadataset"" - ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" - INNER JOIN ""BlocksetEntry"" - ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - "; + await m_dbLock.WaitAsync(token).ConfigureAwait(false); - await using var cmd = Connection.CreateCommand($@" - SELECT - ""Block"".""ID"", - ""Block"".""VolumeID"" - FROM ""BlocksetEntry"" - INNER JOIN ""{m_tempfiletable}"" - ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" - INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - {metadata_query} - ") - .SetTransaction(m_rtr); + try + { + var metadata_query = skipMetadata ? "" : $@" + UNION ALL + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""{m_tempfiletable}"" + INNER JOIN ""Metadataset"" + ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""BlocksetEntry"" + ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + "; - await using var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); - while (await reader.ReadAsync(token).ConfigureAwait(false)) - yield return ( - reader.ConvertValueToInt64(0), - reader.ConvertValueToInt64(1) - ); + await using var cmd = m_connection.CreateCommand($@" + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""BlocksetEntry"" + INNER JOIN ""{m_tempfiletable}"" + ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + {metadata_query} + ") + .SetTransaction(m_rtr); + + await using var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await reader.ReadAsync(token).ConfigureAwait(false)) + yield return ( + reader.ConvertValueToInt64(0), + reader.ConvertValueToInt64(1) + ); + } + finally + { + m_dbLock.Release(); + } } /// @@ -2113,19 +2148,18 @@ namespace Duplicati.Library.Main.Database try { - await using var cmd = connection.CreateCommand(@$" - SELECT - ""Block"".""ID"", - ""Block"".""Hash"", - ""Block"".""Size"", - ""Block"".""VolumeID"" - FROM ""BlocksetEntry"" - INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID - ORDER BY ""BlocksetEntry"".""Index"" - ") + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""BlocksetEntry"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID + ORDER BY ""BlocksetEntry"".""Index"" + ") .SetTransaction(transaction) .SetParameterValue("@BlocksetID", blocksetID); @@ -2158,23 +2192,24 @@ namespace Duplicati.Library.Main.Database { var (connection, transaction) = await GetConnectionFromPool(token) .ConfigureAwait(false); + try { await using var cmd = connection.CreateCommand($@" - SELECT - ""Block"".""ID"", - ""Block"".""Hash"", - ""Block"".""Size"", - ""Block"".""VolumeID"" - FROM ""File"" - INNER JOIN ""Metadataset"" - ON ""File"".""MetadataID"" = ""Metadataset"".""ID"" - INNER JOIN ""BlocksetEntry"" - ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - WHERE ""File"".""ID"" = @FileID - ") + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""File"" + INNER JOIN ""Metadataset"" + ON ""File"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""BlocksetEntry"" + ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE ""File"".""ID"" = @FileID + ") .SetTransaction(transaction) .SetParameterValue("@FileID", fileID); @@ -2207,25 +2242,35 @@ namespace Duplicati.Library.Main.Database /// An asynchronous enumerable of tuples containing the volume name, size, and hash. public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID, [EnumeratorCancellation] CancellationToken token) { - await using var cmd = m_connection.CreateCommand(@" - SELECT - ""Name"", - ""Size"", - ""Hash"" - FROM ""RemoteVolume"" - WHERE ""ID"" = @VolumeID - ") - .SetTransaction(m_rtr) - .SetParameterValue("@VolumeID", VolumeID); - - await using var reader = await cmd.ExecuteReaderAsync(token) + var (connection, transaction) = await GetConnectionFromPool(token) .ConfigureAwait(false); - while (await reader.ReadAsync(token).ConfigureAwait(false)) - yield return ( - reader.ConvertValueToString(0) ?? "", - reader.ConvertValueToInt64(1), - reader.ConvertValueToString(2) ?? "" - ); + + try + { + await using var cmd = connection.CreateCommand(@" + SELECT + ""Name"", + ""Size"", + ""Hash"" + FROM ""RemoteVolume"" + WHERE ""ID"" = @VolumeID + ") + .SetTransaction(transaction) + .SetParameterValue("@VolumeID", VolumeID); + + await using var reader = await cmd.ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await reader.ReadAsync(token).ConfigureAwait(false)) + yield return ( + reader.ConvertValueToString(0) ?? "", + reader.ConvertValueToInt64(1), + reader.ConvertValueToString(2) ?? "" + ); + } + finally + { + m_connection_pool.Add((connection, transaction)); + } } /// From 8c9f129b8a98c28d3648a24ad6f5c4399462b4a3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 24 Jun 2025 12:04:49 +0200 Subject: [PATCH 360/362] Removed unused parameter --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 2e8a8fdd4..12f424c73 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -131,10 +131,9 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Initializes a new instance of the class. /// - /// The database holding information about how many of each block this restore requires. /// Channel for submitting block requests from a volume. /// Number of readers accessing this dictionary. Used during shutdown / cleanup. - private SleepableDictionary(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers) + private SleepableDictionary(IWriteChannel volume_request, Options options, int readers) { m_options = options; m_volume_request = volume_request; @@ -185,7 +184,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers, CancellationToken cancellationToken) { - var sd = new SleepableDictionary(db, volume_request, options, readers); + var sd = new SleepableDictionary(volume_request, options, readers); await foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata, cancellationToken).ConfigureAwait(false)) { From f9111a9dde2708d74ce08b08d67eac43586c21d6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 24 Jun 2025 12:05:15 +0200 Subject: [PATCH 361/362] Made RestoreHandler call the DisposeAsync instead of Dispose on the LocalRestoreDatabase --- Duplicati/Library/Main/Operation/RestoreHandler.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 9766d78af..2334dc0b6 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -147,7 +147,8 @@ namespace Duplicati.Library.Main.Operation } finally { - db?.Dispose(); + if (db != null) + await db.DisposeAsync().ConfigureAwait(false); tmpdb?.Dispose(); } } From bb87e94ffab18df2f422cd2a8813e91e059fd657 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 24 Jun 2025 14:38:58 +0200 Subject: [PATCH 362/362] Post merge master fixes --- .../Main/Database/LocalListDatabase.cs | 54 +++++++++++-------- .../Main/Operation/ListFolderHandler.cs | 5 +- .../Library/Main/Operation/TestHandler.cs | 4 +- Duplicati/UnitTest/DirectListHandlerTests.cs | 23 +++++--- 4 files changed, 55 insertions(+), 31 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index d1d852c57..95fb130ac 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -1099,45 +1099,59 @@ namespace Duplicati.Library.Main.Database /// Gets the minimal unique prefix entries for a given fileset ID. /// This returns the roots of all unique paths in the fileset. /// - /// The fileset id - /// The entries that are unique roots - public IList GetMinimalUniquePrefixEntries(long filesetId) + /// The fileset id. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a list of the entries that are unique roots. + public async IAsyncEnumerable GetMinimalUniquePrefixEntries(long filesetId, [EnumeratorCancellation] CancellationToken token) { - var results = new List(); - - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters(@" WITH AllPaths AS ( - SELECT fl.""ID"" AS ""FileID"", + SELECT + fl.""ID"" AS ""FileID"", pp.""Prefix"" || fl.""Path"" AS ""FullPath"", fl.""BlocksetID"", b.""Length"", fe.""Lastmodified"" FROM ""FilesetEntry"" fe - JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" + JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + JOIN ""PathPrefix"" pp + ON fl.""PrefixID"" = pp.""ID"" + LEFT JOIN ""Blockset"" b + ON fl.""BlocksetID"" = b.""ID"" WHERE fe.""FilesetID"" = @FilesetId ), RootCandidates AS ( SELECT a1.* FROM AllPaths a1 WHERE NOT EXISTS ( - SELECT 1 FROM AllPaths a2 - WHERE a2.""FullPath"" != a1.""FullPath"" - AND a1.""FullPath"" LIKE a2.""FullPath"" || '%' + SELECT 1 + FROM AllPaths a2 + WHERE + a2.""FullPath"" != a1.""FullPath"" + AND a1.""FullPath"" LIKE a2.""FullPath"" || '%' ) ) SELECT ""FullPath"", ""Length"", - CASE WHEN ""BlocksetID"" = -100 THEN 1 ELSE 0 END AS ""IsDirectory"", - CASE WHEN ""BlocksetID"" = -200 THEN 1 ELSE 0 END AS ""IsSymlink"", + CASE + WHEN ""BlocksetID"" = -100 + THEN 1 + ELSE 0 + END AS ""IsDirectory"", + CASE + WHEN ""BlocksetID"" = -200 + THEN 1 + ELSE 0 + END AS ""IsSymlink"", ""Lastmodified"" FROM RootCandidates ORDER BY ""FullPath"" - "); - cmd.SetParameterValue("@FilesetId", filesetId); + ") + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetId); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token)) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(1, -1); @@ -1145,10 +1159,8 @@ namespace Duplicati.Library.Main.Database var isSymlink = rd.GetInt32(3) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(4, 0), DateTimeKind.Utc); - results.Add(new FolderEntry(path, size, isDir, isSymlink, lastModified)); + yield return new FolderEntry(path, size, isDir, isSymlink, lastModified); } - - return results; } /// diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index 8c7aaf569..0305812f2 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -63,7 +63,10 @@ internal static class ListFolderHandler { if (folders != null && folders.Length > 1) throw new UserInformationException("When no folder is specified, only one folder can be listed", "MultipleFoldersFound"); - var rootFolders = db.GetMinimalUniquePrefixEntries(filesetIds[0]); + var rootFolders = await db + .GetMinimalUniquePrefixEntries(filesetIds[0], result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: result.TaskControl.ProgressToken) + .ConfigureAwait(false); result.Entries = new PaginatedResults(0, rootFolders.Count, 1, rootFolders.Count, rootFolders); } else diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 033bc459f..b8b48581d 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -433,9 +433,9 @@ namespace Duplicati.Library.Main.Operation await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(repairdb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await repairdb.Transaction - .CommitAsync("ReplaceFaultyIndexFileCommit") + .CommitAsync("ReplaceFaultyIndexFileCommit", token: cancellationToken) .ConfigureAwait(false); - m_results.RemoveResult(vol.Name); + m_result.RemoveResult(vol.Name); } } catch (Exception ex) diff --git a/Duplicati/UnitTest/DirectListHandlerTests.cs b/Duplicati/UnitTest/DirectListHandlerTests.cs index 39994f8c9..4da12acab 100644 --- a/Duplicati/UnitTest/DirectListHandlerTests.cs +++ b/Duplicati/UnitTest/DirectListHandlerTests.cs @@ -337,7 +337,8 @@ namespace Duplicati.UnitTest ]); var result = await db - .GetMinimalUniquePrefixEntries(1).Select(e => e.Path, CancellationToken.None) + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .Select(e => e.Path) .ToListAsync() .ConfigureAwait(false); @@ -363,7 +364,8 @@ namespace Duplicati.UnitTest ]); var result = await db - .GetMinimalUniquePrefixEntries(1).Select(e => e.Path, CancellationToken.None) + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .Select(e => e.Path) .ToListAsync() .ConfigureAwait(false); @@ -387,7 +389,8 @@ namespace Duplicati.UnitTest ]); var result = await db - .GetMinimalUniquePrefixEntries(1).Select(e => e.Path, CancellationToken.None) + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .Select(e => e.Path) .ToListAsync() .ConfigureAwait(false); @@ -414,7 +417,8 @@ namespace Duplicati.UnitTest ]); var result = await db - .GetMinimalUniquePrefixEntries(1).Select(e => e.Path, CancellationToken.None) + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .Select(e => e.Path) .ToListAsync() .ConfigureAwait(false); @@ -429,7 +433,7 @@ namespace Duplicati.UnitTest } [Test] - public void GetMinimalUniquePrefixEntries_ShouldReturnExpectedMinimalRoots() + public async Task GetMinimalUniquePrefixEntries_ShouldReturnExpectedMinimalRoots() { // Arrange: Prepare prefixes (minimal unique) and contents var testPrefixes = new[] @@ -448,10 +452,15 @@ namespace Duplicati.UnitTest // Act using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + await using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) + .ConfigureAwait(false); SeedTestData(db, testPrefixes); - var resultItems = db.GetMinimalUniquePrefixEntries(1).ToList(); + var resultItems = await db + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .ToListAsync() + .ConfigureAwait(false); + var result = resultItems.Select(e => e.Path).ToList(); Assert.That(result, Does.Contain(@"C:\Downloads\testsource\AA\"));