From fbe79b1ca3a6f74875bcc7ba935531404c623e54 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Mar 2025 10:05:10 +0100 Subject: [PATCH 1/2] Improve SQLite loading This commit changes the way the new restore flow loads the extra database connections so these will also apply the temporary folder settings and custom pragmas. This also changes the logic around the SQLite temporary folder. Previously, this folder would unconditionally be set to the general temporary folder, allowing only an override with a custom pragma. No the setup checks if the system has the environment variable `SQLITE_TMPDIR` set already. If this variable is already set, it will NOT be overwritten. Actual effects of this change depends on how SQLite internally resolves the temporary folder. --- .../Main/Database/LocalRestoreDatabase.cs | 4 +- .../Library/SQLiteHelper/SQLiteLoader.cs | 69 +++++++++++-------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 734885498..68b5b2c4a 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -921,10 +921,12 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""); { if (!m_connection_pool.TryTake(out var connection)) { - connection = (IDbConnection)Activator.CreateInstance(SQLiteHelper.SQLiteLoader.SQLiteConnectionType); + connection = SQLiteHelper.SQLiteLoader.LoadConnection(); connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; connection.Open(); + SQLiteHelper.SQLiteLoader.ApplyCustomPragmas(connection); + using var cmd = connection.CreateCommand(); cmd.ExecuteNonQuery("PRAGMA journal_mode = WAL;"); cmd.ExecuteNonQuery("PRAGMA read_uncommitted = true;"); diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 2f13ba25c..4da9cee89 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -99,6 +99,37 @@ namespace Duplicati.Library.SQLiteHelper return con ?? throw new InvalidOperationException("Failed to load connection"); } + /// + /// Applies user-supplied custom pragmas to the SQLite connection + /// + /// The connection to apply the pragmas to. + /// The connection with the pragmas applied. + public static System.Data.IDbConnection ApplyCustomPragmas(System.Data.IDbConnection con) + { + var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI"); + if (opts == null) + return con; + var topts = opts.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); + if (topts.Length == 0) + return con; + + using (var cmd = con.CreateCommand()) + foreach (var opt in topts) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt); + try + { + cmd.CommandText = string.Format("pragma {0}", opt); + cmd.ExecuteNonQuery(); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "CustomSQLiteOption", ex, @"Error setting custom SQLite option '{0}'.", opt); + } + } + return con; + } + /// /// Loads an SQLite connection instance and opening the database /// @@ -109,7 +140,7 @@ namespace Duplicati.Library.SQLiteHelper if (string.IsNullOrWhiteSpace(targetpath)) throw new ArgumentNullException(nameof(targetpath)); - System.Data.IDbConnection con = LoadConnection(); + var con = LoadConnection(); try { @@ -124,32 +155,7 @@ namespace Duplicati.Library.SQLiteHelper } // set custom Sqlite options - var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI"); - if (opts != null) - { - var topts = opts.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); - if (topts.Length > 0) - { - using (var cmd = con.CreateCommand()) - { - foreach (var opt in topts) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt); - try - { - cmd.CommandText = string.Format("pragma {0}", opt); - cmd.ExecuteNonQuery(); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "CustomSQLiteOption", ex, @"Error setting custom SQLite option '{0}'.", opt); - } - } - } - } - } - - return con; + return ApplyCustomPragmas(con); } /// @@ -189,9 +195,12 @@ namespace Duplicati.Library.SQLiteHelper /// private static void SetEnvironmentVariablesForSQLiteTempDir() { - System.Environment.SetEnvironmentVariable("SQLITE_TMPDIR", Library.Utility.TempFolder.SystemTempPath); - System.Environment.SetEnvironmentVariable("TMP", Library.Utility.TempFolder.SystemTempPath); - System.Environment.SetEnvironmentVariable("TEMP", Library.Utility.TempFolder.SystemTempPath); + // Allow the user to override the temp folder for SQLite + if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("SQLITE_TMPDIR"))) + Environment.SetEnvironmentVariable("SQLITE_TMPDIR", Utility.TempFolder.SystemTempPath); + Environment.SetEnvironmentVariable("TMPDIR", Utility.TempFolder.SystemTempPath); + Environment.SetEnvironmentVariable("TMP", Utility.TempFolder.SystemTempPath); + Environment.SetEnvironmentVariable("TEMP", Utility.TempFolder.SystemTempPath); } /// From d8a15ac7edb69e98574319d7ae55abb0b31c2b2a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Mar 2025 12:05:16 +0100 Subject: [PATCH 2/2] Fixed review comments --- Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 4da9cee89..42760ce8f 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -107,19 +107,16 @@ namespace Duplicati.Library.SQLiteHelper public static System.Data.IDbConnection ApplyCustomPragmas(System.Data.IDbConnection con) { var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI"); - if (opts == null) - return con; - var topts = opts.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); - if (topts.Length == 0) + if (string.IsNullOrWhiteSpace(opts)) return con; using (var cmd = con.CreateCommand()) - foreach (var opt in topts) + foreach (var opt in opts.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries)) { Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt); try { - cmd.CommandText = string.Format("pragma {0}", opt); + cmd.CommandText = string.Format("PRAGMA {0}", opt); cmd.ExecuteNonQuery(); } catch (Exception ex)