From d0f8fffc253f6c4a6831d1b57e567c6739f050d0 Mon Sep 17 00:00:00 2001 From: Dan Stahr Date: Thu, 8 Jun 2017 15:38:27 +0100 Subject: [PATCH 1/4] Added support for executing non-SQL code when performing a database update. Some of the database migrations I'm planning to perform are very slow if using SQL only or entirely impossible using the standard SQLite toolchain. I added a simple interface with "before SQL" and "after SQL" hooks so that a database upgrade can perform custom code operations as well. --- .../DBSchemaUpgrades/DbUpgradesRegistry.cs | 19 + .../DBSchemaUpgrades/IDbSchemaUpgrade.cs | 26 ++ .../Library/SQLiteHelper/DatabaseUpgrader.cs | 32 +- Duplicati/UnitTest/Duplicati.UnitTest.csproj | 348 +++++++++--------- 4 files changed, 251 insertions(+), 174 deletions(-) create mode 100644 Duplicati/Library/SQLiteHelper/DBSchemaUpgrades/DbUpgradesRegistry.cs create mode 100644 Duplicati/Library/SQLiteHelper/DBSchemaUpgrades/IDbSchemaUpgrade.cs diff --git a/Duplicati/Library/SQLiteHelper/DBSchemaUpgrades/DbUpgradesRegistry.cs b/Duplicati/Library/SQLiteHelper/DBSchemaUpgrades/DbUpgradesRegistry.cs new file mode 100644 index 000000000..a0eecc779 --- /dev/null +++ b/Duplicati/Library/SQLiteHelper/DBSchemaUpgrades/DbUpgradesRegistry.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using Duplicati.Library.SQLiteHelper.DBUpdates; + +namespace Duplicati.Library.SQLiteHelper.DBSchemaUpgrades +{ + class DbUpgradesRegistry + { + /// + /// Registry of custom code to be executed along a SQL schema upgrade. The key of the map + /// represents a 1-based version the upgrade code applies to (after performing the update + /// the database will be at that version), the value is an instance of upgrader that + /// implements the hooks code. + /// + public static readonly IDictionary CodeChanges = + new Dictionary() + { + }; + } +} diff --git a/Duplicati/Library/SQLiteHelper/DBSchemaUpgrades/IDbSchemaUpgrade.cs b/Duplicati/Library/SQLiteHelper/DBSchemaUpgrades/IDbSchemaUpgrade.cs new file mode 100644 index 000000000..4033d9242 --- /dev/null +++ b/Duplicati/Library/SQLiteHelper/DBSchemaUpgrades/IDbSchemaUpgrade.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Duplicati.Library.SQLiteHelper.DBUpdates +{ + + // TODO: Consider passing in a transaction and performing everything in one go. + + public interface IDbSchemaUpgrade + { + /// + /// Executed before the textual SQL command is executed. + /// + /// + void BeforeSql(System.Data.IDbConnection connection); + + /// + /// Executed after the textual SQL command is executed. + /// + /// + void AfterSql(System.Data.IDbConnection connection); + } +} diff --git a/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs b/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs index 18d44788f..9bf4fab50 100644 --- a/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs +++ b/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs @@ -23,13 +23,24 @@ using System.Text; using System.Data; using System.Text.RegularExpressions; using System.Linq; +using Duplicati.Library.SQLiteHelper.DBUpdates; +using Duplicati.Library.SQLiteHelper.DBSchemaUpgrades; namespace Duplicati.Library.SQLiteHelper { /// /// This class will read embedded files from the given folder. /// Updates should have the form "1.Sample upgrade.sql". - /// When the database schema changes, simply put a new file into the folder. + /// When the database schema changes, simply put a new file into the folder + /// and set it to be emnbedded in the binary. + /// + /// Additionally, it's possible to execute custom code before and after + /// the SQL is executed. To set up a custom upgrade stage, add your + /// code to DbUpgradesRegistry along with the DB version to apply it with. + /// + /// Even if all the DB upgrade code is handled in C#, you still have to add + /// a dummy SQL file to indicate the version ID is already taken. + /// /// Each upgrade file should ONLY upgrade from the previous version. /// If done correctly, a user may be upgrade from the very first version /// to the very latest. @@ -235,7 +246,9 @@ namespace Duplicati.Library.SQLiteHelper } else if (versions.Count > dbversion) { - string backupfile = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(sourcefile), Strings.DatabaseUpgrader.BackupFilenamePrefix + " " + DateTime.Now.ToString("yyyyMMddhhmmss", System.Globalization.CultureInfo.InvariantCulture) + ".sqlite"); + string backupfile = System.IO.Path.Combine( + System.IO.Path.GetDirectoryName(sourcefile), + Strings.DatabaseUpgrader.BackupFilenamePrefix + " " + DateTime.Now.ToString("yyyyMMddhhmmss", System.Globalization.CultureInfo.InvariantCulture) + ".sqlite"); try { @@ -247,6 +260,16 @@ namespace Duplicati.Library.SQLiteHelper for (int i = dbversion; i < versions.Count; i++) { + IDbSchemaUpgrade dbCodeUpgrade; + + // The versions in the registry are 1-based, the loop index is zero based. + bool hookFound = DbUpgradesRegistry.CodeChanges.TryGetValue(i + 1, out dbCodeUpgrade); + + if (hookFound) + { + dbCodeUpgrade.BeforeSql(connection); + } + //TODO: Find a better way to split SQL statements, as there may be embedded semicolons //in the SQL, like "UPDATE x WHERE y = ';';" @@ -261,6 +284,11 @@ namespace Duplicati.Library.SQLiteHelper cmd.ExecuteNonQuery(); } + if (hookFound) + { + dbCodeUpgrade.AfterSql(connection); + } + // after upgrade, db_version should have changed to i + 1. If logic changes, just requery. preparserVars["db_version"] = i + 1; } diff --git a/Duplicati/UnitTest/Duplicati.UnitTest.csproj b/Duplicati/UnitTest/Duplicati.UnitTest.csproj index b73d49e40..041f4b134 100644 --- a/Duplicati/UnitTest/Duplicati.UnitTest.csproj +++ b/Duplicati/UnitTest/Duplicati.UnitTest.csproj @@ -1,173 +1,177 @@ - - - - Debug - AnyCPU - {ECB63D1C-1724-442D-9228-DEABF14F2EA3} - Library - Duplicati.UnitTest - Duplicati.UnitTest - v4.5 - false - - - true - full - false - bin\Debug - DEBUG; - prompt - 4 - false - - - full - true - bin\Release - prompt - 4 - false - - - - - ..\..\packages\NUnit.3.5.0\lib\net45\nunit.framework.dll - - - - - - - - - - - - - - - - - - - - {81765A64-3661-4E3E-B850-2F6F87A51F74} - Duplicati.CommandLine - - - {E7280DCA-7776-4A73-B9B5-41FD77FC8799} - Duplicati.CommandLine.BackendTester - - - {2AF960C0-357D-4D44-A3D5-8B6E89DB0F11} - Duplicati.CommandLine.BackendTool - - - {17566860-3D98-4604-AA5B-47661F75609F} - Duplicati.GUI.TrayIcon - - - {7E119745-1F62-43F0-936C-F312A1912C0B} - Duplicati.Library.AutoUpdater - - - {8E4CECFB-0413-4B00-AB93-78D1C3902BD5} - Duplicati.Library.Backend.AzureBlob - - - {1BFAE226-8364-4086-825C-BB83F6F3EE4C} - Duplicati.Library.Backend.CloudFiles - - - {FC9B7611-836F-4127-8B44-A7C31F506807} - Duplicati.Library.Backend.File - - - {F61679A9-E5DE-468A-B5A4-05F92D0143D2} - Duplicati.Library.Backend.FTP - - - {CCD76347-7DC7-4B42-B7E1-E500E624CAC3} - Duplicati.Library.Backend.OneDrive - - - {C03F6DFD-805A-4BE0-9338-64870ADDB4A2} - Duplicati.Library.Backend.S3 - - - {FF2BF37C-E502-4C98-BEA0-701671DDFA08} - Duplicati.Library.Backend.SSHv2 - - - {C0270709-2A40-43B5-8CF1-69581B9FA2A1} - Duplicati.Library.Backend.TahoeLAFS - - - {BAE27510-8B5D-44B2-B33E-372A98908041} - Duplicati.Library.Backend.WEBDAV - - - {19ECCE09-B5EB-406C-8C57-BAC66997D469} - Duplicati.Library.Compression - - - {0CA86ECF-5BEC-4909-B4F6-110A03B30B92} - Duplicati.Library.DynamicLoader - - - {94484FDB-2EFA-4CF0-9BE6-A561157B4F87} - Duplicati.Library.Encryption - - - {C5899F45-B0FF-483C-9D38-24A9FCAAB237} - Duplicati.Library.Interface - - - {B68F2214-951F-4F78-8488-66E1ED3F50BF} - Duplicati.Library.Localization - - - {D10A5FC0-11B4-4E70-86AA-8AEA52BD9798} - Duplicati.Library.Logging - - - {52826615-7964-47FE-B4B3-1B2DBDF605B9} - Duplicati.Library.Modules.Builtin - - - {10D2D1B7-C664-41D8-9B3A-00040C3D421B} - Duplicati.Library.Main - - - {D63E53E4-A458-4C2F-914D-92F715F58ACE} - Duplicati.Library.Snapshots - - - {2C838169-B187-4B09-8768-1C24C2521C8D} - Duplicati.Library.SQLiteHelper - - - {DE3E5D4C-51AB-4E5E-BEE8-E636CEBFBA65} - Duplicati.Library.Utility - - - {4D012CB1-4B92-47F4-89B7-BF80A73A2E99} - Duplicati.License - - - {19E661D2-C5DA-4F35-B3EE-7586E5734B5F} - Duplicati.Server - - - {E93F3DE2-FF3A-4709-96A3-8190AA14FA25} - Duplicati.Service - - - {33FD1D24-C28F-4C71-933F-98F1586EA76C} - Duplicati.Server.Serialization - - - - - + + + + Debug + AnyCPU + {ECB63D1C-1724-442D-9228-DEABF14F2EA3} + Library + Duplicati.UnitTest + Duplicati.UnitTest + v4.5 + false + + + true + full + false + bin\Debug + DEBUG; + prompt + 4 + false + + + full + true + bin\Release + prompt + 4 + false + + + + + ..\..\packages\NUnit.3.5.0\lib\net45\nunit.framework.dll + + + + + + + + + + + + + + + + + + + + + {81765A64-3661-4E3E-B850-2F6F87A51F74} + Duplicati.CommandLine + + + {E7280DCA-7776-4A73-B9B5-41FD77FC8799} + Duplicati.CommandLine.BackendTester + + + {2AF960C0-357D-4D44-A3D5-8B6E89DB0F11} + Duplicati.CommandLine.BackendTool + + + {17566860-3D98-4604-AA5B-47661F75609F} + Duplicati.GUI.TrayIcon + + + {7E119745-1F62-43F0-936C-F312A1912C0B} + Duplicati.Library.AutoUpdater + + + {8E4CECFB-0413-4B00-AB93-78D1C3902BD5} + Duplicati.Library.Backend.AzureBlob + + + {1BFAE226-8364-4086-825C-BB83F6F3EE4C} + Duplicati.Library.Backend.CloudFiles + + + {FC9B7611-836F-4127-8B44-A7C31F506807} + Duplicati.Library.Backend.File + + + {F61679A9-E5DE-468A-B5A4-05F92D0143D2} + Duplicati.Library.Backend.FTP + + + {CCD76347-7DC7-4B42-B7E1-E500E624CAC3} + Duplicati.Library.Backend.OneDrive + + + {C03F6DFD-805A-4BE0-9338-64870ADDB4A2} + Duplicati.Library.Backend.S3 + + + {FF2BF37C-E502-4C98-BEA0-701671DDFA08} + Duplicati.Library.Backend.SSHv2 + + + {C0270709-2A40-43B5-8CF1-69581B9FA2A1} + Duplicati.Library.Backend.TahoeLAFS + + + {BAE27510-8B5D-44B2-B33E-372A98908041} + Duplicati.Library.Backend.WEBDAV + + + {19ECCE09-B5EB-406C-8C57-BAC66997D469} + Duplicati.Library.Compression + + + {0CA86ECF-5BEC-4909-B4F6-110A03B30B92} + Duplicati.Library.DynamicLoader + + + {94484FDB-2EFA-4CF0-9BE6-A561157B4F87} + Duplicati.Library.Encryption + + + {C5899F45-B0FF-483C-9D38-24A9FCAAB237} + Duplicati.Library.Interface + + + {B68F2214-951F-4F78-8488-66E1ED3F50BF} + Duplicati.Library.Localization + + + {D10A5FC0-11B4-4E70-86AA-8AEA52BD9798} + Duplicati.Library.Logging + + + {52826615-7964-47FE-B4B3-1B2DBDF605B9} + Duplicati.Library.Modules.Builtin + + + {10D2D1B7-C664-41D8-9B3A-00040C3D421B} + Duplicati.Library.Main + + + {D63E53E4-A458-4C2F-914D-92F715F58ACE} + Duplicati.Library.Snapshots + + + {2C838169-B187-4B09-8768-1C24C2521C8D} + Duplicati.Library.SQLiteHelper + + + {DE3E5D4C-51AB-4E5E-BEE8-E636CEBFBA65} + Duplicati.Library.Utility + + + {4D012CB1-4B92-47F4-89B7-BF80A73A2E99} + Duplicati.License + + + {19E661D2-C5DA-4F35-B3EE-7586E5734B5F} + Duplicati.Server + + + {E93F3DE2-FF3A-4709-96A3-8190AA14FA25} + Duplicati.Service + + + {33FD1D24-C28F-4C71-933F-98F1586EA76C} + Duplicati.Server.Serialization + + + + + + + + \ No newline at end of file From 2703617287ec57f0b8d637d4f6de5aa8aa808505 Mon Sep 17 00:00:00 2001 From: Dan Stahr Date: Thu, 8 Jun 2017 15:52:48 +0100 Subject: [PATCH 2/4] Reverted accidental changes --- Duplicati/UnitTest/Duplicati.UnitTest.csproj | 348 +++++++++---------- 1 file changed, 172 insertions(+), 176 deletions(-) diff --git a/Duplicati/UnitTest/Duplicati.UnitTest.csproj b/Duplicati/UnitTest/Duplicati.UnitTest.csproj index 041f4b134..b73d49e40 100644 --- a/Duplicati/UnitTest/Duplicati.UnitTest.csproj +++ b/Duplicati/UnitTest/Duplicati.UnitTest.csproj @@ -1,177 +1,173 @@ - - - - Debug - AnyCPU - {ECB63D1C-1724-442D-9228-DEABF14F2EA3} - Library - Duplicati.UnitTest - Duplicati.UnitTest - v4.5 - false - - - true - full - false - bin\Debug - DEBUG; - prompt - 4 - false - - - full - true - bin\Release - prompt - 4 - false - - - - - ..\..\packages\NUnit.3.5.0\lib\net45\nunit.framework.dll - - - - - - - - - - - - - - - - - - - - - {81765A64-3661-4E3E-B850-2F6F87A51F74} - Duplicati.CommandLine - - - {E7280DCA-7776-4A73-B9B5-41FD77FC8799} - Duplicati.CommandLine.BackendTester - - - {2AF960C0-357D-4D44-A3D5-8B6E89DB0F11} - Duplicati.CommandLine.BackendTool - - - {17566860-3D98-4604-AA5B-47661F75609F} - Duplicati.GUI.TrayIcon - - - {7E119745-1F62-43F0-936C-F312A1912C0B} - Duplicati.Library.AutoUpdater - - - {8E4CECFB-0413-4B00-AB93-78D1C3902BD5} - Duplicati.Library.Backend.AzureBlob - - - {1BFAE226-8364-4086-825C-BB83F6F3EE4C} - Duplicati.Library.Backend.CloudFiles - - - {FC9B7611-836F-4127-8B44-A7C31F506807} - Duplicati.Library.Backend.File - - - {F61679A9-E5DE-468A-B5A4-05F92D0143D2} - Duplicati.Library.Backend.FTP - - - {CCD76347-7DC7-4B42-B7E1-E500E624CAC3} - Duplicati.Library.Backend.OneDrive - - - {C03F6DFD-805A-4BE0-9338-64870ADDB4A2} - Duplicati.Library.Backend.S3 - - - {FF2BF37C-E502-4C98-BEA0-701671DDFA08} - Duplicati.Library.Backend.SSHv2 - - - {C0270709-2A40-43B5-8CF1-69581B9FA2A1} - Duplicati.Library.Backend.TahoeLAFS - - - {BAE27510-8B5D-44B2-B33E-372A98908041} - Duplicati.Library.Backend.WEBDAV - - - {19ECCE09-B5EB-406C-8C57-BAC66997D469} - Duplicati.Library.Compression - - - {0CA86ECF-5BEC-4909-B4F6-110A03B30B92} - Duplicati.Library.DynamicLoader - - - {94484FDB-2EFA-4CF0-9BE6-A561157B4F87} - Duplicati.Library.Encryption - - - {C5899F45-B0FF-483C-9D38-24A9FCAAB237} - Duplicati.Library.Interface - - - {B68F2214-951F-4F78-8488-66E1ED3F50BF} - Duplicati.Library.Localization - - - {D10A5FC0-11B4-4E70-86AA-8AEA52BD9798} - Duplicati.Library.Logging - - - {52826615-7964-47FE-B4B3-1B2DBDF605B9} - Duplicati.Library.Modules.Builtin - - - {10D2D1B7-C664-41D8-9B3A-00040C3D421B} - Duplicati.Library.Main - - - {D63E53E4-A458-4C2F-914D-92F715F58ACE} - Duplicati.Library.Snapshots - - - {2C838169-B187-4B09-8768-1C24C2521C8D} - Duplicati.Library.SQLiteHelper - - - {DE3E5D4C-51AB-4E5E-BEE8-E636CEBFBA65} - Duplicati.Library.Utility - - - {4D012CB1-4B92-47F4-89B7-BF80A73A2E99} - Duplicati.License - - - {19E661D2-C5DA-4F35-B3EE-7586E5734B5F} - Duplicati.Server - - - {E93F3DE2-FF3A-4709-96A3-8190AA14FA25} - Duplicati.Service - - - {33FD1D24-C28F-4C71-933F-98F1586EA76C} - Duplicati.Server.Serialization - - - - - - - - + + + + Debug + AnyCPU + {ECB63D1C-1724-442D-9228-DEABF14F2EA3} + Library + Duplicati.UnitTest + Duplicati.UnitTest + v4.5 + false + + + true + full + false + bin\Debug + DEBUG; + prompt + 4 + false + + + full + true + bin\Release + prompt + 4 + false + + + + + ..\..\packages\NUnit.3.5.0\lib\net45\nunit.framework.dll + + + + + + + + + + + + + + + + + + + + {81765A64-3661-4E3E-B850-2F6F87A51F74} + Duplicati.CommandLine + + + {E7280DCA-7776-4A73-B9B5-41FD77FC8799} + Duplicati.CommandLine.BackendTester + + + {2AF960C0-357D-4D44-A3D5-8B6E89DB0F11} + Duplicati.CommandLine.BackendTool + + + {17566860-3D98-4604-AA5B-47661F75609F} + Duplicati.GUI.TrayIcon + + + {7E119745-1F62-43F0-936C-F312A1912C0B} + Duplicati.Library.AutoUpdater + + + {8E4CECFB-0413-4B00-AB93-78D1C3902BD5} + Duplicati.Library.Backend.AzureBlob + + + {1BFAE226-8364-4086-825C-BB83F6F3EE4C} + Duplicati.Library.Backend.CloudFiles + + + {FC9B7611-836F-4127-8B44-A7C31F506807} + Duplicati.Library.Backend.File + + + {F61679A9-E5DE-468A-B5A4-05F92D0143D2} + Duplicati.Library.Backend.FTP + + + {CCD76347-7DC7-4B42-B7E1-E500E624CAC3} + Duplicati.Library.Backend.OneDrive + + + {C03F6DFD-805A-4BE0-9338-64870ADDB4A2} + Duplicati.Library.Backend.S3 + + + {FF2BF37C-E502-4C98-BEA0-701671DDFA08} + Duplicati.Library.Backend.SSHv2 + + + {C0270709-2A40-43B5-8CF1-69581B9FA2A1} + Duplicati.Library.Backend.TahoeLAFS + + + {BAE27510-8B5D-44B2-B33E-372A98908041} + Duplicati.Library.Backend.WEBDAV + + + {19ECCE09-B5EB-406C-8C57-BAC66997D469} + Duplicati.Library.Compression + + + {0CA86ECF-5BEC-4909-B4F6-110A03B30B92} + Duplicati.Library.DynamicLoader + + + {94484FDB-2EFA-4CF0-9BE6-A561157B4F87} + Duplicati.Library.Encryption + + + {C5899F45-B0FF-483C-9D38-24A9FCAAB237} + Duplicati.Library.Interface + + + {B68F2214-951F-4F78-8488-66E1ED3F50BF} + Duplicati.Library.Localization + + + {D10A5FC0-11B4-4E70-86AA-8AEA52BD9798} + Duplicati.Library.Logging + + + {52826615-7964-47FE-B4B3-1B2DBDF605B9} + Duplicati.Library.Modules.Builtin + + + {10D2D1B7-C664-41D8-9B3A-00040C3D421B} + Duplicati.Library.Main + + + {D63E53E4-A458-4C2F-914D-92F715F58ACE} + Duplicati.Library.Snapshots + + + {2C838169-B187-4B09-8768-1C24C2521C8D} + Duplicati.Library.SQLiteHelper + + + {DE3E5D4C-51AB-4E5E-BEE8-E636CEBFBA65} + Duplicati.Library.Utility + + + {4D012CB1-4B92-47F4-89B7-BF80A73A2E99} + Duplicati.License + + + {19E661D2-C5DA-4F35-B3EE-7586E5734B5F} + Duplicati.Server + + + {E93F3DE2-FF3A-4709-96A3-8190AA14FA25} + Duplicati.Service + + + {33FD1D24-C28F-4C71-933F-98F1586EA76C} + Duplicati.Server.Serialization + + + + + \ No newline at end of file From 8c3d83f9afdcc76fd3ec075839da1806458019fa Mon Sep 17 00:00:00 2001 From: Dan Stahr Date: Thu, 8 Jun 2017 16:24:06 +0100 Subject: [PATCH 3/4] ...and add the files to the project as well --- .../Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj index 9cf37e422..ec07bfcc7 100644 --- a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj +++ b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj @@ -38,6 +38,8 @@ + + From 377aef5fe1991c0ff1eeff88f9a0c0f667770374 Mon Sep 17 00:00:00 2001 From: Dan Stahr Date: Thu, 8 Jun 2017 23:21:25 +0100 Subject: [PATCH 4/4] Corrected path to the new files. --- .../SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj index ec07bfcc7..06d9deab3 100644 --- a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj +++ b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj @@ -38,8 +38,8 @@ - - + +