From ca1872ce5735dcc5511dc830d62e8fefedc68810 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 14 Aug 2018 15:21:54 +0200 Subject: [PATCH] Added an option to disable filelist consistency checks as it was reported to slow backups with a large number of filesets. This addresses #3194 --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- Duplicati/Library/Main/Operation/BackupHandler.cs | 2 +- Duplicati/Library/Main/Options.cs | 10 ++++++++++ Duplicati/Library/Main/Strings.cs | 2 ++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 385ec4aec..aa6810a36 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -807,7 +807,7 @@ ON using(var cmd2 = m_connection.CreateCommand(transaction)) foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) { - var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS); + var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS); var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid); //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); var storedlist = cmd2.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?", 0, filesetid); diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 1f4e6fb38..7c5eb4af1 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -421,7 +421,7 @@ namespace Duplicati.Library.Main.Operation } // Make sure the database is sane - await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, true); + await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks); // Start the uploader process uploader = Backup.BackendUploader.Run(bk, m_options, db, m_result, m_result.TaskReader, stats); diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 626722bd1..14d9a0e29 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -560,6 +560,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("auto-vacuum", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AutoVacuumShort, Strings.Options.AutoVacuumLong, "false"), new CommandLineArgument("disable-file-scanner", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilescannerShort, Strings.Options.DisablefilescannerLong, "false"), + new CommandLineArgument("disable-filelist-consistency-checks", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilelistconsistencychecksShort, Strings.Options.DisablefilelistconsistencychecksLong, "false"), new CommandLineArgument("disable-on-battery", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableOnBatteryShort, Strings.Options.DisableOnBatteryLong, "false"), new CommandLineArgument("exclude-empty-folders", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ExcludeemptyfoldersShort, Strings.Options.ExcludeemptyfoldersLong, "false"), @@ -1925,6 +1926,15 @@ namespace Duplicati.Library.Main get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-file-scanner"); } } + /// + /// Gets a flag indicating if the filelist consistency checks should be disabled + /// + /// true if the filelist consistency checks should be disabled; otherwise, false. + public bool DisableFilelistConsistencyChecks + { + get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-filelist-consistency-checks"); } + } + /// /// Gets a flag indicating whether the backup should be disabled when on battery power. /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index d0b4b1315..074734f5f 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -245,6 +245,8 @@ namespace Duplicati.Library.Main.Strings public static string AutoVacuumLong { get { return LC.L(@"Some operations that manipulate the local database leave unused entries behind. These entries are not deleted from a hard drive until a VACUUM operation is run. This operation saves disk space in the long run but needs to temporarily create a copy of all valid entries in the database. Setting this to true will allow Duplicati to perform VACUUM operations at its discretion."); } } public static string DisablefilescannerShort { get { return LC.L(@"Disable the read-ahead scanner"); } } public static string DisablefilescannerLong { get { return LC.L(@"When this flag is enabled, the scanner that computes the size of source files is disabled, and instead the reported size is read from the database. Using this flag can speed up the backup by reducing disk access, but will give a less accurate progress indicator."); } } + public static string DisablefilelistconsistencychecksShort { get { return LC.L(@"Disable filelist consistency checks"); } } + public static string DisablefilelistconsistencychecksLong { get { return LC.L(@"In backups with a large number of filesets, the verification can take up a large part of the backup time. If you disable the checks, make sure you run regular check commands to ensure that everything is working as expected."); } } public static string DisableOnBatteryShort { get { return LC.L("Disable the backup when on battery power"); } } public static string DisableOnBatteryLong { get { return LC.L("When this flag is enabled, a scheduled backup will not run if the system is detected to be running on battery power (manual or command line backups will still be run). If the detected power source is mains (i.e., AC) or unknown, then scheduled backups will proceed as normal."); } }