From cfeabfb2dd09318055a080ded2e564d8033b0d06 Mon Sep 17 00:00:00 2001 From: drwtsn32x Date: Mon, 29 Jul 2019 17:56:43 -0700 Subject: [PATCH] Add auto vacuum interval option --- .../Library/Interface/ResultInterfaces.cs | 3 ++ Duplicati/Library/Main/Controller.cs | 34 +++++++++++++--- .../Library/Main/Operation/BackupHandler.cs | 4 +- .../Library/Main/Operation/CompactHandler.cs | 3 +- .../Library/Main/Operation/VacuumHandler.cs | 5 ++- Duplicati/Library/Main/Options.cs | 15 +++++++ Duplicati/Library/Main/ResultClasses.cs | 10 ++++- Duplicati/Library/Main/Strings.cs | 2 + Duplicati/Server/Runner.cs | 40 ++++++++++++++++--- 9 files changed, 98 insertions(+), 18 deletions(-) diff --git a/Duplicati/Library/Interface/ResultInterfaces.cs b/Duplicati/Library/Interface/ResultInterfaces.cs index 588726da7..e11218594 100644 --- a/Duplicati/Library/Interface/ResultInterfaces.cs +++ b/Duplicati/Library/Interface/ResultInterfaces.cs @@ -142,6 +142,7 @@ namespace Duplicati.Library.Interface bool Dryrun { get; } ICompactResults CompactResults { get; } + IVacuumResults VacuumResults { get; } IDeleteResults DeleteResults { get; } IRepairResults RepairResults { get; } } @@ -178,6 +179,8 @@ namespace Duplicati.Library.Interface long DownloadedFileSize { get; } long UploadedFileSize { get; } bool Dryrun { get; } + + IVacuumResults VacuumResults { get; } } public interface ICreateLogDatabaseResults : IBasicResults diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 477e12576..e5a81ada8 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -112,11 +112,8 @@ namespace Duplicati.Library.Main Library.UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule); Library.UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule); - if (!m_options.NoAutoCompact && (LastCompact > DateTime.MinValue) && (LastCompact.Add(m_options.AutoCompactInterval) > DateTime.Now)) - { - Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Skipping auto compaction until {0}", LastCompact.Add(m_options.AutoCompactInterval)); - m_options.RawOptions["no-auto-compact"] = "true"; - } + CheckAutoCompactInterval(); + CheckAutoVacuumInterval(); return RunAction(new BackupResults(), ref inputsources, ref filter, (result) => { @@ -233,6 +230,8 @@ namespace Duplicati.Library.Main public Duplicati.Library.Interface.ICompactResults Compact() { + CheckAutoVacuumInterval(); + return RunAction(new CompactResults(), (result) => { new Operation.CompactHandler(m_backend, m_options, result).Run(); }); @@ -362,7 +361,7 @@ namespace Duplicati.Library.Main public Library.Interface.IVacuumResults Vacuum() { - return RunAction(new VacuumResult(), result => { + return RunAction(new VacuumResults(), result => { new Operation.VacuumHandler(m_options, result).Run(); }); } @@ -1108,6 +1107,29 @@ namespace Duplicati.Library.Main /// public DateTime LastCompact { get; set; } + /// + /// Time of last vacuum operation + /// + public DateTime LastVacuum { get; set; } + + private void CheckAutoCompactInterval() + { + if (!m_options.NoAutoCompact && (LastCompact > DateTime.MinValue) && (LastCompact.Add(m_options.AutoCompactInterval) > DateTime.Now)) + { + Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Skipping auto compaction until {0}", LastCompact.Add(m_options.AutoCompactInterval)); + m_options.RawOptions["no-auto-compact"] = "true"; + } + } + + private void CheckAutoVacuumInterval() + { + if (m_options.AutoVacuum && (LastVacuum > DateTime.MinValue) && (LastVacuum.Add(m_options.AutoVacuumInterval) > DateTime.Now)) + { + Logging.Log.WriteInformationMessage(LOGTAG, "VacuumResults", "Skipping auto vacuum until {0}", LastVacuum.Add(m_options.AutoVacuumInterval)); + m_options.RawOptions["auto-vacuum"] = "false"; + } + } + #region IDisposable Members public void Dispose() diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 030189790..debbda20a 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -536,8 +536,8 @@ namespace Duplicati.Library.Main.Operation m_database.PurgeLogData(m_options.LogRetention); if (m_options.AutoVacuum) { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); - m_database.Vacuum(); + m_result.VacuumResults = new VacuumResults(m_result); + new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).Run(); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); return; diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 297acb129..4f7e7a04e 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -69,7 +69,8 @@ namespace Duplicati.Library.Main.Operation db.WriteResults(); if (m_options.AutoVacuum) { - db.Vacuum(); + m_result.VacuumResults = new VacuumResults(m_result); + new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).Run(); } } } diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index 0b0b790e6..1e68105d6 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -9,9 +9,9 @@ namespace Duplicati.Library.Main.Operation internal class VacuumHandler { private readonly Options m_options; - private readonly VacuumResult m_result; + private readonly VacuumResults m_result; - public VacuumHandler(Options options, VacuumResult result) + public VacuumHandler(Options options, VacuumResults result) { m_options = options; m_result = result; @@ -24,6 +24,7 @@ namespace Duplicati.Library.Main.Operation m_result.SetDatabase(db); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); db.Vacuum(); + m_result.EndTime = DateTime.UtcNow; } } } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index b2c6f8240..06b4dc3b3 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -406,6 +406,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("rebuild-missing-dblock-files", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RebuildmissingdblockfilesShort, Strings.Options.RebuildmissingdblockfilesLong, "false"), new CommandLineArgument("auto-compact-interval", CommandLineArgument.ArgumentType.Timespan, Strings.Options.AutoCompactIntervalShort, Strings.Options.AutoCompactIntervalLong, "0m"), + new CommandLineArgument("auto-vacuum-interval", CommandLineArgument.ArgumentType.Timespan, Strings.Options.AutoVacuumIntervalShort, Strings.Options.AutoVacuumIntervalLong, "0m"), }); return lst; @@ -1805,6 +1806,20 @@ namespace Duplicati.Library.Main get { return GetBool("auto-vacuum"); } } + /// + /// Gets the minimum time that must elapse after last vacuum before running next automatic vacuum + /// + public TimeSpan AutoVacuumInterval + { + get + { + if (!m_options.ContainsKey("auto-vacuum-interval") || string.IsNullOrEmpty(m_options["auto-vacuum-interval"])) + return TimeSpan.Zero; + else + return Library.Utility.Timeparser.ParseTimeSpan(m_options["auto-vacuum-interval"]); + } + } + /// /// Gets a flag indicating if the local filescanner should be disabled /// diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 429622272..8c5d51360 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -579,6 +579,7 @@ namespace Duplicati.Library.Main public override OperationMode MainOperation { get { return OperationMode.Backup; } } public ICompactResults CompactResults { get; internal set; } + public IVacuumResults VacuumResults { get; internal set; } public IDeleteResults DeleteResults { get; internal set; } public IRepairResults RepairResults { get; internal set; } public ITestResults TestResults { get; internal set; } @@ -588,6 +589,7 @@ namespace Duplicati.Library.Main get { if ((CompactResults != null && CompactResults.ParsedResult == ParsedResultType.Error) || + (VacuumResults != null && VacuumResults.ParsedResult == ParsedResultType.Error) || (DeleteResults != null && DeleteResults.ParsedResult == ParsedResultType.Error) || (RepairResults != null && RepairResults.ParsedResult == ParsedResultType.Error) || (TestResults != null && TestResults.ParsedResult == ParsedResultType.Error) || @@ -596,6 +598,7 @@ namespace Duplicati.Library.Main return ParsedResultType.Error; } else if ((CompactResults != null && CompactResults.ParsedResult == ParsedResultType.Warning) || + (VacuumResults != null && VacuumResults.ParsedResult == ParsedResultType.Warning) || (DeleteResults != null && DeleteResults.ParsedResult == ParsedResultType.Warning) || (RepairResults != null && RepairResults.ParsedResult == ParsedResultType.Warning) || (TestResults != null && TestResults.ParsedResult == ParsedResultType.Warning) || @@ -829,6 +832,8 @@ namespace Duplicati.Library.Main public long UploadedFileSize { get; internal set; } public bool Dryrun { get; internal set; } + public IVacuumResults VacuumResults { get; internal set; } + public override OperationMode MainOperation { get { return OperationMode.Compact; } } public CompactResults() : base() { } @@ -969,8 +974,11 @@ namespace Duplicati.Library.Main public IEnumerable Lines { get; set; } } - internal class VacuumResult : BasicResults, IVacuumResults + internal class VacuumResults : BasicResults, IVacuumResults { + public VacuumResults() : base() { } + public VacuumResults(BasicResults p) : base(p) { } + public override OperationMode MainOperation { get { return OperationMode.Vacuum; } } } } diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index d86d7a9ef..da8cc7448 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -282,6 +282,8 @@ namespace Duplicati.Library.Main.Strings public static string AutoCompactIntervalShort { get { return "Minimum time between auto compactions"; } } public static string AutoCompactIntervalLong { get { return "The minimum amount of time that must elapse after the last compaction before another will be automatically triggered at the end of a backup job. Automatic compaction can be a long-running process and may not be desirable to run after every single backup."; } } + public static string AutoVacuumIntervalShort { get { return "Minimum time between auto vacuums"; } } + public static string AutoVacuumIntervalLong { get { return "The minimum amount of time that must elapse after the last vacuum before another will be automatically triggered at the end of a backup job. Automatic vacuum can be a long-running process and may not be desirable to run after every single backup."; } } } internal static class Common diff --git a/Duplicati/Server/Runner.cs b/Duplicati/Server/Runner.cs index 17278701a..128babe0d 100644 --- a/Duplicati/Server/Runner.cs +++ b/Duplicati/Server/Runner.cs @@ -492,6 +492,12 @@ namespace Duplicati.Server ((RunnerData)data).Controller = controller; data.UpdateThrottleSpeed(); + if (backup.Metadata.ContainsKey("LastCompactFinished")) + controller.LastCompact = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]); + + if (backup.Metadata.ContainsKey("LastVacuumFinished")) + controller.LastVacuum = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastVacuumFinished"]); + switch (data.Operation) { case DuplicatiOperation.Backup: @@ -503,9 +509,6 @@ namespace Duplicati.Server where !string.IsNullOrWhiteSpace(p) select p).ToArray(); - if (backup.Metadata.ContainsKey("LastCompactFinished")) - controller.LastCompact = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]); - var r = controller.Backup(sources, filter); UpdateMetadata(backup, r); return r; @@ -701,7 +704,7 @@ namespace Duplicati.Server ); } - private static void UpdateLastCompactMetadata(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.ICompactResults r) + private static void UpdateMetadataLastCompact(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.ICompactResults r) { if (r != null) { @@ -711,6 +714,16 @@ namespace Duplicati.Server } } + private static void UpdateMetadataLastVacuum(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IVacuumResults r) + { + if (r != null) + { + backup.Metadata["LastVacuumDuration"] = r.Duration.ToString(); + backup.Metadata["LastVacuumStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); + backup.Metadata["LastVacuumFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); + } + } + private static void UpdateMetadata(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IParsedBackendStatistics r) { if (r != null) @@ -751,7 +764,19 @@ namespace Duplicati.Server } if (result is Duplicati.Library.Interface.ICompactResults) - UpdateLastCompactMetadata(backup, (Duplicati.Library.Interface.ICompactResults)result); + { + var r = (Duplicati.Library.Interface.ICompactResults)result; + UpdateMetadataLastCompact(backup, r); + + if (r.VacuumResults != null) + UpdateMetadataLastVacuum(backup, r.VacuumResults); + } + + if (result is Duplicati.Library.Interface.IVacuumResults) + { + var r = (Duplicati.Library.Interface.IVacuumResults)result; + UpdateMetadataLastVacuum(backup, r); + } if (result is Duplicati.Library.Interface.IBackupResults) { @@ -764,7 +789,10 @@ namespace Duplicati.Server backup.Metadata["LastBackupDuration"] = r.Duration.ToString(); if (r.CompactResults != null) - UpdateLastCompactMetadata(backup, r.CompactResults); + UpdateMetadataLastCompact(backup, r.CompactResults); + + if (r.VacuumResults != null) + UpdateMetadataLastVacuum(backup, r.VacuumResults); if (r.FilesWithError > 0 || r.Warnings.Any() || r.Errors.Any()) {