From 4a6bf3debace7099c6c56f27102b0d61a8209269 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 18 Sep 2017 11:53:50 +0200 Subject: [PATCH 1/2] Added "DeleteGraceTime" to all files deleted from the backend mananger. This should fix a number of cases where the backup returns a consistency error because the deletion and the backend check happens before the backend is in sync. --- Duplicati/Library/Main/BackendManager.cs | 2 +- Duplicati/Library/Main/Database/LocalDatabase.cs | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/BackendManager.cs b/Duplicati/Library/Main/BackendManager.cs index e8d2e1ae6..2cab8a76e 100644 --- a/Duplicati/Library/Main/BackendManager.cs +++ b/Duplicati/Library/Main/BackendManager.cs @@ -374,7 +374,7 @@ namespace Duplicati.Library.Main db.LogRemoteOperation(((DbOperation)e).Action, ((DbOperation)e).File, ((DbOperation)e).Result, transaction); else if (e is DbUpdate && ((DbUpdate)e).State == RemoteVolumeState.Deleted) { - db.UpdateRemoteVolume(((DbUpdate)e).Remotename, RemoteVolumeState.Deleted, ((DbUpdate)e).Size, ((DbUpdate)e).Hash, true, transaction); + db.UpdateRemoteVolume(((DbUpdate)e).Remotename, RemoteVolumeState.Deleted, ((DbUpdate)e).Size, ((DbUpdate)e).Hash, true, TimeSpan.FromHours(2), transaction); volsRemoved.Add(((DbUpdate)e).Remotename); } else if (e is DbUpdate) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 17c3ca553..35da7c843 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -121,8 +121,8 @@ namespace Duplicati.Library.Main.Database m_selectremotevolumeCommand.CommandText = @"SELECT ""Type"", ""Size"", ""Hash"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" = ?"; m_selectremotevolumeCommand.AddParameter(); - m_removeremotevolumeCommand.CommandText = @"DELETE FROM ""Remotevolume"" WHERE ""Name"" = ?"; - m_removeremotevolumeCommand.AddParameter(); + m_removeremotevolumeCommand.CommandText = @"DELETE FROM ""Remotevolume"" WHERE ""Name"" = ? AND (""DeleteGraceTime"" < ? OR ""State"" != ?)"; + m_removeremotevolumeCommand.AddParameters(3); m_selectremotevolumeIdCommand.CommandText = @"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = ?"; @@ -434,12 +434,14 @@ namespace Duplicati.Library.Main.Database deletecmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", blocksetidstable)); deletecmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", volidstable)); } - catch { /* Ignore, will be deleted on close anyway. */ } - - foreach (var name in names) + catch { /* Ignore, will be deleted on close anyway. */ } + + m_removeremotevolumeCommand.Transaction = tr.Parent; + m_removeremotevolumeCommand.SetParameterValue(1, DateTime.UtcNow.Ticks); + m_removeremotevolumeCommand.SetParameterValue(2, RemoteVolumeState.Deleted.ToString()); + foreach (var name in names) { m_removeremotevolumeCommand.SetParameterValue(0, name); - m_removeremotevolumeCommand.Transaction = tr.Parent; m_removeremotevolumeCommand.ExecuteNonQuery(); } tr.Commit(); From e0d8a57ccc81bba71d805c3d5218eefcd14bc9bc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 18 Sep 2017 11:54:48 +0200 Subject: [PATCH 2/2] Added re-use of the backend across the operations in the backup step. This should fix the AmzCD consistency issue where it does not wait for the listing after performing a delete. --- Duplicati/Library/Main/Operation/BackupHandler.cs | 4 ++-- Duplicati/Library/Main/Operation/CompactHandler.cs | 10 ++++++---- Duplicati/Library/Main/Operation/DeleteHandler.cs | 13 ++++++++----- .../Main/Operation/PurgeBrokenFilesHandler.cs | 2 +- .../Library/Main/Operation/PurgeFilesHandler.cs | 2 +- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 158df0eb6..403e53aca 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -606,7 +606,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); using(var db = new LocalDeleteDatabase(m_database)) - new DeleteHandler(backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall); + new DeleteHandler(backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall, backend); } else if (currentIsSmall && !m_options.NoAutoCompact) @@ -614,7 +614,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); using(var db = new LocalDeleteDatabase(m_database)) - new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref m_transaction); + new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref m_transaction, backend); } } diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 0c4502955..a2cbddc88 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -51,7 +51,7 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyParameters(db, m_options); - var changed = DoCompact(db, false, ref tr); + var changed = DoCompact(db, false, ref tr, null); if (changed && m_options.UploadVerificationFile) FilelistProcessor.UploadVerificationFile(m_backendurl, m_options, m_result.BackendWriter, db, null); @@ -83,15 +83,17 @@ namespace Duplicati.Library.Main.Operation } } - internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction) + internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction, BackendManager sharedBackend) { var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction); report.ReportCompactData(m_result); if (report.ShouldReclaim || report.ShouldCompact) - { - using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) + { + // Workaround where we allow a running backendmanager to be used + using(var bk = sharedBackend == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null) { + var backend = bk ?? sharedBackend; if (!hasVerifiedBackend && !m_options.NoBackendverification) FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter); diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index ce5dac8f2..f16bb9a5e 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -50,7 +50,7 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyParameters(db, m_options); - DoRun(db, ref tr, false, false); + DoRun(db, ref tr, false, false, null); if (!m_options.Dryrun) { @@ -73,10 +73,13 @@ namespace Duplicati.Library.Main.Operation } } - public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBacked, bool forceCompact) - { - using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) + public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBacked, bool forceCompact, BackendManager sharedManager) + { + // Workaround where we allow a running backendmanager to be used + using(var bk = sharedManager == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null) { + var backend = bk ?? sharedManager; + if (!hasVerifiedBacked && !m_options.NoBackendverification) FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter); @@ -142,7 +145,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoAutoCompact && (forceCompact || (toDelete != null && toDelete.Length > 0))) { m_result.CompactResults = new CompactResults(m_result); - new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref transaction); + new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref transaction, sharedManager); } m_result.SetResults( diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 578f12657..6c2755610 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -102,7 +102,7 @@ namespace Duplicati.Library.Main.Operation opts.RawOptions["no-auto-compact"] = "true"; new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults) - .DoRun(rmdb, ref deltr, true, false); + .DoRun(rmdb, ref deltr, true, false, null); if (!m_options.Dryrun) { diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 8fa715daf..fc91ddb64 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -224,7 +224,7 @@ namespace Duplicati.Library.Main.Operation var tr = cdb.BeginTransaction(); try { - new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, ref tr); + new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, ref tr, backend); } catch {