From bc6e32d049012fa2f2f66212a47f8ef856fc5306 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 19 Sep 2017 14:24:59 +0200 Subject: [PATCH] Updated the new backend handler to support the same features as the previous: throttle, error message, and deleteGraceTime. --- .../Main/Operation/Common/BackendHandler.cs | 67 +++++++++++++------ .../Main/Operation/Common/DatabaseCommon.cs | 4 +- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Common/BackendHandler.cs b/Duplicati/Library/Main/Operation/Common/BackendHandler.cs index 40078dd40..c0f94ab0e 100644 --- a/Duplicati/Library/Main/Operation/Common/BackendHandler.cs +++ b/Duplicati/Library/Main/Operation/Common/BackendHandler.cs @@ -172,7 +172,17 @@ namespace Duplicati.Library.Main.Operation.Common m_stats = stats; m_taskreader = taskreader; m_log = new LogWrapper(m_logchannel); - } + m_backend = DynamicLoader.BackendLoader.GetBackend(backendUrl, options.RawOptions); + + var shortname = m_backendurl; + + // Try not to leak hostnames or other information in the error messages + try { shortname = new Library.Utility.Uri(shortname).Scheme; } + catch { } + + if (m_backend == null) + throw new Duplicati.Library.Interface.UserInformationException(string.Format("Backend not supported: {0}", shortname)); + } protected Task RunRetryOnMain(FileEntryItem fe, Func> method) { @@ -254,11 +264,11 @@ namespace Duplicati.Library.Main.Operation.Common await tcs.Task; } - public Task DeleteFileAsync(string remotename, long size) + public Task DeleteFileAsync(string remotename, bool suppressCleanup = false) { var fe = new FileEntryItem(BackendActionType.Delete, remotename); return RunRetryOnMain(fe, () => - DoDelete(fe) + DoDelete(fe, suppressCleanup) ); } @@ -434,7 +444,7 @@ namespace Duplicati.Library.Main.Operation.Common { using (var fs = System.IO.File.OpenRead(item.LocalFilename)) using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) - using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) + using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg))) ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs); } else @@ -450,9 +460,9 @@ namespace Duplicati.Library.Main.Operation.Common if (m_options.ListVerifyUploads) { - var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault(); + var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); if (f == null) - throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", f.Name)); + throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename)); else if (f.Size != item.Size && f.Size >= 0) throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size)); } @@ -489,7 +499,7 @@ namespace Duplicati.Library.Main.Operation.Common return r; } - private async Task DoDelete(FileEntryItem item) + private async Task DoDelete(FileEntryItem item, bool suppressCleanup) { if (m_options.Dryrun) { @@ -538,7 +548,7 @@ namespace Duplicati.Library.Main.Operation.Common await m_database.LogRemoteOperationAsync("delete", item.RemoteFilename, result); } - await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Deleted, -1, null); + await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Deleted, -1, null, suppressCleanup, TimeSpan.FromHours(2)); await m_stats.SendEventAsync(BackendActionType.Delete, BackendEventType.Completed, item.RemoteFilename, item.Size); return true; @@ -581,7 +591,7 @@ namespace Duplicati.Library.Main.Operation.Common { using (var fs = System.IO.File.OpenWrite(tmpfile)) using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) - using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) + using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg))) ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); } else @@ -629,10 +639,10 @@ namespace Duplicati.Library.Main.Operation.Common // Auto-guess the encryption module var ext = (System.IO.Path.GetExtension(item.RemoteFilename) ?? "").TrimStart('.'); - if (!string.Equals(m_options.EncryptionModule, ext, StringComparison.InvariantCultureIgnoreCase)) + if (!string.Equals(m_options.EncryptionModule, ext, StringComparison.OrdinalIgnoreCase)) { // Check if the file is encrypted with something else - if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.InvariantCultureIgnoreCase)) + if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) { using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(ext, m_options.Passphrase, m_options.RawOptions)) if (encmodule != null) @@ -642,7 +652,7 @@ namespace Duplicati.Library.Main.Operation.Common } } // Check if the file is not encrypted - else if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.InvariantCultureIgnoreCase)) + else if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) { await m_log.WriteVerboseAsync("Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, m_options.EncryptionModule); } @@ -686,14 +696,33 @@ namespace Duplicati.Library.Main.Operation.Common { } } - } - - private void HandleProgress(long pg) - { + } + + private string m_lastThrottleUploadValue = null; + private string m_lastThrottleDownloadValue = null; + + private void HandleProgress(ThrottledStream ts, long pg) + { if (!m_taskreader.TransferProgressAsync.WaitForTask().Result) - throw new OperationCanceledException(); - - m_stats.UpdateBackendProgress(pg); + throw new OperationCanceledException(); + + // Update the throttle speeds if they have changed + string tmp; + m_options.RawOptions.TryGetValue("throttle-upload", out tmp); + if (tmp != m_lastThrottleUploadValue) + { + ts.WriteSpeed = m_options.MaxUploadPrSecond; + m_lastThrottleUploadValue = tmp; + } + + m_options.RawOptions.TryGetValue("throttle-download", out tmp); + if (tmp != m_lastThrottleDownloadValue) + { + ts.ReadSpeed = m_options.MaxDownloadPrSecond; + m_lastThrottleDownloadValue = tmp; + } + + m_stats.UpdateBackendProgress(pg); } protected override void Dispose(bool disposing) diff --git a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs index 775d2b861..a5df1c503 100644 --- a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs +++ b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs @@ -46,9 +46,9 @@ namespace Duplicati.Library.Main.Operation.Common return RunOnMain(() => m_db.RegisterRemoteVolume(name, type, state, m_transaction)); } - public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash) + public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan)) { - return RunOnMain(() => m_db.UpdateRemoteVolume(name, state, size, hash, m_transaction)); + return RunOnMain(() => m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, m_transaction)); } public Task CommitTransactionAsync(string message, bool restart = true)