From 4e7ed3458fc66bee5b5c57eb37838b6ee76ea96b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 23 Jul 2013 18:56:01 +0200 Subject: [PATCH] Added auto-rename logic to work around faulty Apache WebDAV backends --- Duplicati/Library/Main/BackendManager.cs | 224 ++++++++++++++---- .../Library/Main/Database/LocalDatabase.cs | 23 ++ Duplicati/Library/Main/Enums.cs | 3 +- .../Main/Operation/FilelistProcessor.cs | 6 + .../Library/Main/Volumes/IndexVolumeReader.cs | 6 + .../Library/Main/Volumes/IndexVolumeWriter.cs | 23 ++ .../Library/Main/Volumes/VolumeWriterBase.cs | 3 +- 7 files changed, 234 insertions(+), 54 deletions(-) diff --git a/Duplicati/Library/Main/BackendManager.cs b/Duplicati/Library/Main/BackendManager.cs index afd362bdf..458de968d 100644 --- a/Duplicati/Library/Main/BackendManager.cs +++ b/Duplicati/Library/Main/BackendManager.cs @@ -56,21 +56,68 @@ namespace Duplicati.Library.Main private class FileEntryItem : IDownloadWaitHandle { + /// + /// The current operation this entry represents + /// public OperationType Operation; + /// + /// The name of the remote file + /// public string RemoteFilename; - public string LocalFilename; + /// + /// The name of the local file + /// + public string LocalFilename { get { return LocalTempfile; } } + /// + /// A reference to a temporary file that is disposed upon + /// failure or completion of the item + /// + public TempFile LocalTempfile; + /// + /// True if the item has been encrypted + /// public bool Encrypted; + /// + /// The result object + /// public object Result; + /// + /// The expected hash value of the file + /// public string Hash; + /// + /// The expected size of the file + /// public long Size; - public IndexVolumeWriter Indexfile; + /// + /// Reference to the index file entry that is updated if this entry changes + /// + public Tuple Indexfile; + /// + /// A flag indicating if the final hash and size of the block volume has been written to the index file + /// + public bool IndexfileUpdated; + /// + /// An exception that this item has caused + /// public Exception Exception; + /// + /// True if an exception ultimately kills the handler, + /// false if the item is returned with an exception + /// public bool ExceptionKillsHandler; + /// + /// A flag indicating if the file is a extra metadata file + /// that has no entry in the database + /// public bool NotTrackedInDb; + /// + /// The event that is signaled once the operation is complete or has failed + /// private System.Threading.ManualResetEvent DoneEvent; - public FileEntryItem(OperationType operation, string remotefilename, IndexVolumeWriter indexfile = null) + public FileEntryItem(OperationType operation, string remotefilename, Tuple indexfile = null) { Operation = operation; RemoteFilename = remotefilename; @@ -81,13 +128,19 @@ namespace Duplicati.Library.Main DoneEvent = new System.Threading.ManualResetEvent(false); } - public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, IndexVolumeWriter indexfile = null) + public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, Tuple indexfile = null) : this(operation, remotefilename, indexfile) { Size = size; Hash = hash; } + public void SetLocalfilename(string name) + { + this.LocalTempfile = Library.Utility.TempFile.WrapExistingFile(name); + this.LocalTempfile.Protected = true; + } + public void SignalComplete() { DoneEvent.Set(); @@ -124,10 +177,10 @@ namespace Duplicati.Library.Main { if (encryption != null && !this.Encrypted) { - var sourcefile = this.LocalFilename + "." + encryption.FilenameExtension; - encryption.Encrypt(this.LocalFilename, sourcefile); + var tempfile = new Library.Utility.TempFile(); + encryption.Encrypt(this.LocalFilename, tempfile); this.DeleteLocalFile(stat); - this.LocalFilename = sourcefile; + this.LocalTempfile = tempfile; this.Hash = null; this.Size = 0; this.Encrypted = true; @@ -154,14 +207,10 @@ namespace Duplicati.Library.Main public void DeleteLocalFile(IBackendWriter stat) { - try - { - System.IO.File.Delete(this.LocalFilename); - } - catch (Exception ex) - { - stat.AddError(string.Format("Failed to delete local file \"{0}\": {1}", this.LocalFilename, ex.Message), ex); - } + if (this.LocalTempfile != null) + try { this.LocalTempfile.Dispose(); } + catch (Exception ex) { stat.AddWarning(string.Format("Failed to dispose temporary file: {0}", this.LocalTempfile), ex); } + finally { this.LocalTempfile = null; } } public BackendActionType BackendActionType @@ -213,6 +262,12 @@ namespace Duplicati.Library.Main public string Hash; } + private class DbRename : IDbEntry + { + public string Oldname; + public string Newname; + } + public DatabaseCollector(LocalDatabase database, IBackendWriter stats) { m_database = database; @@ -235,6 +290,12 @@ namespace Duplicati.Library.Main m_dbqueue.Add(new DbUpdate() { Remotename = remotename, State = state, Size = size, Hash = hash }); } + public void LogDbRename(string oldname, string newname) + { + lock(m_dbqueuelock) + m_dbqueue.Add(new DbRename() { Oldname = oldname, Newname = newname }); + } + public bool FlushDbMessages(bool checkThread = false) { if (m_database != null && (checkThread == false || m_callerThread == System.Threading.Thread.CurrentThread)) @@ -261,6 +322,8 @@ namespace Duplicati.Library.Main db.LogRemoteOperation(((DbOperation)e).Action, ((DbOperation)e).File, ((DbOperation)e).Result, transaction); else if (e is DbUpdate) db.UpdateRemoteVolume(((DbUpdate)e).Remotename, ((DbUpdate)e).State, ((DbUpdate)e).Size, ((DbUpdate)e).Hash, transaction); + else if (e is DbRename) + db.RenameRemoteFile(((DbRename)e).Oldname, ((DbRename)e).Newname, transaction); else if (e != null) m_stats.AddError(string.Format("Queue had element of type: {0}, {1}", e.GetType(), e.ToString()), null); @@ -376,18 +439,22 @@ namespace Duplicati.Library.Main } } + // To work around the Apache WEBDAV issue, we rename the file here + if (item.Operation == OperationType.Put && retries < m_options.NumberOfRetries && !item.NotTrackedInDb) + RenameFileAfterError(item); + if (!recovered) { - try { m_backend.Dispose(); } - catch(Exception dex) { m_statwriter.AddWarning(string.Format("Failed to dispose backend instance: {0}", ex.Message), dex); } - - m_backend = null; - - if (retries < m_options.NumberOfRetries && m_options.RetryDelay.Ticks != 0) - System.Threading.Thread.Sleep(m_options.RetryDelay); - } - + try { m_backend.Dispose(); } + catch(Exception dex) { m_statwriter.AddWarning(string.Format("Failed to dispose backend instance: {0}", ex.Message), dex); } + + m_backend = null; + + if (retries < m_options.NumberOfRetries && m_options.RetryDelay.Ticks != 0) + System.Threading.Thread.Sleep(m_options.RetryDelay); + } } + } while (retries < m_options.NumberOfRetries); @@ -396,7 +463,7 @@ namespace Duplicati.Library.Main item.Exception = lastException; if (item.Operation == OperationType.Put) item.DeleteLocalFile(m_statwriter); - + if (item.ExceptionKillsHandler) { m_lastException = lastException; @@ -417,17 +484,67 @@ namespace Duplicati.Library.Main i.SignalComplete(); } + private void RenameFileAfterError(FileEntryItem item) + { + var p = VolumeBase.ParseFilename(item.RemoteFilename); + var guid = VolumeWriterBase.GenerateGuid(m_options); + var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1); + var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule); + var oldname = item.RemoteFilename; + + m_statwriter.SendEvent(item.BackendActionType, BackendEventType.Rename, oldname, item.Size); + m_statwriter.SendEvent(item.BackendActionType, BackendEventType.Rename, newname, item.Size); + m_statwriter.AddMessage(string.Format("Renaming \"{0}\" to \"{1}\"", oldname, newname)); + m_db.LogDbRename(oldname, newname); + item.RemoteFilename = newname; + + // If there is an index file attached to the block file, + // it references the block filename, so we create a new index file + // which is a copy of the current, but with the new name + if (item.Indexfile != null) + { + if (!item.IndexfileUpdated) + { + item.Indexfile.Item1.FinishVolume(item.Hash, item.Size); + item.Indexfile.Item1.Close(); + item.IndexfileUpdated = true; + } + + IndexVolumeWriter wr = null; + try + { + var hashsize = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm).HashSize / 8; + wr = new IndexVolumeWriter(m_options); + using(var rd = new IndexVolumeReader(p.CompressionModule, item.LocalFilename, m_options, hashsize)) + wr.CopyFrom(rd, x => x == oldname ? newname : x); + item.Indexfile.Item1.Dispose(); + item.Indexfile = new Tuple(wr, item.Indexfile.Item2); + item.Indexfile.Item2.LocalTempfile = wr.TempFile; + wr.Close(); + } + catch + { + if (wr != null) + try { wr.Dispose(); } + catch { } + finally { wr = null; } + + throw; + } + } + } + private void DoPut(FileEntryItem item) { item.Encrypt(m_encryption, m_statwriter); if (item.UpdateHashAndSize(m_options) && !item.NotTrackedInDb) m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash); - if (item.Indexfile != null) + if (item.Indexfile != null && !item.IndexfileUpdated) { - item.Indexfile.FinishVolume(item.Hash, item.Size); - item.Indexfile.Close(); - item.Indexfile = null; + item.Indexfile.Item1.FinishVolume(item.Hash, item.Size); + item.Indexfile.Item1.Close(); + item.IndexfileUpdated = true; } m_db.LogDbOperation("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash })); @@ -611,7 +728,7 @@ namespace Duplicati.Library.Main throw m_lastException; var req = new FileEntryItem(OperationType.Put, remotename, null); - req.LocalFilename = localpath; + req.SetLocalfilename(localpath); req.Encrypted = true; //Prevent encryption req.NotTrackedInDb = true; //Prevent Db updates @@ -633,35 +750,38 @@ namespace Duplicati.Library.Main item.Close(); m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, indexfile); - req.LocalFilename = item.LocalFilename; - - if (m_queue.Enqueue(req) && m_options.SynchronousUpload) - { - req.WaitForComplete(); - if (req.Exception != null) - throw req.Exception; - } - + var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, null); + req.LocalTempfile = item.TempFile; + if (m_lastException != null) throw m_lastException; + FileEntryItem req2 = null; + if (indexfile != null) { m_db.LogDbUpdate(indexfile.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - var req2 = new FileEntryItem(OperationType.Put, indexfile.RemoteFilename); - req2.LocalFilename = indexfile.LocalFilename; - - if (m_queue.Enqueue(req2) && m_options.SynchronousUpload) - { - req2.WaitForComplete(); - if (req2.Exception != null) - throw req2.Exception; - } - - if (m_lastException != null) - throw m_lastException; + req2 = new FileEntryItem(OperationType.Put, indexfile.RemoteFilename); + req2.LocalTempfile = indexfile.TempFile; + req.Indexfile = new Tuple(indexfile, req2); } + + if (m_queue.Enqueue(req) && m_options.SynchronousUpload) + { + req.WaitForComplete(); + if (req.Exception != null) + throw req.Exception; + } + + if (req2 != null && m_queue.Enqueue(req2) && m_options.SynchronousUpload) + { + req2.WaitForComplete(); + if (req2.Exception != null) + throw req2.Exception; + } + + if (m_lastException != null) + throw m_lastException; } public Library.Utility.TempFile Get(string remotename, long size, string hash) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 44fadc840..79b580db4 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -803,6 +803,29 @@ namespace Duplicati.Library.Main.Database } } + public void RenameRemoteFile(string oldname, string newname, System.Data.IDbTransaction transaction) + { + using(var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + using(var cmd = m_connection.CreateCommand()) + { + cmd.Transaction = tr.Parent; + + //Rename the old entry, to preserve ID links + var c = cmd.ExecuteNonQuery(@"UPDATE ""Remotevolume"" SET ""Name"" = ? WHERE ""Name"" = ?", newname, oldname); + if (c != 1) + throw new Exception(string.Format("Unexpected result from renaming \"{0}\" to \"{1}\", expected {2} got {3}", oldname, newname, 1, c)); + + // Grab the type of entry + var type = (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), cmd.ExecuteScalar(@"SELECT ""Type"" FROM ""Remotevolume"" WHERE ""Name"" = ?", newname).ToString(), true); + + //Create a fake new entry with the old name and mark as deleting + // as this ensures we will remove it, if it shows up in some later listing + RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, tr.Parent); + + tr.Commit(); + } + } + /// /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. /// diff --git a/Duplicati/Library/Main/Enums.cs b/Duplicati/Library/Main/Enums.cs index 75874dda4..fdba664b5 100644 --- a/Duplicati/Library/Main/Enums.cs +++ b/Duplicati/Library/Main/Enums.cs @@ -21,7 +21,8 @@ namespace Duplicati.Library.Main Progress, Completed, Retrying, - Failed + Failed, + Rename } /// diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 3ba969456..ad702cb33 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -167,6 +167,12 @@ namespace Duplicati.Library.Main.Operation log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); database.RemoveRemoteVolume(i.Name, null); } + else if (i.State == RemoteVolumeState.Deleting && lookup.ContainsKey(i.Name)) + { + log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); + backend.Delete(i.Name, true); + lookup.Remove(i.Name); + } else { Volumes.IParsedVolume r; diff --git a/Duplicati/Library/Main/Volumes/IndexVolumeReader.cs b/Duplicati/Library/Main/Volumes/IndexVolumeReader.cs index 144c4fe11..489658332 100644 --- a/Duplicati/Library/Main/Volumes/IndexVolumeReader.cs +++ b/Duplicati/Library/Main/Volumes/IndexVolumeReader.cs @@ -14,6 +14,7 @@ namespace Duplicati.Library.Main.Volumes { string Hash { get; } long Length { get; } + Stream Data { get; } IEnumerable Blocklist { get; } } @@ -320,6 +321,11 @@ namespace Duplicati.Library.Main.Volumes { get { return m_size; } } + + public Stream Data + { + get { return m_compression.OpenRead(m_filename); } + } public IEnumerable Blocklist { diff --git a/Duplicati/Library/Main/Volumes/IndexVolumeWriter.cs b/Duplicati/Library/Main/Volumes/IndexVolumeWriter.cs index cea2fa946..98f3e6642 100644 --- a/Duplicati/Library/Main/Volumes/IndexVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/IndexVolumeWriter.cs @@ -75,6 +75,29 @@ namespace Duplicati.Library.Main.Volumes s.Write(data, 0, size); } + public void WriteBlocklist(string hash, Stream source) + { + m_blocklists++; + //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, + using (var s = m_compression.CreateFile(INDEX_BLOCKLIST_FOLDER + Library.Utility.Utility.Base64PlainToBase64Url(hash), CompressionHint.Noncompressible, DateTime.UtcNow)) + Library.Utility.Utility.CopyStream(source, s); + } + + public void CopyFrom(IndexVolumeReader rd, Func filename_mapping) + { + foreach(var n in rd.Volumes) + { + this.StartVolume(filename_mapping(n.Filename)); + foreach(var x in n.Blocks) + this.AddBlock(x.Key, x.Value); + this.FinishVolume(n.Hash, n.Length); + } + + foreach(var b in rd.BlockLists) + using(var s = b.Data) + this.WriteBlocklist(b.Hash, s); + } + public override void Dispose() { if (m_writer != null || m_streamwriter != null) diff --git a/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs b/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs index baa4d4e25..61cc51bc5 100644 --- a/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs +++ b/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs @@ -14,6 +14,7 @@ namespace Duplicati.Library.Main.Volumes protected string m_volumename; public string LocalFilename { get { return m_localfile; } } public string RemoteFilename { get { return m_volumename; } } + public Library.Utility.TempFile TempFile { get { return m_localfile; } } public abstract RemoteVolumeType FileType { get; } @@ -29,7 +30,7 @@ namespace Duplicati.Library.Main.Volumes { } - private static string GenerateGuid(Options options) + public static string GenerateGuid(Options options) { var s = Guid.NewGuid().ToString("N");