diff --git a/Duplicati/Library/Main/BackendManager.cs b/Duplicati/Library/Main/BackendManager.cs
index afd362bdf..458de968d 100644
--- a/Duplicati/Library/Main/BackendManager.cs
+++ b/Duplicati/Library/Main/BackendManager.cs
@@ -56,21 +56,68 @@ namespace Duplicati.Library.Main
private class FileEntryItem : IDownloadWaitHandle
{
+ ///
+ /// The current operation this entry represents
+ ///
public OperationType Operation;
+ ///
+ /// The name of the remote file
+ ///
public string RemoteFilename;
- public string LocalFilename;
+ ///
+ /// The name of the local file
+ ///
+ public string LocalFilename { get { return LocalTempfile; } }
+ ///
+ /// A reference to a temporary file that is disposed upon
+ /// failure or completion of the item
+ ///
+ public TempFile LocalTempfile;
+ ///
+ /// True if the item has been encrypted
+ ///
public bool Encrypted;
+ ///
+ /// The result object
+ ///
public object Result;
+ ///
+ /// The expected hash value of the file
+ ///
public string Hash;
+ ///
+ /// The expected size of the file
+ ///
public long Size;
- public IndexVolumeWriter Indexfile;
+ ///
+ /// Reference to the index file entry that is updated if this entry changes
+ ///
+ public Tuple Indexfile;
+ ///
+ /// A flag indicating if the final hash and size of the block volume has been written to the index file
+ ///
+ public bool IndexfileUpdated;
+ ///
+ /// An exception that this item has caused
+ ///
public Exception Exception;
+ ///
+ /// True if an exception ultimately kills the handler,
+ /// false if the item is returned with an exception
+ ///
public bool ExceptionKillsHandler;
+ ///
+ /// A flag indicating if the file is a extra metadata file
+ /// that has no entry in the database
+ ///
public bool NotTrackedInDb;
+ ///
+ /// The event that is signaled once the operation is complete or has failed
+ ///
private System.Threading.ManualResetEvent DoneEvent;
- public FileEntryItem(OperationType operation, string remotefilename, IndexVolumeWriter indexfile = null)
+ public FileEntryItem(OperationType operation, string remotefilename, Tuple indexfile = null)
{
Operation = operation;
RemoteFilename = remotefilename;
@@ -81,13 +128,19 @@ namespace Duplicati.Library.Main
DoneEvent = new System.Threading.ManualResetEvent(false);
}
- public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, IndexVolumeWriter indexfile = null)
+ public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, Tuple indexfile = null)
: this(operation, remotefilename, indexfile)
{
Size = size;
Hash = hash;
}
+ public void SetLocalfilename(string name)
+ {
+ this.LocalTempfile = Library.Utility.TempFile.WrapExistingFile(name);
+ this.LocalTempfile.Protected = true;
+ }
+
public void SignalComplete()
{
DoneEvent.Set();
@@ -124,10 +177,10 @@ namespace Duplicati.Library.Main
{
if (encryption != null && !this.Encrypted)
{
- var sourcefile = this.LocalFilename + "." + encryption.FilenameExtension;
- encryption.Encrypt(this.LocalFilename, sourcefile);
+ var tempfile = new Library.Utility.TempFile();
+ encryption.Encrypt(this.LocalFilename, tempfile);
this.DeleteLocalFile(stat);
- this.LocalFilename = sourcefile;
+ this.LocalTempfile = tempfile;
this.Hash = null;
this.Size = 0;
this.Encrypted = true;
@@ -154,14 +207,10 @@ namespace Duplicati.Library.Main
public void DeleteLocalFile(IBackendWriter stat)
{
- try
- {
- System.IO.File.Delete(this.LocalFilename);
- }
- catch (Exception ex)
- {
- stat.AddError(string.Format("Failed to delete local file \"{0}\": {1}", this.LocalFilename, ex.Message), ex);
- }
+ if (this.LocalTempfile != null)
+ try { this.LocalTempfile.Dispose(); }
+ catch (Exception ex) { stat.AddWarning(string.Format("Failed to dispose temporary file: {0}", this.LocalTempfile), ex); }
+ finally { this.LocalTempfile = null; }
}
public BackendActionType BackendActionType
@@ -213,6 +262,12 @@ namespace Duplicati.Library.Main
public string Hash;
}
+ private class DbRename : IDbEntry
+ {
+ public string Oldname;
+ public string Newname;
+ }
+
public DatabaseCollector(LocalDatabase database, IBackendWriter stats)
{
m_database = database;
@@ -235,6 +290,12 @@ namespace Duplicati.Library.Main
m_dbqueue.Add(new DbUpdate() { Remotename = remotename, State = state, Size = size, Hash = hash });
}
+ public void LogDbRename(string oldname, string newname)
+ {
+ lock(m_dbqueuelock)
+ m_dbqueue.Add(new DbRename() { Oldname = oldname, Newname = newname });
+ }
+
public bool FlushDbMessages(bool checkThread = false)
{
if (m_database != null && (checkThread == false || m_callerThread == System.Threading.Thread.CurrentThread))
@@ -261,6 +322,8 @@ namespace Duplicati.Library.Main
db.LogRemoteOperation(((DbOperation)e).Action, ((DbOperation)e).File, ((DbOperation)e).Result, transaction);
else if (e is DbUpdate)
db.UpdateRemoteVolume(((DbUpdate)e).Remotename, ((DbUpdate)e).State, ((DbUpdate)e).Size, ((DbUpdate)e).Hash, transaction);
+ else if (e is DbRename)
+ db.RenameRemoteFile(((DbRename)e).Oldname, ((DbRename)e).Newname, transaction);
else if (e != null)
m_stats.AddError(string.Format("Queue had element of type: {0}, {1}", e.GetType(), e.ToString()), null);
@@ -376,18 +439,22 @@ namespace Duplicati.Library.Main
}
}
+ // To work around the Apache WEBDAV issue, we rename the file here
+ if (item.Operation == OperationType.Put && retries < m_options.NumberOfRetries && !item.NotTrackedInDb)
+ RenameFileAfterError(item);
+
if (!recovered)
{
- try { m_backend.Dispose(); }
- catch(Exception dex) { m_statwriter.AddWarning(string.Format("Failed to dispose backend instance: {0}", ex.Message), dex); }
-
- m_backend = null;
-
- if (retries < m_options.NumberOfRetries && m_options.RetryDelay.Ticks != 0)
- System.Threading.Thread.Sleep(m_options.RetryDelay);
- }
-
+ try { m_backend.Dispose(); }
+ catch(Exception dex) { m_statwriter.AddWarning(string.Format("Failed to dispose backend instance: {0}", ex.Message), dex); }
+
+ m_backend = null;
+
+ if (retries < m_options.NumberOfRetries && m_options.RetryDelay.Ticks != 0)
+ System.Threading.Thread.Sleep(m_options.RetryDelay);
+ }
}
+
} while (retries < m_options.NumberOfRetries);
@@ -396,7 +463,7 @@ namespace Duplicati.Library.Main
item.Exception = lastException;
if (item.Operation == OperationType.Put)
item.DeleteLocalFile(m_statwriter);
-
+
if (item.ExceptionKillsHandler)
{
m_lastException = lastException;
@@ -417,17 +484,67 @@ namespace Duplicati.Library.Main
i.SignalComplete();
}
+ private void RenameFileAfterError(FileEntryItem item)
+ {
+ var p = VolumeBase.ParseFilename(item.RemoteFilename);
+ var guid = VolumeWriterBase.GenerateGuid(m_options);
+ var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1);
+ var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule);
+ var oldname = item.RemoteFilename;
+
+ m_statwriter.SendEvent(item.BackendActionType, BackendEventType.Rename, oldname, item.Size);
+ m_statwriter.SendEvent(item.BackendActionType, BackendEventType.Rename, newname, item.Size);
+ m_statwriter.AddMessage(string.Format("Renaming \"{0}\" to \"{1}\"", oldname, newname));
+ m_db.LogDbRename(oldname, newname);
+ item.RemoteFilename = newname;
+
+ // If there is an index file attached to the block file,
+ // it references the block filename, so we create a new index file
+ // which is a copy of the current, but with the new name
+ if (item.Indexfile != null)
+ {
+ if (!item.IndexfileUpdated)
+ {
+ item.Indexfile.Item1.FinishVolume(item.Hash, item.Size);
+ item.Indexfile.Item1.Close();
+ item.IndexfileUpdated = true;
+ }
+
+ IndexVolumeWriter wr = null;
+ try
+ {
+ var hashsize = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm).HashSize / 8;
+ wr = new IndexVolumeWriter(m_options);
+ using(var rd = new IndexVolumeReader(p.CompressionModule, item.LocalFilename, m_options, hashsize))
+ wr.CopyFrom(rd, x => x == oldname ? newname : x);
+ item.Indexfile.Item1.Dispose();
+ item.Indexfile = new Tuple(wr, item.Indexfile.Item2);
+ item.Indexfile.Item2.LocalTempfile = wr.TempFile;
+ wr.Close();
+ }
+ catch
+ {
+ if (wr != null)
+ try { wr.Dispose(); }
+ catch { }
+ finally { wr = null; }
+
+ throw;
+ }
+ }
+ }
+
private void DoPut(FileEntryItem item)
{
item.Encrypt(m_encryption, m_statwriter);
if (item.UpdateHashAndSize(m_options) && !item.NotTrackedInDb)
m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
- if (item.Indexfile != null)
+ if (item.Indexfile != null && !item.IndexfileUpdated)
{
- item.Indexfile.FinishVolume(item.Hash, item.Size);
- item.Indexfile.Close();
- item.Indexfile = null;
+ item.Indexfile.Item1.FinishVolume(item.Hash, item.Size);
+ item.Indexfile.Item1.Close();
+ item.IndexfileUpdated = true;
}
m_db.LogDbOperation("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
@@ -611,7 +728,7 @@ namespace Duplicati.Library.Main
throw m_lastException;
var req = new FileEntryItem(OperationType.Put, remotename, null);
- req.LocalFilename = localpath;
+ req.SetLocalfilename(localpath);
req.Encrypted = true; //Prevent encryption
req.NotTrackedInDb = true; //Prevent Db updates
@@ -633,35 +750,38 @@ namespace Duplicati.Library.Main
item.Close();
m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
- var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, indexfile);
- req.LocalFilename = item.LocalFilename;
-
- if (m_queue.Enqueue(req) && m_options.SynchronousUpload)
- {
- req.WaitForComplete();
- if (req.Exception != null)
- throw req.Exception;
- }
-
+ var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, null);
+ req.LocalTempfile = item.TempFile;
+
if (m_lastException != null)
throw m_lastException;
+ FileEntryItem req2 = null;
+
if (indexfile != null)
{
m_db.LogDbUpdate(indexfile.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
- var req2 = new FileEntryItem(OperationType.Put, indexfile.RemoteFilename);
- req2.LocalFilename = indexfile.LocalFilename;
-
- if (m_queue.Enqueue(req2) && m_options.SynchronousUpload)
- {
- req2.WaitForComplete();
- if (req2.Exception != null)
- throw req2.Exception;
- }
-
- if (m_lastException != null)
- throw m_lastException;
+ req2 = new FileEntryItem(OperationType.Put, indexfile.RemoteFilename);
+ req2.LocalTempfile = indexfile.TempFile;
+ req.Indexfile = new Tuple(indexfile, req2);
}
+
+ if (m_queue.Enqueue(req) && m_options.SynchronousUpload)
+ {
+ req.WaitForComplete();
+ if (req.Exception != null)
+ throw req.Exception;
+ }
+
+ if (req2 != null && m_queue.Enqueue(req2) && m_options.SynchronousUpload)
+ {
+ req2.WaitForComplete();
+ if (req2.Exception != null)
+ throw req2.Exception;
+ }
+
+ if (m_lastException != null)
+ throw m_lastException;
}
public Library.Utility.TempFile Get(string remotename, long size, string hash)
diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs
index 44fadc840..79b580db4 100644
--- a/Duplicati/Library/Main/Database/LocalDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalDatabase.cs
@@ -803,6 +803,29 @@ namespace Duplicati.Library.Main.Database
}
}
+ public void RenameRemoteFile(string oldname, string newname, System.Data.IDbTransaction transaction)
+ {
+ using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
+ using(var cmd = m_connection.CreateCommand())
+ {
+ cmd.Transaction = tr.Parent;
+
+ //Rename the old entry, to preserve ID links
+ var c = cmd.ExecuteNonQuery(@"UPDATE ""Remotevolume"" SET ""Name"" = ? WHERE ""Name"" = ?", newname, oldname);
+ if (c != 1)
+ throw new Exception(string.Format("Unexpected result from renaming \"{0}\" to \"{1}\", expected {2} got {3}", oldname, newname, 1, c));
+
+ // Grab the type of entry
+ var type = (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), cmd.ExecuteScalar(@"SELECT ""Type"" FROM ""Remotevolume"" WHERE ""Name"" = ?", newname).ToString(), true);
+
+ //Create a fake new entry with the old name and mark as deleting
+ // as this ensures we will remove it, if it shows up in some later listing
+ RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, tr.Parent);
+
+ tr.Commit();
+ }
+ }
+
///
/// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
///
diff --git a/Duplicati/Library/Main/Enums.cs b/Duplicati/Library/Main/Enums.cs
index 75874dda4..fdba664b5 100644
--- a/Duplicati/Library/Main/Enums.cs
+++ b/Duplicati/Library/Main/Enums.cs
@@ -21,7 +21,8 @@ namespace Duplicati.Library.Main
Progress,
Completed,
Retrying,
- Failed
+ Failed,
+ Rename
}
///
diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs
index 3ba969456..ad702cb33 100644
--- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs
+++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs
@@ -167,6 +167,12 @@ namespace Duplicati.Library.Main.Operation
log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
database.RemoveRemoteVolume(i.Name, null);
}
+ else if (i.State == RemoteVolumeState.Deleting && lookup.ContainsKey(i.Name))
+ {
+ log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
+ backend.Delete(i.Name, true);
+ lookup.Remove(i.Name);
+ }
else
{
Volumes.IParsedVolume r;
diff --git a/Duplicati/Library/Main/Volumes/IndexVolumeReader.cs b/Duplicati/Library/Main/Volumes/IndexVolumeReader.cs
index 144c4fe11..489658332 100644
--- a/Duplicati/Library/Main/Volumes/IndexVolumeReader.cs
+++ b/Duplicati/Library/Main/Volumes/IndexVolumeReader.cs
@@ -14,6 +14,7 @@ namespace Duplicati.Library.Main.Volumes
{
string Hash { get; }
long Length { get; }
+ Stream Data { get; }
IEnumerable Blocklist { get; }
}
@@ -320,6 +321,11 @@ namespace Duplicati.Library.Main.Volumes
{
get { return m_size; }
}
+
+ public Stream Data
+ {
+ get { return m_compression.OpenRead(m_filename); }
+ }
public IEnumerable Blocklist
{
diff --git a/Duplicati/Library/Main/Volumes/IndexVolumeWriter.cs b/Duplicati/Library/Main/Volumes/IndexVolumeWriter.cs
index cea2fa946..98f3e6642 100644
--- a/Duplicati/Library/Main/Volumes/IndexVolumeWriter.cs
+++ b/Duplicati/Library/Main/Volumes/IndexVolumeWriter.cs
@@ -75,6 +75,29 @@ namespace Duplicati.Library.Main.Volumes
s.Write(data, 0, size);
}
+ public void WriteBlocklist(string hash, Stream source)
+ {
+ m_blocklists++;
+ //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications,
+ using (var s = m_compression.CreateFile(INDEX_BLOCKLIST_FOLDER + Library.Utility.Utility.Base64PlainToBase64Url(hash), CompressionHint.Noncompressible, DateTime.UtcNow))
+ Library.Utility.Utility.CopyStream(source, s);
+ }
+
+ public void CopyFrom(IndexVolumeReader rd, Func filename_mapping)
+ {
+ foreach(var n in rd.Volumes)
+ {
+ this.StartVolume(filename_mapping(n.Filename));
+ foreach(var x in n.Blocks)
+ this.AddBlock(x.Key, x.Value);
+ this.FinishVolume(n.Hash, n.Length);
+ }
+
+ foreach(var b in rd.BlockLists)
+ using(var s = b.Data)
+ this.WriteBlocklist(b.Hash, s);
+ }
+
public override void Dispose()
{
if (m_writer != null || m_streamwriter != null)
diff --git a/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs b/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs
index baa4d4e25..61cc51bc5 100644
--- a/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs
+++ b/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs
@@ -14,6 +14,7 @@ namespace Duplicati.Library.Main.Volumes
protected string m_volumename;
public string LocalFilename { get { return m_localfile; } }
public string RemoteFilename { get { return m_volumename; } }
+ public Library.Utility.TempFile TempFile { get { return m_localfile; } }
public abstract RemoteVolumeType FileType { get; }
@@ -29,7 +30,7 @@ namespace Duplicati.Library.Main.Volumes
{
}
- private static string GenerateGuid(Options options)
+ public static string GenerateGuid(Options options)
{
var s = Guid.NewGuid().ToString("N");