Re-implemented the test-filter method with channels such that it calls the same code as the backup process

# Conflicts:
#	Duplicati/Library/Main/BackendManager.cs
#	Duplicati/Library/Main/Database/ExtensionMethods.cs
#	Duplicati/Library/Main/Operation/BackupHandler.cs
#	Duplicati/Library/Main/Operation/TestFilterHandler.cs
#	Duplicati/Library/Main/Options.cs

Manually ported the `IsSynlink` changes as well as the DNS refresh code

.travis.yml

@@ -25,6 +25,7 @@ env:
   general:
     - USAGEREPORTER_Duplicati_LEVEL=none
     - AUTOUPDATER_Duplicati_SKIP_UPDATE=1
+    - TMPDIR=$HOME/tmp
   matrix:
   - JOB=job1
   - JOB=job2
@@ -54,6 +55,7 @@ script:
 # download and extract testdata
   - echo "travis_fold:start:download_extract_testdata"
   - ls -la .
+  - if [ ! -d ~/tmp ]; then mkdir ~/tmp; fi
   - if [ ! -d ~/download ]; then mkdir ~/download; fi
   - if [ ! -f ~/download/DSMCBE.zip ]; then wget "https://s3.amazonaws.com/duplicati-test-file-hosting/DSMCBE.zip" -O ~/download/DSMCBE.zip; fi
   - if [ ! -f ~/download/data.zip ]; then wget "https://s3.amazonaws.com/duplicati-test-file-hosting/data.zip" -O ~/download/data.zip; fi
@@ -94,4 +96,4 @@ script:
 # start server and run gui tests
   - mono ./Duplicati/GUI/Duplicati.GUI.TrayIcon/bin/Release/Duplicati.Server.exe &
   - if [[ "$JOB" == "job3" ]]; then python guiTests/guiTest.py; fi
-  
+  

Duplicati/CommandLine/ConsoleOutput.cs

@@ -78,7 +78,7 @@ namespace Duplicati.CommandLine
                             Output.WriteLine("  Uploading file ({0}) ...", Library.Utility.Utility.FormatSizeString(size));
                             break;
                         case BackendActionType.Get:
-                            Output.WriteLine("  Downloading file ({0}) ...", size < 0 ? "unknown" : Library.Utility.Utility.FormatSizeString(size));
+                            Output.WriteLine("  Downloading file ({0}) ...", size < 0 ? "unknown size" : Library.Utility.Utility.FormatSizeString(size));
                             break;
                         case BackendActionType.List:
                             Output.WriteLine("  Listing remote folder ...");

Duplicati/Library/Main/AsyncDownloader.cs

@@ -11,141 +11,115 @@ namespace Duplicati.Library.Main
         Library.Utility.TempFile TempFile { get; }
     }
 
-    internal class AsyncDownloader : IEnumerable<IAsyncDownloadedFile>
+    internal class AsyncDownloader
     {
-        private class AsyncDownloaderEnumerator : IEnumerator<IAsyncDownloadedFile>
+        private class AsyncDownloadedFile : IAsyncDownloadedFile
         {
-            private class AsyncDownloadedFile : IAsyncDownloadedFile
+            private Exception m_exception;
+            private Library.Utility.TempFile m_file;
+
+            public string Name { get; private set; }
+            public string Hash { get; private set; }
+            public long Size { get; private set; }
+
+            public void DisposeTempFile()
             {
-                private Exception m_exception;
-                private Library.Utility.TempFile m_file;
-                
-                public string Name { get; private set; }
-                public string Hash { get; private set; }
-                public long Size { get; private set; }
-                
-                public void DisposeTempFile()
-                {
-                    if (m_file != null)
-                        try { m_file.Dispose(); }
-                        finally { m_file = null; }
-                }
-                
-
-                public Library.Utility.TempFile TempFile
-                {
-                    get
-                    {
-                        if (m_exception != null)
-                            throw m_exception;
-
-                        return m_file;
-                    }
-                }
-                
-                public AsyncDownloadedFile(string name, string hash, long size, Library.Utility.TempFile tempfile, Exception exception)
-                {
-                    this.Name = name;
-                    this.Hash = hash;
-                    this.Size = size;
-                    m_exception = exception;
-                    m_file = tempfile;
-                }
-            }
-        
-            private IList<IRemoteVolume> m_volumes;
-            private BackendManager.IDownloadWaitHandle m_handle;
-            private BackendManager m_backend;
-            private int m_index;
-            private AsyncDownloadedFile m_current;
-
-            public AsyncDownloaderEnumerator(IList<IRemoteVolume> volumes, BackendManager backend)
-            {
-                m_volumes = volumes;
-                m_backend = backend;
-                m_index = 0;
+                if (m_file != null)
+                    try { m_file.Dispose(); }
+                    finally { m_file = null; }
             }
 
-            public IAsyncDownloadedFile Current
-            {
-                get { return m_current; }
-            }
 
-            public void Dispose()
+            public Library.Utility.TempFile TempFile
             {
-                if (m_current != null)
+                get
                 {
-                    m_current.DisposeTempFile();
-                    m_current = null;
+                    if (m_exception != null)
+                        throw m_exception;
+
+                    return m_file;
                 }
             }
 
-            object System.Collections.IEnumerator.Current
+            public AsyncDownloadedFile(string name, string hash, long size, Library.Utility.TempFile tempfile, Exception exception)
             {
-                get { return this.Current; }
-            }
-
-            public bool MoveNext()
-            {
-                if (m_current != null)
-                {
-                    m_current.DisposeTempFile();
-                    m_current = null;
-                }
-
-                if (m_index >= m_volumes.Count)
-                    return false;
-
-                if (m_handle == null)
-                    m_handle = m_backend.GetAsync(m_volumes[m_index].Name, m_volumes[m_index].Size, m_volumes[m_index].Hash);
-                
-                string hash = null;
-                long size = -1;
-                Library.Utility.TempFile file = null;
-                Exception exception = null;
-                try
-                {
-                    file = m_handle.Wait(out hash, out size);
-                    
-                }
-                catch (Exception ex)
-                {
-                    exception = ex;
-                }
-                
-                m_current = new AsyncDownloadedFile(m_volumes[m_index].Name, hash, size, file, exception);
-                m_handle = null;
-
-                m_index++;
-                if (m_index < m_volumes.Count)
-                    m_handle = m_backend.GetAsync(m_volumes[m_index].Name, m_volumes[m_index].Size, m_volumes[m_index].Hash);
-
-                return true;
-            }
-
-            public void Reset()
-            {
-                throw new NotSupportedException("Cannot reset " + this.GetType().FullName);
+                this.Name = name;
+                this.Hash = hash;
+                this.Size = size;
+                m_exception = exception;
+                m_file = tempfile;
             }
         }
 
         private IList<IRemoteVolume> m_volumes;
+        private BackendManager.IDownloadWaitHandle m_handle;
         private BackendManager m_backend;
+        private int m_index;
+        private AsyncDownloadedFile m_current;
 
         public AsyncDownloader(IList<IRemoteVolume> volumes, BackendManager backend)
         {
             m_volumes = volumes;
             m_backend = backend;
+            m_index = 0;
         }
 
-        public IEnumerator<IAsyncDownloadedFile> GetEnumerator()
+        public IAsyncDownloadedFile Current
         {
-            return new AsyncDownloaderEnumerator(m_volumes, m_backend);
+            get { return m_current; }
         }
 
-        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+        public void Dispose()
         {
-            return this.GetEnumerator();
+            if (m_current != null)
+            {
+                m_current.DisposeTempFile();
+                m_current = null;
+            }
+        }
+
+        public bool MoveNext()
+        {
+            if (m_current != null)
+            {
+                m_current.DisposeTempFile();
+                m_current = null;
+            }
+
+            if (m_index >= m_volumes.Count)
+                return false;
+
+            if (m_handle == null)
+                m_handle = m_backend.GetAsync(m_volumes[m_index].Name, m_volumes[m_index].Size, m_volumes[m_index].Hash);
+
+            string hash = null;
+            long size = -1;
+            Library.Utility.TempFile file = null;
+            Exception exception = null;
+            try
+            {
+                file = m_handle.Wait(out hash, out size);
+
+            }
+            catch (Exception ex)
+            {
+                exception = ex;
+            }
+
+            m_current = new AsyncDownloadedFile(m_volumes[m_index].Name, hash, size, file, exception);
+            m_handle = null;
+
+            m_index++;
+            if (m_index < m_volumes.Count)
+                m_handle = m_backend.GetAsync(m_volumes[m_index].Name, m_volumes[m_index].Size, m_volumes[m_index].Hash);
+
+            return true;
+        }
+
+        public void Reset()
+        {
+            throw new NotSupportedException("Cannot reset " + this.GetType().FullName);
         }
     }
+
 }

Duplicati/Library/Main/Controller.cs

@@ -405,7 +405,7 @@ namespace Duplicati.Library.Main
             {
                 using (var tf = System.IO.File.Exists(m_options.Dbpath) ? null : new Library.Utility.TempFile())
                 using (var db = new Database.LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true))
-                using (var bk = new BackendManager(m_backend, m_options, result.BackendWriter, null))
+                using (var bk = new BackendManager(m_backend, m_options, result.BackendWriter, null, result))
                     result.SetResult(bk.List());
             });
         }
@@ -417,7 +417,7 @@ namespace Duplicati.Library.Main
                 result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Listing);
                 using (var tf = System.IO.File.Exists(m_options.Dbpath) ? null : new Library.Utility.TempFile())
                 using (var db = new Database.LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true))
-                using (var bk = new BackendManager(m_backend, m_options, result.BackendWriter, null))
+                using (var bk = new BackendManager(m_backend, m_options, result.BackendWriter, null, result))
                 {
                     // Only delete files that match the expected pattern and prefix
                     var list = bk.List()
@@ -617,22 +617,40 @@ namespace Duplicati.Library.Main
                 {
                     m_currentTask = result;
                     m_currentTaskThread = System.Threading.Thread.CurrentThread;
-                    SetupCommonOptions(result, ref paths, ref filter);
+                    using(new Logging.Timer(string.Format("Running {0}", result.MainOperation)))
+                    using(new CoCoL.IsolatedChannelScope())
+                    using(new Operation.Common.LogWrapper())                        
+                    using(m_options.ConcurrencyMaxThreads <= 0 ? null : new CoCoL.CappedThreadedThreadPool(m_options.ConcurrencyMaxThreads))
+                    {
+                        // Start the log handler
+                        var lh = Operation.Common.LogHandler.Run(result);
 
-                    result.WriteLogMessageDirect(Strings.Controller.StartingOperationMessage(m_options.MainAction), Logging.LogMessageType.Information, null);
+                        try
+                        {
+                            m_currentTask = result;
+                            m_currentTaskThread = System.Threading.Thread.CurrentThread;
+                            SetupCommonOptions(result, ref paths, ref filter);
 
-                    using (new Logging.Timer(string.Format("Running {0}", result.MainOperation)))
-                        method(result);
+                            result.WriteLogMessageDirect(Strings.Controller.StartingOperationMessage(m_options.MainAction), Logging.LogMessageType.Information, null);
 
-                    if (result.EndTime.Ticks == 0)
-                        result.EndTime = DateTime.UtcNow;
-                    result.SetDatabase(null);
+                            using (new Logging.Timer(string.Format("Running {0}", result.MainOperation)))
+                                method(result);
 
-                    OnOperationComplete(result);
+                            if (result.EndTime.Ticks == 0)
+                                result.EndTime = DateTime.UtcNow;
+                            result.SetDatabase(null);
 
-                    result.WriteLogMessageDirect(Strings.Controller.CompletedOperationMessage(m_options.MainAction), Logging.LogMessageType.Information, null);
+                            OnOperationComplete(result);
 
-                    return result;
+                            result.WriteLogMessageDirect(Strings.Controller.CompletedOperationMessage(m_options.MainAction), Logging.LogMessageType.Information, null);
+
+                            return result;
+                        }
+                        finally
+                        {
+                            lh.Wait(500);
+                        }
+                    }
                 }
                 catch (Exception ex)
                 {

Duplicati/Library/Main/Database/ExtensionMethods.cs

@@ -33,6 +33,30 @@ namespace Duplicati.Library.Main.Database
             ((System.Data.IDataParameter)self.Parameters[index]).Value = value;
         }
 
+        public static string GetPrintableCommandText(this System.Data.IDbCommand self)
+        {
+            var txt = self.CommandText;
+#if DEBUG
+            foreach(var p in self.Parameters.Cast<System.Data.IDbDataParameter>())
+            {
+                var ix = txt.IndexOf('?');
+                if (ix >= 0)
+                {
+                    string v;
+                    if (p.Value is string)
+                        v = string.Format("\"{0}\"", p.Value);
+                    else if (p.Value == null)
+                        v = "NULL";
+                    else
+                        v = string.Format("{0}", p.Value);
+
+                    txt = txt.Substring(0, ix) + v + txt.Substring(ix + 1);
+                }
+            }
+#endif
+            return txt;
+        }
+
         public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values)
         {
             if (cmd != null)
@@ -45,7 +69,7 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(string.Format("ExecuteNonQuery: {0}", self.CommandText)))
+            using(new Logging.Timer(string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())))
                 return self.ExecuteNonQuery();
         }
 
@@ -61,7 +85,7 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(string.Format("ExecuteScalar: {0}", self.CommandText)))
+            using(new Logging.Timer(string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())))
                 return self.ExecuteScalar();
         }
 
@@ -87,7 +111,7 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(string.Format("ExecuteScalarInt64: {0}", self.CommandText)))
+            using(new Logging.Timer(string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())))
                 using(var rd = self.ExecuteReader())
                     if (rd.Read())
                         return ConvertValueToInt64(rd, 0, defaultvalue);
@@ -107,7 +131,7 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(string.Format("ExecuteReader: {0}", self.CommandText)))
+            using(new Logging.Timer(string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())))
                 return self.ExecuteReader();
         }
 
@@ -197,7 +221,7 @@ namespace Duplicati.Library.Main.Database
                         Console.WriteLine();
                         n++;
                     }
-                    Console.WriteLine("{0} records", n);
+                    Console.WriteLine(string.Format("{0} records", n));
                 }
             }
         }

Duplicati/Library/Main/Database/LocalBackupDatabase.cs

@@ -246,37 +246,52 @@ namespace Duplicati.Library.Main.Database
                 var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
                 if (tc > 0)
                     throw new InvalidDataException("Detected blocks that are not reachable in the block table");
-            }
-
-			if (options.UseBlockCache)
+            }
+
+			if (options.UseBlockCache)
 			{
                 string failedhash = null;
                 try
-                {
-                    var cache = new Dictionary<string, long>();
-                    using (var cmd = m_connection.CreateCommand())
-                    {
-                        cmd.CommandText = @"SELECT ""Hash"", ""Size"" From ""Block""";
-                        using (var rd = cmd.ExecuteReader())
-                            while (rd.Read())
-                                cache.Add(failedhash = rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
+                {
+                    var cache = new Dictionary<string, long>();
+                    using (var cmd = m_connection.CreateCommand())
+                    {
+                        cmd.CommandText = @"SELECT ""Hash"", ""Size"" From ""Block""";
+                        using (var rd = cmd.ExecuteReader())
+                            while (rd.Read())
+                                cache.Add(failedhash = rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
                     }
                     m_blockCache = cache;
                 }
                 catch (Exception ex)
                 {
-                    Logging.Log.WriteMessage(string.Format("Failed to create block cache, this could mean you have hash collisions in your table, the hash that failed is {0}. Error message: {1}.", failedhash, ex.Message), Logging.LogMessageType.Warning);
-					Logging.Log.WriteMessage(string.Format("Disabling block cache due to error"), Logging.LogMessageType.Warning);
-				}
-			}
+                    Logging.Log.WriteMessage(string.Format("Failed to create block cache, this could mean you have hash collisions in your table, the hash that failed is {0}. Error message: {1}.", failedhash, ex.Message), Logging.LogMessageType.Warning);
+					Logging.Log.WriteMessage(string.Format("Disabling block cache due to error"), Logging.LogMessageType.Warning);
+				}
+			}
 		}
 
         /// <summary>
-        /// Adds a block to the local database, returning a value indicating if the value presents a new block
+        /// Probes to see if a block already exists
         /// </summary>
         /// <param name="key">The block key</param>
+        /// <param name="size">The size of the block</param>
         /// <returns>True if the block should be added to the current output</returns>
-        public bool AddBlock (string key, long size, long volumeid, System.Data.IDbTransaction transaction = null)
+        public long FindBlockID (string key, long size)
+        {
+            m_findblockCommand.Transaction = Transaction;
+            m_findblockCommand.SetParameterValue(0, key);
+            m_findblockCommand.SetParameterValue(1, size);
+            return m_findblockCommand.ExecuteScalarInt64(-1);
+        }
+
+		/// <summary>
+		/// Adds a block to the local database, returning a value indicating if the value presents a new block
+		/// </summary>
+		/// <param name="key">The block key</param>
+		/// <param name="size">The size of the block</param>
+		/// <returns>True if the block should be added to the current output</returns>
+		public bool AddBlock (string key, long size, long volumeid)
         {
             long exsize;
 
@@ -289,14 +304,14 @@ namespace Duplicati.Library.Main.Database
                 m_blockCache = null;
             }
 
-            m_findblockCommand.Transaction = transaction;
+            m_findblockCommand.Transaction = Transaction;
             m_findblockCommand.SetParameterValue(0, key);
             m_findblockCommand.SetParameterValue(1, size);
             var r = m_findblockCommand.ExecuteScalarInt64(-1);
 
             if (r == -1L)
             {
-                m_insertblockCommand.Transaction = transaction;
+                m_insertblockCommand.Transaction = Transaction;
                 m_insertblockCommand.SetParameterValue(0, key);
                 m_insertblockCommand.SetParameterValue(1, volumeid);
                 m_insertblockCommand.SetParameterValue(2, size);
@@ -321,108 +336,105 @@ namespace Duplicati.Library.Main.Database
         /// <param name="hashes">The list of hashes</param>
         /// <param name="blocksetid">The id of the blockset, new or old</param>
         /// <returns>True if the blockset was created, false otherwise</returns>
-        public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes, out long blocksetid, System.Data.IDbTransaction transaction = null)
+        public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes, out long blocksetid)
         {
-            m_findblocksetCommand.Transaction = transaction;
+            m_findblocksetCommand.Transaction = Transaction;
             blocksetid = m_findblocksetCommand.ExecuteScalarInt64(null, -1, filehash, size);
             if (blocksetid != -1)
                 return false; //Found it
 
-            using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
+            m_insertblocksetCommand.Transaction = Transaction;
+            m_insertblocksetCommand.SetParameterValue(0, size);
+            m_insertblocksetCommand.SetParameterValue(1, filehash);
+            blocksetid = m_insertblocksetCommand.ExecuteScalarInt64();
+
+            long ix = 0;
+            if (blocklistHashes != null)
             {
-                m_insertblocksetCommand.Transaction = tr.Parent;
-                m_insertblocksetCommand.SetParameterValue(0, size);
-                m_insertblocksetCommand.SetParameterValue(1, filehash);
-                blocksetid = m_insertblocksetCommand.ExecuteScalarInt64();
-
-                long ix = 0;
-                if (blocklistHashes != null)
+                m_insertblocklistHashesCommand.SetParameterValue(0, blocksetid);
+                m_insertblocklistHashesCommand.Transaction = Transaction;
+                foreach(var bh in blocklistHashes)
                 {
-                    m_insertblocklistHashesCommand.SetParameterValue(0, blocksetid);
-                    m_insertblocklistHashesCommand.Transaction = tr.Parent;
-                    foreach(var bh in blocklistHashes)
-                    {
-                        m_insertblocklistHashesCommand.SetParameterValue(1, ix);
-                        m_insertblocklistHashesCommand.SetParameterValue(2, bh);
-                        m_insertblocklistHashesCommand.ExecuteNonQuery();
-                        ix++;
-                    }
-                }
-
-                m_insertblocksetentryCommand.SetParameterValue(0, blocksetid);
-                m_insertblocksetentryCommand.Transaction = tr.Parent;
-
-                m_insertblocksetentryFastCommand.SetParameterValue(0, blocksetid);
-                m_insertblocksetentryFastCommand.Transaction = tr.Parent;
-
-                ix = 0;
-                long remainsize = size;
-                foreach(var h in hashes)
-                {
-                    var exsize = remainsize < blocksize ? remainsize : blocksize;
-                    m_insertblocksetentryCommand.SetParameterValue(1, ix);
-                    m_insertblocksetentryCommand.SetParameterValue(2, h);
-                    m_insertblocksetentryCommand.SetParameterValue(3, exsize);
-                    var c = m_insertblocksetentryCommand.ExecuteNonQuery();
-                    if (c != 1)
-                    {
-                        m_result.AddError(string.Format("Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size), null);
-                        using (var cmd = m_connection.CreateCommand(tr.Parent))
-                        {
-                            var bid = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ?", -1, h);
-                            if (bid == -1)
-                                throw new Exception(string.Format("Could not find any blocks with the given hash: {0}", h));
-                            foreach(var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = ?", h))
-                                m_result.AddError(string.Format("Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)), null);
-                        }
-
-                        throw new Exception(string.Format("Unexpected result count: {0}, expected {1}, check log for more messages", c, 1));
-                    }
-                    
+                    m_insertblocklistHashesCommand.SetParameterValue(1, ix);
+                    m_insertblocklistHashesCommand.SetParameterValue(2, bh);
+                    m_insertblocklistHashesCommand.ExecuteNonQuery();
                     ix++;
-                    remainsize -= blocksize;
                 }
+            }
 
-                tr.Commit();
+            m_insertblocksetentryCommand.SetParameterValue(0, blocksetid);
+            m_insertblocksetentryCommand.Transaction = Transaction;
+
+            m_insertblocksetentryFastCommand.SetParameterValue(0, blocksetid);
+            m_insertblocksetentryFastCommand.Transaction = Transaction;
+
+            ix = 0;
+            long remainsize = size;
+            foreach(var h in hashes)
+            {
+                var exsize = remainsize < blocksize ? remainsize : blocksize;
+                m_insertblocksetentryCommand.SetParameterValue(1, ix);
+                m_insertblocksetentryCommand.SetParameterValue(2, h);
+                m_insertblocksetentryCommand.SetParameterValue(3, exsize);
+                var c = m_insertblocksetentryCommand.ExecuteNonQuery();
+                if (c != 1)
+                {
+                    m_result.AddError(string.Format("Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size), null);
+                    using (var cmd = m_connection.CreateCommand(Transaction))
+                    {
+                        var bid = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ?", -1, h);
+                        if (bid == -1)
+                            throw new Exception(string.Format("Could not find any blocks with the given hash: {0}", h));
+                        foreach(var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = ?", h))
+                            m_result.AddError(string.Format("Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)), null);
+                    }
+
+                    throw new Exception(string.Format("Unexpected result count: {0}, expected {1}, check log for more messages", c, 1));
+                }
+                
+                ix++;
+                remainsize -= blocksize;
             }
 
             return true;
+        }
 
+        /// <summary>
+        /// Gets the metadataset ID from the filehash
+        /// </summary>
+        /// <returns><c>true</c>, if metadataset should be recorded, false if it already exists.</returns>
+        /// <param name="filehash">The metadata hash.</param>
+        /// <param name="size">The size of the metadata.</param>
+        /// <param name="metadataid">The ID of the metadataset.</param>
+        /// <param name="transaction">An optional transaction.</param>
+        public bool GetMetadatasetID(string filehash, long size, out long metadataid)
+        {
+            if (size > 0)
+            {
+                m_findmetadatasetCommand.Transaction = Transaction;
+                metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size);
+                return metadataid != -1;
+            }
+
+            metadataid = -2;
+            return false;
         }
 
         /// <summary>
         /// Adds a metadata set to the database, and returns a value indicating if the record was new
         /// </summary>
-        /// <param name="filehash">The metadata hash</param>
+        /// <param name="hash">The metadata hash</param>
         /// <param name="metadataid">The id of the metadata set</param>
         /// <returns>True if the set was added to the database, false otherwise</returns>
-        public bool AddMetadataset(string filehash, long size, int blocksize, IEnumerable<string> blockhashes, IEnumerable<string> blocklisthashes, out long metadataid, System.Data.IDbTransaction transaction = null)
+        public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid)
         {
-            if (size > 0)
-            {
-                m_findmetadatasetCommand.Transaction = transaction;
-                metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size);
-                if (metadataid != -1)
-                    return false;
-            
-
-                long blocksetid;
-                AddBlockset(filehash, size, blocksize, blockhashes, blocklisthashes, out blocksetid, transaction);
-
-                using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
-                {
-                    m_insertmetadatasetCommand.Transaction = tr.Parent;
-                    m_insertmetadatasetCommand.SetParameterValue(0, blocksetid);
-                    metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64();
-                    tr.Commit();
-                }
-
-                return true;
-            }
-
-            metadataid = -2;
-            return false;
+            if (GetMetadatasetID(filehash, size, out metadataid))
+                return false;
 
+            m_insertmetadatasetCommand.Transaction = Transaction;
+            m_insertmetadatasetCommand.SetParameterValue(0, blocksetid);
+            metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64();
+            return true;
         }
 
         /// <summary>
@@ -433,7 +445,7 @@ namespace Duplicati.Library.Main.Database
         /// <param name="blocksetID">The ID of the hashkey for the file</param>
         /// <param name="metadataID">The ID for the metadata</param>
         /// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
-        public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, System.Data.IDbTransaction transaction)
+        public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID)
         {            
             var fileidobj = -1L;
             PathEntryKeeper entry = null;
@@ -450,7 +462,7 @@ namespace Duplicati.Library.Main.Database
             }
             else
             {
-                m_findfilesetCommand.Transaction = transaction;
+                m_findfilesetCommand.Transaction = Transaction;
                 m_findfilesetCommand.SetParameterValue(0, blocksetID);
                 m_findfilesetCommand.SetParameterValue(1, metadataID);
                 m_findfilesetCommand.SetParameterValue(2, filename);
@@ -459,31 +471,27 @@ namespace Duplicati.Library.Main.Database
             
             if (fileidobj == -1)
             {
-                using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
-                {
-                    m_insertfileCommand.Transaction = tr.Parent;
-                    m_insertfileCommand.SetParameterValue(0, filename);
-                    m_insertfileCommand.SetParameterValue(1, blocksetID);
-                    m_insertfileCommand.SetParameterValue(2, metadataID);
-                    fileidobj = m_insertfileCommand.ExecuteScalarInt64();
-                    tr.Commit();                    
+                m_insertfileCommand.Transaction = Transaction;
+                m_insertfileCommand.SetParameterValue(0, filename);
+                m_insertfileCommand.SetParameterValue(1, blocksetID);
+                m_insertfileCommand.SetParameterValue(2, metadataID);
+                fileidobj = m_insertfileCommand.ExecuteScalarInt64();
 
-                    // We do not need to update this, because we will not ask for the same file twice
-                    if (m_pathLookup != null)
+                // We do not need to update this, because we will not ask for the same file twice
+                if (m_pathLookup != null)
+                {
+                    if (!entryFound)
                     {
-                        if (!entryFound)
-                        {
-                            entry = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
-                            entry.AddFilesetID(blocksetID, metadataID, fileidobj);
-                            m_pathLookup.Insert(filename, entry);
-                        }
-                        else
-                            entry.AddFilesetID(blocksetID, metadataID, fileidobj);
+                        entry = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
+                        entry.AddFilesetID(blocksetID, metadataID, fileidobj);
+                        m_pathLookup.Insert(filename, entry);
                     }
+                    else
+                        entry.AddFilesetID(blocksetID, metadataID, fileidobj);
                 }
             }
-            
-            m_insertfileOperationCommand.Transaction = transaction;
+
+            m_insertfileOperationCommand.Transaction = Transaction;
             m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
             m_insertfileOperationCommand.SetParameterValue(1, fileidobj);
             m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
@@ -491,28 +499,29 @@ namespace Duplicati.Library.Main.Database
 
         }
 
-        public void AddUnmodifiedFile(long fileid, DateTime lastmodified, System.Data.IDbTransaction transaction = null)
+        public void AddUnmodifiedFile(long fileid, DateTime lastmodified)
         {
-            m_insertfileOperationCommand.Transaction = transaction;
+            m_insertfileOperationCommand.Transaction = Transaction;
             m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
             m_insertfileOperationCommand.SetParameterValue(1, fileid);
             m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
             m_insertfileOperationCommand.ExecuteNonQuery();
         }
 
-        public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, System.Data.IDbTransaction transaction = null)
+        public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified)
         {
-            AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID, transaction);
+            AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID);
         }
         
-        public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, System.Data.IDbTransaction transaction = null)
+        public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified)
         {
-            AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction);
+            AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID);
         }
 
         public long GetFileLastModified(string path, long filesetid, out DateTime oldModified)
         {
-            m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
+			m_selectfileHashCommand.Transaction = Transaction;
+			m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
             m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid);
             using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader())
                 if (rd.Read())
@@ -580,11 +589,6 @@ namespace Duplicati.Library.Main.Database
                 
             return r.ToString();
         }
-        
-        public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, System.Data.IDbTransaction transaction)
-        {
-            WriteFileset(filesetvolume, transaction, m_filesetId);
-        }        
 
         public override void Dispose ()
         {
@@ -618,7 +622,7 @@ namespace Duplicati.Library.Main.Database
 
         internal void UpdateChangeStatistics(BackupResults results)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var lastFilesetId = GetPreviousFilesetID(cmd);
                 results.AddedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, FOLDER_BLOCKSET_ID, lastFilesetId);
@@ -655,25 +659,22 @@ namespace Duplicati.Library.Main.Database
             }
         }
 
-        public void AppendFilesFromPreviousSet(System.Data.IDbTransaction transaction, IEnumerable<string> deleted)
+        public void AppendFilesFromPreviousSet(IEnumerable<string> deleted)
         {
-            AppendFilesFromPreviousSet(transaction, deleted, m_filesetId, -1, OperationTimestamp);
+            AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp);
         }
 
-        public void AppendFilesFromPreviousSet(System.Data.IDbTransaction transaction, IEnumerable<string> deleted, long filesetid, long prevId, DateTime timestamp)
+        public void AppendFilesFromPreviousSet(IEnumerable<string> deleted, long filesetid, long prevId, DateTime timestamp)
         {
-            using(var cmd = m_connection.CreateCommand())
-            using(var cmdDelete = m_connection.CreateCommand())
-            using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
+            using(var cmd = m_connection.CreateCommand(Transaction))
+            using(var cmdDelete = m_connection.CreateCommand(Transaction))
             {
                 long lastFilesetId = prevId < 0 ? GetPreviousFilesetID(cmd, timestamp, filesetid) : prevId;
 
-                cmd.Transaction = tr.Parent;
                 cmd.ExecuteNonQuery( @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT ? AS ""FilesetID"", ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)) ", filesetid, lastFilesetId, filesetid);
 
                 if (deleted != null)
                 {
-                    cmdDelete.Transaction = tr.Parent;
                     cmdDelete.CommandText = @"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) ";
                     cmdDelete.AddParameters(2);
                     cmdDelete.SetParameterValue(0, filesetid);
@@ -684,8 +685,6 @@ namespace Duplicati.Library.Main.Database
                         cmdDelete.ExecuteNonQuery();
                     }
                 }
-
-                tr.Commit();
             }
         }
     
@@ -695,16 +694,15 @@ namespace Duplicati.Library.Main.Database
         /// <param name="volumeid">The ID of the fileset volume to update</param>
         /// <param name="timestamp">The timestamp of the operation to create</param>
         /// <param name="transaction">An optional external transaction</param>
-        public override long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null)
+        public override long CreateFileset(long volumeid, DateTime timestamp)
         {
-            return m_filesetId = base.CreateFileset(volumeid, timestamp, transaction);
+            return m_filesetId = base.CreateFileset(volumeid, timestamp);
         }
                                 
-        public IEnumerable<KeyValuePair<long, DateTime>> GetIncompleteFilesets(System.Data.IDbTransaction transaction)
+        public IEnumerable<KeyValuePair<long, DateTime>> GetIncompleteFilesets()
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
-                cmd.Transaction = transaction;
                 using(var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"")  AND (""RemoteVolume"".""State"" = ""Uploading"" OR ""RemoteVolume"".""State"" = ""Temporary"")"))
                     while(rd.Read())
                     {
@@ -718,7 +716,7 @@ namespace Duplicati.Library.Main.Database
 
         public IRemoteVolume GetRemoteVolumeFromName(string name)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             using(var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Name"" = ?", name))
                 if (rd.Read())
                     return new RemoteVolume(rd.GetValue(0).ToString(), rd.GetValue(1).ToString(), rd.ConvertValueToInt64(2));
@@ -728,10 +726,11 @@ namespace Duplicati.Library.Main.Database
 
         public RemoteVolumeEntry GetRemoteVolumeFromID(long id)
         {
-            using (var cmd = m_connection.CreateCommand())
+            using (var cmd = m_connection.CreateCommand(Transaction))
             using (var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""RemoteVolume"" WHERE ""ID"" = ?", id))
                 if (rd.Read())
                     return new RemoteVolumeEntry(
+                        id,
                         rd.GetValue(0).ToString(),
                         (rd.GetValue(3) == null || rd.GetValue(3) == DBNull.Value) ? null : rd.GetValue(3).ToString(),
                         rd.ConvertValueToInt64(2, -1),
@@ -745,26 +744,62 @@ namespace Duplicati.Library.Main.Database
 
         public IEnumerable<string> GetMissingIndexFiles()
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             using(var rd = cmd.ExecuteReader(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND NOT ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"") AND ""State"" IN (?,?)", RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()))
                 while (rd.Read())
                     yield return rd.GetValue(0).ToString();
         }
         
-        public void LinkFilesetToVolume(long filesetid, long volumeid, System.Data.IDbTransaction transaction)
+        public void LinkFilesetToVolume(long filesetid, long volumeid)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
-                cmd.Transaction = transaction;
                 var c = cmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""VolumeID"" = ? WHERE ""ID"" = ?", volumeid, filesetid);
                 if (c != 1)
                     throw new Exception(string.Format("Failed to link filesetid {0} to volumeid {1}", filesetid, volumeid));
             }            
         }
 
+        public void MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid)
+        {
+            using(var cmd = m_connection.CreateCommand(Transaction))
+            {
+                var c = cmd.ExecuteNonQuery(@"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""VolumeID"" = ? ", targetvolumeid, blockkey, size, sourcevolumeid);
+                if (c != 1)
+                    throw new Exception(string.Format("Failed to move block {0}:{1} from volume {2}, count: {3}", blockkey, size, sourcevolumeid, c));
+            }
+        }
+
+        public void SafeDeleteRemoteVolume(string name)
+        {
+            var volumeid = GetRemoteVolumeID(name);
+
+            using(var cmd = m_connection.CreateCommand(Transaction))
+            {
+                var c = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = ? ", -1, volumeid);
+                if (c != 0)
+                    throw new Exception(string.Format("Failed to safe-delete volume {0}, blocks: {1}", name, c));
+
+                RemoveRemoteVolume(name);
+            }            
+        }
+
+        public IEnumerable<string> GetBlocklistHashes(string name)
+        {
+            var volumeid = GetRemoteVolumeID(name);
+            using(var cmd = m_connection.CreateCommand(Transaction))
+            {
+                // Grab the strings and return as array to avoid concurrent access to the IEnumerable
+                return cmd.ExecuteReaderEnumerable(
+                    @"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE ""Block"".""VolumeID"" = ? AND ""Block"".""Hash"" IN (SELECT ""Hash"" FROM ""BlocklistHash"")", volumeid)
+                    .Select(x => x.ConvertValueToString(0))
+                    .ToArray();
+            }            
+        }
+
         public string GetFirstPath()
         {
-            using (var cmd = m_connection.CreateCommand())
+            using (var cmd = m_connection.CreateCommand(Transaction))
             {
                 cmd.CommandText = string.Format(@"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1");
                 var v0 = cmd.ExecuteScalar();
@@ -774,6 +809,5 @@ namespace Duplicati.Library.Main.Database
                 return v0.ToString();
             }
         }
-
     }
 }

Duplicati/Library/Main/Database/LocalBugReportDatabase.cs

@@ -29,16 +29,12 @@ namespace Duplicati.Library.Main.Database
 
         public void Fix()
         {
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
-                cmd.Transaction = tr;
                 var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                 
-                using(var upcmd = m_connection.CreateCommand())
+                using(var upcmd = m_connection.CreateCommand(Transaction))
                 {
-                
-                    upcmd.Transaction = tr;
                     upcmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL)", tablename));
                     upcmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" ", tablename));
                     upcmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Obfuscated"" = ? || length(""RealPath"") || ? || ""ID"" || (CASE WHEN substr(""RealPath"", length(""RealPath"")) = ? THEN ? ELSE ? END) ", tablename), Library.Utility.Utility.IsClientLinux ? "/" : "X:\\", Library.Utility.Utility.DirectorySeparatorString, Library.Utility.Utility.DirectorySeparatorString, Library.Utility.Utility.DirectorySeparatorString, ".bin");
@@ -58,11 +54,8 @@ namespace Duplicati.Library.Main.Database
                 cmd.ExecuteNonQuery(string.Format(@"UPDATE ""File"" SET ""Path"" = (SELECT ""Obfuscated"" FROM ""{0}"" WHERE ""Path"" = ""RealPath"") ", tablename));
                 
                 cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", tablename));
-                
-                using(new Logging.Timer("CommitUpdateBugReport"))
-                    tr.Commit();
-                
-                cmd.Transaction = null;
+
+                CommitTransaction("CommitUpdateBugReport", false);
             }
         }
     }

Duplicati/Library/Main/Database/LocalDatabase.cs

@@ -36,6 +36,11 @@ namespace Duplicati.Library.Main.Database
 
         public bool ShouldCloseConnection { get; set; }
 
+        private System.Data.IDbTransaction m_transaction;
+        private LocalDatabase m_parent;
+
+        internal System.Data.IDbTransaction Transaction { get { return m_parent?.Transaction ?? m_transaction; } }
+
         protected static System.Data.IDbConnection CreateConnection(string path)
         {
             path = System.IO.Path.GetFullPath(path);
@@ -58,6 +63,7 @@ namespace Duplicati.Library.Main.Database
             : this(CreateConnection(path), operation)
         {
             ShouldCloseConnection = shouldclose;
+            this.m_transaction = m_connection.BeginTransaction();
         }
 
         /// <summary>
@@ -70,13 +76,14 @@ namespace Duplicati.Library.Main.Database
             this.m_connection = db.m_connection;
             this.m_operationid = db.m_operationid;
             this.m_result = db.m_result;
+            this.m_parent = db;
         }
         
         /// <summary>
         /// Creates a new database instance and starts a new operation
         /// </summary>
         /// <param name="operation">The name of the operation</param>
-        public LocalDatabase(System.Data.IDbConnection connection, string operation)
+        private LocalDatabase(System.Data.IDbConnection connection, string operation)
             : this(connection)
         {
             this.OperationTimestamp = DateTime.UtcNow;
@@ -111,11 +118,11 @@ namespace Duplicati.Library.Main.Database
             m_updateremotevolumeCommand.CommandText = @"UPDATE ""Remotevolume"" SET ""OperationID"" = ?, ""State"" = ?, ""Hash"" = ?, ""Size"" = ? WHERE ""Name"" = ?";
             m_updateremotevolumeCommand.AddParameters(5);
 
-            m_selectremotevolumesCommand.CommandText = @"SELECT ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""Remotevolume""";
+            m_selectremotevolumesCommand.CommandText = @"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""Remotevolume""";
 
+            m_selectremotevolumeCommand.CommandText = m_selectremotevolumesCommand.CommandText + @" WHERE ""Name"" = ?";
             m_selectduplicateRemoteVolumesCommand.CommandText = string.Format(@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN (""{0}"", ""{1}"")) AND NOT ""State"" IN (""{0}"", ""{1}"")", RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString());
 
-            m_selectremotevolumeCommand.CommandText = @"SELECT ""Type"", ""Size"", ""Hash"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" = ?";
             m_selectremotevolumeCommand.AddParameter();
 
             m_removeremotevolumeCommand.CommandText = @"DELETE FROM ""Remotevolume"" WHERE ""Name"" = ? AND (""DeleteGraceTime"" < ? OR ""State"" != ?)";
@@ -160,19 +167,19 @@ namespace Duplicati.Library.Main.Database
             return Library.Utility.Utility.EPOCH.AddSeconds(seconds);
         }
 
-        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, System.Data.IDbTransaction transaction = null)
+        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash)
         { 
-            UpdateRemoteVolume(name, state, size, hash, false, transaction); 
+            UpdateRemoteVolume(name, state, size, hash, false); 
         }
 
-        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, System.Data.IDbTransaction transaction = null)
+        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup)
         {
-            UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), transaction); 
+            UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0)); 
         }
 
-        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, TimeSpan deleteGraceTime, System.Data.IDbTransaction transaction = null)
+        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, TimeSpan deleteGraceTime)
         {
-            m_updateremotevolumeCommand.Transaction = transaction;
+            m_updateremotevolumeCommand.Transaction = Transaction;
             m_updateremotevolumeCommand.SetParameterValue(0, m_operationid);
             m_updateremotevolumeCommand.SetParameterValue(1, state.ToString());
             m_updateremotevolumeCommand.SetParameterValue(2, hash);
@@ -183,20 +190,20 @@ namespace Duplicati.Library.Main.Database
                 throw new Exception(string.Format("Unexpected number of remote volumes detected: {0}!", c));
 
             if (deleteGraceTime.Ticks > 0)
-                using(var cmd = m_connection.CreateCommand(transaction))
+                using(var cmd = m_connection.CreateCommand(Transaction))
                     if ((c = cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = ? WHERE ""Name"" = ? ", (DateTime.UtcNow + deleteGraceTime).Ticks, name)) != 1)
                         throw new Exception(string.Format("Unexpected number of updates when recording remote volume updates: {0}!", c));
 
 
             if (!suppressCleanup && state == RemoteVolumeState.Deleted)
-                   RemoveRemoteVolume(name, transaction);
+                   RemoveRemoteVolume(name);
         }
         
         public IEnumerable<KeyValuePair<long, DateTime>> FilesetTimes
         { 
             get 
             {
-                using(var cmd = m_connection.CreateCommand())
+                using(var cmd = m_connection.CreateCommand(Transaction))
                 using(var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"))
                     while (rd.Read())
                         yield return new KeyValuePair<long, DateTime>(rd.GetInt64(0), ParseFromEpochSeconds(rd.GetInt64(1)).ToLocalTime());
@@ -255,34 +262,34 @@ namespace Duplicati.Library.Main.Database
             return new Tuple<string, object[]>(query, args.ToArray());
         }
 
-        public long GetRemoteVolumeID(string file, System.Data.IDbTransaction transaction = null)
+        public long GetRemoteVolumeID(string file)
         {
-            m_selectremotevolumeIdCommand.Transaction = transaction;
+            m_selectremotevolumeIdCommand.Transaction = Transaction;
             return m_selectremotevolumeIdCommand.ExecuteScalarInt64(null, -1, file);
         }
 
-        public bool GetRemoteVolume(string file, out string hash, out long size, out RemoteVolumeType type, out RemoteVolumeState state)
+        public RemoteVolumeEntry GetRemoteVolume(string file)
         {
+            m_selectremotevolumeCommand.Transaction = Transaction;
             m_selectremotevolumeCommand.SetParameterValue(0, file);
-            using (var rd = m_selectremotevolumeCommand.ExecuteReader())
+            using(var rd = m_selectremotevolumeCommand.ExecuteReader())
                 if (rd.Read())
-                {
-                    type = (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(0).ToString());
-                    size = (rd.GetValue(1) == null || rd.GetValue(1) == DBNull.Value) ? -1 : rd.GetInt64(1);
-                    hash = (rd.GetValue(2) == null || rd.GetValue(2) == DBNull.Value) ? null : rd.GetValue(2).ToString();
-                    state = (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(3).ToString());
-                    return true;
-                }
-
-            hash = null;
-            size = -1;
-            type = (RemoteVolumeType)(-1);
-            state = (RemoteVolumeState)(-1);
-            return false;
+                    return new RemoteVolumeEntry(
+                        rd.ConvertValueToInt64(0),
+                        rd.GetValue(1).ToString(),
+                        (rd.GetValue(4) == null || rd.GetValue(4) == DBNull.Value) ? null : rd.GetValue(4).ToString(),
+                        rd.ConvertValueToInt64(3, -1),
+                        (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(2).ToString()),
+                        (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(5).ToString()),
+                        new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc)
+                    );
+            
+            return RemoteVolumeEntry.Empty;
         }
 
         public IEnumerable<KeyValuePair<string, RemoteVolumeState>> DuplicateRemoteVolumes()
         {
+            m_selectduplicateRemoteVolumesCommand.Transaction = Transaction;
             foreach(var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerable(null))
             {
                 yield return new KeyValuePair<string, RemoteVolumeState>(
@@ -294,17 +301,19 @@ namespace Duplicati.Library.Main.Database
 
         public IEnumerable<RemoteVolumeEntry> GetRemoteVolumes()
         {
+            m_selectremotevolumesCommand.Transaction = Transaction;
             using (var rd = m_selectremotevolumesCommand.ExecuteReader())
             {
                 while (rd.Read())
                 {
                     yield return new RemoteVolumeEntry(
-                        rd.GetValue(0).ToString(),
-                        (rd.GetValue(3) == null || rd.GetValue(3) == DBNull.Value) ? null : rd.GetValue(3).ToString(),
-                        rd.ConvertValueToInt64(2, -1),
-                        (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(1).ToString()),
-                        (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(4).ToString()),
-                        new DateTime(rd.ConvertValueToInt64(5, 0), DateTimeKind.Utc)
+                        rd.ConvertValueToInt64(0),
+                        rd.GetValue(1).ToString(),
+                        (rd.GetValue(4) == null || rd.GetValue(4) == DBNull.Value) ? null : rd.GetValue(4).ToString(),
+                        rd.ConvertValueToInt64(3, -1),
+                        (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(2).ToString()),
+                        (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(5).ToString()),
+                        new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc)
                     );
                 }
             }
@@ -316,9 +325,9 @@ namespace Duplicati.Library.Main.Database
         /// <param name="operation">The operation performed</param>
         /// <param name="path">The path involved</param>
         /// <param name="data">Any data relating to the operation</param>
-        public void LogRemoteOperation(string operation, string path, string data, System.Data.IDbTransaction transaction)
+        public void LogRemoteOperation(string operation, string path, string data)
         {
-            m_insertremotelogCommand.Transaction = transaction;
+            m_insertremotelogCommand.Transaction = Transaction;
             m_insertremotelogCommand.SetParameterValue(0, m_operationid);
             m_insertremotelogCommand.SetParameterValue(1, NormalizeDateTimeToEpochSeconds(DateTime.UtcNow));
             m_insertremotelogCommand.SetParameterValue(2, operation);
@@ -333,9 +342,9 @@ namespace Duplicati.Library.Main.Database
         /// <param name="type">The message type</param>
         /// <param name="message">The message</param>
         /// <param name="exception">An optional exception</param>
-        public void LogMessage(string type, string message, Exception exception, System.Data.IDbTransaction transaction)
+        public void LogMessage(string type, string message, Exception exception)
         {
-            m_insertlogCommand.Transaction = transaction;
+            m_insertlogCommand.Transaction = Transaction;
             m_insertlogCommand.SetParameterValue(0, m_operationid);
             m_insertlogCommand.SetParameterValue(1, NormalizeDateTimeToEpochSeconds(DateTime.UtcNow));
             m_insertlogCommand.SetParameterValue(2, type);
@@ -344,33 +353,29 @@ namespace Duplicati.Library.Main.Database
             m_insertlogCommand.ExecuteNonQuery();
         }
 
-        public void UnlinkRemoteVolume(string name, RemoteVolumeState state, System.Data.IDbTransaction transaction = null)
+        public void UnlinkRemoteVolume(string name, RemoteVolumeState state)
         {
-            using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
             using(var cmd = m_connection.CreateCommand())
             {
-                cmd.Transaction = tr.Parent;
+                cmd.Transaction = Transaction;
                 var c = cmd.ExecuteNonQuery(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = ? AND ""State"" = ? ", name, state.ToString());
                 if (c != 1)
                     throw new Exception(string.Format("Unexpected number of remote volumes deleted: {0}, expected {1}", c, 1));
-
-                tr.Commit();
             }
         }
 
-        public void RemoveRemoteVolume(string name, System.Data.IDbTransaction transaction = null)
+        public void RemoveRemoteVolume(string name)
         {
-            RemoveRemoteVolumes(new string[] { name }, transaction);
+            RemoveRemoteVolumes(new string[] { name });
         }
 
-        public void RemoveRemoteVolumes(IEnumerable<string> names, System.Data.IDbTransaction transaction = null)
+        public void RemoveRemoteVolumes(IEnumerable<string> names)
         {
             if (names == null || !names.Any()) return;
 
-            using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
             using (var deletecmd = m_connection.CreateCommand())
             {
-                deletecmd.Transaction = tr.Parent;
+                deletecmd.Transaction = Transaction;
 
                 string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                 var volidstable = "DelVolSetIds-" + temptransguid;
@@ -383,7 +388,7 @@ namespace Duplicati.Library.Main.Database
                 deletecmd.AddParameters(1);
                 foreach (var name in names)
                 {
-                    var volumeid = GetRemoteVolumeID(name, tr.Parent);
+                    var volumeid = GetRemoteVolumeID(name);
                     deletecmd.SetParameterValue(0, volumeid);
                     deletecmd.ExecuteNonQuery();
                 }
@@ -433,7 +438,7 @@ namespace Duplicati.Library.Main.Database
                 }
                 catch { /* Ignore, will be deleted on close anyway. */ }
 
-				m_removeremotevolumeCommand.Transaction = tr.Parent;
+                m_removeremotevolumeCommand.Transaction = Transaction;
                 m_removeremotevolumeCommand.SetParameterValue(1, DateTime.UtcNow.Ticks);
 				m_removeremotevolumeCommand.SetParameterValue(2, RemoteVolumeState.Deleted.ToString());
 				foreach (var name in names)
@@ -441,7 +446,6 @@ namespace Duplicati.Library.Main.Database
                     m_removeremotevolumeCommand.SetParameterValue(0, name);
                     m_removeremotevolumeCommand.ExecuteNonQuery();
                 }
-                tr.Commit();
             }
         }
         
@@ -453,40 +457,35 @@ namespace Duplicati.Library.Main.Database
 
         public long RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state)
         {
-            return RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), null);
+            return RegisterRemoteVolume(name, type, state, size, new TimeSpan(0));
         }
 
-        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, System.Data.IDbTransaction transaction)
+        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state)
         {
-            return RegisterRemoteVolume(name, type, state, new TimeSpan(0), transaction);
+            return RegisterRemoteVolume(name, type, state, new TimeSpan(0));
         }
 
-        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, System.Data.IDbTransaction transaction)
+        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime)
         {
-            return RegisterRemoteVolume(name, type, state, -1, deleteGraceTime, transaction);
+            return RegisterRemoteVolume(name, type, state, -1, deleteGraceTime);
         }
             
-        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, System.Data.IDbTransaction transaction)
+        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime)
         {
-            using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
-            {
-                m_createremotevolumeCommand.SetParameterValue(0, m_operationid);
-                m_createremotevolumeCommand.SetParameterValue(1, name);
-                m_createremotevolumeCommand.SetParameterValue(2, type.ToString());
-                m_createremotevolumeCommand.SetParameterValue(3, state.ToString());
-                m_createremotevolumeCommand.SetParameterValue(4, size);
-                m_createremotevolumeCommand.SetParameterValue(5, 0);
+            m_createremotevolumeCommand.SetParameterValue(0, m_operationid);
+            m_createremotevolumeCommand.SetParameterValue(1, name);
+            m_createremotevolumeCommand.SetParameterValue(2, type.ToString());
+            m_createremotevolumeCommand.SetParameterValue(3, state.ToString());
+            m_createremotevolumeCommand.SetParameterValue(4, size);
+            m_createremotevolumeCommand.SetParameterValue(5, 0);
 
-                if (deleteGraceTime.Ticks <= 0)
-                    m_createremotevolumeCommand.SetParameterValue(6, 0);
-                else
-                    m_createremotevolumeCommand.SetParameterValue(6, (DateTime.UtcNow + deleteGraceTime).Ticks);
-                
-                m_createremotevolumeCommand.Transaction = tr.Parent;
-                var r = m_createremotevolumeCommand.ExecuteScalarInt64();
-                tr.Commit();
-                return r;
-            }
+            if (deleteGraceTime.Ticks <= 0)
+                m_createremotevolumeCommand.SetParameterValue(6, 0);
+            else
+                m_createremotevolumeCommand.SetParameterValue(6, (DateTime.UtcNow + deleteGraceTime).Ticks);
+            
+            m_createremotevolumeCommand.Transaction = Transaction;
+            return m_createremotevolumeCommand.ExecuteScalarInt64();
         }
 
         public long GetFilesetID(DateTime restoretime, long[] versions)
@@ -545,55 +544,39 @@ namespace Duplicati.Library.Main.Database
             return res;
         }
 
-
-        public System.Data.IDbTransaction BeginTransaction()
+        /// <summary>
+        /// Writes the current changes to the database
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="message">The message to use for logging the time spent in this operation.</param>
+        /// <param name="restart">If set to <c>true</c>, a transaction will be started again after this call.</param>
+        public void CommitTransaction(string message, bool restart = true)
         {
-            return m_connection.BeginTransaction();
-        }
-
-        protected class TemporaryTransactionWrapper : IDisposable
-        {
-            private System.Data.IDbTransaction m_parent;
-            private bool m_isTemporary;
-
-            public TemporaryTransactionWrapper(System.Data.IDbConnection connection, System.Data.IDbTransaction transaction)
+            if (m_parent != null)
+                m_parent.CommitTransaction(message, restart);
+            else
             {
-                if (transaction != null)
+                using (new Logging.Timer(message))
                 {
-                    m_parent = transaction;
-                    m_isTemporary = false;
-                }
-                else
-                {
-                    m_parent = connection.BeginTransaction();
-                    m_isTemporary = true;
+                    m_transaction.Commit();
+                    m_transaction = restart ? m_connection.BeginTransaction() : null;
                 }
             }
-
-            public System.Data.IDbConnection Connection { get { return m_parent.Connection; } }
-            public System.Data.IsolationLevel IsolationLevel { get { return m_parent.IsolationLevel; } }
-
-            public void Commit() 
-            { 
-                if (m_isTemporary) 
-                    m_parent.Commit(); 
-            }
-
-            public void Rollback()
-            {
-                if (m_isTemporary)
-                    m_parent.Rollback(); 
-            }
-
-            public void Dispose() 
-            {
-                if (m_isTemporary)
-                    m_parent.Dispose();
-            }
-
-            public System.Data.IDbTransaction Parent { get { return m_parent; } }
         }
-        
+
+        public void RollbackTransaction(bool restart = true)
+        {
+            if (m_parent != null)
+                m_parent.RollbackTransaction();
+            else
+            {
+                m_transaction.Rollback();
+                if (restart)
+                    m_transaction = m_connection.BeginTransaction();
+            }
+        }
+
+
         private class LocalFileEntry : ILocalFileEntry
         {
             private System.Data.IDataReader m_reader;
@@ -652,17 +635,17 @@ namespace Duplicati.Library.Main.Database
                 yield return new LocalFileEntry(rd);
         }
 
-        private IEnumerable<KeyValuePair<string, string>> GetDbOptionList(System.Data.IDbTransaction transaction = null)
+        private IEnumerable<KeyValuePair<string, string>> GetDbOptionList()
         {
-            using(var cmd = m_connection.CreateCommand(transaction))
+            using(var cmd = m_connection.CreateCommand(Transaction))
             using(var rd = cmd.ExecuteReader(@"SELECT ""Key"", ""Value"" FROM ""Configuration"" "))
             while(rd.Read())
                 yield return new KeyValuePair<string, string>(rd.GetValue(0).ToString(), rd.GetValue(1).ToString());
         }
                     
-        public IDictionary<string, string> GetDbOptions(System.Data.IDbTransaction transaction = null)
+        public IDictionary<string, string> GetDbOptions()
         {
-            return GetDbOptionList(transaction).ToDictionary(x => x.Key, x => x.Value);    
+            return GetDbOptionList().ToDictionary(x => x.Key, x => x.Value);    
         }
 
         public bool RepairInProgress
@@ -703,17 +686,14 @@ namespace Duplicati.Library.Main.Database
             }
         }
         
-        public void SetDbOptions(IDictionary<string, string> options, System.Data.IDbTransaction transaction = null)
+        public void SetDbOptions(IDictionary<string, string> options)
         {
-            using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
             using(var cmd = m_connection.CreateCommand())
             {
-                cmd.Transaction = tr.Parent;
+                cmd.Transaction = Transaction;
                 cmd.ExecuteNonQuery(@"DELETE FROM ""Configuration"" ");
                 foreach(var kp in options)
                     cmd.ExecuteNonQuery(@"INSERT INTO ""Configuration"" (""Key"", ""Value"") VALUES (?, ?) ", kp.Key, kp.Value);
-                
-                tr.Commit();
             }
         }
 
@@ -723,9 +703,9 @@ namespace Duplicati.Library.Main.Database
                 return cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > ?", -1, fhblocksize);
         }
 
-        public void VerifyConsistency(System.Data.IDbTransaction transaction, long blocksize, long hashsize, bool verifyfilelists)
+        public void VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists)
         {
-            using (var cmd = m_connection.CreateCommand(transaction))
+            using (var cmd = m_connection.CreateCommand(Transaction))
             {
                 // Calculate the lengths for each blockset                
                 var combinedLengths = @"
@@ -786,16 +766,21 @@ ON
                 if (itemswithnoblocklisthash != 0)
                     throw new InvalidDataException(string.Format("Found {0} file(s) with missing blocklist hashes", itemswithnoblocklisthash));
 
-                if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0)
-                    throw new Exception("Detected file entries with not associated blocks");
+                if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Blockset"" WHERE ""Length"" > 0 AND ""ID"" NOT IN (SELECT ""BlocksetId"" FROM ""BlocksetEntry"")") != 0)
+                {
+                    throw new Exception("Detected non-empty blocksets with no associated blocks!");
+                }
 
+                if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0)
+                    throw new Exception("Detected files associated with non-existing blocksets!");
 
                 if (verifyfilelists)
                 {
-                    using(var cmd2 = m_connection.CreateCommand(transaction))
+                    using(var cmd2 = m_connection.CreateCommand(Transaction))
                     foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1)))
                     {
-                        var expandedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid);
+                            var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS);
+                        var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid);
                         //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
                         var storedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?"), 0, filesetid);
 
@@ -806,29 +791,29 @@ ON
             }
         }
 
-        public interface IBlock
-        {
-            string Hash { get; }
-            long Size { get; }
-        }
-        
-        internal class Block : IBlock
-        {
-            public string Hash { get; private set; }
-            public long Size { get; private set; }
-            
-            public Block(string hash, long size)
-            {
-                this.Hash = hash;
-                this.Size = size;
-            }
-        }        
+		public interface IBlock
+		{
+			string Hash { get; }
+			long Size { get; }
+		}
+		
+		internal class Block : IBlock
+		{
+			public string Hash { get; private set; }
+			public long Size { get; private set; }
+			
+			public Block(string hash, long size)
+			{
+				this.Hash = hash;
+				this.Size = size;
+			}
+		}		
 
         public IEnumerable<IBlock> GetBlocks(long volumeid)
-        {
-            using(var cmd = m_connection.CreateCommand())
-            using(var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = ?", volumeid))
-                while (rd.Read())
+		{
+            using(var cmd = m_connection.CreateCommand(Transaction))
+			using(var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = ?", volumeid))
+				while (rd.Read())
                     yield return new Block(rd.GetValue(0).ToString(), rd.GetInt64(1));
         }
 
@@ -952,40 +937,41 @@ FROM
     FROM 
         (
         SELECT 
-            ""A"".""Path"" AS ""Path"", 
-            ""D"".""Lastmodified"" AS ""Lastmodified"", 
-            ""B"".""Length"" AS ""Filelength"", 
-            ""B"".""FullHash"" AS ""Filehash"", 
-            ""E"".""FullHash"" AS ""Metahash"", 
-            ""E"".""Length"" AS ""Metalength"", 
-            ""A"".""BlocksetID"" AS ""BlocksetID"",
-            ""F"".""Hash"" AS ""FirstBlockHash"",
-            ""F"".""Size"" AS ""FirstBlockSize"",
-            ""H"".""Hash"" AS ""FirstMetaBlockHash"",
-            ""H"".""Size"" AS ""FirstMetaBlockSize"",
-            ""C"".""BlocksetID"" AS ""MetablocksetID""
+	        ""A"".""Path"" AS ""Path"", 
+	        ""D"".""Lastmodified"" AS ""Lastmodified"", 
+	        ""B"".""Length"" AS ""Filelength"", 
+	        ""B"".""FullHash"" AS ""Filehash"", 
+	        ""E"".""FullHash"" AS ""Metahash"", 
+	        ""E"".""Length"" AS ""Metalength"",
+	        ""A"".""BlocksetID"" AS ""BlocksetID"",
+	        ""F"".""Hash"" AS ""FirstBlockHash"",
+	        ""F"".""Size"" AS ""FirstBlockSize"",
+	        ""H"".""Hash"" AS ""FirstMetaBlockHash"",
+	        ""H"".""Size"" AS ""FirstMetaBlockSize"",
+	        ""C"".""BlocksetID"" AS ""MetablocksetID""
         FROM 
-            ""File"" A, 
-            ""Blockset"" B, 
-            ""Metadataset"" C, 
-            ""FilesetEntry"" D, 
-            ""Blockset"" E, 
-            ""Block"" F,
-            ""BlocksetEntry"" G,
-            ""Block"" H,
-            ""BlocksetEntry"" I
+	        ""File"" A	
+        LEFT JOIN ""Blockset"" B
+          ON ""A"".""BlocksetID"" = ""B"".""ID"" 
+        LEFT JOIN ""Metadataset"" C  
+          ON ""A"".""MetadataID"" = ""C"".""ID""
+        LEFT JOIN ""FilesetEntry"" D
+          ON ""A"".""ID"" = ""D"".""FileID""
+        LEFT JOIN ""Blockset"" E
+          ON ""E"".""ID"" = ""C"".""BlocksetID""
+        LEFT JOIN ""BlocksetEntry"" G
+          ON ""B"".""ID"" = ""G"".""BlocksetID""
+        LEFT JOIN ""Block"" F 
+          ON ""G"".""BlockID"" = ""F"".""ID""  
+        LEFT JOIN ""BlocksetEntry"" I
+          ON ""E"".""ID"" = ""I"".""BlocksetID""
+        LEFT JOIN ""Block"" H 
+          ON ""I"".""BlockID"" = ""H"".""ID""
         WHERE 
-            ""A"".""ID"" = ""D"".""FileID"" 
-            AND ""D"".""FilesetID"" = ? 
-            AND ""A"".""BlocksetID"" = ""B"".""ID"" 
-            AND ""A"".""MetadataID"" = ""C"".""ID"" 
-            AND ""E"".""ID"" = ""C"".""BlocksetID""
-            AND ""B"".""ID"" = ""G"".""BlocksetID""
-            AND ""G"".""BlockID"" = ""F"".""ID""
-            AND ""G"".""Index"" = 0
-            AND ""I"".""BlocksetID"" = ""E"".""ID""
-            AND ""I"".""BlockID"" = ""H"".""ID""
-            AND ""I"".""Index"" = 0
+          ""A"".""BlocksetId"" >= 0 AND
+          ""D"".""FilesetID"" = ? AND
+          (""I"".""Index"" = 0 OR ""I"".""Index"" IS NULL) AND  
+          (""G"".""Index"" = 0 OR ""G"".""Index"" IS NULL)
         ) J
     LEFT OUTER JOIN 
         ""BlocklistHash"" K 
@@ -1047,11 +1033,10 @@ ORDER BY
 
 ";
 
-        public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, System.Data.IDbTransaction transaction, long filesetId)
+        public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId)
         {
-            using (var cmd = m_connection.CreateCommand())
+            using (var cmd = m_connection.CreateCommand(Transaction))
             {
-            	cmd.Transaction = transaction;
                 cmd.CommandText = LIST_FOLDERS_AND_SYMLINKS;
                 cmd.AddParameter(FOLDER_BLOCKSET_ID);
                 cmd.AddParameter(SYMLINK_BLOCKSET_ID);
@@ -1144,30 +1129,22 @@ ORDER BY
                 
                 if (type == Library.Utility.FilterType.Regexp)
                 {
-                    using(var cmd = m_connection.CreateCommand())
+                    using(var cmd = m_connection.CreateCommand(transaction))
                     {
-                        cmd.Transaction = transaction;
                         cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename));
-                        using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
-                        {
-                            cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", Tablename);
-                            cmd.AddParameter();
-                            cmd.Transaction = tr.Parent;
-                            using(var c2 = m_connection.CreateCommand())
-                            using(var rd = c2.ExecuteReader(@"SELECT DISTINCT ""Path"" FROM ""File"" "))
-                                while(rd.Read())
+                        cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", Tablename);
+                        cmd.AddParameter();
+                        using(var c2 = m_connection.CreateCommand(transaction))
+                        using(var rd = c2.ExecuteReader(@"SELECT DISTINCT ""Path"" FROM ""File"" "))
+                            while(rd.Read())
+                            {
+                                var p = rd.GetValue(0).ToString();
+                                if(Library.Utility.FilterExpression.Matches(filter, p))
                                 {
-                                    var p = rd.GetValue(0).ToString();
-                                    if(Library.Utility.FilterExpression.Matches(filter, p))
-                                    {
-                                        cmd.SetParameterValue(0, p);
-                                        cmd.ExecuteNonQuery();
-                                    }
+                                    cmd.SetParameterValue(0, p);
+                                    cmd.ExecuteNonQuery();
                                 }
-                            
-                            
-                            tr.Commit();
-                        }
+                            }
                     }
                 }
                 else
@@ -1189,14 +1166,11 @@ ORDER BY
                     }
                     
                     sb.Length = sb.Length - " OR ".Length;
-                    
-                    using(var cmd = m_connection.CreateCommand())
-                    using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
+
+                    using (var cmd = m_connection.CreateCommand(transaction))
                     {
-                        cmd.Transaction = tr.Parent;
                         cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename));
                         cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE " + sb, Tablename), args.ToArray());
-                        tr.Commit();
                     }
                 }
             }
@@ -1214,13 +1188,10 @@ ORDER BY
             }                
         }
         
-        public void RenameRemoteFile(string oldname, string newname, System.Data.IDbTransaction transaction)
+        public void RenameRemoteFile(string oldname, string newname)
         {
-            using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
-                cmd.Transaction = tr.Parent;
-                
                 //Rename the old entry, to preserve ID links
                 var c = cmd.ExecuteNonQuery(@"UPDATE ""Remotevolume"" SET ""Name"" = ? WHERE ""Name"" = ?", newname, oldname);
                 if (c != 1)
@@ -1231,9 +1202,7 @@ ORDER BY
                 
                 //Create a fake new entry with the old name and mark as deleting
                 // as this ensures we will remove it, if it shows up in some later listing
-                RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, tr.Parent);
-                
-                tr.Commit();
+                RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting);
             }
         }
         
@@ -1243,21 +1212,15 @@ ORDER BY
         /// <param name="volumeid">The ID of the fileset volume to update</param>
         /// <param name="timestamp">The timestamp of the operation to create</param>
         /// <param name="transaction">An optional external transaction</param>
-        public virtual long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null)
+        public virtual long CreateFileset(long volumeid, DateTime timestamp)
         {
-            using (var cmd = m_connection.CreateCommand())
-            using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
-            {
-                cmd.Transaction = tr.Parent;                
-                var id = cmd.ExecuteScalarInt64(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", -1, m_operationid, NormalizeDateTimeToEpochSeconds(timestamp), volumeid);
-                tr.Commit();
-                return id;
-            }
+            using (var cmd = m_connection.CreateCommand(Transaction))
+                return cmd.ExecuteScalarInt64(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", -1, m_operationid, NormalizeDateTimeToEpochSeconds(timestamp), volumeid);
         }
         
-        public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction)
+        public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID)
         {
-            m_insertIndexBlockLink.Transaction = transaction;
+            m_insertIndexBlockLink.Transaction = Transaction;
             m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID);
             m_insertIndexBlockLink.SetParameterValue(1, blockVolumeID);
             m_insertIndexBlockLink.ExecuteNonQuery();
@@ -1265,7 +1228,7 @@ ORDER BY
 
         public IEnumerable<Tuple<string, byte[], int>> GetBlocklists(long volumeid, long blocksize, int hashsize)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var sql = string.Format(@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM " + 
                     @"(SELECT ""BlocklistHash"".""BlocksetID"", ""Block"".""Hash"", * FROM  ""BlocklistHash"",""Block"" WHERE  ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""Block"".""VolumeID"" = ?) A, " + 
@@ -1288,12 +1251,14 @@ ORDER BY
                             yield return new Tuple<string, byte[], int>(curHash, buffer, index);
                             curHash = null;
                             index = 0;
+                            buffer = new byte[blocksize];
                         }
 
                         var hash = Convert.FromBase64String(rd.GetValue(1).ToString());
                         Array.Copy(hash, 0, buffer, index, hashsize);
                         curHash = blockhash;
                         index += hashsize;
+
                     }
 
                 if (curHash != null)
@@ -1305,14 +1270,11 @@ ORDER BY
 
         public void PurgeLogData(DateTime threshold)
         {
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var t = NormalizeDateTimeToEpochSeconds(threshold);
                 cmd.ExecuteNonQuery(@"DELETE FROM ""LogData"" WHERE ""Timestamp"" < ?", t);
                 cmd.ExecuteNonQuery(@"DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < ?", t);
-
-                tr.Commit();
             }
         }
         
@@ -1402,10 +1364,11 @@ ORDER BY
                         recurseobjects: true, 
                         collectionlimit: 5
                     ).ToString(),
-                    null,
                     null
-                );
+              );
             }
+
+            CommitTransaction("WriteMessage");
         }
     }
 }

Duplicati/Library/Main/Database/LocalDeleteDatabase.cs

@@ -58,12 +58,10 @@ namespace Duplicati.Library.Main.Database
         /// <param name="toDelete">The fileset entries to delete</param>
         /// <param name="transaction">The transaction to execute the commands in</param>
         /// <returns>A list of filesets to delete</returns>
-        public IEnumerable<KeyValuePair<string, long>> DropFilesetsFromTable(DateTime[] toDelete, System.Data.IDbTransaction transaction)
+        public IEnumerable<KeyValuePair<string, long>> DropFilesetsFromTable(DateTime[] toDelete)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
-                cmd.Transaction = transaction;
-
                 var q = "";
                 foreach(var n in toDelete)
                     if (q.Length == 0)
@@ -123,7 +121,7 @@ namespace Duplicati.Library.Main.Database
         /// The sizes are the uncompressed values.
         /// </summary>
         /// <returns>A list of tuples with name, datasize, wastedbytes.</returns>
-        private IEnumerable<VolumeUsage> GetWastedSpaceReport(System.Data.IDbTransaction transaction)
+        private IEnumerable<VolumeUsage> GetWastedSpaceReport()
         {
             var tmptablename = "UsageReport-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
             
@@ -140,9 +138,8 @@ namespace Duplicati.Library.Main.Database
             var collected = @"SELECT ""VolumeID"" AS ""VolumeID"", SUM(""ActiveSize"") AS ""ActiveSize"", SUM(""InactiveSize"") AS ""InactiveSize"", MAX(""Sortime"") AS ""Sorttime"" FROM (" + combined + @") GROUP BY ""VolumeID"" ";
             var createtable = @"CREATE TEMPORARY TABLE """ + tmptablename + @""" AS " + collected;
                         
-            using (var cmd = m_connection.CreateCommand())
+            using (var cmd = m_connection.CreateCommand(Transaction))
             {
-                cmd.Transaction = transaction;
                 try
                 {
                     cmd.ExecuteNonQuery(createtable, RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
@@ -259,34 +256,36 @@ namespace Duplicati.Library.Main.Database
             }
         }
         
-        public ICompactReport GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, System.Data.IDbTransaction transaction)
+        public ICompactReport GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount)
         {
-            return new CompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, GetWastedSpaceReport(transaction).ToList());
+            return new CompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, GetWastedSpaceReport().ToList());
         }
         
                 
         public interface IBlockQuery : IDisposable
         {
-            bool UseBlock(string hash, long size, System.Data.IDbTransaction transaction);
+            bool UseBlock(string hash, long size);
         }
         
         private class BlockQuery : IBlockQuery
         {
             private System.Data.IDbCommand m_command;
+            private LocalDatabase m_parent;
 
-            public BlockQuery(System.Data.IDbConnection con, Options options, System.Data.IDbTransaction transaction)
+            public BlockQuery(System.Data.IDbConnection con, Options options, LocalDatabase parent)
             {
+                m_parent = parent;
                 m_command = con.CreateCommand();
-                m_command.Transaction = transaction;
+                m_command.Transaction = m_parent.Transaction;
 
                 m_command.Parameters.Clear();
                 m_command.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
                 m_command.AddParameters(2);
             }
             
-            public bool UseBlock(string hash, long size, System.Data.IDbTransaction transaction)
+            public bool UseBlock(string hash, long size)
             {                
-                m_command.Transaction = transaction;
+                m_command.Transaction = m_parent.Transaction;
                 m_command.SetParameterValue(0, hash);    
                 m_command.SetParameterValue(1, size);
                 var r = m_command.ExecuteScalar();
@@ -304,17 +303,17 @@ namespace Duplicati.Library.Main.Database
         /// <summary>
         /// Builds a lookup table to enable faster response to block queries
         /// </summary>
-        public IBlockQuery CreateBlockQueryHelper(Options options, System.Data.IDbTransaction transaction)
+        public IBlockQuery CreateBlockQueryHelper(Options options)
         {
-            return new BlockQuery(m_connection, options, transaction);
+            return new BlockQuery(m_connection, options, this);
         }
 
-        public void MoveBlockToNewVolume(string hash, long size, long volumeID, System.Data.IDbTransaction tr)
+        public void MoveBlockToNewVolume(string hash, long size, long volumeID)
         {
             m_moveBlockToNewVolumeCommand.SetParameterValue(0, volumeID);
             m_moveBlockToNewVolumeCommand.SetParameterValue(1, hash);
             m_moveBlockToNewVolumeCommand.SetParameterValue(2, size);
-            m_moveBlockToNewVolumeCommand.Transaction = tr;
+            m_moveBlockToNewVolumeCommand.Transaction = Transaction;
             var c = m_moveBlockToNewVolumeCommand.ExecuteNonQuery();
             if (c != 1)
                 throw new Exception("Unexpected update result");
@@ -325,18 +324,16 @@ namespace Duplicati.Library.Main.Database
         /// </summary>
         /// <returns>The deletable volumes.</returns>
         /// <param name="deleteableVolumes">Block volumes slated for deletion.</param>
-        public IEnumerable<IRemoteVolume> GetDeletableVolumes(IEnumerable<IRemoteVolume> deleteableVolumes, System.Data.IDbTransaction transaction)
+        public IEnumerable<IRemoteVolume> GetDeletableVolumes(IEnumerable<IRemoteVolume> deleteableVolumes)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 // Although the generated index volumes are always in pairs,
                 // this code handles many-to-many relations between
                 // index files and block volumes, should this be added later
                 var lookupBlock = new Dictionary<string, List<IRemoteVolume>>();
                 var lookupIndexfiles = new Dictionary<string, List<string>>();
-                
-                cmd.Transaction = transaction;
-                
+
                 using(var rd = cmd.ExecuteReader(@"SELECT ""C"".""Name"", ""B"".""Name"", ""B"".""Hash"", ""B"".""Size"" FROM ""IndexBlockLink"" A, ""RemoteVolume"" B, ""RemoteVolume"" C WHERE ""A"".""IndexVolumeID"" = ""B"".""ID"" AND ""A"".""BlockVolumeID"" = ""C"".""ID"" AND ""B"".""Hash"" IS NOT NULL AND ""B"".""Size"" IS NOT NULL "))
                     while(rd.Read())
                     {

Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs

@@ -55,7 +55,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""
             ShouldCloseConnection = false;
         }
 
-        public IEnumerable<Tuple<DateTime, long, long>> GetBrokenFilesets(DateTime time, long[] versions, System.Data.IDbTransaction transaction)
+        public IEnumerable<Tuple<DateTime, long, long>> GetBrokenFilesets(DateTime time, long[] versions)
         {
             var query = string.Format(BROKEN_FILE_SETS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
             var clause = GetFilelistWhereClause(time, versions);
@@ -64,33 +64,33 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""
 
             query += @" GROUP BY ""A"".""FilesetID""";
 
-            using (var cmd = Connection.CreateCommand(transaction))
+            using (var cmd = Connection.CreateCommand(Transaction))
                  foreach (var rd in cmd.ExecuteReaderEnumerable(query, clause.Item2))
                     if (!rd.IsDBNull(0))
                         yield return new Tuple<DateTime, long, long>(ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0));
         }
 
-        public IEnumerable<Tuple<string, long>> GetBrokenFilenames(long filesetid, System.Data.IDbTransaction transaction)
+        public IEnumerable<Tuple<string, long>> GetBrokenFilenames(long filesetid)
         {
-            using (var cmd = Connection.CreateCommand(transaction))
+            using (var cmd = Connection.CreateCommand(Transaction))
                 foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(BROKEN_FILE_NAMES, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID), filesetid))
                     if (!rd.IsDBNull(0))
                         yield return new Tuple<string, long>(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
         }
 
-        public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, System.Data.IDbTransaction transaction)
+        public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname)
         {
-            using (var cmd = Connection.CreateCommand(transaction))
+            using (var cmd = Connection.CreateCommand(Transaction))
                 cmd.ExecuteNonQuery(string.Format(INSERT_BROKEN_IDS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, tablename, IDfieldname), filesetid);
         }
 
-        public void RemoveMissingBlocks(IEnumerable<string> names, System.Data.IDbTransaction transaction)
+        public void RemoveMissingBlocks(IEnumerable<string> names)
         {
             if (names == null || !names.Any()) return;
-            if (transaction == null)
+            if (Transaction == null)
                 throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state");
 
-            using (var deletecmd = m_connection.CreateCommand(transaction))
+            using (var deletecmd = m_connection.CreateCommand(Transaction))
             {
                 string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                 var volidstable = "DelVolSetIds-" + temptransguid;
@@ -102,7 +102,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""
                 deletecmd.AddParameters(1);
                 foreach (var name in names)
                 {
-                    var volumeid = GetRemoteVolumeID(name, transaction);
+                    var volumeid = GetRemoteVolumeID(name);
                     deletecmd.SetParameterValue(0, volumeid);
                     deletecmd.ExecuteNonQuery();
                 }
@@ -125,9 +125,9 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""
             }
         }
 
-        public long GetFilesetFileCount(long filesetid, System.Data.IDbTransaction transaction)
+        public long GetFilesetFileCount(long filesetid)
         {
-            using (var cmd = m_connection.CreateCommand(transaction))
+            using (var cmd = m_connection.CreateCommand(Transaction))
                 return cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?", 0, filesetid);
         }
     }

Duplicati/Library/Main/Database/LocalListChangesDatabase.cs

@@ -90,37 +90,36 @@ namespace Duplicati.Library.Main.Database
         private class StorageHelper : IStorageHelper
         {
             private System.Data.IDbConnection m_connection;
-            private System.Data.IDbTransaction m_transaction;
+            private LocalDatabase m_parent;
             
             private System.Data.IDbCommand m_insertPreviousElementCommand;
             private System.Data.IDbCommand m_insertCurrentElementCommand;
             
             private string m_previousTable;
             private string m_currentTable;
-            
-            public StorageHelper(System.Data.IDbConnection con)
+
+            public StorageHelper(System.Data.IDbConnection con, LocalDatabase parent)
             {
                 m_connection = con;
+                m_parent = parent;
                 m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                 m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
-                
-                m_transaction = m_connection.BeginTransaction();
-                
+
                 using(var cmd = m_connection.CreateCommand())
                 {
-                    cmd.Transaction = m_transaction;
+                    cmd.Transaction = m_parent.Transaction;
                     
                     cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_previousTable));
                     cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_currentTable));
                 }
                 
                 m_insertPreviousElementCommand = m_connection.CreateCommand();
-                m_insertPreviousElementCommand.Transaction = m_transaction;
+                m_insertPreviousElementCommand.Transaction = m_parent.Transaction;
                 m_insertPreviousElementCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_previousTable);
                 m_insertPreviousElementCommand.AddParameters(5);
                 
                 m_insertCurrentElementCommand = m_connection.CreateCommand();
-                m_insertCurrentElementCommand.Transaction = m_transaction;
+                m_insertCurrentElementCommand.Transaction = m_parent.Transaction;
                 m_insertCurrentElementCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_currentTable);
                 m_insertCurrentElementCommand.AddParameters(5);
             }
@@ -137,7 +136,7 @@ namespace Duplicati.Library.Main.Database
                 
                 using(var cmd = m_connection.CreateCommand())
                 {
-                    cmd.Transaction = m_transaction;
+                    cmd.Transaction = m_parent.Transaction;
                     if (filter == null || filter.Empty)
                     {
                         // Simple case, select everything
@@ -171,7 +170,7 @@ namespace Duplicati.Library.Main.Database
                         {
                             cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", tablename);
                             cmd2.AddParameters(5);
-                            cmd2.Transaction = m_transaction;
+                            cmd2.Transaction = m_parent.Transaction;
     
                             using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {0} A WHERE ""A"".""FilesetID"" = ?", combined), filesetId))
                                 while (rd.Read())
@@ -234,7 +233,7 @@ namespace Duplicati.Library.Main.Database
                 
                 using(var cmd = m_connection.CreateCommand())
                 {
-                    cmd.Transaction = m_transaction;
+                    cmd.Transaction = m_parent.Transaction;
                     
                     var result = new ChangeSizeReport();
                     
@@ -257,7 +256,7 @@ namespace Duplicati.Library.Main.Database
                 
                 using(var cmd = m_connection.CreateCommand())
                 {
-                    cmd.Transaction = m_transaction;
+                    cmd.Transaction = m_parent.Transaction;
                     
                     var result = new ChangeCountReport();
                     result.AddedFolders = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.Folder);
@@ -285,7 +284,7 @@ namespace Duplicati.Library.Main.Database
                 
                 using(var cmd = m_connection.CreateCommand())
                 {
-                    cmd.Transaction = m_transaction;
+                    cmd.Transaction = m_parent.Transaction;
                     foreach(var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Folder)))
                         yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.Folder, s);                        
                     foreach(var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Symlink)))
@@ -324,24 +323,12 @@ namespace Duplicati.Library.Main.Database
                     catch {}
                     finally { m_insertCurrentElementCommand = null; }
                 }
-                
-                if (m_transaction != null)
-                {
-                    try { m_transaction.Rollback(); }
-                    catch {}
-                    finally
-                    {
-                        m_previousTable = null;
-                        m_currentTable= null;
-                        m_transaction = null;
-                    }
-                }
             }
         }
         
         public IStorageHelper CreateStorageHelper()
         {
-            return new StorageHelper(m_connection);
+            return new StorageHelper(m_connection, this);
         }
     }
 }

Duplicati/Library/Main/Database/LocalPurgeDatabase.cs

@@ -32,14 +32,14 @@ namespace Duplicati.Library.Main.Database
         {
         }
 
-        public ITemporaryFileset CreateTemporaryFileset(long parentid, System.Data.IDbTransaction transaction)
+        public ITemporaryFileset CreateTemporaryFileset(long parentid)
         {
-            return new TemporaryFileset(parentid, this, m_connection, transaction);
+            return new TemporaryFileset(parentid, this, m_connection);
         }
 
-        public string GetRemoteVolumeNameForFileset(long id, System.Data.IDbTransaction transaction)
+        public string GetRemoteVolumeNameForFileset(long id)
         {
-            using (var cmd = m_connection.CreateCommand(transaction))
+            using (var cmd = m_connection.CreateCommand(Transaction))
             using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = ? "), id))
                 if (!rd.Read())
                     throw new Exception(string.Format("No remote volume found for fileset with id {0}", id));
@@ -47,9 +47,9 @@ namespace Duplicati.Library.Main.Database
                     return rd.ConvertValueToString(0);
         }
 
-        internal long CountOrphanFiles(System.Data.IDbTransaction transaction)
+        internal long CountOrphanFiles()
         {
-            using (var cmd = m_connection.CreateCommand(transaction))
+            using (var cmd = m_connection.CreateCommand(Transaction))
             using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""File"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")"))
                 if (rd.Read())
                     return rd.ConvertValueToInt64(0, 0);
@@ -72,7 +72,6 @@ namespace Duplicati.Library.Main.Database
         private class TemporaryFileset : ITemporaryFileset
         {
             private readonly System.Data.IDbConnection m_connection;
-            private readonly System.Data.IDbTransaction m_transaction;
             private readonly string m_tablename;
             private LocalPurgeDatabase m_parentdb;
 
@@ -80,21 +79,20 @@ namespace Duplicati.Library.Main.Database
             public long RemovedFileCount { get; private set; }
             public long RemovedFileSize { get; private set; }
 
-            public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, System.Data.IDbConnection connection, System.Data.IDbTransaction transaction)
+            public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, System.Data.IDbConnection connection)
             {
                 this.ParentID = parentid;
                 m_parentdb = parentdb;
                 m_connection = connection;
-                m_transaction = transaction;
                 m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
 
-                using (var cmd = m_connection.CreateCommand(m_transaction))
+                using (var cmd = m_connection.CreateCommand(m_parentdb.Transaction))
                     cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER PRIMARY KEY) ", m_tablename));
             }
 
             public void ApplyFilter(Action<System.Data.IDbCommand, long, string> filtercommand)
             {
-                using (var cmd = m_connection.CreateCommand(m_transaction))
+                using (var cmd = m_connection.CreateCommand(m_parentdb.Transaction))
                     filtercommand(cmd, ParentID, m_tablename);
                 
                 PostFilterChecks();
@@ -109,7 +107,7 @@ namespace Duplicati.Library.Main.Database
                     // SQLite only supports ASCII compares
                     var p = (filter as Library.Utility.FilterExpression).GetSimpleList();
                     var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
-                    using (var cmd = m_connection.CreateCommand(m_transaction))
+                    using (var cmd = m_connection.CreateCommand(m_parentdb.Transaction))
                     {
                         cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", filenamestable));
                         cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filenamestable);
@@ -129,8 +127,8 @@ namespace Duplicati.Library.Main.Database
                 {
                     // Do row-wise iteration
                     object[] values = new object[2];
-                    using (var cmd = m_connection.CreateCommand(m_transaction))
-                    using (var cmd2 = m_connection.CreateCommand(m_transaction))
+                    using (var cmd = m_connection.CreateCommand(m_parentdb.Transaction))
+                    using (var cmd2 = m_connection.CreateCommand(m_parentdb.Transaction))
                     {
                         cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"") VALUES (?)", m_tablename);
                         cmd2.AddParameters(1);
@@ -153,7 +151,7 @@ namespace Duplicati.Library.Main.Database
 
             private void PostFilterChecks()
             {
-                using (var cmd = m_connection.CreateCommand(m_transaction))
+                using (var cmd = m_connection.CreateCommand(m_parentdb.Transaction))
                 {
                     RemovedFileCount = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""{0}""", m_tablename), 0);
                     RemovedFileSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""C"".""Length"") FROM ""{0}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename), 0);
@@ -165,9 +163,9 @@ namespace Duplicati.Library.Main.Database
 
             public Tuple<long, long> ConvertToPermanentFileset(string name, DateTime timestamp)
             {                
-                var remotevolid = m_parentdb.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
-                var filesetid = m_parentdb.CreateFileset(remotevolid, timestamp, m_transaction);
-                using (var cmd = m_connection.CreateCommand(m_transaction))
+                var remotevolid = m_parentdb.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
+                var filesetid = m_parentdb.CreateFileset(remotevolid, timestamp);
+                using (var cmd = m_connection.CreateCommand(m_parentdb.Transaction))
                     cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT ?, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN ""{0}"" ", m_tablename), filesetid, ParentID);
 
                 return new Tuple<long, long>(remotevolid, filesetid);
@@ -175,7 +173,7 @@ namespace Duplicati.Library.Main.Database
 
             public IEnumerable<KeyValuePair<string, long>> ListAllDeletedFiles()
             {
-                using (var cmd = m_connection.CreateCommand(m_transaction))
+                using (var cmd = m_connection.CreateCommand(m_parentdb.Transaction))
                 using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{0}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename)))
                     while (rd.Read())
                         yield return new KeyValuePair<string, long>(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
@@ -185,7 +183,7 @@ namespace Duplicati.Library.Main.Database
             {
                 try
                 {
-                    using (var cmd = m_connection.CreateCommand(m_transaction))
+                    using (var cmd = m_connection.CreateCommand(m_parentdb.Transaction))
                         cmd.ExecuteNonQuery(@"DROP TABLE IF EXISTS ""{0}""", m_tablename);
                 }
                 catch

Duplicati/Library/Main/Database/LocalRecreateDatabase.cs

@@ -197,12 +197,10 @@ namespace Duplicati.Library.Main.Database
                 m_filesetLookup = new PathLookupHelper<PathEntryKeeper>();
         }
 
-        public void FindMissingBlocklistHashes(long hashsize, long blocksize, System.Data.IDbTransaction transaction)
+        public void FindMissingBlocklistHashes(long hashsize, long blocksize)
         {
-            using(var cmd = m_connection.CreateCommand())
-            {
-                cmd.Transaction = transaction;
-                
+            using(var cmd = m_connection.CreateCommand(Transaction))
+            {                
                 //Update all small blocklists and matching blocks
 
                 var selectSmallBlocks = string.Format(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist);
@@ -287,22 +285,22 @@ namespace Duplicati.Library.Main.Database
             }
         }
         
-        public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
+        public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid)
         {
-            AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction);
+            AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid);
         }
 
-        public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
+        public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid)
         {
-            AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
+            AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid);
         }
         
-        public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
+        public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid)
         {
-            AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metadataid, transaction);
+            AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metadataid);
         }
         
-        private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
+        private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, long metadataid)
         {
             var fileid = -1L;
                         
@@ -314,7 +312,7 @@ namespace Duplicati.Library.Main.Database
             }
             else
             {
-                m_findFilesetCommand.Transaction = transaction;
+                m_findFilesetCommand.Transaction = Transaction;
                 m_findFilesetCommand.SetParameterValue(0, path);
                 m_findFilesetCommand.SetParameterValue(1, blocksetid);
                 m_findFilesetCommand.SetParameterValue(2, metadataid);
@@ -323,7 +321,7 @@ namespace Duplicati.Library.Main.Database
             
             if (fileid < 0)
             {
-                m_insertFileCommand.Transaction = transaction;
+                m_insertFileCommand.Transaction = Transaction;
                 m_insertFileCommand.SetParameterValue(0, path);
                 m_insertFileCommand.SetParameterValue(1, blocksetid);
                 m_insertFileCommand.SetParameterValue(2, metadataid);
@@ -342,45 +340,45 @@ namespace Duplicati.Library.Main.Database
                 }
             }
             
-            m_insertFilesetEntryCommand.Transaction = transaction;
+            m_insertFilesetEntryCommand.Transaction = Transaction;
             m_insertFilesetEntryCommand.SetParameterValue(0, filesetid);
             m_insertFilesetEntryCommand.SetParameterValue(1, fileid);
             m_insertFilesetEntryCommand.SetParameterValue(2, time.ToUniversalTime().Ticks);
             m_insertFilesetEntryCommand.ExecuteNonQuery();
         }
         
-        public long AddMetadataset(string metahash, long metahashsize, IEnumerable<string> metablocklisthashes, long expectedmetablocklisthashes, System.Data.IDbTransaction transaction)
+        public long AddMetadataset(string metahash, long metahashsize, IEnumerable<string> metablocklisthashes, long expectedmetablocklisthashes)
         {
             var metadataid = -1L;
             if (metahash == null)
                 return metadataid;
                                 
-            m_findMetadatasetCommand.Transaction = transaction;
+            m_findMetadatasetCommand.Transaction = Transaction;
             m_findMetadatasetCommand.SetParameterValue(0, metahash);
             m_findMetadatasetCommand.SetParameterValue(1, metahashsize);
             metadataid = m_findMetadatasetCommand.ExecuteScalarInt64(-1);
             if (metadataid != -1)
                 return metadataid;
 
-            var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction);
+            var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes);
             
-            m_insertMetadatasetCommand.Transaction = transaction;
+            m_insertMetadatasetCommand.Transaction = Transaction;
             m_insertMetadatasetCommand.SetParameterValue(0, blocksetid);
             metadataid = m_insertMetadatasetCommand.ExecuteScalarInt64(-1);
                             
             return metadataid;
         }
         
-        public long AddBlockset(string fullhash, long size, IEnumerable<string> blocklisthashes, long expectedblocklisthashes, System.Data.IDbTransaction transaction)
+        public long AddBlockset(string fullhash, long size, IEnumerable<string> blocklisthashes, long expectedblocklisthashes)
         {
-            m_findBlocksetCommand.Transaction = transaction;
+            m_findBlocksetCommand.Transaction = Transaction;
             m_findBlocksetCommand.SetParameterValue(0, size);
             m_findBlocksetCommand.SetParameterValue(1, fullhash);
             var blocksetid = m_findBlocksetCommand.ExecuteScalarInt64(-1);
             if (blocksetid != -1)
                 return blocksetid;                        
             
-            m_insertBlocksetCommand.Transaction = transaction;
+            m_insertBlocksetCommand.Transaction = Transaction;
             m_insertBlocksetCommand.SetParameterValue(0, size);
             m_insertBlocksetCommand.SetParameterValue(1, fullhash);
             blocksetid = m_insertBlocksetCommand.ExecuteScalarInt64(-1);
@@ -389,7 +387,7 @@ namespace Duplicati.Library.Main.Database
             if (blocklisthashes != null)
             {
                 var index = 0L;
-                m_insertBlocklistHashCommand.Transaction = transaction;
+                m_insertBlocklistHashCommand.Transaction = Transaction;
                 m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid);
 
                 foreach(var hash in blocklisthashes)
@@ -413,9 +411,9 @@ namespace Duplicati.Library.Main.Database
             return blocksetid;
         }
 
-        public bool UpdateBlock(string hash, long size, long volumeID, System.Data.IDbTransaction transaction)
+        public bool UpdateBlock(string hash, long size, long volumeID)
         {
-            m_findHashBlockCommand.Transaction = transaction;
+            m_findHashBlockCommand.Transaction = Transaction;
             m_findHashBlockCommand.SetParameterValue(0, hash);
             m_findHashBlockCommand.SetParameterValue(1, size);
             var currentVolumeId = m_findHashBlockCommand.ExecuteScalarInt64(-2);
@@ -426,7 +424,7 @@ namespace Duplicati.Library.Main.Database
             if (currentVolumeId == -2)
             {
                 //Insert
-                m_insertBlockCommand.Transaction = transaction;
+                m_insertBlockCommand.Transaction = Transaction;
                 m_insertBlockCommand.SetParameterValue(0, hash);
                 m_insertBlockCommand.SetParameterValue(1, size);
                 m_insertBlockCommand.SetParameterValue(2, volumeID);
@@ -437,7 +435,7 @@ namespace Duplicati.Library.Main.Database
             else if (currentVolumeId == -1)
             {
                 //Update
-                m_updateBlockVolumeCommand.Transaction = transaction;
+                m_updateBlockVolumeCommand.Transaction = Transaction;
                 m_updateBlockVolumeCommand.SetParameterValue(0, volumeID);
                 m_updateBlockVolumeCommand.SetParameterValue(1, hash);
                 m_updateBlockVolumeCommand.SetParameterValue(2, size);
@@ -449,7 +447,7 @@ namespace Duplicati.Library.Main.Database
             }
             else
             {
-                m_insertDuplicateBlockCommand.Transaction = transaction;
+                m_insertDuplicateBlockCommand.Transaction = Transaction;
                 m_insertDuplicateBlockCommand.SetParameterValue(0, hash);
                 m_insertDuplicateBlockCommand.SetParameterValue(1, size);
                 m_insertDuplicateBlockCommand.SetParameterValue(2, volumeID);
@@ -459,24 +457,24 @@ namespace Duplicati.Library.Main.Database
             }            
         }
 
-        public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, System.Data.IDbTransaction transaction)
+        public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize)
         {
-            m_insertSmallBlockset.Transaction = transaction;
+            m_insertSmallBlockset.Transaction = Transaction;
             m_insertSmallBlockset.SetParameterValue(0, filehash);
             m_insertSmallBlockset.SetParameterValue(1, blockhash);
             m_insertSmallBlockset.SetParameterValue(2, blocksize);
             m_insertSmallBlockset.ExecuteNonQuery();
         }
         
-        public bool UpdateBlockset(string hash, IEnumerable<string> blocklisthashes, System.Data.IDbTransaction transaction)
+        public bool UpdateBlockset(string hash, IEnumerable<string> blocklisthashes)
         {
-            m_findblocklisthashCommand.Transaction = transaction;
+            m_findblocklisthashCommand.Transaction = Transaction;
             m_findblocklisthashCommand.SetParameterValue(0, hash);
             var r = m_findblocklisthashCommand.ExecuteScalar();
             if (r != null && r != DBNull.Value)
                 return false;
         
-            m_insertBlockset.Transaction = transaction;                
+            m_insertBlockset.Transaction = Transaction;                
             m_insertBlockset.SetParameterValue(0, hash);
             
             var index = 0L;

Duplicati/Library/Main/Database/LocalRepairDatabase.cs

@@ -31,7 +31,7 @@ namespace Duplicati.Library.Main.Database
         
         public long GetFilesetIdFromRemotename(string filelist)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var filesetid = cmd.ExecuteScalarInt64(@"SELECT ""Fileset"".""ID"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = ?", -1, filelist);
                 if (filesetid == -1)
@@ -131,20 +131,20 @@ namespace Duplicati.Library.Main.Database
         private class MissingBlockList : IMissingBlockList
         {
             private System.Data.IDbConnection m_connection;
-            private TemporaryTransactionWrapper m_transaction;
             private System.Data.IDbCommand m_insertCommand;
             private string m_tablename;
             private string m_volumename;
-            
-            public MissingBlockList(string volumename, System.Data.IDbConnection connection, System.Data.IDbTransaction transaction)
+            private LocalDatabase m_parent;
+
+            public MissingBlockList(string volumename, System.Data.IDbConnection connection, LocalDatabase parent)
             {
+                m_parent = parent;
                 m_connection = connection;
-                m_transaction = new TemporaryTransactionWrapper(m_connection, transaction);
                 m_volumename = volumename;
                 var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                 using(var cmd = m_connection.CreateCommand())
                 {
-                    cmd.Transaction = m_transaction.Parent;
+                    cmd.Transaction = m_parent.Transaction;
                     cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ", tablename));
                     m_tablename = tablename;
                     
@@ -156,7 +156,7 @@ namespace Duplicati.Library.Main.Database
                 }
 
                 m_insertCommand = m_connection.CreateCommand();
-                m_insertCommand.Transaction = m_transaction.Parent;
+                m_insertCommand.Transaction = m_parent.Transaction;
                 m_insertCommand.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""Restored"" = ? ", tablename);
                 m_insertCommand.AddParameters(4);
             }
@@ -172,7 +172,7 @@ namespace Duplicati.Library.Main.Database
             
             public IEnumerable<IBlockWithSources> GetSourceFilesWithBlocks(long blocksize)
             {
-                using(var cmd = m_connection.CreateCommand(m_transaction.Parent))
+                using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
                     using(var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""{0}"".""Hash"", ""{0}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {1} FROM  ""{0}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{0}"".""Hash"" = ""Block"".""Hash"" AND ""{0}"".""Size"" = ""Block"".""Size"" AND ""{0}"".""Restored"" = ? ", m_tablename, blocksize), 0))
                         if (rd.Read())
                         {
@@ -184,7 +184,7 @@ namespace Duplicati.Library.Main.Database
                     
             public IEnumerable<KeyValuePair<string, long>> GetMissingBlocks()
             {
-                using(var cmd = m_connection.CreateCommand(m_transaction.Parent))
+                using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
                     foreach(var rd in cmd.ExecuteReaderEnumerable(string.Format(@"SELECT ""{0}"".""Hash"", ""{0}"".""Size"" FROM ""{0}"" WHERE ""{0}"".""Restored"" = ? ", m_tablename), 0))
                         yield return new KeyValuePair<string, long>(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
             }
@@ -196,14 +196,14 @@ namespace Duplicati.Library.Main.Database
             
                 var cmdtxt = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = ? AND ""FilesetEntry"".""FileID"" IN  (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))";
             
-                using(var cmd = m_connection.CreateCommand(m_transaction.Parent))
+                using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
                     foreach(var rd in cmd.ExecuteReaderEnumerable(string.Format(cmdtxt, m_tablename), RemoteVolumeType.Files.ToString()))
                         yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2));
             }
             
             public IEnumerable<IRemoteVolume> GetMissingBlockSources()
             {
-                using(var cmd = m_connection.CreateCommand(m_transaction.Parent))
+                using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
                     foreach(var rd in cmd.ExecuteReaderEnumerable(string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{0}"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != ? ", m_tablename), m_volumename))
                         yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2));
             }
@@ -214,8 +214,8 @@ namespace Duplicati.Library.Main.Database
                 {
                     try
                     {
-                        using(var cmd = m_connection.CreateCommand(m_transaction.Parent))
-                            cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_transaction));
+                        using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
+                            cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_parent.Transaction));
                     }
                     catch { }
                     finally { m_tablename = null; }
@@ -228,18 +228,15 @@ namespace Duplicati.Library.Main.Database
             }
         }
         
-        public IMissingBlockList CreateBlockList(string volumename, System.Data.IDbTransaction transaction = null)
+        public IMissingBlockList CreateBlockList(string volumename)
         {
-            return new MissingBlockList(volumename, m_connection, transaction);
+            return new MissingBlockList(volumename, m_connection, this);
         }
 
         public void FixDuplicateMetahash()
         {
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
-                cmd.Transaction = tr;
-
                 var sql_count = 
                     @"SELECT COUNT(*) FROM (" +
                     @" SELECT DISTINCT c1 FROM (" +
@@ -258,7 +255,7 @@ namespace Duplicati.Library.Main.Database
 
                     var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID""";
 
-                    using(var c2 = m_connection.CreateCommand(tr))
+                    using(var c2 = m_connection.CreateCommand(Transaction))
                     {
                         c2.CommandText = string.Format(@"UPDATE ""{0}"" SET ""MetadataID"" = ? WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = ?)", tablename);
                         c2.CommandText += @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = ? AND ""ID"" != ?";
@@ -273,7 +270,7 @@ namespace Duplicati.Library.Main.Database
 
                     using(var c2 = m_connection.CreateCommand())
                     {
-                        c2.Transaction = tr;
+                        c2.Transaction = Transaction;
                         c2.CommandText = string.Format(@"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)", tablename);
                         c2.CommandText += string.Format(@"; DELETE FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?", tablename);
                         foreach(var rd in cmd.ExecuteReaderEnumerable(sql))
@@ -291,15 +288,14 @@ namespace Duplicati.Library.Main.Database
                         throw new Duplicati.Library.Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!");
 
                     m_result.AddMessage("Duplicate metadatahashes repaired succesfully");
-                    tr.Commit();
+                    CommitTransaction("Repaird duplicate metadatahashes");
                 }
             }
         }
 
         public void FixDuplicateFileentries()
         {
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var sql_count = @"SELECT COUNT(*) FROM (SELECT ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""File"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1";
 
@@ -312,7 +308,7 @@ namespace Duplicati.Library.Main.Database
                             SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""File"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") 
                             WHERE ""Entries"" > 1 ORDER BY ""ID""";
 
-                    using(var c2 = m_connection.CreateCommand(tr))
+                    using(var c2 = m_connection.CreateCommand(Transaction))
                     {
                         c2.CommandText = @"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)";
                         c2.CommandText += @"; DELETE FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?";
@@ -326,7 +322,7 @@ namespace Duplicati.Library.Main.Database
                         throw new Duplicati.Library.Interface.UserInformationException("Repair failed, there are still duplicate file entries!");
 
                     m_result.AddMessage("Duplicate file entries repaired succesfully");
-                    tr.Commit();
+                    CommitTransaction("Repaired duplicate file entries");
                 }
             }
 
@@ -344,18 +340,17 @@ namespace Duplicati.Library.Main.Database
             var sql = string.Format(@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual""", blocksize / hashsize);
             var countsql = @"SELECT COUNT(*) FROM (" + sql + @")";
 
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0);
                 if (itemswithnoblocklisthash != 0)
                 {
                     m_result.AddMessage(string.Format("Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash));
-                    using(var c2 = m_connection.CreateCommand(tr))
-                    using(var c3 = m_connection.CreateCommand(tr))
-                    using(var c4 = m_connection.CreateCommand(tr))
-                    using(var c5 = m_connection.CreateCommand(tr))
-                    using(var c6 = m_connection.CreateCommand(tr))
+                    using(var c2 = m_connection.CreateCommand(Transaction))
+                    using(var c3 = m_connection.CreateCommand(Transaction))
+                    using(var c4 = m_connection.CreateCommand(Transaction))
+                    using(var c5 = m_connection.CreateCommand(Transaction))
+                    using(var c6 = m_connection.CreateCommand(Transaction))
                     {
                         c3.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?, ?, ?) ";
                         c4.CommandText = @"SELECT COUNT(*) FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?";
@@ -432,7 +427,7 @@ namespace Duplicati.Library.Main.Database
                         throw new Duplicati.Library.Interface.UserInformationException(string.Format("Failed to repair, after repair {0} blocklisthashes were missing", itemswithnoblocklisthash));
 
                     m_result.AddMessage("Missing blocklisthashes repaired succesfully");
-                    tr.Commit();
+                    CommitTransaction("Repaied blocklisthashes");
                 }
             }
 
@@ -440,8 +435,7 @@ namespace Duplicati.Library.Main.Database
 
         public void FixDuplicateBlocklistHashes(long blocksize, long hashsize)
         {
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1";
 
@@ -454,7 +448,7 @@ namespace Duplicati.Library.Main.Database
 
                     var unique_count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0);
 
-                    using(var c2 = m_connection.CreateCommand(tr))
+                    using(var c2 = m_connection.CreateCommand(Transaction))
                     {
                         c2.CommandText = @"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? AND ""Index"" = ? LIMIT ?)";
                         foreach(var rd in cmd.ExecuteReaderEnumerable(dup_sql))
@@ -478,7 +472,7 @@ namespace Duplicati.Library.Main.Database
 
                     try
                     {
-                        VerifyConsistency(tr, blocksize, hashsize, true);
+                        VerifyConsistency(blocksize, hashsize, true);
                     }
                     catch(Exception ex)
                     {
@@ -486,15 +480,14 @@ namespace Duplicati.Library.Main.Database
                     }
 
                     m_result.AddMessage("Duplicate blocklisthashes repaired succesfully");
-                    tr.Commit();
+                    CommitTransaction("Repaired duplicate blocklisthashes");
                 }
             }
         }
 
         public void CheckAllBlocksAreInVolume(string filename, IEnumerable<KeyValuePair<string, long>> blocks)
         {
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
 
@@ -521,7 +514,7 @@ namespace Duplicati.Library.Main.Database
 
         public void CheckBlocklistCorrect(string hash, long length, IEnumerable<string> blocklist, long blocksize, long blockhashlength)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var query = string.Format(@"
 SELECT 

Duplicati/Library/Main/Database/LocalRestoreDatabase.cs

@@ -180,7 +180,7 @@ namespace Duplicati.Library.Main.Database
             m_tempfiletable = "Fileset-" + guid;
             m_tempblocktable = "Blocks-" + guid;
 
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(Transaction))
             {
                 var filesetIds = GetFilesetIDs(NormalizeDateTime(restoretime), versions).ToList();
                 while(filesetIds.Count > 0)
@@ -220,46 +220,40 @@ namespace Duplicati.Library.Main.Database
                         // If we get a list of filenames, the lookup table is faster
                         // unfortunately we cannot do this if the filesystem is case sensitive as
                         // SQLite only supports ASCII compares
-                        using(var tr = m_connection.BeginTransaction())
+                        var p = (filter as Library.Utility.FilterExpression).GetSimpleList();
+                        var m_filenamestable = "Filenames-" + guid;
+                        cmd.Transaction = Transaction;
+                        cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", m_filenamestable));
+                        cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable);
+                        cmd.AddParameter();
+                        
+                        foreach(var s in p)
                         {
-                            var p = (filter as Library.Utility.FilterExpression).GetSimpleList();
-                            var m_filenamestable = "Filenames-" + guid;
-                            cmd.Transaction = tr;
-                            cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", m_filenamestable));
-                            cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable);
-                            cmd.AddParameter();
-                            
-                            foreach(var s in p)
-                            {
-                                cmd.SetParameterValue(0, s);
-                                cmd.ExecuteNonQuery();
-                            }
-
-                            cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable);
-                            cmd.SetParameterValue(0, filesetId);
-                            var c = cmd.ExecuteNonQuery();
-                            
-                            cmd.Parameters.Clear();
-                            
-                            if (c != p.Length && c != 0)
-                            {
-                                var sb = new StringBuilder();
-                                sb.AppendLine();
-                                
-                                using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable)))
-                                    while (rd.Read())
-                                        sb.AppendLine(rd.GetValue(0).ToString());
-    
-                                var actualrestoretime = ParseFromEpochSeconds(cmd.ExecuteScalarInt64(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = ?", 0, filesetId));
-                                log.AddWarning(string.Format("{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb), null);
-                                cmd.Parameters.Clear();
-                            }
-                            
-                            cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_filenamestable));
-                            
-                            using(new Logging.Timer("CommitPrepareFileset"))
-                                tr.Commit();
+                            cmd.SetParameterValue(0, s);
+                            cmd.ExecuteNonQuery();
                         }
+
+                        cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable);
+                        cmd.SetParameterValue(0, filesetId);
+                        var c = cmd.ExecuteNonQuery();
+                        
+                        cmd.Parameters.Clear();
+                        
+                        if (c != p.Length && c != 0)
+                        {
+                            var sb = new StringBuilder();
+                            sb.AppendLine();
+                            
+                            using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable)))
+                                while (rd.Read())
+                                    sb.AppendLine(rd.GetValue(0).ToString());
+
+                            var actualrestoretime = ParseFromEpochSeconds(cmd.ExecuteScalarInt64(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = ?", 0, filesetId));
+                            log.AddWarning(string.Format("{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb), null);
+                            cmd.Parameters.Clear();
+                        }
+                        
+                        cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_filenamestable));
                     }
                     else
                     {
@@ -998,7 +992,6 @@ namespace Duplicati.Library.Main.Database
             void SetFileDataVerified(long targetfileid);
             void Commit(ILogWriter log);
             void UpdateProcessed(IOperationProgressUpdater writer);
-            System.Data.IDbTransaction Transaction { get; }
         }
 
         /// <summary>
@@ -1013,26 +1006,27 @@ namespace Duplicati.Library.Main.Database
             private System.Data.IDbCommand m_updateAsRestoredCommand;
             private System.Data.IDbCommand m_updateFileAsDataVerifiedCommand;
             private System.Data.IDbCommand m_statUpdateCommand;
+            private LocalDatabase m_parent;
             private bool m_hasUpdates = false;
 
             private string m_blocktablename;
             private string m_filetablename;
 
-            public System.Data.IDbTransaction Transaction { get { return m_insertblockCommand.Transaction; } }
-
-            public DirectBlockMarker(System.Data.IDbConnection connection, string blocktablename, string filetablename, string statstablename)
+            public DirectBlockMarker(System.Data.IDbConnection connection, string blocktablename, string filetablename, string statstablename, LocalDatabase parent)
             {
+                m_parent = parent;
                 m_insertblockCommand = connection.CreateCommand();
+
                 m_resetfileCommand = connection.CreateCommand();
                 m_updateAsRestoredCommand = connection.CreateCommand();
                 m_updateFileAsDataVerifiedCommand = connection.CreateCommand();
                 m_statUpdateCommand = connection.CreateCommand();
 
-                m_insertblockCommand.Transaction = connection.BeginTransaction();
-                m_resetfileCommand.Transaction = m_insertblockCommand.Transaction;
-                m_updateAsRestoredCommand.Transaction = m_insertblockCommand.Transaction;
-                m_updateFileAsDataVerifiedCommand.Transaction = m_insertblockCommand.Transaction;
-                m_statUpdateCommand.Transaction = m_insertblockCommand.Transaction;
+                m_insertblockCommand.Transaction = m_parent.Transaction;
+                m_resetfileCommand.Transaction = m_parent.Transaction;
+                m_updateAsRestoredCommand.Transaction = m_parent.Transaction;
+                m_updateFileAsDataVerifiedCommand.Transaction = m_parent.Transaction;
+                m_statUpdateCommand.Transaction = m_parent.Transaction;
 
                 m_blocktablename = blocktablename;
                 m_filetablename = filetablename;
@@ -1134,12 +1128,7 @@ namespace Duplicati.Library.Main.Database
 
             public void Commit(ILogWriter log)
             {
-                var tr = m_insertblockCommand.Transaction;
-                m_insertblockCommand.Dispose();
-                m_insertblockCommand = null;
-                using (new Logging.Timer("CommitBlockMarker"))
-                    tr.Commit();
-                tr.Dispose();
+                m_parent.CommitTransaction("CommitBlockMarker");
             }
 
             public void Dispose()
@@ -1173,7 +1162,7 @@ namespace Duplicati.Library.Main.Database
 
         public IBlockMarker CreateBlockMarker()
         {
-            return new DirectBlockMarker(m_connection, m_tempblocktable, m_tempfiletable, m_totalprogtable);
+            return new DirectBlockMarker(m_connection, m_tempblocktable, m_tempfiletable, m_totalprogtable, this);
         }
 
         public override void Dispose()

Duplicati/Library/Main/Database/LocalTestDatabase.cs

@@ -164,32 +164,32 @@ namespace Duplicati.Library.Main.Database
         
         private abstract class Basiclist : IDisposable
         {
-              protected System.Data.IDbConnection m_connection;
-              protected string m_volumename;
-              protected string m_tablename;
-              protected System.Data.IDbTransaction m_transaction;
-              protected System.Data.IDbCommand m_insertCommand;
-              protected abstract string TABLE_PREFIX { get; }
-              protected abstract string TABLEFORMAT { get; }
-              protected abstract string INSERTCOMMAND { get; }
-              protected abstract int INSERTARGUMENTS { get; }
+            protected System.Data.IDbConnection m_connection;
+            protected string m_volumename;
+            protected string m_tablename;
+            protected LocalDatabase m_parent;
+            protected System.Data.IDbCommand m_insertCommand;
+            protected abstract string TABLE_PREFIX { get; }
+            protected abstract string TABLEFORMAT { get; }
+            protected abstract string INSERTCOMMAND { get; }
+            protected abstract int INSERTARGUMENTS { get; }
               
-              public Basiclist(System.Data.IDbConnection connection, string volumename)
+            public Basiclist(System.Data.IDbConnection connection, string volumename, LocalDatabase parent)
               {
                 m_connection = connection;
                 m_volumename = volumename;
-                m_transaction = m_connection.BeginTransaction();
+                m_parent = parent;
+
                 var tablename = TABLE_PREFIX + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
 
-                using(var cmd = m_connection.CreateCommand())
+                using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
                 {
-                    cmd.Transaction = m_transaction;
                     cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" {1}", tablename, TABLEFORMAT));
                     m_tablename = tablename;
                 }
                 
                 m_insertCommand = m_connection.CreateCommand();
-                m_insertCommand.Transaction = m_transaction;
+                m_insertCommand.Transaction = m_parent.Transaction;
                 m_insertCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" {1}", m_tablename, INSERTCOMMAND);
                 m_insertCommand.AddParameters(INSERTARGUMENTS);
             }
@@ -201,7 +201,7 @@ namespace Duplicati.Library.Main.Database
                     { 
                         using(var cmd = m_connection.CreateCommand())
                         {
-                            cmd.Transaction = m_transaction;
+                            cmd.Transaction = m_parent.Transaction;
                             cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tablename));
                         }
                     }
@@ -212,11 +212,6 @@ namespace Duplicati.Library.Main.Database
                     try { m_insertCommand.Dispose(); }
                     catch {}
                     finally { m_insertCommand = null; }
-                    
-                if (m_transaction != null)
-                    try { m_transaction.Rollback(); }
-                    catch {}
-                    finally { m_transaction = null; }
             }
         }
         
@@ -233,8 +228,8 @@ namespace Duplicati.Library.Main.Database
             protected override string INSERTCOMMAND { get { return @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (?,?,?,?,?)"; } }
             protected override int INSERTARGUMENTS { get { return 5; } }
 
-            public Filelist(System.Data.IDbConnection connection, string volumename)
-                : base(connection, volumename)
+            public Filelist(System.Data.IDbConnection connection, string volumename, LocalDatabase parent)
+                : base(connection, volumename, parent)
             { 
             }
             
@@ -258,10 +253,8 @@ namespace Duplicati.Library.Main.Database
                 var modified = @"SELECT ? AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{0}"" E, ""{1}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"")  ";
                 var drop = @"DROP TABLE IF EXISTS ""{1}"" ";
                 
-                using(var cmd = m_connection.CreateCommand())
+                using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
                 {
-                    cmd.Transaction = m_transaction;
-                    
                     try
                     {
                         cmd.ExecuteNonQuery(string.Format(create, m_tablename, cmpName), m_volumename);
@@ -292,8 +285,8 @@ namespace Duplicati.Library.Main.Database
             protected override string INSERTCOMMAND { get { return @"(""Name"", ""Hash"", ""Size"") VALUES (?,?,?)"; } }
             protected override int INSERTARGUMENTS { get { return 3; } }
             
-            public Indexlist(System.Data.IDbConnection connection, string volumename)
-                : base(connection, volumename)
+            public Indexlist(System.Data.IDbConnection connection, string volumename, LocalDatabase parent)
+                : base(connection, volumename, parent)
             {
             }
                     
@@ -314,10 +307,8 @@ namespace Duplicati.Library.Main.Database
                 var modified = @"SELECT ? AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{0}"" E, ""{1}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") ";
                 var drop = @"DROP TABLE IF EXISTS ""{1}"" ";
                 
-                using(var cmd = m_connection.CreateCommand())
+                using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
                 {
-                    cmd.Transaction = m_transaction;
-                    
                     try
                     {
                         cmd.ExecuteNonQuery(string.Format(create, m_tablename, cmpName), m_volumename);
@@ -348,8 +339,8 @@ namespace Duplicati.Library.Main.Database
             protected override string INSERTCOMMAND { get { return @"(""Hash"", ""Size"") VALUES (?,?)"; } }
             protected override int INSERTARGUMENTS { get { return 2; } }
             
-            public Blocklist(System.Data.IDbConnection connection, string volumename)
-                : base(connection, volumename)
+            public Blocklist(System.Data.IDbConnection connection, string volumename, LocalDatabase parent)
+                : base(connection, volumename, parent)
             { }            
         
             public void AddBlock(string hash, long size)
@@ -370,10 +361,8 @@ namespace Duplicati.Library.Main.Database
                 var modified = @"SELECT ? AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{0}"" E, ""{1}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size""  ";
                 var drop = @"DROP TABLE IF EXISTS ""{1}"" ";
                 
-                using(var cmd = m_connection.CreateCommand())
+                using(var cmd = m_connection.CreateCommand(m_parent.Transaction))
                 {
-                    cmd.Transaction = m_transaction;
-                    
                     try
                     {
                         cmd.ExecuteNonQuery(string.Format(create, cmpName, curBlocks, delBlocks), m_volumename, m_volumename);
@@ -393,17 +382,17 @@ namespace Duplicati.Library.Main.Database
         
         public IFilelist CreateFilelist(string name)
         {
-            return new Filelist(m_connection, name);
+            return new Filelist(m_connection, name, this);
         }
 
         public IIndexlist CreateIndexlist(string name)
         {
-            return new Indexlist(m_connection, name);
+            return new Indexlist(m_connection, name, this);
         }
 
         public IBlocklist CreateBlocklist(string name)
         {
-            return new Blocklist(m_connection, name);
+            return new Blocklist(m_connection, name, this);
         }
     }
 }

Duplicati/Library/Main/Database/RemoteVolumeEntry.cs

@@ -7,28 +7,25 @@ namespace Duplicati.Library.Main.Database
 {
     public struct RemoteVolumeEntry : IRemoteVolume
     {
-        private readonly string m_name;
-        private readonly string m_hash;
-        private readonly long m_size;
-        private readonly RemoteVolumeType m_type;
-        private readonly RemoteVolumeState m_state;
-        private readonly DateTime m_deleteGracePeriod;
-        
-        public string Name { get { return m_name; } }
-        public string Hash { get { return m_hash; } }
-        public long Size { get { return m_size; } }
-        public RemoteVolumeType Type { get { return m_type; } }
-        public RemoteVolumeState State { get { return m_state; } }
-        public DateTime deleteGracePeriod { get { return m_deleteGracePeriod; } }
+        public long ID { get; private set; }
+        public string Name { get; private set; }
+        public string Hash { get; private set; }
+        public long Size { get; private set; }
+        public RemoteVolumeType Type { get; private set; }
+        public RemoteVolumeState State { get; private set; }
+        public DateTime DeleteGracePeriod { get; private set; }
 
-        public RemoteVolumeEntry(string name, string hash, long size, RemoteVolumeType type, RemoteVolumeState state, DateTime deleteGracePeriod)
+        public static readonly RemoteVolumeEntry Empty = new RemoteVolumeEntry(-1, null, null, -1, (RemoteVolumeType)(-1), (RemoteVolumeState)(-1), default(DateTime));
+
+        public RemoteVolumeEntry(long id, string name, string hash, long size, RemoteVolumeType type, RemoteVolumeState state, DateTime deleteGracePeriod)
         {
-            m_name = name;
-            m_size = size;
-            m_type = type;
-            m_state = state;
-            m_hash = hash;
-            m_deleteGracePeriod = deleteGracePeriod;
+            ID = id;
+            Name = name;
+            Size = size;
+            Type = type;
+            State = state;
+            Hash = hash;
+            DeleteGracePeriod = deleteGracePeriod;
         }
     }
 }

Duplicati/Library/Main/Duplicati.Library.Main.csproj

@@ -39,9 +39,13 @@
     <Reference Include="System" />
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
+    <Reference Include="System.Core" />
     <Reference Include="Newtonsoft.Json">
       <HintPath>..\..\..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
+    <Reference Include="CoCoL">
+      <HintPath>..\..\..\packages\CoCoL.1.5.1\lib\net45\CoCoL.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Operation\VacuumHandler.cs" />
@@ -106,11 +110,45 @@
     <Compile Include="Strings.cs" />
     <Compile Include="Operation\RestoreHandlerMetadataStorage.cs" />
     <Compile Include="Operation\SystemInfoHandler.cs" />
+    <Compile Include="Operation\Backup\FileEnumerationProcess.cs" />
+    <Compile Include="Operation\Backup\MetadataPreProcess.cs" />
+    <Compile Include="Operation\Backup\MetadataGenerator.cs" />
+    <Compile Include="Operation\Common\LogMessage.cs" />
+    <Compile Include="Operation\Common\ProgressEvent.cs" />
+    <Compile Include="Operation\Backup\FileBlockProcessor.cs" />
+    <Compile Include="Operation\Backup\DataBlock.cs" />
+    <Compile Include="Operation\Backup\FilePreFilterProcess.cs" />
+    <Compile Include="Operation\Backup\DataBlockProcessor.cs" />
+    <Compile Include="Operation\Backup\SpillCollectorProcess.cs" />
+    <Compile Include="Operation\Backup\BackupDatabase.cs" />
+    <Compile Include="Operation\Common\DatabaseCommon.cs" />
+    <Compile Include="Operation\Common\SingleRunner.cs" />
+    <Compile Include="Operation\Common\BackendHandler.cs" />
+    <Compile Include="Operation\Common\StatsCollector.cs" />
+    <Compile Include="Operation\Backup\BackupStatsCollector.cs" />
+    <Compile Include="Operation\Common\LogHandler.cs" />
+    <Compile Include="Operation\Backup\ProgressHandler.cs" />
+    <Compile Include="Operation\Backup\BackendUploader.cs" />
+    <Compile Include="Operation\Common\IndexVolumeCreator.cs" />
+    <Compile Include="Operation\Backup\UploadSyntheticFilelist.cs" />
+    <Compile Include="Operation\Backup\Channels.cs" />
+    <Compile Include="Operation\Common\Channels.cs" />
+    <Compile Include="Operation\Common\LogWrapper.cs" />
+    <Compile Include="Operation\Backup\CountFilesHandler.cs" />
+    <Compile Include="Operation\Backup\UploadRealFilelist.cs" />
+    <Compile Include="Operation\Common\TaskControl.cs" />
+    <Compile Include="Operation\Backup\StreamBlockSplitter.cs" />
+    <Compile Include="Operation\Backup\StreamBlock.cs" />
     <Compile Include="Operation\PurgeFilesHandler.cs" />
     <Compile Include="Database\LocalPurgeDatabase.cs" />
     <Compile Include="Database\LocalListBrokenFilesDatabase.cs" />
     <Compile Include="Operation\ListBrokenFilesHandler.cs" />
     <Compile Include="Operation\PurgeBrokenFilesHandler.cs" />
+    <Compile Include="Operation\Backup\RecreateMissingIndexFiles.cs" />
+    <Compile Include="Operation\Common\IBackendHandlerDatabase.cs" />
+    <Compile Include="Operation\Common\HashMismatchException.cs" />
+    <Compile Include="Operation\Common\BackendHandlerDatabaseGuard.cs" />
+    <Compile Include="Operation\Common\IIndexVolumeCreatorDatabase.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Utility\Duplicati.Library.Utility.csproj">
@@ -178,4 +216,8 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
-</Project>
+  <ItemGroup>
+    <Folder Include="Operation\Backup\" />
+    <Folder Include="Operation\Common\" />
+  </ItemGroup>
+</Project>

Duplicati/Library/Main/Operation/Backup/BackendUploader.cs

@@ -0,0 +1,205 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Operation.Common;
+using Duplicati.Library.Main.Volumes;
+using System.Collections.Generic;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    internal interface IUploadRequest
+    {
+    }
+
+    internal class FlushRequest : IUploadRequest
+    {
+        public Task<long> LastWriteSizeAync { get { return m_tcs.Task; } }
+        private TaskCompletionSource<long> m_tcs = new TaskCompletionSource<long>();
+        public void SetFlushed(long size)
+        {
+            m_tcs.TrySetResult(size);
+        }
+
+    }
+
+    internal class IndexVolumeUploadRequest : IUploadRequest
+    {
+        public IndexVolumeWriter IndexVolume { get; private set; }
+
+        public IndexVolumeUploadRequest(IndexVolumeWriter indexVolume)
+        {
+            IndexVolume = indexVolume;
+        }
+    }
+
+    internal class FilesetUploadRequest : IUploadRequest
+    {
+        public FilesetVolumeWriter Fileset { get; private set; }
+
+        public FilesetUploadRequest(FilesetVolumeWriter fileset)
+        {
+            Fileset = fileset;
+        }
+    }
+
+    internal class VolumeUploadRequest : IUploadRequest
+    {
+        public readonly BlockVolumeWriter BlockVolume;
+        public readonly bool CreateIndexVolume;
+        public readonly IEnumerable<string> BlocklistData;
+
+        public VolumeUploadRequest(BlockVolumeWriter blockvolume, bool createindexvolume, IEnumerable<string> blocklistdata)
+        {
+            BlockVolume = blockvolume;
+            CreateIndexVolume = createindexvolume;
+            BlocklistData = blocklistdata;
+        }
+
+        public static string EncodeBlockListEntry(string hash, long size, byte[] data)
+        {
+            return hash + ":" + size + ":" + Convert.ToBase64String(data);
+        }
+
+        public static string DecodeBlockListEntryHash(string entry)
+        {
+            var ix = entry.IndexOf(':');
+            return entry.Substring(0, ix);
+        }
+
+        public static Tuple<string, long, byte[]> DecodeBlockListEntry(string entry)
+        {
+            var els = entry.Split(new char[] { ':' }, 3);
+            return new Tuple<string, long, byte[]>(
+                els[0],
+                long.Parse(els[1]),
+                Convert.FromBase64String(els[2])
+            );
+        }
+    }
+   
+    /// <summary>
+    /// This class encapsulates all requests to the backend
+    /// and ensures that the <code>AsynchronousUploadLimit</code> is honored
+    /// </summary>
+    internal static class BackendUploader
+    {
+        public static Task Run(Common.BackendHandler backend, Options options, Common.DatabaseCommon database, BackupResults results, Common.ITaskReader taskreader, StatsCollector stats)
+        {
+            return AutomationExtensions.RunTask(new
+            {
+                Input = Channels.BackendRequest.ForRead,
+            },
+
+            async self =>
+            {
+                var inProgress = new Queue<KeyValuePair<int, Task>>();
+                var max_pending = options.AsynchronousUploadLimit == 0 ? long.MaxValue : options.AsynchronousUploadLimit;
+                var noIndexFiles = options.IndexfilePolicy == Options.IndexFileStrategy.None;
+                var active = 0;
+                var lastSize = -1L;
+                
+                while(!await self.Input.IsRetiredAsync && await taskreader.ProgressAsync)
+                {
+                    try
+                    {
+                        var req = await self.Input.ReadAsync();
+
+                        if (!await taskreader.ProgressAsync)
+                            continue;
+                        
+                        KeyValuePair<int, Task> task = default(KeyValuePair<int, Task>);
+                        if (req is VolumeUploadRequest)
+                        {
+                            lastSize = ((VolumeUploadRequest)req).BlockVolume.SourceSize;
+
+                            if (noIndexFiles || !((VolumeUploadRequest)req).CreateIndexVolume)
+                                task = new KeyValuePair<int, Task>(1, backend.UploadFileAsync(((VolumeUploadRequest)req).BlockVolume, null));
+                            else
+                                task = new KeyValuePair<int, Task>(2, backend.UploadFileAsync(((VolumeUploadRequest)req).BlockVolume, name => IndexVolumeCreator.CreateIndexVolume(name, options, database, ((VolumeUploadRequest)req).BlocklistData)));
+                        }
+                        else if (req is FilesetUploadRequest)
+                            task = new KeyValuePair<int, Task>(1, backend.UploadFileAsync(((FilesetUploadRequest)req).Fileset));
+                        else if (req is IndexVolumeUploadRequest)
+                            task = new KeyValuePair<int, Task>(1, backend.UploadFileAsync(((IndexVolumeUploadRequest)req).IndexVolume));
+                        else if (req is FlushRequest)
+                        {
+                            try
+                            {
+                                while(inProgress.Count > 0)
+                                    await inProgress.Dequeue().Value;
+                                active = 0;
+                            }
+                            finally
+                            {
+                                ((FlushRequest)req).SetFlushed(lastSize);
+                            }
+                        }
+
+                        if (task.Value != null)
+                        {
+                            inProgress.Enqueue(task);
+                            active += task.Key;
+                        }
+                    }
+                    catch(Exception ex)
+                    {
+                        if (!ex.IsRetiredException())
+                            throw;
+                    }
+
+                    while(active >= max_pending)
+                    {
+                        var top = inProgress.Dequeue();
+
+                        // See if we are done
+                        if (await Task.WhenAny(top.Value, Task.Delay(500)) != top.Value)
+                        {
+                            try
+                            {
+                                stats.SetBlocking(true);
+                                await top.Value;
+                            }
+                            finally
+                            {
+                                stats.SetBlocking(false);
+                            }
+                        }
+
+						active -= top.Key;
+                    }
+                }
+
+                results.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
+
+                try
+                {
+                    stats.SetBlocking(true);
+                    while (inProgress.Count > 0)
+                        await inProgress.Dequeue().Value;
+                }
+                finally
+                {
+                    stats.SetBlocking(false);
+                }
+            });
+                                
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs

@@ -0,0 +1,236 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Database;
+using CoCoL;
+using Duplicati.Library.Main.Operation.Common;
+using System.Collections.Generic;
+using Duplicati.Library.Main.Volumes;
+using System.Linq;
+
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// Asynchronous interface that ensures all requests
+    /// to the database are performed in a sequential manner
+    /// </summary>
+    internal class BackupDatabase : DatabaseCommon
+    {
+        private LocalBackupDatabase m_database;
+
+        public class FileEntryData
+        {
+            public long id;
+            public DateTime modified;
+            public long filesize;
+            public string metahash;
+            public long metasize;
+        }
+
+        public BackupDatabase(LocalBackupDatabase database, Options options)
+            : base(database, options)
+        {
+            m_database = database;
+        }
+
+        public Task<long> FindBlockIDAsync(string hash, long size)
+        {
+            return RunOnMain(() => m_database.FindBlockID(hash, size));
+        }
+
+        public Task<bool> AddBlockAsync(string hash, long size, long volumeid)
+        {
+            return RunOnMain(() => m_database.AddBlock(hash, size, volumeid));
+        }
+
+        public Task<string> GetFileHashAsync(long fileid)
+        {
+            return RunOnMain(() => m_database.GetFileHash(fileid));
+        }
+
+        public Task<Tuple<bool, long>> AddMetadatasetAsync(string hash, long size, long blocksetid)
+        {
+            return RunOnMain(() =>
+            {
+                long metadataid;
+                var n = m_database.AddMetadataset(hash, size, blocksetid, out metadataid);
+                return new Tuple<bool, long>(n, metadataid);
+            });
+        }
+
+        public Task<Tuple<bool, long>> GetMetadataIDAsync(string hash, long size)
+        {
+            return RunOnMain(() =>
+            {
+                long metadataid;
+                var r = m_database.GetMetadatasetID(hash, size, out metadataid);
+                return new Tuple<bool, long>(r, metadataid);
+            });
+        }
+
+        public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified)
+        {
+            return RunOnMain(() => m_database.AddDirectoryEntry(filename, metadataid, lastModified));
+        }
+
+        public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified)
+        {
+            return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified));
+        }
+
+        public Task<KeyValuePair<long, DateTime>> GetFileLastModifiedAsync(string path, long lastfilesetid)
+        {
+            return RunOnMain(() =>
+            {
+                DateTime lastModified;
+                var id = m_database.GetFileLastModified(path, lastfilesetid, out lastModified);
+
+                return new KeyValuePair<long, DateTime>(id, lastModified);
+            });
+		}
+
+		public Task<FileEntryData> GetFileEntryAsync(string path, long lastfilesetid)
+        {
+            return RunOnMain(() => { 
+                DateTime oldModified;
+                long lastFileSize;
+                string oldMetahash;
+                long oldMetasize;
+
+                var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize);
+                return
+                    id < 0 ?
+                    null :
+                    new FileEntryData() { id = id, modified = oldModified, filesize = lastFileSize, metahash = oldMetahash, metasize = oldMetasize };
+            });
+        }
+
+        public Task<long> AddBlocksetAsync(string filehash, long size, int blocksize, IEnumerable<string> hashlist, IEnumerable<string> blocklisthashes)
+        {
+            return RunOnMain(() => {
+                long blocksetid;
+
+                m_database.AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes, out blocksetid);
+                return blocksetid;
+            });
+        }
+
+        public Task AddFileAsync(string filename, DateTime lastmodified, long blocksetid, long metadataid)
+        {
+            return RunOnMain(() => m_database.AddFile(filename, lastmodified, blocksetid, metadataid));
+        }
+
+        public Task AddUnmodifiedAsync(long fileid, DateTime lastModified)
+        {
+            return RunOnMain(() => m_database.AddUnmodifiedFile(fileid, lastModified));
+        }
+            
+
+        public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid)
+        {
+            return RunOnMain(() => m_database.MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid));
+        }
+
+        public Task SafeDeleteRemoteVolumeAsync(string remotename)
+        {
+            return RunOnMain(() => m_database.SafeDeleteRemoteVolume(remotename));
+        }
+
+        public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume)
+        {
+            if (indexvolume != null)
+                return Task.FromResult<bool>(true);
+
+            return RunOnMain(() =>
+            {
+                m_database.AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID);
+                indexvolume.StartVolume(blockvolume.RemoteFilename);
+
+                foreach(var b in m_database.GetBlocks(blockvolume.VolumeID))
+                    indexvolume.AddBlock(b.Hash, b.Size);
+
+                m_database.UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+            });
+        }
+
+        public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist)
+        {
+            return RunOnMain(() => m_database.AppendFilesFromPreviousSet(deletedFilelist));
+        }
+
+        public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp)
+        {
+            return RunOnMain(() => m_database.AppendFilesFromPreviousSet(deletedFilelist, filesetid, prevId, timestamp));
+        }
+
+        public Task<KeyValuePair<long, DateTime>[]> GetIncompleteFilesetsAsync()
+        {
+            // TODO: Consider AsyncEnumerable
+            return RunOnMain(() => { return m_database.GetIncompleteFilesets().OrderBy(x => x.Value).ToArray(); });
+        }
+
+        public Task<KeyValuePair<long, DateTime>[]> GetFilesetTimesAsync()
+        {
+            // TODO: Consider AsyncEnumerable
+            return RunOnMain(() => m_database.FilesetTimes.ToArray());
+        }
+
+        public Task<long> CreateFilesetAsync(long volumeID, DateTime fileTime)
+        {
+            return RunOnMain(() => m_database.CreateFileset(volumeID, fileTime));
+        }
+
+        public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid)
+        {
+            return RunOnMain(() => m_database.LinkFilesetToVolume(filesetid, volumeid));
+        }
+
+        public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid)
+        {
+            return RunOnMain(() => m_database.WriteFileset(fsw, filesetid));
+        }
+
+        public Task<string[]> GetMissingIndexFilesAsync()
+        {
+            // TODO: Consider AsyncEnumerable
+            return RunOnMain(() => m_database.GetMissingIndexFiles().ToArray());
+        }
+
+        public Task UpdateChangeStatisticsAsync(BackupResults result)
+        {
+            return RunOnMain(() => m_database.UpdateChangeStatistics(result));
+        }
+
+        public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists)
+        {
+            return RunOnMain(() => m_database.VerifyConsistency(blocksize, blockhashSize, verifyfilelists));
+        }
+
+        public Task RemoveRemoteVolumeAsync(string remoteFilename)
+        {
+            return RunOnMain(() => m_database.RemoveRemoteVolume(remoteFilename));
+        }
+
+        public Task<RemoteVolumeEntry> GetRemoteVolumeFromIDAsync(long fileid)
+        {
+            return RunOnMain(() => m_database.GetRemoteVolumeFromID(fileid));
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/BackupStatsCollector.cs

@@ -0,0 +1,70 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using Duplicati.Library.Main.Operation.Common;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// Asynchronous interface that ensures all stat requests
+    /// are performed in a sequential manner
+    /// </summary>
+    internal class BackupStatsCollector : StatsCollector
+    {
+        private BackupResults m_res;
+
+        public BackupStatsCollector(BackupResults res)
+            : base(res.BackendWriter)
+        {
+            m_res = res;
+        }
+
+        public Task AddOpenedFile(long size)
+        {
+            return RunOnMain(() => {
+                m_res.SizeOfOpenedFiles += size;
+                m_res.OpenedFiles++;
+            });
+        }
+
+        public Task AddAddedFile(long size)
+        {
+            return RunOnMain(() => {
+                m_res.SizeOfAddedFiles += size;
+                m_res.AddedFiles++;
+            });
+        }
+
+        public Task AddModifiedFile(long size)
+        {
+            return RunOnMain(() => {
+                m_res.SizeOfModifiedFiles += size;
+                m_res.ModifiedFiles++;
+            });
+        }
+
+        public Task AddExaminedFile(long size)
+        {
+            return RunOnMain(() => {
+                m_res.SizeOfExaminedFiles += size;
+                m_res.ExaminedFiles++;
+            });
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/Channels.cs

@@ -0,0 +1,61 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// Static typed definitions of channels used for the backup process
+    /// </summary>
+    internal static class Channels
+    {
+        /// <summary>
+        /// Requests to the backend are send over this channel and picked up by the <see cref="BackendUploader" /> class
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<IUploadRequest> BackendRequest = new ChannelMarkerWrapper<IUploadRequest>(new ChannelNameAttribute("BackendRequests"));
+        /// <summary>
+        /// When the backup completes, all in-progress archives are sent from the <see cref="DataBlockProcessor"/> to the <see cref="SpillCollector"/>
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<VolumeUploadRequest> SpillPickup = new ChannelMarkerWrapper<VolumeUploadRequest>(new ChannelNameAttribute("SpillPickup"));
+        /// <summary>
+        /// All data blocks are sent during the scanning to the <see cref="DataBlockProcessor"/> who bundles them in compressed archives
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<DataBlock> OutputBlocks = new ChannelMarkerWrapper<DataBlock>(new ChannelNameAttribute("OutputBlocks"));
+        /// <summary>
+        /// If a file has changes in the metadata, it is sent to the <see cref="FileBlockProcessor"/> where it is read
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<MetadataPreProcess.FileEntry> AcceptedChangedFile = new ChannelMarkerWrapper<MetadataPreProcess.FileEntry>(new ChannelNameAttribute("AcceptedChangedFile"));
+        /// <summary>
+        /// If a file has changes in the metadata, it is sent to the <see cref="FileBlockProcessor"/> where it is read
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<StreamBlock> StreamBlock = new ChannelMarkerWrapper<StreamBlock>(new ChannelNameAttribute("StreamBlockSplitter"));
+        /// <summary>
+        /// After metadata has been processed and collected, the <see cref="MetadataPreProcess"/> sends the file data to the <see cref="FilePreFilterProcess"/>
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<MetadataPreProcess.FileEntry> ProcessedFiles = new ChannelMarkerWrapper<MetadataPreProcess.FileEntry>(new ChannelNameAttribute("ProcessedFiles"));
+        /// <summary>
+        /// When enumerating the source folders, all discovered paths are sent by the <see cref="FileEnumerationProcess"/> to the <see cref="MetadataPreProcess"/> 
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<string> SourcePaths = new ChannelMarkerWrapper<string>(new ChannelNameAttribute("SourcePaths"));
+        /// <summary>
+        /// All progress events are communicated over this channel, to ensure that parallel progress is reported as a if it was sequential
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<Common.ProgressEvent> ProgressEvents = new ChannelMarkerWrapper<Common.ProgressEvent>(new ChannelNameAttribute("ProgressEvents"));
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs

@@ -0,0 +1,75 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using Duplicati.Library.Utility;
+using System.Threading;
+using CoCoL;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    internal static class CountFilesHandler
+    {
+        public static Task Run(Snapshots.ISnapshotService snapshot, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, Common.ITaskReader taskreader, System.Threading.CancellationToken token)
+        {
+            // Make sure we create the enumeration process in a seperate scope,
+            // but keep the log channel from the parent scope
+            using(new IsolatedChannelScope(Common.Channels.LogChannel))
+            {
+                var enumeratorTask = Backup.FileEnumerationProcess.Run(snapshot, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ChangedFilelist, taskreader);
+
+                var counterTask = AutomationExtensions.RunTask(new 
+                {
+                    Input = Backup.Channels.SourcePaths.ForRead
+                },
+                    
+                async self =>
+                {
+                    var count = 0L;
+                    var size = 0L;
+
+                    try
+                    {
+                        while (await taskreader.ProgressAsync && !token.IsCancellationRequested)
+                        {
+                            var path = await self.Input.ReadAsync();
+
+                            count++;
+
+                            try
+                            {
+                                size += snapshot.GetFileSize(path);
+                            }
+                            catch
+                            {
+                            }
+
+                            result.OperationProgressUpdater.UpdatefileCount(count, size, false);                    
+                        }
+                    }
+                    finally
+                    {
+                        result.OperationProgressUpdater.UpdatefileCount(count, size, true);
+                    }
+                });
+
+                return Task.WhenAll(enumeratorTask, counterTask);
+            }
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/DataBlock.cs

@@ -0,0 +1,56 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using Duplicati.Library.Interface;
+using System.Threading.Tasks;
+using CoCoL;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// The data block represents a single blob of data read from a file
+    /// </summary>
+    internal struct DataBlock
+    {
+        public string HashKey;
+        public byte[] Data;
+        public int Offset;
+        public long Size;
+        public CompressionHint Hint;
+        public bool IsBlocklistHashes;
+        public TaskCompletionSource<bool> TaskCompletion;
+
+        public static async Task<bool> AddBlockToOutputAsync(IWriteChannel<DataBlock> channel, string hash, byte[] data, int offset, long size, CompressionHint hint, bool isBlocklistHashes)
+        {
+            var tcs = new TaskCompletionSource<bool>();
+
+            await channel.WriteAsync(new DataBlock() {
+                HashKey = hash,
+                Data = data,
+                Offset = offset,
+                Size = size,
+                Hint = hint,
+                IsBlocklistHashes = isBlocklistHashes,
+                TaskCompletion = tcs
+            });
+
+            var r = await tcs.Task;
+            return r;
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs

@@ -0,0 +1,175 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Database;
+using Duplicati.Library.Main.Volumes;
+using Duplicati.Library.Main.Operation.Common;
+using System.IO;
+using System.Linq;
+using System.Collections.Generic;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// This class receives data blocks, registers then in the database.
+    /// New blocks are added to a compressed archive and sent
+    /// to the uploader
+    /// </summary>
+    internal static class DataBlockProcessor
+    {
+        /// <summary>
+        /// The number of bytes to reserve for the file header
+        /// in the compressed volume
+        /// </summary>
+        public const int BlockCompressionOverhead = 1024;
+
+        /// <summary>
+        /// A multiplier for protecting against block data
+        /// expanding during compression
+        /// </summary>
+        public const float NonCompressibleExpansionFactor = 1.02f;
+
+
+        public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader)
+        {
+            return AutomationExtensions.RunTask(
+            new
+            {
+                LogChannel = Common.Channels.LogChannel.ForWrite,
+                Input = Channels.OutputBlocks.ForRead,
+                Output = Channels.BackendRequest.ForWrite,
+                SpillPickup = Channels.SpillPickup.ForWrite,
+            },
+
+            async self =>
+            {
+                BlockVolumeWriter blockvolume = null;
+                var useindex = options.IndexfilePolicy == Options.IndexFileStrategy.Full;
+                var indexdata = useindex ? new Library.Utility.FileBackedStringList() : null;
+
+                // The limit for a single volume size
+                var maxvolumesize = options.VolumeSize - BlockCompressionOverhead;
+
+                try
+                {
+                    while(true)
+                    {
+                        var b = await self.Input.ReadAsync();
+
+                        // Lazy-start a new block volume
+                        if (blockvolume == null)
+                        {
+                            // Before we start a new volume, probe to see if it exists
+                            // This will delay creation of volumes for differential backups
+                            // There can be a race, such that two workers determine that
+                            // the block is missing, but this will be solved by the AddBlock call
+                            // which runs atomically
+                            if (await database.FindBlockIDAsync(b.HashKey, b.Size) >= 0)
+                            {
+                                b.TaskCompletion.TrySetResult(false);
+                                continue;
+                            }
+
+                            blockvolume = new BlockVolumeWriter(options);
+                            blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
+                        }
+
+                        var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID);
+                        b.TaskCompletion.TrySetResult(newBlock);
+
+                        if (newBlock)
+                        {
+                            // At this point we have registered the block as belonging to the current
+                            // volume, but it is possible that there is not enough space to put it in
+                            if (blockvolume.Filesize + (b.Size * NonCompressibleExpansionFactor) > maxvolumesize)
+                            {
+                                BlockVolumeWriter tmpvolume = null;
+                                try
+                                {
+                                    // Start a new volume 
+                                    tmpvolume = new BlockVolumeWriter(options);
+                                    tmpvolume.VolumeID = await database.RegisterRemoteVolumeAsync(tmpvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
+
+                                    // Move the current block to the new volume
+                                    await database.MoveBlockToVolumeAsync(b.HashKey, b.Size, blockvolume.VolumeID, tmpvolume.VolumeID);
+
+                                    // Close this volume, and send it to upload
+                                    blockvolume.Close();
+                                    await database.CommitTransactionAsync("CommitAddBlockToOutputFlush");
+                                    await self.Output.WriteAsync(new VolumeUploadRequest(blockvolume, true, indexdata));
+
+                                    // Continue with the freshly created volume
+                                    blockvolume = tmpvolume;
+                                    if (useindex)
+                                        indexdata = new Library.Utility.FileBackedStringList();
+                                }
+                                catch
+                                {
+                                    // If something goes wrong, we need to clear the temp volume
+                                    if (tmpvolume != null && tmpvolume != blockvolume)
+                                        try { tmpvolume.Dispose(); }
+                                        catch { } // Ignore this and report the original error
+
+                                    throw;
+                                }
+                            }
+
+#if DEBUG
+                            var presize = blockvolume.Filesize;
+#endif
+
+                            // Now add the block to the current volume, as we know there is space for it
+                            blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint);
+                            if (b.IsBlocklistHashes && useindex)
+                                indexdata.Add(VolumeUploadRequest.EncodeBlockListEntry(b.HashKey, b.Size, b.Data));
+                                
+#if DEBUG
+                            var volumesizeincrease = blockvolume.Filesize - presize;
+                            var expectedincrease = (b.Size * NonCompressibleExpansionFactor) + BlockCompressionOverhead;
+
+                            if (volumesizeincrease > expectedincrease)
+                                Logging.Log.WriteMessage(string.Format("Size increased {0} bytes more than expected when adding {1} to volume", volumesizeincrease - expectedincrease, b.HashKey), Logging.LogMessageType.Warning);
+#endif
+                        }
+
+                        // We ignore the stop signal, but not the pause and terminate
+                        await taskreader.ProgressAsync;
+                    }
+                }
+                catch(Exception ex)
+                {
+                    if (ex.IsRetiredException())
+                    {
+                        // If we have collected data, merge all pending volumes into a single volume
+                        if (blockvolume != null && blockvolume.SourceSize > 0)
+                        {
+                            await self.SpillPickup.WriteAsync(new VolumeUploadRequest(blockvolume, true, indexdata));
+                        }
+                    }
+
+                    throw;
+                }
+            });
+        }
+
+
+
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs

@@ -0,0 +1,138 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using Duplicati.Library.Main.Operation.Common;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using Duplicati.Library.Utility;
+using System.Linq;
+using Duplicati.Library.Interface;
+using System.IO;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// This class runs a process which opens a file and outputs data blocks for processing
+    /// </summary>
+    internal static class FileBlockProcessor
+    {
+        public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader)
+        {
+            return AutomationExtensions.RunTask(
+            new 
+            {
+                Input = Channels.AcceptedChangedFile.ForRead,
+                StreamBlockChannel = Channels.StreamBlock.ForWrite,
+                LogChannel = Common.Channels.LogChannel.ForWrite
+            },
+
+            async self =>
+            {
+                var log = new LogWrapper(self.LogChannel);
+                var blocksize = options.Blocksize;
+
+                while (await taskreader.ProgressAsync)
+                {
+                    var e = await self.Input.ReadAsync();
+                    var filesize = 0L;
+
+                    try
+                    {
+                        var hint = options.GetCompressionHintFromFilename(e.Path);
+                        var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId);
+
+                        StreamProcessResult filestreamdata;
+
+                        // Process metadata and actual data in parallel
+                        var metatask =
+                            Task.Run(async () =>
+                            {
+                                // If we have determined that metadata has not changed, just grab the ID
+                                if (!e.MetadataChanged)
+                                {
+                                    var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length);
+                                    if (!res.Item1)
+                                        return res.Item2;
+
+                                    await log.WriteWarningAsync(string.Format("Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2), null);
+                                    e.MetadataChanged = true;
+                                }
+
+                                return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Path, e.MetaHashAndSize, database, log, self.StreamBlockChannel)).Item2;
+                            });
+
+                        using (var fs = snapshot.OpenRead(e.Path))                            
+                            filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Path, fs, false, hint);
+
+                        await stats.AddOpenedFile(filestreamdata.Streamlength);
+
+                        var metadataid = await metatask;
+                        var filekey = filestreamdata.Streamhash;
+
+                        if (oldHash != filekey)
+                        {
+                            if (oldHash == null)
+                                await log.WriteVerboseAsync("New file {0}", e.Path);
+                            else
+                                await log.WriteVerboseAsync("File has changed {0}", e.Path);
+
+                            if (e.OldId < 0)
+                            {
+                                await stats.AddAddedFile(filesize);
+
+                                if (options.Dryrun)
+                                    await log.WriteDryRunAsync("Would add new file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
+                            }
+                            else
+                            {
+                                await stats.AddModifiedFile(filesize);
+
+                                if (options.Dryrun)
+                                    await log.WriteDryRunAsync("Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
+                            }
+
+                            await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid);
+                        }
+                        else if (e.MetadataChanged)
+                        {
+                            await log.WriteVerboseAsync("File has only metadata changes {0}", e.Path);
+                            await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid);
+                        }
+                        else /*if (e.OldId >= 0)*/
+                        {
+                            // When we write the file to output, update the last modified time
+                            await log.WriteVerboseAsync("File has not changed {0}", e.Path);
+                            await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                        }
+                    }
+                    catch(Exception ex)
+                    {
+                        if (ex.IsRetiredException())
+                            return;
+                        else
+                            await log.WriteWarningAsync(string.Format("Failed to process file {0}", e.Path), ex);
+                    }
+                }
+            }
+            );
+
+
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs

@@ -0,0 +1,217 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Threading.Tasks;
+using System.IO;
+using System.Collections.Generic;
+using System.Linq;
+using Duplicati.Library.Main.Operation.Common;
+using Duplicati.Library.Snapshots;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// The file enumeration process takes a list of source folders as input,
+    /// applies all filters requested and emits the filtered set of filenames
+    /// to its output channel
+    /// </summary>
+    internal static class FileEnumerationProcess 
+    {
+        public static Task Run(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, string[] changedfilelist, Common.ITaskReader taskreader)
+        {
+            return AutomationExtensions.RunTask(
+            new {
+                LogChannel = Common.Channels.LogChannel.ForWrite,
+                Output = Backup.Channels.SourcePaths.ForWrite
+            },
+
+            async self =>
+            {
+                var log = new LogWrapper(self.LogChannel);
+                var hardlinkmap = new Dictionary<string, string>();
+                var mixinqueue = new Queue<string>();
+                Duplicati.Library.Utility.IFilter enumeratefilter = emitfilter;
+
+                bool includes;
+                bool excludes;
+                Library.Utility.FilterExpression.AnalyzeFilters(emitfilter, out includes, out excludes);
+                if (includes && !excludes)
+                    enumeratefilter = Library.Utility.FilterExpression.Combine(emitfilter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
+
+                // If we have a specific list, use that instead of enumerating the filesystem
+                IEnumerable<string> worklist;
+                if (changedfilelist != null && changedfilelist.Length > 0)
+                {
+                    worklist = changedfilelist.Where(x =>
+                    {
+                        var fa = FileAttributes.Normal;
+                        try
+                        {
+                            fa = snapshot.GetAttributes(x);
+                        }
+                        catch
+                        {
+                        }
+
+                        return AttributeFilterAsync(null, x, fa, snapshot, log, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, attributeFilter, enumeratefilter, mixinqueue).WaitForTask().Result;
+                    });
+                }
+                else
+                {
+                    worklist = snapshot.EnumerateFilesAndFolders((root, path, attr) => {
+                        return AttributeFilterAsync(root, path, attr, snapshot, log, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, attributeFilter, enumeratefilter, mixinqueue).WaitForTask().Result;                        
+                    }, (rootpath, path, ex) => {
+                        log.WriteWarningAsync(string.Format("Error reported while accessing path {0}", path), ex).WaitForTaskOrThrow();
+                    });
+                }
+
+
+                // Process each path, and dequeue the mixins with symlinks as we go
+                foreach(var s in worklist)
+                {
+                    if (!await taskreader.ProgressAsync)
+                        return;
+                    
+                    while (mixinqueue.Count > 0)
+                        await self.Output.WriteAsync(mixinqueue.Dequeue());
+
+                    Library.Utility.IFilter m;
+                    if (emitfilter != enumeratefilter && !Library.Utility.FilterExpression.Matches(emitfilter, s, out m))
+                        continue;
+
+                    await self.Output.WriteAsync(s);
+                }
+
+                // Trailing symlinks are caught here
+                while (mixinqueue.Count > 0)
+                    await self.Output.WriteAsync(mixinqueue.Dequeue());
+            });
+        }
+
+  
+        /// <summary>
+        /// Plugin filter for enumerating a list of files.
+        /// </summary>
+        /// <returns>True if the path should be returned, false otherwise.</returns>
+        /// <param name="rootpath">The root path that initiated this enumeration.</param>
+        /// <param name="path">The current path.</param>
+        /// <param name="attributes">The file or folder attributes.</param>
+        private static async Task<bool> AttributeFilterAsync(string rootpath, string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, LogWrapper log, Library.Utility.IFilter sourcefilter, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary<string, string> hardlinkmap, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, Queue<string> mixinqueue)
+        {
+			// Step 1, exclude block devices
+			try
+            {
+                if (snapshot.IsBlockDevice(path))
+                {
+                    await log.WriteVerboseAsync("Excluding block device: {0}", path);
+                    return false;
+                }
+            }
+            catch (Exception ex)
+            {
+                await log.WriteWarningAsync(string.Format("Failed to process path: {0}", path), ex);
+                return false;
+            }
+
+            // Check if we explicitly include this entry
+            Duplicati.Library.Utility.IFilter sourcematch;
+            bool sourcematches;
+            if (sourcefilter.Matches(path, out sourcematches, out sourcematch) && sourcematches)
+            {
+                await log.WriteVerboseAsync("Including source path: {0}", path);
+                return true;
+            }
+
+            // If we have a hardlink strategy, obey it
+            if (hardlinkPolicy != Options.HardlinkStrategy.All)
+            {
+                try
+                {
+                    var id = snapshot.HardlinkTargetID(path);
+                    if (id != null)
+                    {
+                        if (hardlinkPolicy == Options.HardlinkStrategy.None)
+                        {
+                            await log.WriteVerboseAsync("Excluding hardlink: {0} ({1})", path, id);
+                            return false;
+                        }
+                        else if (hardlinkPolicy == Options.HardlinkStrategy.First)
+                        {
+                            string prevPath;
+                            if (hardlinkmap.TryGetValue(id, out prevPath))
+                            {
+                                await log.WriteVerboseAsync("Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath);
+                                return false;
+                            }
+                            else
+                            {
+                                hardlinkmap.Add(id, path);
+                            }
+                        }
+                    }
+                }
+                catch (Exception ex)
+                {
+                    await log.WriteWarningAsync(string.Format("Failed to process path: {0}", path), ex);
+                    return false;
+                }                    
+            }
+
+            // If we exclude files based on attributes, filter that
+            if ((attributeFilter & attributes) != 0)
+            {
+                await log.WriteVerboseAsync("Excluding path due to attribute filter: {0}", path);
+                return false;
+            }
+
+            // Then check if the filename is not explicitly excluded by a filter
+            Library.Utility.IFilter match;
+            if (!Library.Utility.FilterExpression.Matches(enumeratefilter, path, out match))
+            {
+                await log.WriteVerboseAsync("Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString());
+                return false;
+            }
+            else if (match != null)
+            {
+                await log.WriteVerboseAsync("Including path due to filter: {0} => {1}", path, match.ToString());
+            }
+
+            // If the file is a symlink, apply special handling
+            var isSymlink = snapshot.IsSymlink(path, attributes);
+            if (isSymlink && symlinkPolicy == Options.SymlinkStrategy.Ignore)
+            {
+                await log.WriteVerboseAsync("Excluding symlink: {0}", path);
+                return false;
+            }
+
+            if (isSymlink && symlinkPolicy == Options.SymlinkStrategy.Store)
+            {
+                await log.WriteVerboseAsync("Storing symlink: {0}", path);
+
+                // We return false because we do not want to recurse into the path,
+                // but we add the symlink to the mixin so we process the symlink itself
+                mixinqueue.Enqueue(path);
+                return false;
+            }
+
+            // All the way through, yes!
+            return true;
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs

@@ -0,0 +1,90 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+using System.Collections.Generic;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// This process takes files that are processed for metadata, 
+    /// and checks if anything indicates that the file has changed
+    /// and submits potentially changed files for scanning
+    /// </summary>
+    internal static class FilePreFilterProcess
+    {
+        public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database)
+        {
+            return AutomationExtensions.RunTask(
+            new
+            {
+                LogChannel = Common.Channels.LogChannel.ForWrite,
+                Input = Channels.ProcessedFiles.ForRead,
+                Output = Channels.AcceptedChangedFile.ForWrite
+            },
+
+            async self =>
+            {
+                var EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);
+                var blocksize = options.Blocksize;
+                var log = new LogWrapper(self.LogChannel);
+
+                while (true)
+                {
+                    var e = await self.Input.ReadAsync();
+
+                    long filestatsize = -1;
+                    try
+                    {
+                        filestatsize = snapshot.GetFileSize(e.Path);
+                    }
+                    catch
+                    {
+                    }
+
+                    await stats.AddExaminedFile(filestatsize);
+
+                    e.MetaHashAndSize = options.StoreMetadata ? Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot, log), options) : EMPTY_METADATA;
+
+                    var timestampChanged = e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
+                    var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
+                    var tooLargeFile = options.SkipFilesLargerThan != long.MaxValue && options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > options.SkipFilesLargerThan;
+                    e.MetadataChanged = !options.CheckFiletimeOnly && !options.SkipMetadata && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
+
+                    if ((e.OldId < 0 || options.DisableFiletimeCheck || timestampChanged || filesizeChanged || e.MetadataChanged) && !tooLargeFile)
+                    {
+                        await log.WriteVerboseAsync("Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, e.OldId <= 0, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
+                        await self.Output.WriteAsync(e);
+                    }
+                    else
+                    {
+                        if (tooLargeFile)
+                            await log.WriteVerboseAsync("Skipped checking file, because the size exceeds limit {0}", e.Path);
+                        else
+                            await log.WriteVerboseAsync("Skipped checking file, because timestamp was not updated {0}", e.Path);
+
+                        await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                    }
+                }
+            });
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs

@@ -0,0 +1,85 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using CoCoL;
+using Duplicati.Library.Main.Operation.Common;
+using Duplicati.Library.Snapshots;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// This class encasuplates the generation of metadata for a filesystem entry
+    /// </summary>
+    internal static class MetadataGenerator
+    {
+        public static async Task<Dictionary<string, string>> GenerateMetadataAsync(string path, System.IO.FileAttributes attributes, Options options, Snapshots.ISnapshotService snapshot, LogWrapper log)
+        {
+            try
+            {
+                Dictionary<string, string> metadata;
+
+                if (options.StoreMetadata)
+                {
+                    metadata = snapshot.GetMetadata(path, snapshot.IsSymlink(path, attributes), options.SymlinkPolicy == Options.SymlinkStrategy.Follow);
+                    if (metadata == null)
+                        metadata = new Dictionary<string, string>();
+
+                    if (!metadata.ContainsKey("CoreAttributes"))
+                        metadata["CoreAttributes"] = attributes.ToString();
+
+                    if (!metadata.ContainsKey("CoreLastWritetime"))
+                    {
+                        try
+                        {
+                            metadata["CoreLastWritetime"] = snapshot.GetLastWriteTimeUtc(path).Ticks.ToString();
+                        }
+                        catch (Exception ex)
+                        {
+                            await log.WriteWarningAsync(string.Format("Failed to read timestamp on \"{0}\"", path), ex);
+                        }
+                    }
+
+                    if (!metadata.ContainsKey("CoreCreatetime"))
+                    {
+                        try
+                        {
+                            metadata["CoreCreatetime"] = snapshot.GetCreationTimeUtc(path).Ticks.ToString();
+                        }
+                        catch (Exception ex)
+                        {
+                            await log.WriteWarningAsync(string.Format("Failed to read timestamp on \"{0}\"", path), ex);
+                        }
+                    }
+                }
+                else
+                {
+                    metadata = new Dictionary<string, string>();
+                }
+
+                return metadata;
+            }
+            catch(Exception ex)
+            {
+                await log.WriteWarningAsync(string.Format("Failed to process metadata for \"{0}\", storing empty metadata", path), ex);
+                return new Dictionary<string, string>();
+            }
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs

@@ -0,0 +1,290 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Threading.Tasks;
+using System.IO;
+using System.Collections.Generic;
+using Duplicati.Library.Interface;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// This class processes paths for metadata and emits the metadata blocks for storage.
+    /// Folders and symlinks in the database, and paths are forwarded to be scanned for changes
+    /// </summary>
+    internal static class MetadataPreProcess
+    {
+        public class FileEntry
+        {
+            // From input
+            public string Path;
+
+            // From database
+            public long OldId;
+            public DateTime OldModified;
+            public long LastFileSize;
+            public string OldMetaHash;
+            public long OldMetaSize;
+
+            // From filedata
+            public DateTime LastWrite;
+            public FileAttributes Attributes;
+
+            // After processing metadata
+            public IMetahash MetaHashAndSize;
+            public bool MetadataChanged;
+        }
+
+        public static Task RunOnlyRecord(Snapshots.ISnapshotService snapshot, Options options)
+        {
+            return Run(snapshot, options, null, -1, false);
+        }
+
+        public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid)
+        {
+            return Run(snapshot, options, database, lastfilesetid, true);
+        }
+
+        private static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, bool storeData)
+        {
+            return AutomationExtensions.RunTask(new
+            {
+                Input = Backup.Channels.SourcePaths.ForRead,
+                StreamBlockChannel = Channels.StreamBlock.ForWrite,
+                LogChannel = Common.Channels.LogChannel.ForWrite,
+                Output = Backup.Channels.ProcessedFiles.ForWrite,
+            },
+
+            async self =>
+            {
+                var log = new LogWrapper(self.LogChannel);
+                var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), options);
+
+                while (true)
+                {
+                    var path = await self.Input.ReadAsync();
+
+                    var lastwrite = new DateTime(0, DateTimeKind.Utc);
+                    var attributes = default(FileAttributes);
+                    try
+                    {
+                        lastwrite = snapshot.GetLastWriteTimeUtc(path);
+                    }
+                    catch (Exception ex)
+                    {
+                        await log.WriteWarningAsync(string.Format("Failed to read timestamp on \"{0}\"", path), ex);
+                    }
+
+                    try
+                    {
+                        attributes = snapshot.GetAttributes(path);
+                    }
+                    catch (Exception ex)
+                    {
+                        await log.WriteWarningAsync(string.Format("Failed to read attributes on \"{0}\"", path), ex);
+                    }
+
+                    if (storeData)
+                    {
+                        // If we only have metadata, stop here
+                        if (await ProcessMetadata(path, attributes, lastwrite, options, log, snapshot, emptymetadata, database, self.StreamBlockChannel))
+                        {
+                            try
+                            {
+                                if (options.CheckFiletimeOnly || options.DisableFiletimeCheck)
+                                {
+                                    var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid);
+                                    await self.Output.WriteAsync(new FileEntry()
+                                    {
+                                        OldId = tmp.Key < 0 ? -1 : tmp.Key,
+                                        Path = path,
+                                        Attributes = attributes,
+                                        LastWrite = lastwrite,
+                                        OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value,
+                                        LastFileSize = -1,
+                                        OldMetaHash = null,
+                                        OldMetaSize = -1
+                                    });
+                                }
+                                else
+                                {
+                                    var res = await database.GetFileEntryAsync(path, lastfilesetid);
+                                    await self.Output.WriteAsync(new FileEntry()
+                                    {
+                                        OldId = res == null ? -1 : res.id,
+                                        Path = path,
+                                        Attributes = attributes,
+                                        LastWrite = lastwrite,
+                                        OldModified = res == null ? new DateTime(0) : res.modified,
+                                        LastFileSize = res == null ? -1 : res.filesize,
+                                        OldMetaHash = res == null ? null : res.metahash,
+                                        OldMetaSize = res == null ? -1 : res.metasize
+                                    });
+                                }
+                            }
+                            catch (Exception ex)
+                            {
+                                await log.WriteErrorAsync(string.Format("Failed to process entry, path: {0}", path), ex);
+                            }
+                        }
+                    }
+                    else
+                    {
+                        // Write a dummy output
+                        await self.Output.WriteAsync(new FileEntry()
+                        {
+                            OldId = -1,
+                            Path = path,
+                            Attributes = attributes,
+                            LastWrite = lastwrite,
+                            OldModified = new DateTime(0),
+                            LastFileSize = -1,
+                            OldMetaHash = null,
+                            OldMetaSize = -1
+                        });
+
+                    }
+                }
+            });
+        }
+
+        /// <summary>
+        /// Processes the metadata for the given path.
+        /// </summary>
+        /// <returns><c>True</c> if the path should be submitted to more analysis, <c>false</c> if there is nothing else to do</returns>
+        private static async Task<bool> ProcessMetadata(string path, FileAttributes attributes, DateTime lastwrite, Options options, LogWrapper log, Snapshots.ISnapshotService snapshot, IMetahash emptymetadata, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
+        {
+            if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint)
+            {
+                // Not all reparse points are symlinks.
+                // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders)
+                // are reparse points, which allows the folder to hook into the OneDrive service and download things on-demand.
+                // If we can't find a symlink target for the current path, we won't treat it as a symlink.
+                string symlinkTarget = snapshot.GetSymlinkTarget(path);
+                if (!string.IsNullOrWhiteSpace(symlinkTarget))
+                {
+                    if (options.SymlinkPolicy == Options.SymlinkStrategy.Ignore)
+                    {
+                        await log.WriteVerboseAsync("Ignoring symlink {0}", path);
+                        return false;
+                    }
+
+                    if (options.SymlinkPolicy == Options.SymlinkStrategy.Store)
+                    {
+                        var metadata = await MetadataGenerator.GenerateMetadataAsync(path, attributes, options, snapshot, log);
+
+                        if (!metadata.ContainsKey("CoreSymlinkTarget"))
+                        {
+                            var p = snapshot.GetSymlinkTarget(path);
+
+                            if (string.IsNullOrWhiteSpace(p))
+                                await log.WriteVerboseAsync("Ignoring empty symlink {0}", path);
+                            else
+                                metadata["CoreSymlinkTarget"] = p;
+                        }
+
+                        var metahash = Utility.WrapMetadata(metadata, options);
+                        await AddSymlinkToOutputAsync(path, DateTime.UtcNow, metahash, log, database, streamblockchannel);
+
+                        await log.WriteVerboseAsync("Stored symlink {0}", path);
+                        // Don't process further
+                        return false;
+                    }
+                }
+                else
+                {
+                    await log.WriteVerboseAsync("Treating empty symlink as regular path {0}", path);
+                }
+
+            }
+
+            if ((attributes & FileAttributes.Directory) == FileAttributes.Directory)
+            {
+                IMetahash metahash;
+
+                if (options.StoreMetadata)
+                {
+                    metahash = Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(path, attributes, options, snapshot, log), options);
+                }
+                else
+                {
+                    metahash = emptymetadata;
+                }
+
+                await log.WriteVerboseAsync("Adding directory {0}", path);
+                await AddFolderToOutputAsync(path, lastwrite, metahash, log, database, streamblockchannel);
+                return false;
+            }
+
+            // Regular file, keep going
+            return true;
+        }
+
+        /// <summary>
+        /// Adds metadata to output, and returns the metadataset ID
+        /// </summary>
+        /// <returns>The metadataset ID.</returns>
+        /// <param name="path">The path for which metadata is processed.</param>
+        /// <param name="meta">The metadata entry.</param>
+        /// <param name="maxmetadatasize">The maximum size of metadata to process.</param>
+        /// <param name="database">The database connection.</param>
+        /// <param name="log">The log instance.</param>
+        /// <param name="streamblockchannel">The channel to write streams to.</param>
+        internal static async Task<Tuple<bool, long>> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, LogWrapper log, IWriteChannel<StreamBlock> streamblockchannel)
+        {
+            StreamProcessResult res;
+            using (var ms = new MemoryStream(meta.Blob))
+                res = await StreamBlock.ProcessStream(streamblockchannel, path, ms, true, CompressionHint.Default); 
+
+            return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid);
+        }
+
+        /// <summary>
+        /// Adds a file to the output, 
+        /// </summary>
+        /// <param name="filename">The name of the file to record</param>
+        /// <param name="lastModified">The value of the lastModified timestamp</param>
+        /// <param name="hashlist">The list of hashes that make up the file</param>
+        /// <param name="size">The size of the file</param>
+        /// <param name="fragmentoffset">The offset into a fragment block where the last few bytes are stored</param>
+        /// <param name="metadata">A lookup table with various metadata values describing the file</param>
+        private static async Task AddFolderToOutputAsync(string filename, DateTime lastModified, IMetahash meta, LogWrapper log, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
+        {
+            var metadataid = await AddMetadataToOutputAsync(filename, meta, database, log, streamblockchannel);
+            await database.AddDirectoryEntryAsync(filename, metadataid.Item2, lastModified);
+        }
+
+        /// <summary>
+        /// Adds a file to the output, 
+        /// </summary>
+        /// <param name="filename">The name of the file to record</param>
+        /// <param name="lastModified">The value of the lastModified timestamp</param>
+        /// <param name="hashlist">The list of hashes that make up the file</param>
+        /// <param name="size">The size of the file</param>
+        /// <param name="fragmentoffset">The offset into a fragment block where the last few bytes are stored</param>
+        /// <param name="metadata">A lookup table with various metadata values describing the file</param>
+        private static async Task AddSymlinkToOutputAsync(string filename, DateTime lastModified, IMetahash meta, LogWrapper log, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
+        {
+            var metadataid = await AddMetadataToOutputAsync(filename, meta, database, log, streamblockchannel);
+            await database.AddSymlinkEntryAsync(filename, metadataid.Item2, lastModified);
+        }
+
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/ProgressHandler.cs

@@ -0,0 +1,89 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Linq;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Operation.Common;
+using System.Collections.Generic;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// The progress handler reads all progress messages and reports progress as if the processing is sequential
+    /// </summary>
+    internal static class ProgressHandler
+    {
+        public static Task Run(BackupResults stat)
+        {
+            return AutomationExtensions.RunTask(new
+            {
+                Input = Channels.ProgressEvents.ForRead
+            },
+            
+            async self =>
+            {
+                var filesStarted = new Dictionary<string, long>();
+                var fileProgress = new Dictionary<string, long>();
+                string current = null;
+
+                while(true)
+                {
+                    var t = await self.Input.ReadAsync();
+                    switch(t.Type)
+                    {
+                        case EventType.FileStarted:
+                            filesStarted[t.Filepath] = t.Length;
+                            fileProgress[t.Filepath] = 0;
+                            break;
+                        case EventType.FileProgressUpdate:
+                            if (t.Filepath == current)
+                                stat.OperationProgressUpdater.UpdateFileProgress(t.Length);
+                            break;
+                        case EventType.FileClosed:
+                            if (fileProgress.ContainsKey(t.Filepath))
+                                fileProgress[t.Filepath] = t.Length;
+                        
+                            if (t.Filepath == current)
+                            {
+                                stat.OperationProgressUpdater.UpdateFileProgress(t.Length);
+                                current = null;
+                            }
+                            filesStarted.Remove(t.Filepath);
+                            fileProgress.Remove(t.Filepath);
+                            break;
+                    }
+
+                    if (current == null)
+                    {
+                        current = filesStarted.OrderByDescending(x => x.Value).Select(x => x.Key).FirstOrDefault();
+                        if (current != null)
+                        {
+                            stat.OperationProgressUpdater.StartFile(current, filesStarted[current]);
+                            if (fileProgress.ContainsKey(current) && fileProgress[current] > 0)
+                                stat.OperationProgressUpdater.UpdateFileProgress(fileProgress[current]);       
+                        }
+                    }
+                }
+            });
+
+        }
+
+
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs

@@ -0,0 +1,60 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using CoCoL;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    internal static class RecreateMissingIndexFiles
+    {
+        public static Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader)
+        {
+            return AutomationExtensions.RunTask(new
+            {
+                LogChannel = Common.Channels.LogChannel.ForWrite,
+                UploadChannel = Channels.BackendRequest.ForWrite
+            },
+
+			async self =>
+			{
+				var log = new LogWrapper(self.LogChannel);
+
+				if (options.IndexfilePolicy != Options.IndexFileStrategy.None)
+				{
+				    foreach (var blockfile in await database.GetMissingIndexFilesAsync())
+				    {
+				        if (!await taskreader.ProgressAsync)
+				            return;
+
+				        await log.WriteInformationAsync(string.Format("Re-creating missing index file for {0}", blockfile));
+				        var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database, null);
+
+				        if (!await taskreader.ProgressAsync)
+				            return;
+
+				        await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+				        await self.UploadChannel.WriteAsync(new IndexVolumeUploadRequest(w));
+				    }
+				}
+			});
+
+		}
+
+	}
+}

Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs

@@ -0,0 +1,150 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Operation.Common;
+using System.Collections.Generic;
+using System.Linq;
+using Duplicati.Library.Main.Volumes;
+using System.IO;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// This process just waits until all block processes are terminated
+    /// and collects the non-written volumes.
+    /// All remaining volumes are re-packed into one or more filled
+    /// volumes and uploaded
+    /// </summary>
+    internal static class SpillCollectorProcess
+    {
+        public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader)
+        {
+            return AutomationExtensions.RunTask(
+            new
+            {
+                Input = Channels.SpillPickup.ForRead,
+                Output = Channels.BackendRequest.ForWrite,
+            },
+
+            async self => 
+            {
+                // The limit for a single volume size
+                var maxvolumesize = options.VolumeSize - DataBlockProcessor.BlockCompressionOverhead;
+
+                var useindex = options.IndexfilePolicy == Options.IndexFileStrategy.Full;
+                var lst = new List<VolumeUploadRequest>();
+
+                while(!await self.Input.IsRetiredAsync)
+                    try
+                    {
+                        lst.Add((VolumeUploadRequest)await self.Input.ReadAsync());
+                    }
+                    catch (Exception ex)
+                    {
+                        if (ex.IsRetiredException())
+                            break;
+                        throw;
+                    }
+
+
+                while(lst.Count > 1)
+                {
+                    // We ignore the stop signal, but not the pause and terminate
+                    await taskreader.ProgressAsync;
+
+                    // The top-level item is the output target
+                    var target = lst[0];
+                    lst.RemoveAt(0);
+
+                    // The next item is copied into the target
+                    var source = lst[0];
+                    source.BlockVolume.Close();
+                    lst.RemoveAt(0);
+
+                    var buffer = new byte[options.Blocksize];
+                    var indexblocks = useindex ? new HashSet<string>() : null;
+
+                    using(var rd = new BlockVolumeReader(options.CompressionModule, source.BlockVolume.LocalFilename, options))
+                    {
+                        foreach(var file in rd.Blocks)
+                        {
+                            var len = rd.ReadBlock(file.Key, buffer);
+
+                            // If we do not have enough space, finish this one
+                            if (target.BlockVolume.Filesize + (len * DataBlockProcessor.NonCompressibleExpansionFactor) > maxvolumesize)
+                            {
+                                if (indexblocks != null && target.BlocklistData != null && source.BlocklistData != null)
+                                {
+                                    foreach(var s in source.BlocklistData.Where(x => indexblocks.Contains(VolumeUploadRequest.DecodeBlockListEntryHash(x))))
+                                        ((Library.Utility.FileBackedStringList)target.BlocklistData).Add(s);
+                                    indexblocks = new HashSet<string>();
+                                }
+
+                                target.BlockVolume.Close();
+                                await self.Output.WriteAsync(target);
+
+                                if (lst.Count == 0)
+                                {
+                                    // No more targets, make one
+                                    target = new VolumeUploadRequest(new BlockVolumeWriter(options), true, useindex ? new Library.Utility.FileBackedStringList() : null);
+                                    target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
+                                }
+                                else
+                                {
+                                    // Grab the next target
+                                    target = lst[0];
+                                    lst.RemoveAt(0);
+                                }
+                            }
+
+                            target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default);
+                            await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID);
+                            if (indexblocks != null)
+                                indexblocks.Add(file.Key);
+                        }
+                    }
+
+                    // Make sure they are out of the database
+                    System.IO.File.Delete(source.BlockVolume.LocalFilename);
+                    await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename);
+
+                    if (indexblocks != null && target.BlocklistData != null && source.BlocklistData != null)
+                    {
+                        foreach(var s in source.BlocklistData.Where(x => indexblocks.Contains(VolumeUploadRequest.DecodeBlockListEntryHash(x))))
+                            ((Library.Utility.FileBackedStringList)target.BlocklistData).Add(s);
+                    }
+
+                    // Re-inject the target
+                    lst.Insert(0, target);
+                }
+
+                foreach(var n in lst)
+                {
+                    // We ignore the stop signal, but not the pause and terminate
+                    await taskreader.ProgressAsync;
+
+                    n.BlockVolume.Close();
+                    await self.Output.WriteAsync(n);
+                }
+
+            });
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/StreamBlock.cs

@@ -0,0 +1,55 @@
+//  Copyright (C) 2016, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.IO;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    internal struct StreamProcessResult
+    {
+        public string Streamhash;
+        public long Streamlength;
+        public long Blocksetid; 
+    }
+
+    internal struct StreamBlock
+    {
+        public string Path;
+        public Stream Stream;
+        public bool IsMetadata;
+        public CompressionHint Hint;
+        public TaskCompletionSource<StreamProcessResult> Result;
+
+        public static async Task<StreamProcessResult> ProcessStream(IWriteChannel<StreamBlock> channel, string path, Stream stream, bool isMetadata, CompressionHint hint)
+        {
+            var tcs = new TaskCompletionSource<StreamProcessResult>();
+            await channel.WriteAsync(new StreamBlock()
+            {
+                Path = path,
+                Stream = stream,
+                IsMetadata = isMetadata,
+                Hint = hint,
+                Result = tcs
+            });
+
+            return await tcs.Task;
+        }
+    }
+}

Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs

@@ -0,0 +1,189 @@
+//  Copyright (C) 2016, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using Duplicati.Library.Main.Operation.Common;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using Duplicati.Library.Utility;
+using System.Linq;
+using Duplicati.Library.Interface;
+using System.IO;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    internal static class StreamBlockSplitter
+    {
+        public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader)
+        {
+            return AutomationExtensions.RunTask(
+            new
+            {
+                Input = Channels.StreamBlock.ForRead,
+                LogChannel = Common.Channels.LogChannel.ForWrite,
+                ProgressChannel = Channels.ProgressEvents.ForWrite,
+                BlockOutput = Channels.OutputBlocks.ForWrite
+            },
+
+            async self =>
+            {
+                var log = new LogWrapper(self.LogChannel);
+                var blocksize = options.Blocksize;
+                var filehasher = Duplicati.Library.Utility.HashAlgorithmHelper.Create(options.FileHashAlgorithm);
+                var blockhasher = Duplicati.Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm);
+                var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), options);
+                var maxmetadatasize = (options.Blocksize / (long)options.BlockhashSize) * options.Blocksize;
+
+                 if (blockhasher == null)
+                    throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm));
+                 if (filehasher == null)
+                     throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(options.FileHashAlgorithm));
+ 
+                 if (!blockhasher.CanReuseTransform)
+                     throw new UserInformationException(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm));
+                 if (!filehasher.CanReuseTransform)
+                     throw new UserInformationException(Strings.Common.InvalidCryptoSystem(options.FileHashAlgorithm));
+
+                using (var empty_metadata_stream = new MemoryStream(emptymetadata.Blob))
+                while (await taskreader.ProgressAsync)
+                {
+                    var send_close = false;
+                    var filesize = 0L;
+
+                    var e = await self.Input.ReadAsync();
+                    var cur = e.Result;
+
+                    try
+                    {
+                        var stream = e.Stream;
+
+                        using (var blocklisthashes = new Library.Utility.FileBackedStringList())
+                        using (var hashcollector = new Library.Utility.FileBackedStringList())
+                        {
+                            var blocklistbuffer = new byte[blocksize];
+                            var blocklistoffset = 0L;
+
+                            long fslen = -1;
+                            try { fslen = stream.Length; }
+                            catch (Exception ex) { await log.WriteWarningAsync(string.Format("Failed to read file length for file {0}", e.Path), ex); }
+
+                            if (e.IsMetadata && fslen > maxmetadatasize)
+                            {
+                                //TODO: To fix this, the "WriteFileset" method in BackupHandler needs to
+                                // be updated such that it can select sets even when there are multiple
+                                // blocklist hashes for the metadata.
+                                // This could be done such that an extra query is made if the metadata
+                                // spans multiple blocklist hashes, as it is not expected to be common
+
+                                await log.WriteWarningAsync(string.Format("Metadata size is {0}, but the largest accepted size is {1}, recording empty metadata for {2}", fslen, maxmetadatasize, e.Path), null);
+                                empty_metadata_stream.Position = 0;
+                                stream = empty_metadata_stream;
+                                fslen = stream.Length;
+                            }
+
+                            await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = fslen, Type = EventType.FileStarted });
+                            send_close = true;
+
+                            filehasher.Initialize();
+                            var lastread = 0;
+                            var buf = new byte[blocksize];
+                            var lastupdate = DateTime.Now;
+
+                            // Core processing loop, read blocks of data and hash individually
+                            while (((lastread = await stream.ForceStreamReadAsync(buf, blocksize)) != 0))
+                            {
+                                // Run file hashing concurrently to squeeze a little extra concurrency out of it
+                                var pftask = Task.Run(() => filehasher.TransformBlock(buf, 0, lastread, buf, 0));
+
+                                var hashdata = blockhasher.ComputeHash(buf, 0, lastread);
+                                var hashkey = Convert.ToBase64String(hashdata);
+
+                                // If we have too many hashes, flush the blocklist
+                                if (blocklistbuffer.Length - blocklistoffset < hashdata.Length)
+                                {
+                                    var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, (int)blocklistoffset));
+                                    blocklisthashes.Add(blkey);
+                                    await DataBlock.AddBlockToOutputAsync(self.BlockOutput, blkey, blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
+                                    blocklistoffset = 0;
+                                    blocklistbuffer = new byte[blocksize];
+                                }
+
+                                // Store the current hash in the blocklist
+                                Array.Copy(hashdata, 0, blocklistbuffer, blocklistoffset, hashdata.Length);
+                                blocklistoffset += hashdata.Length;
+                                hashcollector.Add(hashkey);
+                                filesize += lastread;
+
+                                // Don't spam updates
+                                if ((DateTime.Now - lastupdate).TotalSeconds > 10)
+                                {
+                                    await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filesize, Type = EventType.FileProgressUpdate });
+                                    lastupdate = DateTime.Now;
+                                }
+
+                                // Make sure the filehasher is done with the buf instance before we pass it on
+                                await pftask;
+                                await DataBlock.AddBlockToOutputAsync(self.BlockOutput, hashkey, buf, 0, lastread, e.Hint, false);
+                                buf = new byte[blocksize];
+                            }
+
+                            // If we have more than a single block of data, output the (trailing) blocklist
+                            if (hashcollector.Count > 1)
+                            {
+                                var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, (int)blocklistoffset));
+                                blocklisthashes.Add(blkey);
+                                await DataBlock.AddBlockToOutputAsync(self.BlockOutput, blkey, blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
+                            }
+
+                            filehasher.TransformFinalBlock(new byte[0], 0, 0);
+                            var filehash = Convert.ToBase64String(filehasher.Hash);
+                            var blocksetid = await database.AddBlocksetAsync(filehash, filesize, blocksize, hashcollector, blocklisthashes);
+                            cur.SetResult(new StreamProcessResult() { Streamlength = filesize, Streamhash = filehash, Blocksetid = blocksetid });
+                            cur = null;
+                        }
+                    }
+                    catch (Exception ex)
+                    {                        
+                        try
+                        {
+                            if (cur != null)
+                                cur.TrySetException(ex);
+                        }
+                        catch { }
+
+                        // Rethrow
+                        if (ex.IsRetiredException())
+                            throw;
+                    }
+                    finally
+                    {
+                        if (cur != null)
+                        {
+                            try { cur.TrySetCanceled(); }
+                            catch { }
+                            cur = null;
+                        }
+
+                        if (send_close)
+                            await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filesize, Type = EventType.FileClosed });
+                        send_close = false;
+                    }
+                }
+            });
+        }
+    }
+}

Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs

@@ -0,0 +1,85 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using CoCoL;
+using Duplicati.Library.Main.Volumes;
+using System.IO;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    internal static class UploadRealFilelist
+    {
+        public static Task Run(BackupResults result, BackupDatabase db, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader)
+        {
+            return AutomationExtensions.RunTask(new
+            {
+                Output = Channels.BackendRequest.ForWrite,
+                LogChannel = Common.Channels.LogChannel.ForWrite
+            },
+
+            async self =>
+            {
+                if (!await taskreader.ProgressAsync)
+                    return;
+                
+                var log = new Common.LogWrapper(self.LogChannel);
+
+                // Update the reported source and backend changes
+                using(new Logging.Timer("UpdateChangeStatistics"))
+                    await db.UpdateChangeStatisticsAsync(result);
+
+                var changeCount = 
+                    result.AddedFiles + result.ModifiedFiles + result.DeletedFiles +
+                    result.AddedFolders + result.ModifiedFolders + result.DeletedFolders +
+                    result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks;
+
+                //Changes in the filelist triggers a filelist upload
+                if (options.UploadUnchangedBackups || changeCount > 0)
+                {
+                    using(new Logging.Timer("Uploading a new fileset"))
+                    {
+                        if (!string.IsNullOrEmpty(options.ControlFiles))
+                            foreach(var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
+                                filesetvolume.AddControlFile(p, options.GetCompressionHintFromFilename(p));
+
+                        if (!await taskreader.ProgressAsync)
+                            return;
+
+                        await db.WriteFilesetAsync(filesetvolume, filesetid);
+                        filesetvolume.Close();
+
+                        if (!await taskreader.ProgressAsync)
+                            return;
+                        
+                        await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                        await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
+                        await self.Output.WriteAsync(new FilesetUploadRequest(filesetvolume));
+                    }
+                }
+                else
+                {
+                    await log.WriteVerboseAsync("removing temp files, as no data needs to be uploaded");
+                    await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename);
+                }
+
+                await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
+            });
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs

@@ -0,0 +1,143 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using Duplicati.Library.Main.Volumes;
+using System.Threading.Tasks;
+using System.Linq;
+using Duplicati.Library.Main.Operation.Common;
+using System.IO;
+
+namespace Duplicati.Library.Main.Operation.Backup
+{
+    /// <summary>
+    /// This class encapsulates generation of synthetic file list
+    /// </summary>
+    internal static class UploadSyntheticFilelist
+    {
+        public static Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, string lasttempfilelist, long lasttempfileid)
+        {
+            return AutomationExtensions.RunTask(new
+            {
+                LogChannel = Common.Channels.LogChannel.ForWrite,
+                UploadChannel = Channels.BackendRequest.ForWrite
+            },
+
+            async self => 
+            {                
+                var log = new LogWrapper(self.LogChannel);
+
+                // Check if we should upload a synthetic filelist
+                if (options.DisableSyntheticFilelist || string.IsNullOrWhiteSpace(lasttempfilelist) || lasttempfileid < 0)
+                    return;
+
+                // Check that we still need to process this after the cleanup has performed its duties
+                var syntbase = await database.GetRemoteVolumeFromIDAsync(lasttempfileid);
+
+                // If we do not have a valid entry, warn and quit
+                if (syntbase.Name == null || syntbase.State != RemoteVolumeState.Uploaded)
+                {
+                    // TODO: If the repair succeeds, this could give a false warning?
+                    await log.WriteWarningAsync(string.Format("Expected there to be a temporary fileset for synthetic filelist ({0}, {1}, {2}), but none was found?", lasttempfileid, lasttempfilelist, syntbase.State), null);
+                    return;
+                }
+
+                // Files is missing or repaired
+                if (syntbase.Name == null || (syntbase.State != RemoteVolumeState.Uploading && syntbase.State != RemoteVolumeState.Temporary))
+                {
+                    await log.WriteInformationAsync(string.Format("Skipping synthetic upload because temporary fileset appers to be complete: ({0}, {1}, {2})", lasttempfileid, lasttempfilelist, syntbase.State), null);
+                    return;
+                }
+
+                // Ready to build and upload the synthetic list
+                await database.CommitTransactionAsync("PreSyntheticFilelist");
+                var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList();
+
+                result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize);
+                await log.WriteInformationAsync("Uploading filelist from previous interrupted backup");
+
+                if (!await taskreader.ProgressAsync)
+                    return;
+
+                var incompleteSet = incompleteFilesets.Last();
+                var badIds = from n in incompleteFilesets select n.Key;
+
+                var prevs = (from n in await database.GetFilesetTimesAsync()
+                    where 
+                    n.Key < incompleteSet.Key
+                    &&
+                    !badIds.Contains(n.Key)
+                    orderby n.Key                                                
+                    select n.Key).ToArray();
+
+                var prevId = prevs.Length == 0 ? -1 : prevs.Last();
+
+                    FilesetVolumeWriter fsw = null;
+                    try
+                    {
+                        var s = 1;
+                        var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);
+                        var oldFilesetID = incompleteSet.Key;
+
+                        // Probe for an unused filename
+                        while (s < 60)
+                        {
+                            var id = await database.GetRemoteVolumeIDAsync(VolumeBase.GenerateFilename(RemoteVolumeType.Files, options, null, fileTime));
+                            if (id < 0)
+                                break;
+
+                            fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s);
+                        }
+
+                        fsw = new FilesetVolumeWriter(options, fileTime);
+                        fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
+
+                        if (!string.IsNullOrEmpty(options.ControlFiles))
+                            foreach(var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
+                                fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p));
+
+                        var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime);
+                        await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID);
+                        await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime);
+
+                        await database.WriteFilesetAsync(fsw, newFilesetID);
+
+                        if (!await taskreader.ProgressAsync)
+                            return;
+                        
+                        await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                        await database.CommitTransactionAsync("CommitUpdateFilelistVolume");
+                        await self.UploadChannel.WriteAsync(new FilesetUploadRequest(fsw));
+                        fsw = null;
+                    }
+                    catch
+                    {
+                        await database.RollbackTransactionAsync();
+                        throw;
+                    }
+                    finally
+                    {
+                        if (fsw != null)
+                            try { fsw.Dispose(); }
+                        catch { fsw = null; }
+                    }                          
+                }
+            );
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Common/BackendHandler.cs

@@ -0,0 +1,775 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using Duplicati.Library.Interface;
+using Duplicati.Library.Main.Volumes;
+using Duplicati.Library.Utility;
+using CoCoL;
+using Duplicati.Library.Localization.Short;
+using Newtonsoft.Json;
+using System.Linq;
+using System.Text;
+using System.IO;
+
+namespace Duplicati.Library.Main.Operation.Common
+{    
+    /// <summary>
+    /// This class encapsulates access to the backend and ensures at most one connection is active at a time
+    /// </summary>
+    internal class BackendHandler : SingleRunner
+    {
+        public const string VOLUME_HASH = "SHA256";
+
+        /// <summary>
+        /// Data storage for an ongoing backend operation
+        /// </summary>
+        protected class FileEntryItem
+        {
+            /// <summary>
+            /// The current operation this entry represents
+            /// </summary>
+            public BackendActionType Operation;
+            /// <summary>
+            /// The name of the remote file
+            /// </summary>
+            public string RemoteFilename;
+            /// <summary>
+            /// The name of the local file
+            /// </summary>
+            public string LocalFilename { get { return LocalTempfile; } }
+            /// <summary>
+            /// A reference to a temporary file that is disposed upon
+            /// failure or completion of the item
+            /// </summary>
+            public TempFile LocalTempfile;
+            /// <summary>
+            /// True if the item has been encrypted
+            /// </summary>
+            public bool Encrypted;
+            /// <summary>
+            /// The expected hash value of the file
+            /// </summary>
+            public string Hash;
+            /// <summary>
+            /// The expected size of the file
+            /// </summary>
+            public long Size;
+            /// <summary>
+            /// A flag indicating if the file is a extra metadata file
+            /// that has no entry in the database
+            /// </summary>
+            public bool TrackedInDb = true;
+            /// <summary>
+            /// A flag that indicates that the download is only checked for the hash and the file is not decrypted or returned
+            /// </summary>            
+            public bool VerifyHashOnly;
+            /// <summary>
+            /// A flag indicating if the operation is a retry run
+            /// </summary>
+            public bool IsRetry;
+
+            public FileEntryItem(BackendActionType operation, string remotefilename)
+            {
+                Operation = operation;
+                RemoteFilename = remotefilename;
+                Size = -1;
+            }
+
+            public FileEntryItem(BackendActionType operation, string remotefilename, long size, string hash)
+                : this(operation, remotefilename)
+            {
+                Size = size;
+                Hash = hash;
+            }
+
+            public void SetLocalfilename(string name)
+            {
+                this.LocalTempfile = Library.Utility.TempFile.WrapExistingFile(name);
+                this.LocalTempfile.Protected = true;
+            }
+                
+            public async Task Encrypt(Options options, LogWrapper log)
+            {
+                if (!this.Encrypted && !options.NoEncryption)
+                {
+                    var tempfile = new Library.Utility.TempFile();
+                    using (var enc = DynamicLoader.EncryptionLoader.GetModule(options.EncryptionModule, options.Passphrase, options.RawOptions))
+                        enc.Encrypt(this.LocalFilename, tempfile);
+
+                    await this.DeleteLocalFile(log);
+
+                    this.LocalTempfile = tempfile;
+                    this.Hash = null;
+                    this.Size = 0;
+                    this.Encrypted = true;
+                }
+            }
+
+            public static string CalculateFileHash(string filename)
+            {
+                using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
+                using (var hasher = Duplicati.Library.Utility.HashAlgorithmHelper.Create(VOLUME_HASH))
+                    return Convert.ToBase64String(hasher.ComputeHash(fs));
+            }
+
+
+            public bool UpdateHashAndSize(Options options)
+            {
+                if (Hash == null || Size < 0)
+                {
+                    Hash = CalculateFileHash(this.LocalFilename);
+                    Size = new System.IO.FileInfo(this.LocalFilename).Length;
+                    return true;
+                }
+
+                return false;
+            }
+
+            public async Task DeleteLocalFile(LogWrapper log)
+            {
+                if (this.LocalTempfile != null)
+                    try { this.LocalTempfile.Dispose(); }
+                catch (Exception ex) { await log.WriteWarningAsync(string.Format("Failed to dispose temporary file: {0}", this.LocalTempfile), ex); }
+                finally { this.LocalTempfile = null; }
+            }
+        }
+
+
+        private LogWrapper m_log;
+
+        private IBackendHandlerDatabase m_database;
+        private IBackend m_backend;
+        private Options m_options;
+        private string m_backendurl;
+        private bool m_uploadSuccess;
+        private StatsCollector m_stats;
+        private ITaskReader m_taskreader;
+
+        public BackendHandler(Options options, string backendUrl, IBackendHandlerDatabase database, StatsCollector stats, ITaskReader taskreader)
+            : base()
+        {
+            m_backendurl = backendUrl;
+            m_database = database;
+            m_options = options;
+            m_backendurl = backendUrl;
+            m_stats = stats;
+            m_taskreader = taskreader;
+            m_log = new LogWrapper();
+            m_backend = DynamicLoader.BackendLoader.GetBackend(backendUrl, options.RawOptions);
+			
+            var shortname = m_backendurl;
+
+			// Try not to leak hostnames or other information in the error messages
+			try { shortname = new Library.Utility.Uri(shortname).Scheme; }
+			catch { }
+
+			if (m_backend == null)
+			    throw new Duplicati.Library.Interface.UserInformationException(string.Format("Backend not supported: {0}", shortname));
+		}
+            
+        protected Task<T> RunRetryOnMain<T>(FileEntryItem fe, Func<Task<T>> method)
+        {
+            return RunOnMain<T>(() =>
+                DoWithRetry<T>(fe, method)
+            );
+        }
+
+        public IQuotaInfo QuotaInfo { get { return (m_backend as IQuotaEnabledBackend)?.Quota; } }
+
+        public Task PutUnencryptedAsync(string remotename, string localpath)
+        {
+            var fe = new FileEntryItem(BackendActionType.Put, remotename);
+            fe.SetLocalfilename(localpath);
+            fe.Encrypted = true; //Prevent encryption
+            fe.TrackedInDb = false; //Prevent Db updates
+
+            return RunRetryOnMain<bool>(fe, async () =>
+            {
+                await DoPut(fe);
+                m_uploadSuccess = true;
+                return true;
+            });
+
+        }
+            
+        public async Task UploadFileAsync(VolumeWriterBase item, Func<string, Task<IndexVolumeWriter>> createIndexFile = null)
+        {
+            var fe = new FileEntryItem(BackendActionType.Put, item.RemoteFilename);
+            fe.SetLocalfilename(item.LocalFilename);
+            item.Close();
+
+            var tcs = new TaskCompletionSource<bool>();
+
+            var backgroundhashAndEncrypt = Task.Run(async () =>
+            {
+                await fe.Encrypt(m_options, m_log).ConfigureAwait(false);
+                return fe.UpdateHashAndSize(m_options);
+            });
+
+            await RunOnMain(async () =>
+            {
+                try
+                {
+                    await DoWithRetry(fe, async () => {
+                        if (fe.IsRetry)
+                            await RenameFileAfterErrorAsync(fe);
+
+                        // Make sure the encryption and hashing has completed
+                        await backgroundhashAndEncrypt;
+
+                        return await DoPut(fe);
+                    });
+
+                    if (createIndexFile != null)
+                    {
+                        var ix = await createIndexFile(fe.RemoteFilename);
+                        var indexFile = new FileEntryItem(BackendActionType.Put, ix.RemoteFilename);
+                        indexFile.SetLocalfilename(ix.LocalFilename);
+
+                        await m_database.UpdateRemoteVolumeAsync(indexFile.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+
+                        await DoWithRetry(indexFile, async () => {
+                            if (indexFile.IsRetry)
+                                await RenameFileAfterErrorAsync(indexFile);
+
+                            return await DoPut(indexFile);
+                        });
+                    }
+
+                    tcs.TrySetResult(true);
+                }
+                catch(Exception ex)
+                {
+                    if (ex is System.Threading.ThreadAbortException)
+                        tcs.TrySetCanceled();
+                    else
+                        tcs.TrySetException(ex);
+                }
+            });
+
+            await tcs.Task;
+        }
+
+        public Task DeleteFileAsync(string remotename, bool suppressCleanup = false)
+        {
+            var fe = new FileEntryItem(BackendActionType.Delete, remotename);
+            return RunRetryOnMain(fe, () =>
+                DoDelete(fe, suppressCleanup)
+            );
+        }
+
+        public Task CreateFolder(string remotename)
+        {
+            var fe = new FileEntryItem(BackendActionType.CreateFolder, remotename);
+            return RunRetryOnMain(fe, () =>
+                DoCreateFolder(fe)
+            );
+        }
+
+
+        public Task<IList<Library.Interface.IFileEntry>> ListFilesAsync()
+        {
+            var fe = new FileEntryItem(BackendActionType.List, null);
+            return RunRetryOnMain(fe, () => 
+                DoList(fe)
+            );
+        }
+
+        public Task<Library.Utility.TempFile> GetFileAsync(string remotename, long size, string remotehash)
+        {
+            var fe = new FileEntryItem(BackendActionType.Get, remotename, size, remotehash);
+            return RunRetryOnMain(fe, () => DoGet(fe) );
+        }
+
+        public Task<Tuple<Library.Utility.TempFile, long, string>> GetFileWithInfoAsync(string remotename)
+        {
+            var fe = new FileEntryItem(BackendActionType.Get, remotename);
+            return RunRetryOnMain(fe, async () => {
+                var res = await DoGet(fe);
+                return new Tuple<Library.Utility.TempFile, long, string>(
+                    res,
+                    fe.Size,
+                    fe.Hash
+                );
+            });
+        }
+
+        public Task<Library.Utility.TempFile> GetFileForTestingAsync(string remotename, long size, string remotehash)
+        {
+            var fe = new FileEntryItem(BackendActionType.Get, remotename);
+            fe.VerifyHashOnly = true;
+            return RunRetryOnMain(fe, () => DoGet(fe));
+        }
+
+        private async Task ResetBackendAsync(Exception ex)
+        {
+            try
+            {
+                if (m_backend != null)
+                    m_backend.Dispose();
+            }
+            catch (Exception dex)
+            {
+                await m_log.WriteWarningAsync(LC.L("Failed to dispose backend instance: {0}", (ex ?? dex).Message), dex);
+            }
+
+            m_backend = null;
+
+        }
+
+        private async Task<T> DoWithRetry<T>(FileEntryItem item, Func<Task<T>> method)
+        {
+            item.IsRetry = false;
+            Exception lastException = null;
+
+            if (!await m_taskreader.TransferProgressAsync)
+                throw new OperationCanceledException();
+
+            if (m_workerSource.IsCancellationRequested)
+                throw new OperationCanceledException();
+            
+            for(var i = 0; i < m_options.NumberOfRetries; i++)
+            {
+                if (m_options.RetryDelay.Ticks != 0 && i != 0)
+                    await Task.Delay(m_options.RetryDelay);
+
+                if (!await m_taskreader.TransferProgressAsync)
+                    throw new OperationCanceledException();
+
+                if (m_workerSource.IsCancellationRequested)
+                    throw new OperationCanceledException();
+
+                try
+                {
+                    if (m_backend == null)
+                        m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions);
+                    if (m_backend == null)
+                        throw new Exception("Backend failed to re-load");
+                    
+                    var r = await method();
+                    return r;
+                }
+                catch (Exception ex)
+                {
+                    item.IsRetry = true;
+                    lastException = ex;
+                    await m_log.WriteRetryAttemptAsync(string.Format("Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", item.Operation, item.RemoteFilename, i + 1, m_options.NumberOfRetries, ex.Message), ex);
+
+                    // If the thread is aborted, we exit here
+                    if (ex is System.Threading.ThreadAbortException || ex is OperationCanceledException)
+                        break;
+
+                    if (ex is System.Net.WebException && ((System.Net.WebException)ex).Status == System.Net.WebExceptionStatus.NameResolutionFailure)
+                    {
+                        try
+                        {
+                            var names = m_backend.DNSName ?? new string[0];
+                            foreach (var name in names)
+                                if (!string.IsNullOrWhiteSpace(name))
+                                    try
+                                    {
+                                        System.Net.Dns.GetHostEntry(name);
+                                    }
+                                    catch (Exception dnsex)
+                                    {
+                                        await m_log.WriteVerboseAsync("Failed to refresh DNS record for {0}: {1}", name, dnsex);
+                                    }
+                        }
+                        catch (Exception bkdnsex)
+                        {
+                            await m_log.WriteWarningAsync("Failed to get DNS names from the backend", bkdnsex);
+                        }
+                    }
+
+                    await m_stats.SendEventAsync(item.Operation, i < m_options.NumberOfRetries ? BackendEventType.Retrying : BackendEventType.Failed, item.RemoteFilename, item.Size);
+
+                    bool recovered = false;
+                    if (!m_uploadSuccess && ex is Duplicati.Library.Interface.FolderMissingException && m_options.AutocreateFolders)
+                    {
+                        try
+                        { 
+                            // If we successfully create the folder, we can re-use the connection
+                            m_backend.CreateFolder(); 
+                            recovered = true;
+                        }
+                        catch (Exception dex)
+                        { 
+                            await m_log.WriteWarningAsync(string.Format("Failed to create folder: {0}", ex.Message), dex); 
+                        }
+                    }
+                        
+                    if (!recovered)
+                        await ResetBackendAsync(ex);
+                }
+                finally
+                {
+                    if (m_options.NoConnectionReuse)
+                        await ResetBackendAsync(null);
+                }
+            }
+
+            throw lastException;
+        }
+
+        private async Task RenameFileAfterErrorAsync(FileEntryItem item)
+        {
+            var p = VolumeBase.ParseFilename(item.RemoteFilename);
+            var guid = VolumeWriterBase.GenerateGuid(m_options);
+            var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1);
+            var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule);
+            var oldname = item.RemoteFilename;
+
+            await m_stats.SendEventAsync(item.Operation, BackendEventType.Rename, oldname, item.Size);
+            await m_stats.SendEventAsync(item.Operation, BackendEventType.Rename, newname, item.Size);
+            await m_log.WriteInformationAsync(string.Format("Renaming \"{0}\" to \"{1}\"", oldname, newname));
+            await m_database.RenameRemoteFileAsync(oldname, newname);
+            item.RemoteFilename = newname;
+        }
+
+        private async Task<bool> DoPut(FileEntryItem item, bool updatedHash = false)
+        {
+            // If this is not already encrypted, do it now
+            await item.Encrypt(m_options, m_log);
+
+            updatedHash |= item.UpdateHashAndSize(m_options);
+
+            if (updatedHash && item.TrackedInDb)
+                await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
+
+            if (m_options.Dryrun)
+            {
+                await m_log.WriteDryRunAsync("Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length));
+                await item.DeleteLocalFile(m_log);
+                return true;
+            }
+            
+            await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
+            await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size);
+
+            var begin = DateTime.Now;
+
+            if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
+            {
+                using (var fs = System.IO.File.OpenRead(item.LocalFilename))
+                using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
+                using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
+                    ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs);
+            }
+            else
+                m_backend.Put(item.RemoteFilename, item.LocalFilename);
+
+            var duration = DateTime.Now - begin;
+            await m_log.WriteProfilingAsync(string.Format("Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds))), null);
+
+            if (item.TrackedInDb)
+                await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);
+
+            await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);
+
+            if (m_options.ListVerifyUploads)
+            {
+                var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
+                if (f == null)
+                    throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
+                else if (f.Size != item.Size && f.Size >= 0)
+                    throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
+            }
+                
+            await item.DeleteLocalFile(m_log);
+            await m_database.CommitTransactionAsync("CommitAfterUpload");
+
+            return true;
+        }
+
+        private async Task<IList<Library.Interface.IFileEntry>> DoList(FileEntryItem item)
+        {
+            await m_stats.SendEventAsync(BackendActionType.List, BackendEventType.Started, null, -1);
+
+            var r = m_backend.List().ToList();
+
+            var sb = new StringBuilder();
+            sb.AppendLine("[");
+            long count = 0;
+            foreach (var e in r)
+            {
+                if (count != 0)
+                    sb.AppendLine(",");
+                count++;
+                sb.Append(JsonConvert.SerializeObject(e));
+            }
+
+            sb.AppendLine();
+            sb.Append("]");
+            await m_database.LogRemoteOperationAsync("list", "", sb.ToString());
+
+            await m_stats.SendEventAsync(BackendActionType.List, BackendEventType.Completed, null, r.Count);
+
+            return r;
+        }
+
+        private async Task<bool> DoDelete(FileEntryItem item, bool suppressCleanup)
+        {
+            if (m_options.Dryrun)
+            {
+                await m_log.WriteDryRunAsync("Would upload delete remote volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(item.Size));
+                return true;
+            }
+
+            await m_stats.SendEventAsync(BackendActionType.Delete, BackendEventType.Started, item.RemoteFilename, item.Size);
+
+            string result = null;
+            try
+            {
+                m_backend.Delete(item.RemoteFilename);
+            }
+            catch (Exception ex)
+            {
+                var isFileMissingException = ex is Library.Interface.FileMissingException || ex is System.IO.FileNotFoundException;
+                var wr = ex as System.Net.WebException == null ? null : (ex as System.Net.WebException).Response as System.Net.HttpWebResponse;
+
+                if (isFileMissingException || (wr != null && wr.StatusCode == System.Net.HttpStatusCode.NotFound))
+                {
+                    await m_log.WriteWarningAsync(LC.L("Delete operation failed for {0} with FileNotFound, listing contents", item.RemoteFilename), ex);
+                    bool success = false;
+
+                    try
+                    {
+                        success = !m_backend.List().Select(x => x.Name).Contains(item.RemoteFilename);
+                    }
+                    catch
+                    {
+                    }
+
+                    if (success)
+                    {
+                        await m_log.WriteInformationAsync(LC.L("Listing indicates file {0} is deleted correctly", item.RemoteFilename));
+                        return true;
+                    }
+
+                }
+
+                result = ex.ToString();
+                throw;
+            }
+            finally
+            {
+                await m_database.LogRemoteOperationAsync("delete", item.RemoteFilename, result);
+            }
+
+            await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Deleted, -1, null, suppressCleanup, TimeSpan.FromHours(2));
+            await m_stats.SendEventAsync(BackendActionType.Delete, BackendEventType.Completed, item.RemoteFilename, item.Size);
+
+            return true;
+        }
+
+        private async Task<bool> DoCreateFolder(FileEntryItem item)
+        {
+            await m_stats.SendEventAsync(BackendActionType.CreateFolder, BackendEventType.Started, null, -1);
+
+            string result = null;
+            try
+            {
+                m_backend.CreateFolder();
+            } 
+            catch (Exception ex)
+            {
+                result = ex.ToString();
+                throw;
+            }
+            finally
+            {
+                await m_database.LogRemoteOperationAsync("createfolder", item.RemoteFilename, result);
+            }
+
+            await m_stats.SendEventAsync(BackendActionType.CreateFolder, BackendEventType.Completed, null, -1);
+            return true;
+        }
+
+        private async Task<Library.Utility.TempFile> DoGet(FileEntryItem item)
+        {
+            Library.Utility.TempFile tmpfile = null;
+
+            // Grab size from the database if we do not know it,
+            // as this makes the output prettier
+            // Don't care if we fail to read it for whatever reason
+            var size = item.Size;
+            if (size < 0)
+                try { size = (await m_database.GetVolumeInfoAsync(item.RemoteFilename)).Size; }
+                catch { }
+
+            await m_stats.SendEventAsync(BackendActionType.Get, BackendEventType.Started, item.RemoteFilename, size);
+
+            try
+            {
+                var begin = DateTime.Now;
+
+                tmpfile = new Library.Utility.TempFile();
+                if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
+                {
+                    using (var fs = System.IO.File.OpenWrite(tmpfile))
+                    using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
+                    using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
+                        ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
+                }
+                else
+                    m_backend.Get(item.RemoteFilename, tmpfile);
+
+                var duration = DateTime.Now - begin;
+                var filehash = FileEntryItem.CalculateFileHash(tmpfile);
+                var filesize = new FileInfo(tmpfile).Length;
+
+                await m_log.WriteProfilingAsync(string.Format("Downloaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(filesize), duration, Library.Utility.Utility.FormatSizeString((long)(filesize / duration.TotalSeconds))), null);
+                await m_database.LogRemoteOperationAsync("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = filesize, Hash = filehash }));
+                await m_stats.SendEventAsync(BackendActionType.Get, BackendEventType.Completed, item.RemoteFilename, filesize);
+
+                if (!m_options.SkipFileHashChecks)
+                {
+                    if (item.Size >= 0)
+                    {
+                        if (filesize != item.Size)
+                            throw new Exception(Strings.Controller.DownloadedFileSizeError(item.RemoteFilename, filesize, item.Size));
+                    }
+                    else
+                        item.Size = filesize;
+
+                    if (!string.IsNullOrEmpty(item.Hash))
+                    {
+                        if (filehash != item.Hash)
+                            throw new Duplicati.Library.Main.Operation.Common.HashMismatchException(Strings.Controller.HashMismatchError(tmpfile, item.Hash, filehash));
+                    }
+                    else
+                        item.Hash = filehash;
+                }
+
+                // Fast exit
+                if (item.VerifyHashOnly)
+                    return null;
+                
+                // Decrypt before returning
+                if (!m_options.NoEncryption)
+                {
+                    try
+                    {
+                        using(var tmpfile2 = tmpfile)
+                        { 
+                            tmpfile = new Library.Utility.TempFile();
+
+                            // Auto-guess the encryption module
+                            var ext = (System.IO.Path.GetExtension(item.RemoteFilename) ?? "").TrimStart('.');
+                            if (!string.Equals(m_options.EncryptionModule, ext, StringComparison.OrdinalIgnoreCase))
+                            {
+                                // Check if the file is encrypted with something else
+                                if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase))
+                                {
+                                    using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(ext, m_options.Passphrase, m_options.RawOptions))
+                                        if (encmodule != null)
+                                        {
+                                            await m_log.WriteVerboseAsync("Filename extension \"{0}\" does not match encryption module \"{1}\", using matching encryption module", ext, m_options.EncryptionModule);
+                                            encmodule.Decrypt(tmpfile2, tmpfile);
+                                        }
+                                }
+                                // Check if the file is not encrypted
+                                else if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase))
+                                {
+                                    await m_log.WriteVerboseAsync("Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, m_options.EncryptionModule);
+                                }
+                                // Fallback, lets see what happens...
+                                else
+                                {
+                                    await m_log.WriteVerboseAsync("Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, m_options.EncryptionModule);
+                                    using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions))
+                                        encmodule.Decrypt(tmpfile2, tmpfile);
+                                }
+                            }
+                            else
+                            {
+                                using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions))
+                                    encmodule.Decrypt(tmpfile2, tmpfile);
+                            }
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        //If we fail here, make sure that we throw a crypto exception
+                        if (ex is System.Security.Cryptography.CryptographicException)
+                            throw;
+                        else
+                            throw new System.Security.Cryptography.CryptographicException(ex.Message, ex);
+                    }
+                }
+
+                var res = tmpfile;
+                tmpfile = null;
+                return res;
+            }
+            finally
+            {
+                try 
+                { 
+                    if (tmpfile != null) 
+                        tmpfile.Dispose();
+                }
+                catch
+                {
+                }
+            }
+        }
+
+		private string m_lastThrottleUploadValue = null;
+		private string m_lastThrottleDownloadValue = null;
+
+		private void HandleProgress(ThrottledStream ts, long pg)    
+		{
+            if (!m_taskreader.TransferProgressAsync.WaitForTask().Result)
+                throw new OperationCanceledException();
+
+			// Update the throttle speeds if they have changed
+			string tmp;
+			m_options.RawOptions.TryGetValue("throttle-upload", out tmp);
+			if (tmp != m_lastThrottleUploadValue)
+			{
+				ts.WriteSpeed = m_options.MaxUploadPrSecond;
+				m_lastThrottleUploadValue = tmp;
+			}
+
+			m_options.RawOptions.TryGetValue("throttle-download", out tmp);
+			if (tmp != m_lastThrottleDownloadValue)
+			{
+				ts.ReadSpeed = m_options.MaxDownloadPrSecond;
+				m_lastThrottleDownloadValue = tmp;
+			}
+
+			m_stats.UpdateBackendProgress(pg);
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            base.Dispose(disposing);
+
+            if (m_backend != null)
+                try { m_backend.Dispose(); }
+                catch {}
+                finally { m_backend = null; }
+
+            if (m_log != null)
+                m_log.Dispose();
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Common/BackendHandlerDatabaseGuard.cs

@@ -0,0 +1,284 @@
+//  Copyright (C) 2018, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// This class provides a thread-safe wrapper around a non-thread safe
+    /// database instance, such that the database can be safely used within
+    /// the <see cref="BackendHandler"/> without needing to rewrite all the
+    /// code that does not handle concurrent access.
+    /// </summary>
+    internal class BackendHandlerDatabaseGuard : IDisposable, IBackendHandlerDatabase, IIndexVolumeCreatorDatabase
+    {
+        /// <summary>
+        /// The database we are wrapping
+        /// </summary>
+        private readonly Database.LocalDatabase m_db;
+        /// <summary>
+        /// A flag indicating if this is a dry-run
+        /// </summary>
+        private readonly bool m_dryrun;
+        /// <summary>
+        /// The list of pending work
+        /// </summary>
+        private List<Tuple<Action, TaskCompletionSource<bool>>> m_workQueue = new List<Tuple<Action, TaskCompletionSource<bool>>>();
+        /// <summary>
+        /// The lock object protecting the queue.
+        /// </summary>
+        private readonly object m_lock = new object();
+        /// <summary>
+        /// The main thread, used to guard against execution from another thread
+        /// </summary>
+        private readonly System.Threading.Thread m_mainthread;
+
+        /// <summary>
+        /// Initializes a new instance of the
+        /// <see cref="T:Duplicati.Library.Main.Operation.Common.BackendHandlerDatabaseGuard"/> class.
+        /// </summary>
+        /// <param name="db">The database to wrap.</param>
+        /// <param name="dryrun">A flag indicating if this is a dry-run</param>
+        public BackendHandlerDatabaseGuard(Database.LocalDatabase db, bool dryrun)
+        {
+            if (db == null)
+                throw new ArgumentNullException(nameof(db));
+            m_db = db; 
+            m_mainthread = System.Threading.Thread.CurrentThread;
+            m_dryrun = dryrun;
+        }
+
+        /// <summary>
+        /// Adds a pending work item to the queue
+        /// </summary>
+        /// <returns>The task wait handle.</returns>
+        /// <param name="action">The method to run when ready.</param>
+        private Task AddToQueue(Action action)
+        {
+            if (m_workQueue == null)
+                throw new ObjectDisposedException("This database guard instance has been shut down");
+
+
+            var tcs = new TaskCompletionSource<bool>();
+            lock (m_lock)
+                m_workQueue.Add(new Tuple<Action, TaskCompletionSource<bool>>(action, tcs));
+            return tcs.Task;
+        }
+
+        /// <summary>
+        /// Writes the current changes to the database
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="message">The message to use for logging the time spent in this operation.</param>
+        /// <param name="restart">If set to <c>true</c>, a transaction will be started again after this call.</param>
+        public Task CommitTransactionAsync(string message, bool restart = true)
+        {
+            return AddToQueue(() =>
+            {
+                if (m_dryrun)
+                {
+                    if (!restart)
+                        m_db.RollbackTransaction();
+                    return;
+                }
+                m_db.CommitTransaction(message, restart);
+            });
+        }
+
+        /// <summary>
+        /// Writes remote operation log data to the database
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="operation">The operation performed.</param>
+        /// <param name="path">The remote path used.</param>
+        /// <param name="data">Any data reported by the operation.</param>
+        public Task LogRemoteOperationAsync(string operation, string path, string data)
+        {
+            return AddToQueue(() => { m_db.LogRemoteOperation(operation, path, data); });
+        }
+
+        /// <summary>
+        /// Renames a remote file in the database
+        /// </summary>
+        /// <returns>The remote file to rename.</returns>
+        /// <param name="oldname">The old filename.</param>
+        /// <param name="newname">The new filename.</param>
+        public Task RenameRemoteFileAsync(string oldname, string newname)
+        {
+            return AddToQueue(() => { m_db.RenameRemoteFile(oldname, newname); });
+        }
+
+        /// <summary>
+        /// Updates the remote volume information.
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="name">The name of the remote volume to update.</param>
+        /// <param name="state">The new volume state.</param>
+        /// <param name="size">The new volume size.</param>
+        /// <param name="hash">The new volume hash.</param>
+        /// <param name="suppressCleanup">If set to <c>true</c> suppress cleanup operation.</param>
+        /// <param name="deleteGraceTime">The new delete grace time.</param>
+        public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan))
+        {
+            return AddToQueue(() => { m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime); });
+        }
+
+        /// <summary>
+        /// Gets the volume information for a remote file, given the name.
+        /// </summary>
+        /// <returns>The remote volume information.</returns>
+        /// <param name="remotename">The name of the remote file to query.</param>
+        public async Task<Database.RemoteVolumeEntry> GetVolumeInfoAsync(string remotename)
+        {
+            var res = default(Database.RemoteVolumeEntry);
+            await AddToQueue(() => res = m_db.GetRemoteVolume(remotename));
+            return res;
+        }
+
+
+        /// <summary>
+        /// Creates and registers a remote volume
+        /// </summary>
+        /// <returns>The newly created volume ID.</returns>
+        /// <param name="name">The name of the remote file.</param>
+        /// <param name="type">The type of the remote file.</param>
+        /// <param name="state">The state of the remote file.</param>
+        public async Task<long> RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state)
+        {
+            var res = default(long);
+            await AddToQueue(() => res = m_db.RegisterRemoteVolume(name, type, state));
+            return res;
+        }
+
+        /// <summary>
+        /// Gets a list of all blocks associated with a given volume
+        /// </summary>
+        /// <returns>The blocks found in the volume.</returns>
+        /// <param name="volumeid">The ID of the volume to examine.</param>
+        public async Task GetBlocksAsync(long volumeid, Action<Database.LocalDatabase.IBlock> callback)
+        {
+            await AddToQueue(() => {
+                foreach (var n in m_db.GetBlocks(volumeid))
+                    callback(n);
+                });
+        }
+
+        /// <summary>
+        /// Gets the blocklists contained in a remote volume
+        /// </summary>
+        /// <returns>The blocklists.</returns>
+        /// <param name="volumeid">The ID of the volume to get the blocklists for.</param>
+        /// <param name="blocksize">The blocksize setting.</param>
+        /// <param name="hashsize">The size of the hash in bytes.</param>
+        /// <param name="callback">The callback method called for each result</param>
+        public Task GetBlocklistsAsync(long volumeid, int blocksize, int hashsize, Action<Tuple<string, byte[], int>> callback)
+        {
+            return AddToQueue(() => {
+                foreach (var e in m_db.GetBlocklists(volumeid, blocksize, hashsize))
+                    callback(e);
+            });
+        }
+
+        /// <summary>
+        /// Adds a link between a block volume and an index volume
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="indexVolumeID">The index volume ID.</param>
+        /// <param name="blockVolumeID">The block volume ID.</param>
+        public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID)
+        {
+            return AddToQueue(() => m_db.AddIndexBlockLink(indexVolumeID, blockVolumeID));
+        }
+
+        /// <summary>
+        /// Processes all pending operations into the database.
+        /// </summary>
+        public void ProcessAllPendingOperations()
+        {
+            //if (m_mainthread != System.Threading.Thread.CurrentThread)
+            //    throw new InvalidOperationException("Attempted to flush database work queue from another thread");
+
+            // If we are shut down, just exit
+            if (m_workQueue == null)
+                return;
+
+            var prevqueue = m_workQueue;
+            lock (m_lock)
+            {
+                // No need to fiddle if nothing has happened
+                if (prevqueue.Count == 0)
+                    return;
+
+                m_workQueue = new List<Tuple<Action, TaskCompletionSource<bool>>>();
+            }
+
+            // We repeat here to allow quicker progress in case the backend is blocked
+            // on a log message, and emits a new one immediately after having one handled
+            while (prevqueue.Count != 0)
+            {
+                foreach (var op in prevqueue)
+                {
+                    try
+                    {
+                        op.Item1();
+                        op.Item2.TrySetResult(true);
+                    }
+                    catch (Exception ex)
+                    {
+                        op.Item2.TrySetException(ex);
+                    }
+                }
+
+                // Inject a sleep here to allow the BackendHandler to progress
+                // Not pretty, but this entire class should be deleted once the code is 
+                // rewritten to use thread-safe database access
+                System.Threading.Thread.Sleep(100);
+
+                prevqueue = m_workQueue;
+                lock (m_lock)
+                    m_workQueue = new List<Tuple<Action, TaskCompletionSource<bool>>>();
+            }
+        }
+
+        /// <summary>
+        /// Releases all resource used by the
+        /// <see cref="T:Duplicati.Library.Main.Operation.Common.BackendHandlerDatabaseGuard"/> object.
+        /// </summary>
+        /// <remarks>Call <see cref="Dispose"/> when you are finished using the
+        /// <see cref="T:Duplicati.Library.Main.Operation.Common.BackendHandlerDatabaseGuard"/>. The
+        /// <see cref="Dispose"/> method leaves the
+        /// <see cref="T:Duplicati.Library.Main.Operation.Common.BackendHandlerDatabaseGuard"/> in an unusable state.
+        /// After calling <see cref="Dispose"/>, you must release all references to the
+        /// <see cref="T:Duplicati.Library.Main.Operation.Common.BackendHandlerDatabaseGuard"/> so the garbage collector
+        /// can reclaim the memory that the
+        /// <see cref="T:Duplicati.Library.Main.Operation.Common.BackendHandlerDatabaseGuard"/> was occupying.</remarks>
+        public void Dispose()
+        {
+            lock (m_lock)
+            {
+                if (m_workQueue != null)
+                    foreach (var n in m_workQueue)
+                        n.Item2.TrySetCanceled();
+                m_workQueue = null;
+            }
+        }
+
+
+    }
+}

Duplicati/Library/Main/Operation/Common/Channels.cs

@@ -0,0 +1,29 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    internal static class Channels
+    {
+        /// <summary>
+        /// The log channel coordinats log messages
+        /// </summary>
+        public static readonly ChannelMarkerWrapper<LogMessage> LogChannel = new ChannelMarkerWrapper<LogMessage>(new ChannelNameAttribute("LogChannel"));
+    }
+}

Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs

@@ -0,0 +1,187 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using CoCoL;
+using Duplicati.Library.Main.Database;
+using System.Collections.Generic;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Asynchronous interface that ensures all requests
+    /// to the database are performed in a sequential manner
+    /// </summary>
+    internal class DatabaseCommon : SingleRunner, IBackendHandlerDatabase, IIndexVolumeCreatorDatabase, IDisposable
+    {
+        protected LocalDatabase m_db;
+        protected Options m_options;
+
+        public DatabaseCommon(LocalDatabase db, Options options)
+            : base()
+        {
+            m_db = db;
+            m_options = options;
+        }
+
+        /// <summary>
+        /// Creates and registers a remote volume
+        /// </summary>
+        /// <returns>The newly created volume ID.</returns>
+        /// <param name="name">The name of the remote file.</param>
+        /// <param name="type">The type of the remote file.</param>
+        /// <param name="state">The state of the remote file.</param>
+        public Task<long> RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state)
+        {
+            return RunOnMain(() => m_db.RegisterRemoteVolume(name, type, state));
+        }
+
+        /// <summary>
+        /// Updates the remote volume information.
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="name">The name of the remote volume to update.</param>
+        /// <param name="state">The new volume state.</param>
+        /// <param name="size">The new volume size.</param>
+        /// <param name="hash">The new volume hash.</param>
+        /// <param name="suppressCleanup">If set to <c>true</c> suppress cleanup operation.</param>
+        /// <param name="deleteGraceTime">The new delete grace time.</param>
+        public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan))
+        {
+            return RunOnMain(() => m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime));
+        }
+
+        /// <summary>
+        /// Writes the current changes to the database
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="message">The message to use for logging the time spent in this operation.</param>
+        /// <param name="restart">If set to <c>true</c>, a transaction will be started again after this call.</param>
+        public Task CommitTransactionAsync(string message, bool restart = true)
+        {
+            return RunOnMain(() => 
+            {
+                if (m_options.Dryrun)
+                    m_db.RollbackTransaction(restart);
+                else
+                    m_db.CommitTransaction(message, restart);
+            });
+        }
+
+        public Task RollbackTransactionAsync(bool restart = true)
+        {
+            return RunOnMain(() =>
+            {
+                m_db.RollbackTransaction(restart);
+            });
+        }
+
+        /// <summary>
+        /// Renames a remote file in the database
+        /// </summary>
+        /// <returns>The remote file to rename.</returns>
+        /// <param name="oldname">The old filename.</param>
+        /// <param name="newname">The new filename.</param>
+        public Task RenameRemoteFileAsync(string oldname, string newname)
+        {
+            return RunOnMain(() => m_db.RenameRemoteFile(oldname, newname));
+        }
+
+        /// <summary>
+        /// Writes remote operation log data to the database
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="operation">The operation performed.</param>
+        /// <param name="path">The remote path used.</param>
+        /// <param name="data">Any data reported by the operation.</param>
+        public Task LogRemoteOperationAsync(string operation, string path, string data)
+        {
+            return RunOnMain(() => m_db.LogRemoteOperation(operation, path, data));
+        }
+
+        /// <summary>
+        /// Gets a list of all blocks associated with a given volume
+        /// </summary>
+        /// <returns>The blocks found in the volume.</returns>
+        /// <param name="volumeid">The ID of the volume to examine.</param>
+        /// <param name="callback">The method called for each item</param>
+        public Task GetBlocksAsync(long volumeid, Action<LocalDatabase.IBlock> callback)
+        {
+            return RunOnMain(() => {
+                foreach (var n in m_db.GetBlocks(volumeid))
+                    callback(n);
+            });
+        }
+
+        /// <summary>
+        /// Gets the volume information for a remote file, given the name.
+        /// </summary>
+        /// <returns>The remote volume information.</returns>
+        /// <param name="remotename">The name of the remote file to query.</param>
+        public Task<RemoteVolumeEntry> GetVolumeInfoAsync(string remotename)
+        {
+            return RunOnMain(() => m_db.GetRemoteVolume(remotename));
+        }
+
+        /// <summary>
+        /// Gets the blocklists contained in a remote volume
+        /// </summary>
+        /// <returns>The blocklists.</returns>
+        /// <param name="volumeid">The ID of the volume to get the blocklists for.</param>
+        /// <param name="blocksize">The blocksize setting.</param>
+        /// <param name="hashsize">The size of the hash in bytes.</param>
+        /// <param name="callback">The callback method called for each result</param>
+        public Task GetBlocklistsAsync(long volumeid, int blocksize, int hashsize, Action<Tuple<string, byte[], int>> callback)
+        {
+            return RunOnMain(() =>
+            {
+                foreach (var n in m_db.GetBlocklists(volumeid, blocksize, hashsize))
+                    callback(n);
+            });
+        }
+
+        /// <summary>
+        /// Gets the ID for a remote volume.
+        /// </summary>
+        /// <returns>The remote volume ID.</returns>
+        /// <param name="remotename">The remote file name.</param>
+        public Task<long> GetRemoteVolumeIDAsync(string remotename)
+        {
+            return RunOnMain(() => m_db.GetRemoteVolumeID(remotename));
+        }
+
+        /// <summary>
+        /// Adds a link between a block volume and an index volume
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="indexVolumeID">The index volume ID.</param>
+        /// <param name="blockVolumeID">The block volume ID.</param>
+        public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID)
+        {
+            return RunOnMain(() => m_db.AddIndexBlockLink(indexVolumeID, blockVolumeID));
+        }
+
+
+        protected override void Dispose(bool isDisposing)
+        {
+            base.Dispose(isDisposing);
+        }
+
+    }
+}
+

Duplicati/Library/Main/Operation/Common/HashMismatchException.cs

@@ -0,0 +1,44 @@
+//  Copyright (C) 2018, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Class to represent hash failures
+    /// </summary>
+    [Serializable]
+    public class HashMismatchException : Exception
+    {
+        /// <summary>
+        /// Default constructor, sets a generic string as the message
+        /// </summary>
+        public HashMismatchException() : base() { }
+
+        /// <summary>
+        /// Constructor with non-default message
+        /// </summary>
+        /// <param name="message">The exception message</param>
+        public HashMismatchException(string message) : base(message) { }
+
+        /// <summary>
+        /// Constructor with non-default message and inner exception details
+        /// </summary>
+        /// <param name="message">The exception message</param>
+        /// <param name="innerException">The exception that caused this exception</param>
+        public HashMismatchException(string message, Exception innerException) : base(message, innerException) { }
+    }
+}

Duplicati/Library/Main/Operation/Common/IBackendHandlerDatabase.cs

@@ -0,0 +1,71 @@
+//  Copyright (C) 2018, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Interface for just the operations needed by the backendhandler
+    /// </summary>
+    internal interface IBackendHandlerDatabase
+    {
+        /// <summary>
+        /// Updates the remote volume information.
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="name">The name of the remote volume to update.</param>
+        /// <param name="state">The new volume state.</param>
+        /// <param name="size">The new volume size.</param>
+        /// <param name="hash">The new volume hash.</param>
+        /// <param name="suppressCleanup">If set to <c>true</c> suppress cleanup operation.</param>
+        /// <param name="deleteGraceTime">The new delete grace time.</param>
+        Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan));
+
+        /// <summary>
+        /// Renames a remote file in the database
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="oldname">The old filename.</param>
+        /// <param name="newname">The new filename.</param>
+        Task RenameRemoteFileAsync(string oldname, string newname);
+
+        /// <summary>
+        /// Writes remote operation log data to the database
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="operation">The operation performed.</param>
+        /// <param name="path">The remote path used.</param>
+        /// <param name="data">Any data reported by the operation.</param>
+        Task LogRemoteOperationAsync(string operation, string path, string data);
+
+        /// <summary>
+        /// Writes the current changes to the database
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="message">The message to use for logging the time spent in this operation.</param>
+        /// <param name="restart">If set to <c>true</c>, a transaction will be started again after this call.</param>
+        Task CommitTransactionAsync(string message, bool restart = true);
+
+        /// <summary>
+        /// Gets the volume information for a remote file, given the name.
+        /// </summary>
+        /// <returns>The remote volume information.</returns>
+        /// <param name="remotename">The name of the remote file to query.</param>
+        Task<Database.RemoteVolumeEntry> GetVolumeInfoAsync(string remotename);
+    }
+}

Duplicati/Library/Main/Operation/Common/IIndexVolumeCreatorDatabase.cs

@@ -0,0 +1,70 @@
+//  Copyright (C) 2018, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// An interface for the operations needed by the <see cref="IndexVolumeCreator"/> routine
+    /// </summary>
+    internal interface IIndexVolumeCreatorDatabase
+    {
+        /// <summary>
+        /// Creates and registers a remote volume
+        /// </summary>
+        /// <returns>The newly created volume ID.</returns>
+        /// <param name="name">The name of the remote file.</param>
+        /// <param name="type">The type of the remote file.</param>
+        /// <param name="state">The state of the remote file.</param>
+        Task<long> RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state);
+
+        /// <summary>
+        /// Gets a list of all blocks associated with a given volume
+        /// </summary>
+        /// <returns>The blocks found in the volume.</returns>
+        /// <param name="volumeid">The ID of the volume to examine.</param>
+        /// <param name="callback">The callback method called for each result</param>
+        Task GetBlocksAsync(long volumeid, Action<Database.LocalDatabase.IBlock> callback);
+
+        /// <summary>
+        /// Gets the blocklists contained in a remote volume
+        /// </summary>
+        /// <returns>The blocklists.</returns>
+        /// <param name="volumeid">The ID of the volume to get the blocklists for.</param>
+        /// <param name="blocksize">The blocksize setting.</param>
+        /// <param name="hashsize">The size of the hash in bytes.</param>
+        /// <param name="callback">The callback method called for each result</param>
+        Task GetBlocklistsAsync(long volumeid, int blocksize, int hashsize, Action<Tuple<string, byte[], int>> callback);
+
+        /// <summary>
+        /// Adds a link between a block volume and an index volume
+        /// </summary>
+        /// <returns>An awaitable task.</returns>
+        /// <param name="indexVolumeID">The index volume ID.</param>
+        /// <param name="blockVolumeID">The block volume ID.</param>
+        Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID);        
+
+        /// <summary>
+        /// Gets the volume information for a remote file, given the name.
+        /// </summary>
+        /// <returns>The remote volume information.</returns>
+        /// <param name="remotename">The name of the remote file to query.</param>
+        Task<Database.RemoteVolumeEntry> GetVolumeInfoAsync(string remotename);
+    }
+}

Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs

@@ -0,0 +1,86 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Volumes;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Encapsulates creating an index file from the database contents
+    /// </summary>
+    internal static class IndexVolumeCreator
+    {
+        /// <summary>
+        /// Creates an index volume by reading the associated data from the database
+        /// </summary>
+        /// <returns>The index volume.</returns>
+        /// <param name="blockname">The name of the remote volume to build the index for.</param>
+        /// <param name="options">The job options.</param>
+        /// <param name="database">The database to use.</param>
+        /// <param name="blocklist">An optional pre-collected list of blocklist hashes</param>
+        public static async Task<IndexVolumeWriter> CreateIndexVolume(string blockname, Options options, Common.IIndexVolumeCreatorDatabase database, IEnumerable<string> blocklist)
+        {
+            using(var h = Duplicati.Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm))
+            {
+                var w = new IndexVolumeWriter(options);
+                w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);
+
+                var blockvolume = await database.GetVolumeInfoAsync(blockname);
+
+                var st = DateTime.Now;
+                w.StartVolume(blockname);
+                await database.GetBlocksAsync(blockvolume.ID, b => w.AddBlock(b.Hash, b.Size));
+                w.FinishVolume(blockvolume.Hash, blockvolume.Size);
+
+                if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
+                    using (new Library.Logging.Timer("Create index file"))
+                    {
+                        if (blocklist != null)
+                        {
+                            foreach (var b in blocklist.Select(x => Backup.VolumeUploadRequest.DecodeBlockListEntry(x)))
+                            {
+                                var bh = Convert.ToBase64String(h.ComputeHash(b.Item3, 0, (int)b.Item2));
+                                if (bh != b.Item1)
+                                    throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1));
+                                w.WriteBlocklist(b.Item1, b.Item3, 0, (int)b.Item2);
+                            }
+                        }
+                        else
+                        {
+                            await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize, b =>
+                            {
+                                var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
+                                if (bh != b.Item1)
+                                    throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1));
+                                w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
+                            });
+                        }
+                    }
+
+                w.Close();
+
+                await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID);
+
+                return w;
+            }
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Common/LogHandler.cs

@@ -0,0 +1,63 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using CoCoL;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Handles writing incomming messages to the current log interface
+    /// </summary>
+    internal static class LogHandler
+    {
+        public static Task Run(ILogWriter log)
+        {
+            return AutomationExtensions.RunTask(
+                new
+                {
+                    LogChannel = ChannelManager.GetChannel(Channels.LogChannel.ForRead)
+                },
+
+                async self =>
+                {
+                    while(true)
+                    {
+                        var msg = await self.LogChannel.ReadAsync();
+
+                        if (msg.IsDryRun)
+                            log.AddDryrunMessage(msg.Message);
+                        else if (msg.IsVerbose)
+                            log.AddVerboseMessage(msg.Message);
+                        else if (msg.IsRetry && log is IBackendWriter)
+                            ((IBackendWriter)log).AddRetryAttempt(msg.Message, msg.Exception);
+                        else if (msg.Level == Duplicati.Library.Logging.LogMessageType.Error)
+                            log.AddError(msg.Message, msg.Exception);
+                        else if (msg.Level == Duplicati.Library.Logging.LogMessageType.Warning)
+                            log.AddWarning(msg.Message, msg.Exception);
+                        else if (msg.Level == Duplicati.Library.Logging.LogMessageType.Profiling)
+                            log.AddVerboseMessage(msg.Message);
+                        else
+                            log.AddMessage(msg.Message);
+
+                    }
+                }
+            );
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Common/LogMessage.cs

@@ -0,0 +1,36 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Representation of a log message
+    /// </summary>
+    public struct LogMessage
+    {
+        public Logging.LogMessageType Level;
+        public string Message;
+        public Exception Exception;
+        public bool IsVerbose;
+        public bool IsDryRun;
+        public bool IsRetry;
+    }
+}
+

Duplicati/Library/Main/Operation/Common/LogWrapper.cs

@@ -0,0 +1,122 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Helper method to reduce code-text size when writing log messages
+    /// </summary>
+    public class LogWrapper : IDisposable
+    {
+        private bool m_autoLeave;
+        private IWriteChannel<LogMessage> m_channel;
+
+        //TODO: Fix this so the verbose messages are not sent unless required
+
+        public LogWrapper()
+        {
+            m_channel = ChannelManager.GetChannel(Channels.LogChannel.ForWrite);
+            m_autoLeave = true;
+        }
+
+        public LogWrapper(IWriteChannel<LogMessage> channel)
+        {
+            m_channel = channel;
+            m_autoLeave = false;
+        }
+
+        public Task WriteWarningAsync(string message, Exception ex)
+        {
+            return m_channel.WriteAsync(new LogMessage() { 
+                Level = Duplicati.Library.Logging.LogMessageType.Warning,
+                Message = message,
+                Exception = ex
+            });
+        }
+
+        public Task WriteErrorAsync(string message, Exception ex)
+        {
+            return m_channel.WriteAsync(new LogMessage() { 
+                Level = Duplicati.Library.Logging.LogMessageType.Error,
+                Message = message,
+                Exception = ex
+            });
+        }
+
+        public Task WriteProfilingAsync(string message, Exception ex)
+        {
+            return m_channel.WriteAsync(new LogMessage() { 
+                Level = Duplicati.Library.Logging.LogMessageType.Profiling,
+                Message = message,
+                Exception = ex
+            });
+        }
+
+        public Task WriteInformationAsync(string message, Exception ex = null)
+        {
+            return m_channel.WriteAsync(new LogMessage() { 
+                Level = Duplicati.Library.Logging.LogMessageType.Information,
+                Message = message,
+                Exception = ex
+            });
+        }
+
+        public Task WriteVerboseAsync(string message, params object[] args)
+        {
+            return m_channel.WriteAsync(new LogMessage() { 
+                Level = Duplicati.Library.Logging.LogMessageType.Information,
+                Message = string.Format(message, args),
+                IsVerbose = true
+            });
+        }
+
+        public Task WriteDryRunAsync(string message, params object[] args)
+        {
+            return m_channel.WriteAsync(new LogMessage() { 
+                Level = Duplicati.Library.Logging.LogMessageType.Information,
+                Message = string.Format(message, args),
+                IsDryRun = true
+            });
+        }
+
+        public Task WriteRetryAttemptAsync(string message, params object[] args)
+        {
+            return m_channel.WriteAsync(new LogMessage() { 
+                Level = Duplicati.Library.Logging.LogMessageType.Information,
+                Message = string.Format(message, args),
+                IsRetry = true
+            });
+        }
+
+        public void Dispose()
+        {
+            if (m_autoLeave && m_channel != null)
+            {
+                m_autoLeave = false;
+                if (m_channel is IJoinAbleChannel)
+                    ((IJoinAbleChannel)m_channel).Leave(false);
+                else if (m_channel is IJoinAbleChannelEnd)
+                    ((IJoinAbleChannelEnd)m_channel).Leave();
+            }
+            m_channel = null;
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Common/ProgressEvent.cs

@@ -0,0 +1,41 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// The types of events being sent
+    /// </summary>
+    public enum EventType
+    {
+        FileStarted,
+        FileProgressUpdate,
+        FileClosed
+    }
+
+    /// <summary>
+    /// Description of a particular event
+    /// </summary>
+    public struct ProgressEvent
+    {
+        public EventType Type { get; set; }
+        public string Filepath { get; set; }
+        public long Length { get; set; }
+    }
+}
+

Duplicati/Library/Main/Operation/Common/SingleRunner.cs

@@ -0,0 +1,162 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using CoCoL;
+using System.Threading;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Provides mutually exclusive access to a resource,
+    /// by ensuring all methods are executed sequentially
+    /// </summary>
+    internal abstract class SingleRunner : IDisposable
+    {
+        protected IChannel<Func<Task>> m_channel;
+        protected readonly Task m_worker;
+        protected CancellationTokenSource m_workerSource;
+
+        public SingleRunner()
+        {
+            AutomationExtensions.AutoWireChannels(this, null);
+            m_channel = ChannelManager.CreateChannel<Func<Task>>();
+            m_workerSource = new System.Threading.CancellationTokenSource();
+            m_worker = AutomationExtensions.RunProtected(this, Start);
+        }
+
+        private async Task Start()
+        {
+            try
+            {
+                var ct = m_workerSource.Token;
+                while (!ct.IsCancellationRequested)
+                {
+                    // Grab next task
+                    var nextTask = await m_channel.ReadAsync();
+
+                    // Execute it
+                    await nextTask();
+                }
+            }
+            catch(Exception ex)
+            {
+                this.Dispose();
+                if (!ex.IsRetiredException())
+                    Logging.Log.WriteMessage($"Unepected shutdown of worker {this.GetType().FullName}", Logging.LogMessageType.Warning, ex);
+            }
+        }
+
+        protected Task<T> DoRunOnMain<T>(Func<Task<T>> method)
+        {
+            var res = new TaskCompletionSource<T>();
+
+            Task.Run(async () =>
+            {
+                try
+                {
+                    if (m_workerSource.IsCancellationRequested)
+                    {
+                        res.TrySetCanceled();
+                        return;
+                    }
+
+                    await m_channel.WriteAsync(async () =>
+                    {
+                        if (m_workerSource.IsCancellationRequested)
+                        {
+                            res.TrySetCanceled();
+                            return;
+                        }
+
+                        try
+                        {
+                            var r = await method().ConfigureAwait(false);
+
+                            // We cannot await this, as it can cause deadlocks
+                            Task.Run(() => res.SetResult(r)).FireAndForget(); 
+                        }
+                        catch (Exception ex)
+                        {
+                            if (ex is System.Threading.ThreadAbortException)
+                                res.TrySetCanceled();
+                            else
+                                res.TrySetException(ex);
+                        }
+                    }).ConfigureAwait(false);
+                }
+                catch (Exception ex)
+                {
+                    if (ex is System.Threading.ThreadAbortException)
+                        res.TrySetCanceled();
+                    else
+                        res.TrySetException(ex);
+                }
+            });
+
+            return res.Task;
+        }
+
+        protected Task RunOnMain(Action method)
+        {
+            return DoRunOnMain<bool>(() =>
+            {
+                method();
+                return Task.FromResult(true);
+            });
+        }
+
+        protected Task<T> RunOnMain<T>(Func<T> method)
+        {
+            return DoRunOnMain(() =>
+            {
+                return Task.FromResult(method());
+            });
+        }
+
+        protected Task RunOnMain(Func<Task> method)
+        {
+            return DoRunOnMain(async () => {
+                await method().ConfigureAwait(false);
+                return true;
+            });
+        }
+
+        protected Task<T> RunOnMain<T>(Func<Task<T>> method)
+        {
+            return DoRunOnMain(method);
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+
+        protected virtual void Dispose(bool isDisposing)
+        {
+            m_workerSource.Cancel();
+
+            if (m_channel != null)
+                try { m_channel.Retire(); }
+                catch { }
+                finally { }
+
+            AutomationExtensions.RetireAllChannels(this);
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Common/StatsCollector.cs

@@ -0,0 +1,51 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Ensures sequentially emitted events
+    /// </summary>
+    internal class StatsCollector : SingleRunner, IDisposable
+    {
+        protected IBackendWriter m_bw;
+
+        public StatsCollector(IBackendWriter bw)
+        {
+            m_bw = bw;
+        }
+
+        public Task SendEventAsync(BackendActionType action, BackendEventType type, string path, long size)
+        {
+            return RunOnMain(() => m_bw.SendEvent(action, type, path, size));
+        }
+
+        public void UpdateBackendProgress(long pg)
+        {
+            m_bw.BackendProgressUpdater.UpdateProgress(pg);
+        }
+
+        public void SetBlocking(bool isBlocked)
+        {
+            if (m_bw.BackendProgressUpdater != null)
+                m_bw.BackendProgressUpdater.SetBlocking(isBlocked);
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/Common/TaskControl.cs

@@ -0,0 +1,251 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    /// <summary>
+    /// Interface for async access to the current control state
+    /// </summary>
+    public interface ITaskReader
+    {
+        /// <summary>
+        /// Processing tasks can regularly await this,
+        /// which will pause the task or stop it if required.
+        /// The return value is <c>true</c> if the program should continue and <c>false</c> if a stop is requested
+        /// </summary>
+        Task<bool> ProgressAsync { get; }
+        /// <summary>
+        /// Gets the transfer progress async control.
+        /// The transfer handler should await this instead of the <see cref="ProgressAsync"/> event.
+        /// </summary>
+        Task<bool> TransferProgressAsync { get; }
+    }
+
+    /// <summary>
+    /// Interface for controlling the progress
+    /// </summary>
+    public interface ITaskCommander
+    {
+        /// <summary>
+        /// Requests that progress should be paused
+        /// </summary>
+        /// <param name="alsoTransfers">If set to <c>true</c> transfer are also suspended.</param>
+        void Pause(bool alsoTransfers = false);
+        /// <summary>
+        /// Resumes running a paused process
+        /// </summary>
+        void Resume();
+        /// <summary>
+        /// Requests that the progress should be stopped in an orderly manner,
+        /// which allows current transfers to be completed.
+        /// </summary>
+        /// <param name="alsoTransfers">If set to <c>true</c> active transfer are also stopped.</param>
+        void Stop(bool alsoTransfers = false);
+        /// <summary>
+        /// Terminates the progress without allowing a flush
+        /// </summary>
+        void Terminate();
+    }
+
+    /// <summary>
+    /// Implementation of the task control
+    /// </summary>
+    public class TaskControl : ITaskReader, ITaskCommander, IDisposable
+    {
+        /// <summary>
+        /// Internal state tracking to avoid invalid requests
+        /// </summary>
+        private enum State
+        {
+            /// <summary>
+            /// The task is running
+            /// </summary>
+            Active,
+            /// <summary>
+            /// The task is paused
+            /// </summary>
+            Paused,
+            /// <summary>
+            /// The task is stopped
+            /// </summary>
+            Stopped,
+            /// <summary>
+            /// The task is terminated
+            /// </summary>
+            Terminated
+        }
+
+        /// <summary>
+        /// Internal control state for progress event
+        /// </summary>
+        private TaskCompletionSource<bool> m_progress;
+        /// <summary>
+        /// Internal control state for the transfer event
+        /// </summary>
+        private TaskCompletionSource<bool> m_transfer;
+
+        /// <summary>
+        /// The control lock instance
+        /// </summary>
+        private readonly object m_lock = new object();
+
+        /// <summary>
+        /// The current progress state
+        /// </summary>
+        private State m_progressstate = State.Paused;
+        /// <summary>
+        /// The current transfer state
+        /// </summary>
+        private State m_transferstate = State.Paused;
+
+        /// <summary>
+        /// Processing tasks can regularly await this,
+        /// which will pause the task or stop it if required.
+        /// The return value is <c>true</c> if the program should continue and <c>false</c> if a stop is requested
+        /// </summary>
+        public Task<bool> ProgressAsync { get { return m_progress.Task; } }
+        /// <summary>
+        /// Gets the transfer progress async control.
+        /// The transfer handler should await this instead of the <see cref="ProgressAsync"/> event.
+        /// </summary>
+        public Task<bool> TransferProgressAsync { get { return m_transfer.Task; } }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Duplicati.Library.Main.Operation.Common.TaskControl"/> class.
+        /// </summary>
+        public TaskControl()
+        {
+            m_progress = new TaskCompletionSource<bool>();
+            m_transfer = new TaskCompletionSource<bool>();
+            Resume();
+        }
+
+        /// <summary>
+        /// Resumes running a paused process
+        /// </summary>
+        public void Resume()
+        {
+            lock(m_lock)
+            {
+                if (m_progressstate == State.Paused)
+                {
+                    m_progress.SetResult(true);
+                    m_progressstate = State.Active;
+                }
+                if (m_transferstate == State.Paused)
+                {
+                    m_transfer.SetResult(true);
+                    m_transferstate = State.Active;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Requests that progress should be paused
+        /// </summary>
+        /// <param name="alsoTransfers">If set to <c>true</c> also transfers.</param>
+        public void Pause(bool alsoTransfers = false)
+        {
+            lock(m_lock)
+            {
+                if (m_progressstate == State.Active)
+                {
+                    m_progress = new TaskCompletionSource<bool>();
+                    m_progressstate = State.Paused;
+                }
+
+                if (alsoTransfers && m_transferstate == State.Active)
+                {
+                    m_transfer = new TaskCompletionSource<bool>();
+                    m_transferstate = State.Paused;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Requests that the progress should be stopped in an orderly manner,
+        /// which allows current transfers to be completed.
+        /// </summary>
+        /// <param name="alsoTransfers">If set to <c>true</c> also transfers.</param>
+        public void Stop(bool alsoTransfers = false)
+        {
+            lock(m_lock)
+            {
+                if (m_progressstate == State.Active || m_progressstate == State.Paused)
+                {
+                    if (m_progressstate != State.Paused)
+                        m_progress = new TaskCompletionSource<bool>();
+                    
+                    m_progress.SetResult(false);
+                    m_progressstate = State.Stopped;
+                }
+
+                if (alsoTransfers && (m_transferstate == State.Active || m_transferstate == State.Paused))
+                {
+                    if (m_transferstate != State.Paused)
+                        m_transfer = new TaskCompletionSource<bool>();
+                    
+                    m_transfer.SetResult(false);
+                    m_transferstate = State.Stopped;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Terminates the progress without allowing a flush
+        /// </summary>
+        public void Terminate()
+        {
+            lock(m_lock)
+            {
+                if (m_progressstate != State.Terminated)
+                {
+                    if (m_progressstate != State.Paused)
+                        m_progress = new TaskCompletionSource<bool>();
+                    
+                    m_progress.SetCanceled();
+                    m_progressstate = State.Terminated;
+                }
+                if (m_transferstate != State.Terminated)
+                {
+                    if (m_transferstate != State.Paused)
+                        m_transfer = new TaskCompletionSource<bool>();
+                    
+                    m_transfer.SetCanceled();
+                    m_transferstate = State.Terminated;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Releases all resource used by the <see cref="Duplicati.Library.Main.Operation.Common.TaskControl"/> object.
+        /// </summary>
+        /// <remarks>Call <see cref="Dispose"/> when you are finished using the
+        /// <see cref="Duplicati.Library.Main.Operation.Common.TaskControl"/>. The <see cref="Dispose"/> method leaves
+        /// the <see cref="Duplicati.Library.Main.Operation.Common.TaskControl"/> in an unusable state. After calling
+        /// <see cref="Dispose"/>, you must release all references to the
+        /// <see cref="Duplicati.Library.Main.Operation.Common.TaskControl"/> so the garbage collector can reclaim the
+        /// memory that the <see cref="Duplicati.Library.Main.Operation.Common.TaskControl"/> was occupying.</remarks>
+        public void Dispose()
+        {
+            Terminate();
+        }
+    }
+}
+

Duplicati/Library/Main/Operation/CompactHandler.cs

@@ -21,6 +21,7 @@ using System.Collections.Generic;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Main.Volumes;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace Duplicati.Library.Main.Operation
 {
@@ -43,158 +44,141 @@ namespace Duplicati.Library.Main.Operation
                 throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
             
             using(var db = new LocalDeleteDatabase(m_options.Dbpath, "Compact"))
+            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result))
             {
-                var tr = db.BeginTransaction();
-                try
+                m_result.SetDatabase(db);
+                Utility.UpdateOptionsFromDb(db, m_options);
+                Utility.VerifyParameters(db, m_options);
+                
+                var changed = DoCompact(db, false, backend);
+                
+                if (changed && m_options.UploadVerificationFile)
+                    FilelistProcessor.UploadVerificationFile(backend, m_options, m_result.BackendWriter, db);
+                
+                if (!m_options.Dryrun)
                 {
-                    m_result.SetDatabase(db);
-                    Utility.UpdateOptionsFromDb(db, m_options);
-                    Utility.VerifyParameters(db, m_options);
-                    
-                    var changed = DoCompact(db, false, ref tr, null);
-                    
-                    if (changed && m_options.UploadVerificationFile)
-                        FilelistProcessor.UploadVerificationFile(m_backendurl, m_options, m_result.BackendWriter, db, null);
-                    
-                    if (!m_options.Dryrun)
+                    if (changed)
                     {
-                        using(new Logging.Timer("CommitCompact"))
-                            tr.Commit();
-                        if (changed)
+                        db.WriteResults();
+                        db.CommitTransaction("CommitCompact", false);
+                        if (m_options.AutoVacuum)
                         {
-                            db.WriteResults();
-                            if (m_options.AutoVacuum)
-                            {
-                                db.Vacuum();
-                            }
+                            db.Vacuum();
                         }
                     }
                     else
-                        tr.Rollback();
-                    
-                    tr = null;
-                }
-                finally
-                {
-                    if (tr != null)
-                        try { tr.Rollback(); }
-                        catch { }
+                        db.CommitTransaction("CommitCompact", false);
+
                 }
+                else
+                    db.RollbackTransaction(false);
             }
         }
         
-        internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction, BackendManager sharedBackend)
+        internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, BackendManager backend)
         {
-            var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction);
+            var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount);
             report.ReportCompactData(m_result);
             
             if (report.ShouldReclaim || report.ShouldCompact)
             {
-				// Workaround where we allow a running backendmanager to be used
-				using(var bk = sharedBackend == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
-                {
-                    var backend = bk ?? sharedBackend;
-                    if (!hasVerifiedBackend && !m_options.NoBackendverification)
-                        FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
-        
-                    BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
-                    newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
+                if (!hasVerifiedBackend && !m_options.NoBackendverification)
+                    FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
     
-                    IndexVolumeWriter newvolindex = null;
+                BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
+                newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
+
+                long blocksInVolume = 0;
+                long discardedBlocks = 0;
+                long discardedSize = 0;
+                byte[] buffer = new byte[m_options.Blocksize];
+                var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
+                
+                //These are for bookkeeping
+                var uploadedVolumes = new List<KeyValuePair<string, long>>();
+                var uploadedIndexVolumes = new Dictionary<string, KeyValuePair<string, long>>();
+                var deletedVolumes = new List<KeyValuePair<string, long>>();
+                var downloadedVolumes = new List<KeyValuePair<string, long>>();
+                
+                //We start by deleting unused volumes to save space before uploading new stuff
+                var fullyDeleteable = (from v in remoteList
+                                       where report.DeleteableVolumes.Contains(v.Name)
+                                       select (IRemoteVolume)v).ToList();                    
+                deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable));
+
+                // This list is used to pick up unused volumes,
+                // so they can be deleted once the upload of the
+                // required fragments is complete
+                var deleteableVolumes = new List<IRemoteVolume>();
+
+                if (report.ShouldCompact)
+                {
+                    Func<string, Task<IndexVolumeWriter>> indexCreator = null;
                     if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
                     {
-                        newvolindex = new IndexVolumeWriter(m_options);
-                        newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
-                        db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
-                        newvolindex.StartVolume(newvol.RemoteFilename);
+                        indexCreator = async (name) => {
+                            var tmp = await Common.IndexVolumeCreator.CreateIndexVolume(name, m_options, backend.Database, null);
+                            // We assign to a dictionary, to prevent counting multiple items if the upload fails
+                            // and the index file is recreated
+                            uploadedIndexVolumes[name] = new KeyValuePair<string, long>(tmp.RemoteFilename, new System.IO.FileInfo(tmp.LocalFilename).Length);
+                            return tmp;
+                        };
                     }
-                    
-                    long blocksInVolume = 0;
-                    long discardedBlocks = 0;
-                    long discardedSize = 0;
-                    byte[] buffer = new byte[m_options.Blocksize];
-                    var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
-                    
-                    //These are for bookkeeping
-                    var uploadedVolumes = new List<KeyValuePair<string, long>>();
-                    var deletedVolumes = new List<KeyValuePair<string, long>>();
-                    var downloadedVolumes = new List<KeyValuePair<string, long>>();
-                    
-                    //We start by deleting unused volumes to save space before uploading new stuff
-                    var fullyDeleteable = (from v in remoteList
-                                           where report.DeleteableVolumes.Contains(v.Name)
-                                           select (IRemoteVolume)v).ToList();                    
-                    deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable, ref transaction));
 
-                    // This list is used to pick up unused volumes,
-                    // so they can be deleted once the upload of the
-                    // required fragments is complete
-                    var deleteableVolumes = new List<IRemoteVolume>();
+                    var volumesToDownload = (from v in remoteList
+                                             where report.CompactableVolumes.Contains(v.Name)
+                                             select (IRemoteVolume)v).ToList();
 
-                    if (report.ShouldCompact)
+                    var dl = new AsyncDownloader(volumesToDownload, backend);
+                    using (var q = db.CreateBlockQueryHelper(m_options))
                     {
-                        var volumesToDownload = (from v in remoteList
-                                                 where report.CompactableVolumes.Contains(v.Name)
-                                                 select (IRemoteVolume)v).ToList();
-                        
-                        using(var q = db.CreateBlockQueryHelper(m_options, transaction))
+                        while(dl.MoveNext())
                         {
-                            foreach(var entry in new AsyncDownloader(volumesToDownload, backend))
-                            using(var tmpfile = entry.TempFile)
+                            var entry = dl.Current;
+                            using (var tmpfile = entry.TempFile)
                             {
                                 if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                                 {
-                                    backend.WaitForComplete(db, transaction);
+                                    backend.WaitForComplete();
                                     return false;
                                 }
-                                    
+
                                 downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
                                 var inst = VolumeBase.ParseFilename(entry.Name);
-                                using(var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
+                                using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
                                 {
-                                    foreach(var e in f.Blocks)
+                                    foreach (var e in f.Blocks)
                                     {
-                                        if (q.UseBlock(e.Key, e.Value, transaction))
+                                        if (q.UseBlock(e.Key, e.Value))
                                         {
                                             //TODO: How do we get the compression hint? Reverse query for filename in db?
                                             var s = f.ReadBlock(e.Key, buffer);
                                             if (s != e.Value)
                                                 throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
-                                                
+
                                             newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
-                                            if (newvolindex != null)
-                                                newvolindex.AddBlock(e.Key, e.Value);
-                                                
-                                            db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID, transaction);
+
+                                            db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID);
                                             blocksInVolume++;
-                                            
+
                                             if (newvol.Filesize > m_options.VolumeSize)
                                             {
                                                 uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, newvol.Filesize));
-                                                if (newvolindex != null)
-                                                    uploadedVolumes.Add(new KeyValuePair<string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
-    
+
+                                                newvol.Close();
                                                 if (!m_options.Dryrun)
-                                                    backend.Put(newvol, newvolindex);
+                                                    backend.Put(newvol, indexCreator);
                                                 else
                                                     m_result.AddDryrunMessage(string.Format("Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)));
-                                                
-                                                
+
+
                                                 newvol = new BlockVolumeWriter(m_options);
-                                                newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
-                
-                                                if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
-                                                {
-                                                    newvolindex = new IndexVolumeWriter(m_options);
-                                                    newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
-                                                    db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
-                                                    newvolindex.StartVolume(newvol.RemoteFilename);
-                                                }
-                                                
+                                                newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
+
                                                 blocksInVolume = 0;
-                                                
+
                                                 //After we upload this volume, we can delete all previous encountered volumes
-                                                deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
+                                                deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes));
                                                 deleteableVolumes = new List<IRemoteVolume>();
                                             }
                                         }
@@ -205,79 +189,75 @@ namespace Duplicati.Library.Main.Operation
                                         }
                                     }
                                 }
-    
+
                                 deleteableVolumes.Add(entry);
                             }
-                            
-                            if (blocksInVolume > 0)
-                            {
-                                uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, newvol.Filesize));
-                                if (newvolindex != null)
-                                    uploadedVolumes.Add(new KeyValuePair<string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
-                                if (!m_options.Dryrun)
-                                    backend.Put(newvol, newvolindex);
-                                else
-                                    m_result.AddDryrunMessage(string.Format("Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)));
-                            }
+                        }
+                        
+                        if (blocksInVolume > 0)
+                        {
+                            uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, newvol.Filesize));
+                            newvol.Close();
+                            if (!m_options.Dryrun)
+                                backend.Put(newvol, indexCreator);
                             else
-                            {
-                                db.RemoveRemoteVolume(newvol.RemoteFilename, transaction);
-                                if (newvolindex != null)
-                                {
-                                    db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction);
-                                    newvolindex.FinishVolume(null, 0);
-                                }
-                            }
+                                m_result.AddDryrunMessage(string.Format("Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)));
+                        }
+                        else
+                        {
+                            db.RemoveRemoteVolume(newvol.RemoteFilename);
                         }
                     }
-                    
-                    deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
-                                        
-                    var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
-                    var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
-                    var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
-                    
-                    m_result.DeletedFileCount = deletedVolumes.Count;
-                    m_result.DownloadedFileCount = downloadedVolumes.Count;
-                    m_result.UploadedFileCount = uploadedVolumes.Count;
-                    m_result.DeletedFileSize = deletedSize;
-                    m_result.DownloadedFileSize = downloadSize;
-                    m_result.UploadedFileSize = uploadSize;
-                    m_result.Dryrun = m_options.Dryrun;
-                    
-                    if (m_result.Dryrun)
-                    {
-                        if (downloadedVolumes.Count == 0)
-                            m_result.AddDryrunMessage(string.Format("Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)));
-                        else
-                            m_result.AddDryrunMessage(string.Format("Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}", 
-                                                                    m_result.DownloadedFileCount, 
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize), 
-                                                                    m_result.DeletedFileCount, 
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount, 
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), 
-                                                                    m_result.DeletedFileCount - m_result.UploadedFileCount, 
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)));
-                    }
-                    else
-                    {
-                        if (m_result.DownloadedFileCount == 0)
-                            m_result.AddMessage(string.Format("Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)));
-                        else
-                            m_result.AddMessage(string.Format("Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}", 
-                                                              m_result.DownloadedFileCount, 
-                                                              Library.Utility.Utility.FormatSizeString(downloadSize), 
-                                                              m_result.DeletedFileCount, 
-                                                              Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), 
-                                                              m_result.UploadedFileCount, 
-                                                              Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), 
-                                                              m_result.DeletedFileCount - m_result.UploadedFileCount, 
-                                                              Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)));
-                    }
-                            
-                    backend.WaitForComplete(db, transaction);
                 }
+                
+                deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes));
+                backend.WaitForEmpty();
 
+                uploadedVolumes.AddRange(uploadedIndexVolumes.Values);
+                                    
+                var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
+                var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
+                var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
+                
+                m_result.DeletedFileCount = deletedVolumes.Count;
+                m_result.DownloadedFileCount = downloadedVolumes.Count;
+                m_result.UploadedFileCount = uploadedVolumes.Count;
+                m_result.DeletedFileSize = deletedSize;
+                m_result.DownloadedFileSize = downloadSize;
+                m_result.UploadedFileSize = uploadSize;
+                m_result.Dryrun = m_options.Dryrun;
+                
+                if (m_result.Dryrun)
+                {
+                    if (downloadedVolumes.Count == 0)
+                        m_result.AddDryrunMessage(string.Format("Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)));
+                    else
+                        m_result.AddDryrunMessage(string.Format("Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}", 
+                                                                m_result.DownloadedFileCount, 
+                                                                Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize), 
+                                                                m_result.DeletedFileCount, 
+                                                                Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount, 
+                                                                Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), 
+                                                                m_result.DeletedFileCount - m_result.UploadedFileCount, 
+                                                                Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)));
+                }
+                else
+                {
+                    if (m_result.DownloadedFileCount == 0)
+                        m_result.AddMessage(string.Format("Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)));
+                    else
+                        m_result.AddMessage(string.Format("Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}", 
+                                                          m_result.DownloadedFileCount, 
+                                                          Library.Utility.Utility.FormatSizeString(downloadSize), 
+                                                          m_result.DeletedFileCount, 
+                                                          Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), 
+                                                          m_result.UploadedFileCount, 
+                                                          Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), 
+                                                          m_result.DeletedFileCount - m_result.UploadedFileCount, 
+                                                          Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)));
+                }
+                        
+                backend.WaitForEmpty();
                 m_result.EndTime = DateTime.UtcNow;
                 return (m_result.DeletedFileCount + m_result.UploadedFileCount) > 0;
             }
@@ -288,27 +268,25 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        private IEnumerable<KeyValuePair<string, long>> DoDelete(LocalDeleteDatabase db, BackendManager backend, IEnumerable<IRemoteVolume> deleteableVolumes, ref System.Data.IDbTransaction transaction)
+        private IEnumerable<KeyValuePair<string, long>> DoDelete(LocalDeleteDatabase db, BackendManager backend, IEnumerable<IRemoteVolume> deleteableVolumes)
         {
             // Mark all volumes as disposable
             foreach(var f in deleteableVolumes)
-                db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
+                db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash);
 
             // Before we commit the current state, make sure the backend has caught up
-            backend.WaitForEmpty(db, transaction);
+            backend.WaitForEmpty();
 
             if (!m_options.Dryrun)
-            {
-                transaction.Commit();
-                transaction = db.BeginTransaction();
-            }
+                db.CommitTransaction("AfterDeleteRemotes");
 
-            return PerformDelete(backend, db.GetDeletableVolumes(deleteableVolumes, transaction));
+            return PerformDelete(backend, db.GetDeletableVolumes(deleteableVolumes));
         }
             
         
         private IEnumerable<KeyValuePair<string, long>> PerformDelete(BackendManager backend, IEnumerable<IRemoteVolume> list)
         {
+            var res = new List<KeyValuePair<string, long>>();
             foreach(var f in list)
             {
                 if (!m_options.Dryrun)
@@ -316,8 +294,10 @@ namespace Duplicati.Library.Main.Operation
                 else
                     m_result.AddDryrunMessage(string.Format("Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)));
 
-                yield return new KeyValuePair<string, long>(f.Name, f.Size);
-            }               
+                res.Add(new KeyValuePair<string, long>(f.Name, f.Size));
+            }
+
+            return res;
         }
     }
 }

Duplicati/Library/Main/Operation/DeleteHandler.cs

@@ -43,40 +43,27 @@ namespace Duplicati.Library.Main.Operation
 
             using(var db = new Database.LocalDeleteDatabase(m_options.Dbpath, "Delete"))
             {
-                var tr = db.BeginTransaction();
-                try
+                m_result.SetDatabase(db);
+                Utility.UpdateOptionsFromDb(db, m_options);
+                Utility.VerifyParameters(db, m_options);
+                
+                DoRun(db, false, false, null);
+                
+                if (!m_options.Dryrun)
                 {
-                    m_result.SetDatabase(db);
-                    Utility.UpdateOptionsFromDb(db, m_options);
-                    Utility.VerifyParameters(db, m_options);
-                    
-                    DoRun(db, ref tr, false, false, null);
-                    
-                    if (!m_options.Dryrun)
-                    {
-                        using(new Logging.Timer("CommitDelete"))
-                            tr.Commit();
-
-                        db.WriteResults();
-                    }
-                    else
-                        tr.Rollback();
-
-                    tr = null;
-                }
-                finally
-                {
-                    if (tr != null)
-                        try { tr.Rollback(); }
-                        catch { }
+                    db.WriteResults();
+                    using (new Logging.Timer("CommitDelete"))
+                        db.CommitTransaction("After delete operation", false);
                 }
+                else
+                    db.RollbackTransaction(false);
             }
         }
 
-        public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBacked, bool forceCompact, BackendManager sharedManager)
+        public void DoRun(Database.LocalDeleteDatabase db, bool hasVerifiedBacked, bool forceCompact, BackendManager sharedManager)
         {
             // Workaround where we allow a running backendmanager to be used
-            using(var bk = sharedManager == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
+            using(var bk = sharedManager == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result) : null)
             {
                 var backend = bk ?? sharedManager;
 
@@ -96,21 +83,18 @@ namespace Duplicati.Library.Main.Operation
                 if (toDelete != null && toDelete.Length > 0)
                     m_result.AddMessage(string.Format("Deleting {0} remote fileset(s) ...", toDelete.Length));
 
-                var lst = db.DropFilesetsFromTable(toDelete, transaction).ToArray();
+                var lst = db.DropFilesetsFromTable(toDelete).ToArray();
                 foreach(var f in lst)
-                    db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction);
+                    db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null);
 
                 if (!m_options.Dryrun)
-                {
-                    transaction.Commit();
-                    transaction = db.BeginTransaction();
-                }
+                    db.CommitTransaction("Removed filelists");
 
                 foreach(var f in lst)
                 {
                     if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                     {
-                        backend.WaitForComplete(db, transaction);
+                        backend.WaitForComplete();
                         return;
                     }
 
@@ -120,10 +104,7 @@ namespace Duplicati.Library.Main.Operation
                         m_result.AddDryrunMessage(string.Format("Would delete remote fileset: {0}", f.Key));
                 }
 
-                if (sharedManager == null)
-                    backend.WaitForComplete(db, transaction);
-                else
-                    backend.WaitForEmpty(db, transaction);
+                backend.WaitForEmpty();
                 
                 var count = lst.Length;
                 if (!m_options.Dryrun)
@@ -148,7 +129,7 @@ namespace Duplicati.Library.Main.Operation
                 if (!m_options.NoAutoCompact && (forceCompact || (toDelete != null && toDelete.Length > 0)))
                 {
                     m_result.CompactResults = new CompactResults(m_result);
-                    new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref transaction, sharedManager);
+                    new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, sharedManager);
                 }
                 
                 m_result.SetResults(

Duplicati/Library/Main/Operation/FilelistProcessor.cs

@@ -150,14 +150,12 @@ namespace Duplicati.Library.Main.Operation
         /// <summary>
         /// Uploads the verification file.
         /// </summary>
-        /// <param name="backendurl">The backend url</param>
+        /// <param name="backend">The backend to use</param>
         /// <param name="options">The options to use</param>
         /// <param name="result">The result writer</param>
         /// <param name="db">The attached database</param>
-        /// <param name="transaction">An optional transaction object</param>
-        public static void UploadVerificationFile(string backendurl, Options options, IBackendWriter result, LocalDatabase db, System.Data.IDbTransaction transaction)
+        public static void UploadVerificationFile(BackendManager backend, Options options, IBackendWriter result, LocalDatabase db)
         {
-            using(var backend = new BackendManager(backendurl, options, result, db))
             using(var tempfile = new Library.Utility.TempFile())
             {
                 var remotename = options.Prefix + "-verification.json";
@@ -171,7 +169,7 @@ namespace Duplicati.Library.Main.Operation
                 else
                 {
                     backend.PutUnencrypted(remotename, tempfile);
-                    backend.WaitForComplete(db, transaction);
+                    backend.WaitForEmpty();
                 }
             }
         }
@@ -293,9 +291,9 @@ namespace Duplicati.Library.Main.Operation
                         }
                         else
                         {
-                            if (i.deleteGracePeriod > DateTime.UtcNow)
+                            if (i.DeleteGracePeriod > DateTime.UtcNow)
                             {
-                                log.AddMessage(string.Format("keeping delete request for {0} until {1}", i.Name, i.deleteGracePeriod.ToLocalTime()));
+                                log.AddMessage(string.Format("keeping delete request for {0} until {1}", i.Name, i.DeleteGracePeriod.ToLocalTime()));
                             }
                             else
                             {
@@ -323,13 +321,13 @@ namespace Duplicati.Library.Main.Operation
                             if (string.Equals(i.Name, protectedfile))
                             {
                                 log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
-                                database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null);
+                                database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0));
                             }
                             else
                             {
                                 log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name));
                                 cleanupRemovedRemoteVolumes.Add(i.Name);
-                                database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);
+                                database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2));
                             }
                         }
                         else
@@ -373,7 +371,7 @@ namespace Duplicati.Library.Main.Operation
             }
 
             // cleanup deleted volumes in DB en block
-            database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null);
+            database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes);
 
             foreach(var i in missingHash)
                 log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null);

Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs

@@ -41,14 +41,13 @@ namespace Duplicati.Library.Main.Operation
                 throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath));
 
             using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath))
-            using (var tr = db.BeginTransaction())
-                DoRun(db, tr, filter, callbackhandler);
+                DoRun(db, filter, callbackhandler);
         }
 
-        public static Tuple<DateTime, long, long>[] GetBrokenFilesetsFromRemote(string backendurl, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options, out List<Database.RemoteVolumeEntry> missing)
+        public static Tuple<DateTime, long, long>[] GetBrokenFilesetsFromRemote(string backendurl, BasicResults result, Database.LocalListBrokenFilesDatabase db, Options options, out List<Database.RemoteVolumeEntry> missing)
         {
             missing = null;
-            var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray();
+            var brokensets = db.GetBrokenFilesets(options.Time, options.Version).ToArray();
 
             if (brokensets.Length == 0)
             {
@@ -57,7 +56,7 @@ namespace Duplicati.Library.Main.Operation
 
                 result.AddMessage("No broken filesets found in database, checking for missing remote files");
 
-                using (var backend = new BackendManager(backendurl, options, result.BackendWriter, db))
+                using (var backend = new BackendManager(backendurl, options, result.BackendWriter, db, result))
                 {
                     var remotestate = FilelistProcessor.RemoteListAnalysis(backend, options, db, result.BackendWriter, null);
                     if (!remotestate.ParsedVolumes.Any())
@@ -72,20 +71,20 @@ namespace Duplicati.Library.Main.Operation
 
                     // Mark all volumes as disposable
                     foreach (var f in missing)
-                        db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
+                        db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash);
 
                     result.AddMessage(string.Format("Marked {0} remote files for deletion", missing.Count));
 
                     // Drop all content from tables
-                    db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction);
+                    db.RemoveMissingBlocks(missing.Select(x => x.Name));
                 }
-                brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray();
+                brokensets = db.GetBrokenFilesets(options.Time, options.Version).ToArray();
             }
 
             return brokensets;
         }
 
-        private void DoRun(Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Library.Utility.IFilter filter, Func<long, DateTime, long, string, long, bool> callbackhandler)
+        private void DoRun(Database.LocalListBrokenFilesDatabase db, Library.Utility.IFilter filter, Func<long, DateTime, long, string, long, bool> callbackhandler)
         {
             if (filter != null && !filter.Empty)
                 throw new UserInformationException("Filters are not supported for this operation");
@@ -94,7 +93,7 @@ namespace Duplicati.Library.Main.Operation
                 throw new UserInformationException("The command does not work on partially recreated databases");
 
             List<Database.RemoteVolumeEntry> missing;
-            var brokensets = GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, transaction, m_options, out missing);
+            var brokensets = GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, m_options, out missing);
             if (brokensets == null)
                 return;
 
@@ -125,7 +124,7 @@ namespace Duplicati.Library.Main.Operation
                                 x.Version, 
                                 x.Timestamp, 
                                 callbackhandler == null && !m_options.ListSetsOnly
-                                    ? db.GetBrokenFilenames(x.FilesetID, transaction).ToArray().AsEnumerable()
+                                    ? db.GetBrokenFilenames(x.FilesetID).ToArray().AsEnumerable()
                                     : new MockList<Tuple<string, long>>((int)x.BrokenCount)
                     ))
                 .ToArray();
@@ -133,7 +132,7 @@ namespace Duplicati.Library.Main.Operation
 
             if (callbackhandler != null)
                 foreach (var bs in brokenfilesets)
-                    foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, transaction))
+                    foreach (var fe in db.GetBrokenFilenames(bs.FilesetID))
                         if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2))
                             break;
         }

Duplicati/Library/Main/Operation/ListChangesHandler.cs

@@ -68,10 +68,10 @@ namespace Duplicati.Library.Main.Operation
             
             DateTime baseVersionTime = new DateTime(0);
             DateTime compareVersionTime = new DateTime(0);
-            
+
             using(var tmpdb = useLocalDb ? null : new Library.Utility.TempFile())
             using(var db = new Database.LocalListChangesDatabase(useLocalDb ? m_options.Dbpath : (string)tmpdb))
-            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result))
             using(var storageKeeper = db.CreateStorageHelper())
             {
                 m_result.SetDatabase(db);

Duplicati/Library/Main/Operation/ListControlFilesHandler.cs

@@ -38,7 +38,7 @@ namespace Duplicati.Library.Main.Operation
         {
             using (var tmpdb = new Library.Utility.TempFile())
             using (var db = new Database.LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result))
             {
                 m_result.SetDatabase(db);
                 
@@ -59,15 +59,10 @@ namespace Duplicati.Library.Main.Operation
                                 return;
                         
                             var file = fileversion.Value.File;
-                            long size;
-                            string hash;
-                            RemoteVolumeType type;
-                            RemoteVolumeState state;
-                            if (!db.GetRemoteVolume(file.Name, out hash, out size, out type, out state))
-                                size = file.Size;
+                            var entry = db.GetRemoteVolume(file.Name);
     
                             var files = new List<Library.Interface.IListResultFile>();
-                            using (var tmpfile = backend.Get(file.Name, size, hash))
+                        using (var tmpfile = backend.Get(file.Name, entry.Size < 0 ? file.Size : entry.Size, entry.Hash))
                             using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
                                 foreach (var cf in tmp.ControlFiles)
                                     if (Library.Utility.FilterExpression.Matches(filter, cf.Key))
@@ -89,9 +84,10 @@ namespace Duplicati.Library.Main.Operation
                 }
                 finally
                 {
-                    backend.WaitForComplete(db, null);
+                    backend.WaitForEmpty();
                 }
             }
-        }    }
+        }    
+    }
 }
 

Duplicati/Library/Main/Operation/ListFilesHandler.cs

@@ -24,7 +24,7 @@ namespace Duplicati.Library.Main.Operation
             var parsedfilter = new Library.Utility.FilterExpression(filterstrings);
             var filter = Library.Utility.JoinedFilterExpression.Join(parsedfilter, compositefilter);
             var simpleList = !((filter is Library.Utility.FilterExpression && ((Library.Utility.FilterExpression)filter).Type == Library.Utility.FilterType.Simple) || m_options.AllVersions);
-        
+
             //Use a speedy local query
             if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath))
                 using(var db = new Database.LocalListDatabase(m_options.Dbpath))
@@ -78,7 +78,7 @@ namespace Duplicati.Library.Main.Operation
             // Otherwise, grab info from remote location
             using (var tmpdb = new Library.Utility.TempFile())
             using (var db = new Database.LocalDatabase(tmpdb, "List", true))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result))
             {
                 m_result.SetDatabase(db);
                 

Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs

@@ -45,12 +45,11 @@ namespace Duplicati.Library.Main.Operation
             List<Database.RemoteVolumeEntry> missing = null;
             
             using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath))
-            using (var tr = db.BeginTransaction())
             {
                 if (db.PartiallyRecreated)
                     throw new UserInformationException("The command does not work on partially recreated databases");
 
-                var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, tr, m_options, out missing);
+                var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, m_options, out missing);
                 if (sets == null)
                     return;
                 
@@ -77,7 +76,7 @@ namespace Duplicati.Library.Main.Operation
                     Timestamp = x.Item1,
                     RemoveCount = x.Item3,
                     Version = filesets.FindIndex(y => y.Key == x.Item2),
-                    SetCount = db.GetFilesetFileCount(x.Item2, tr)
+                    SetCount = db.GetFilesetFileCount(x.Item2)
                 }).ToArray();
 
                 var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
@@ -93,34 +92,21 @@ namespace Duplicati.Library.Main.Operation
                     m_result.DeleteResults = new DeleteResults(m_result);
                     using (var rmdb = new Database.LocalDeleteDatabase(db))
                     {
-                        var deltr = rmdb.BeginTransaction();
-                        try
+                        var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
+                        opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
+                        opts.RawOptions.Remove("time");
+                        opts.RawOptions["no-auto-compact"] = "true";
+
+                        new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults)
+                            .DoRun(rmdb, true, false, null);
+
+                        if (!m_options.Dryrun)
                         {
-                            var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
-                            opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
-                            opts.RawOptions.Remove("time");
-                            opts.RawOptions["no-auto-compact"] = "true";
-
-                            new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults)
-                                .DoRun(rmdb, ref deltr, true, false, null);
-
-                            if (!m_options.Dryrun)
-                            {
-                                using (new Logging.Timer("CommitDelete"))
-                                    deltr.Commit();
-
-                                rmdb.WriteResults();
-                            }
-                            else
-                                deltr.Rollback();
+                            rmdb.CommitTransaction("CommitDelete");
+                            rmdb.WriteResults();
                         }
-                        finally
-                        {
-                            if (deltr != null)
-                                try { deltr.Rollback(); }
-                                catch { }
-                        }
-
+                        else
+                            rmdb.RollbackTransaction();
                     }
 
                     pgoffset += (pgspan * fully_emptied.Length);
@@ -152,7 +138,7 @@ namespace Duplicati.Library.Main.Operation
                             {
                                 if (filesetid != bs.FilesetID)
                                     throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
-                                db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction);
+                                db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID");
                             });
                         }
 
@@ -162,15 +148,15 @@ namespace Duplicati.Library.Main.Operation
                 }
 
                 if (m_options.Dryrun)
-                    tr.Rollback();
+                    db.RollbackTransaction();
                 else
-                    tr.Commit();
+                    db.CommitTransaction("AfterCompact");
 
                 m_result.OperationProgressUpdater.UpdateProgress(0.95f);
 
                 if (missing != null && missing.Count > 0)
                 {
-                    using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+                    using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result))
                     {
                         foreach (var f in missing)
                             if (m_options.Dryrun)
@@ -183,7 +169,7 @@ namespace Duplicati.Library.Main.Operation
                 if (!m_options.Dryrun && db.RepairInProgress)
                 {                    
                     m_result.AddMessage("Database was previously marked as in-progress, checking if it is valid after purging files");
-                    db.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, true);
+                    db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true);
                     m_result.AddMessage("Purge completed, and consistency checks completed, marking database as complete");
                     db.RepairInProgress = false;
                 }

Duplicati/Library/Main/Operation/PurgeFilesHandler.cs

@@ -57,7 +57,7 @@ namespace Duplicati.Library.Main.Operation
 
             var doCompactStep = !m_options.NoAutoCompact && filtercommand == null;
 
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result))
             {
                 if (db.PartiallyRecreated)
                     throw new UserInformationException("The purge command does not work on partially recreated databases");
@@ -69,7 +69,7 @@ namespace Duplicati.Library.Main.Operation
                 if (versions.Length <= 0)
                     throw new UserInformationException("No filesets matched the supplied time or versions");
 
-                var orphans = db.CountOrphanFiles(null);
+                var orphans = db.CountOrphanFiles();
                 if (orphans != 0)
                     throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans));
 
@@ -78,7 +78,7 @@ namespace Duplicati.Library.Main.Operation
 
                 if (filtercommand == null)
                 {
-                    db.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, false);
+                    db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false);
 
                     if (m_options.NoBackendverification)
                         FilelistProcessor.VerifyLocalList(backend, m_options, db, m_result.BackendWriter);
@@ -97,107 +97,110 @@ namespace Duplicati.Library.Main.Operation
                 // Reverse makes sure we re-write the old versions first
                 foreach (var versionid in versions.Reverse())
                 {
-                    using (var tr = db.BeginTransaction())
-                    {
-                        var ix = -1;
-                        for (var i = 0; i < filesets.Length; i++)
-                            if (filesets[i].Key == versionid)
-                            {
-                                ix = i;
-                                break;
-                            }
+                    if (m_options.Dryrun)
+                        db.RollbackTransaction();
+                    else
+                        db.CommitTransaction("StartNewVersionRemoval");
 
-                        if (ix < 0)
-                            throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid));
-
-                        var secs = 0;
-                        while (secs < 60)
+                    var ix = -1;
+                    for (var i = 0; i < filesets.Length; i++)
+                        if (filesets[i].Key == versionid)
                         {
-                            secs++;
-                            var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs));
-                            if (db.GetRemoteVolumeID(tfn, tr) < 0)
-                                break;
+                            ix = i;
+                            break;
                         }
 
-                        var ts = filesets[ix].Value.AddSeconds(secs);
-                        var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr);
+                    if (ix < 0)
+                        throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid));
 
-                        if (secs >= 60)
-                            throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts));
+                    var secs = 0;
+                    while (secs < 60)
+                    {
+                        secs++;
+                        var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs));
+                        if (db.GetRemoteVolumeID(tfn) < 0)
+                            break;
+                    }
 
-                        if (ix != 0 && filesets[ix - 1].Value <= ts)
-                            throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
+                    var ts = filesets[ix].Value.AddSeconds(secs);
+                    var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key);
 
-                        using (var tempset = db.CreateTemporaryFileset(versionid, tr))
+                    if (secs >= 60)
+                        throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts));
+
+                    if (ix != 0 && filesets[ix - 1].Value <= ts)
+                        throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
+
+                    using (var tempset = db.CreateTemporaryFileset(versionid))
+                    {
+                        if (filtercommand == null)
+                            tempset.ApplyFilter(filter);
+                        else
+                            tempset.ApplyFilter(filtercommand);
+
+                        if (tempset.RemovedFileCount == 0)
                         {
-                            if (filtercommand == null)
-                                tempset.ApplyFilter(filter);
-                            else
-                                tempset.ApplyFilter(filtercommand);
+                            m_result.AddMessage(string.Format("Not writing a new fileset for {0} as it was not changed", prevfilename));
+                            currentprogress += versionprogress;
+                            db.RollbackTransaction();
+                            continue;
+                        }
+                        else
+                        {
+                            using (var tf = new Library.Utility.TempFile())
+                            using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts))
+                            {
+                                var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts);
+                                vol.VolumeID = newids.Item1;
 
-                            if (tempset.RemovedFileCount == 0)
-                            {
-                                m_result.AddMessage(string.Format("Not writing a new fileset for {0} as it was not changed", prevfilename));
-                                currentprogress += versionprogress;
-                                tr.Rollback();
-                                continue;
-                            }
-                            else
-                            {
-                                using (var tf = new Library.Utility.TempFile())
-                                using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts))
+                                m_result.AddMessage(string.Format("Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)));
+
+                                db.WriteFileset(vol, newids.Item2);
+
+                                m_result.RemovedFileSize += tempset.RemovedFileSize;
+                                m_result.RemovedFileCount += tempset.RemovedFileCount;
+                                m_result.RewrittenFileLists++;
+
+                                currentprogress += (versionprogress / 2);
+                                m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
+
+                                if (m_options.Dryrun || m_options.Verbose || Logging.Log.LogLevel == Logging.LogMessageType.Profiling)
                                 {
-                                    var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts);
-                                    vol.VolumeID = newids.Item1;
-
-                                    m_result.AddMessage(string.Format("Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)));
-
-                                    db.WriteFileset(vol, tr, newids.Item2);
-
-                                    m_result.RemovedFileSize += tempset.RemovedFileSize;
-                                    m_result.RemovedFileCount += tempset.RemovedFileCount;
-                                    m_result.RewrittenFileLists++;
-
-                                    currentprogress += (versionprogress / 2);
-                                    m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
-
-                                    if (m_options.Dryrun || m_options.Verbose || Logging.Log.LogLevel == Logging.LogMessageType.Profiling)
+                                    foreach (var fe in tempset.ListAllDeletedFiles())
                                     {
-                                        foreach (var fe in tempset.ListAllDeletedFiles())
-                                        {
-                                            var msg = string.Format("  Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value));
+                                        var msg = string.Format("  Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value));
 
-                                            if (Logging.Log.LogLevel == Logging.LogMessageType.Profiling)
-                                                Logging.Log.WriteMessage(msg, Logging.LogMessageType.Profiling);
-
-                                            if (m_options.Dryrun)
-                                                m_result.AddDryrunMessage(msg);
-                                            else if (m_options.Verbose)
-                                                m_result.AddVerboseMessage(msg);
-                                        }
+                                        if (Logging.Log.LogLevel == Logging.LogMessageType.Profiling)
+                                            Logging.Log.WriteMessage(msg, Logging.LogMessageType.Profiling);
 
                                         if (m_options.Dryrun)
-                                            m_result.AddDryrunMessage("Writing files to remote storage");
+                                            m_result.AddDryrunMessage(msg);
                                         else if (m_options.Verbose)
-                                            m_result.AddVerboseMessage("Writing files to remote storage");
+                                            m_result.AddVerboseMessage(msg);
                                     }
 
                                     if (m_options.Dryrun)
-                                    {
-                                        m_result.AddDryrunMessage(string.Format("Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount));
-                                        tr.Rollback();
-                                    }
-                                    else
-                                    {
-                                        var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray();
-                                        foreach (var f in lst)
-                                            db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr);
+                                        m_result.AddDryrunMessage("Writing files to remote storage");
+                                    else if (m_options.Verbose)
+                                        m_result.AddVerboseMessage("Writing files to remote storage");
+                                }
 
-                                        tr.Commit();
-                                        backend.Put(vol, synchronous: true);
-                                        backend.Delete(prevfilename, -1, true);
-                                        backend.FlushDbMessages();
-                                    }
+                                if (m_options.Dryrun)
+                                {
+                                    m_result.AddDryrunMessage(string.Format("Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount));
+                                    db.RollbackTransaction();
+                                }
+                                else
+                                {
+                                    var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }).ToArray();
+                                    foreach (var f in lst)
+                                        db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null);
+
+                                    db.CommitTransaction("BeforeFileRemoval");
+                                    backend.Put(vol, null, synchronous: true);
+                                    backend.Delete(prevfilename, -1, synchronous: true);
+                                    backend.FlushDbMessages();
+                                    db.CommitTransaction("AfterFileRemoval");
                                 }
                             }
                         }
@@ -221,21 +224,7 @@ namespace Duplicati.Library.Main.Operation
                         m_result.CompactResults = new CompactResults(m_result);
                         using (var cdb = new Database.LocalDeleteDatabase(db))
                         {
-                            var tr = cdb.BeginTransaction();
-                            try
-                            {
-                                new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, ref tr, backend);
-                            }
-                            catch
-                            {
-                                try { tr.Rollback(); }
-                                catch { }
-                            }
-                            finally
-                            {
-                                try { tr.Commit(); }
-                                catch { }
-                            }
+                            new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, backend);
                         }
                     }
 
@@ -243,7 +232,7 @@ namespace Duplicati.Library.Main.Operation
                     m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete);
                 }
 
-                backend.WaitForComplete(db, null);
+                backend.WaitForEmpty();
             }
         }
     }

Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs

@@ -64,7 +64,7 @@ namespace Duplicati.Library.Main.Operation
                 // Mark as incomplete
                 db.PartiallyRecreated = true;
 
-                Utility.UpdateOptionsFromDb(db, m_options, null);
+                Utility.UpdateOptionsFromDb(db, m_options);
                 DoRun(db, true, filter, filelistfilter, blockprocessor);
                 db.WriteResults();
             }
@@ -84,7 +84,7 @@ namespace Duplicati.Library.Main.Operation
 
             //We build a local database in steps.
             using(var restoredb = new LocalRecreateDatabase(dbparent, m_options))
-            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb))
+            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb, m_result))
             {
                 restoredb.RepairInProgress = true;
 
@@ -155,9 +155,9 @@ namespace Duplicati.Library.Main.Operation
                     Utility.VerifyParameters(restoredb, m_options);
                 }
 
-                //Record all blocksets and files needed
-                using(var tr = restoredb.BeginTransaction())
+                try
                 {
+                    restoredb.CommitTransaction("AfterRebuild");
                     var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList();
                     m_result.AddMessage(string.Format("Rebuild database started, downloading {0} filelists", filelistWork.Count));
 
@@ -166,41 +166,45 @@ namespace Duplicati.Library.Main.Operation
                     // Register the files we are working with, if not already updated
                     if (updating)
                     {
-                        foreach(var n in filelists)
+                        foreach (var n in filelists)
                             if (volumeIds[n.File.Name] == -1)
-                                volumeIds[n.File.Name] = restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), tr);
+                                volumeIds[n.File.Name] = restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0));
                     }
-                                
+
                     var isFirstFilelist = true;
                     var blocksize = m_options.Blocksize;
                     var hashes_pr_block = blocksize / m_options.BlockhashSize;
+                    var dl = new AsyncDownloader(filelistWork, backend);
 
-                    foreach(var entry in new AsyncDownloader(filelistWork, backend))
+
+                    while(dl.MoveNext())
+                    {
+                        var entry = dl.Current;
                         try
                         {
                             if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                             {
-                                backend.WaitForComplete(restoredb, null);
+                                backend.WaitForComplete();
                                 m_result.EndTime = DateTime.UtcNow;
                                 return;
-                            }    
-                        
+                            }
+
                             progress++;
                             if (filelistWork.Count == 1 && m_options.RepairOnlyPaths)
                                 m_result.OperationProgressUpdater.UpdateProgress(0.5f);
                             else
                                 m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f));
 
-                            using(var tmpfile = entry.TempFile)
+                            using (var tmpfile = entry.TempFile)
                             {
                                 isFirstFilelist = false;
 
                                 if (entry.Hash != null && entry.Size > 0)
-                                    restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash, tr);
+                                    restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash);
 
                                 var parsed = VolumeBase.ParseFilename(entry.Name);
 
-                                if (!hasUpdatedOptions && !updating) 
+                                if (!hasUpdatedOptions && !updating)
                                 {
                                     VolumeReaderBase.UpdateOptionsFromManifest(parsed.CompressionModule, tmpfile, m_options);
                                     hasUpdatedOptions = true;
@@ -211,13 +215,13 @@ namespace Duplicati.Library.Main.Operation
 
 
                                 // Create timestamped operations based on the file timestamp
-                                var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr);
-                                using(var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options))
-                                    foreach(var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path)))
+                                var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time);
+                                using (var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options))
+                                    foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path)))
                                     {
                                         try
                                         {
-                                            var expectedmetablocks = (fe.Metasize + blocksize - 1)  / blocksize;
+                                            var expectedmetablocks = (fe.Metasize + blocksize - 1) / blocksize;
                                             var expectedmetablocklisthashes = (expectedmetablocks + hashes_pr_block - 1) / hashes_pr_block;
                                             if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0;
 
@@ -225,32 +229,32 @@ namespace Duplicati.Library.Main.Operation
                                             switch (fe.Type)
                                             {
                                                 case FilelistEntryType.Folder:
-                                                    metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
-                                                    restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
+                                                    metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes);
+                                                    restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, metadataid);
                                                     break;
                                                 case FilelistEntryType.File:
                                                     var expectedblocks = (fe.Size + blocksize - 1) / blocksize;
                                                     var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block;
                                                     if (expectedblocks <= 1) expectedblocklisthashes = 0;
 
-                                                    var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr);
-                                                    metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
-                                                    restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, metadataid, tr);
+                                                    var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes);
+                                                    metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes);
+                                                    restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, metadataid);
 
                                                     if (fe.Size <= blocksize)
                                                     {
                                                         if (!string.IsNullOrWhiteSpace(fe.Blockhash))
-                                                            restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, tr);
+                                                            restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize);
                                                         else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
-                                                            restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, tr);
+                                                            restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size);
                                                         else
                                                             m_result.AddWarning(string.Format("No block hash found for file: {0}", fe.Path), null);
                                                     }
 
                                                     break;
                                                 case FilelistEntryType.Symlink:
-                                                    metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
-                                                    restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
+                                                    metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes);
+                                                    restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, metadataid);
                                                     break;
                                                 default:
                                                     m_result.AddWarning(string.Format("Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path), null);
@@ -260,11 +264,11 @@ namespace Duplicati.Library.Main.Operation
                                             if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink))
                                             {
                                                 if (!string.IsNullOrWhiteSpace(fe.Metablockhash))
-                                                    restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, tr);
+                                                    restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize);
                                                 else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
-                                                    restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, tr);
+                                                    restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize);
                                                 else
-                                                    m_result.AddWarning(string.Format("No block hash found for file metadata: {0}", fe.Path), null);                                                
+                                                    m_result.AddWarning(string.Format("No block hash found for file metadata: {0}", fe.Path), null);
                                             }
                                         }
                                         catch (Exception ex)
@@ -289,13 +293,16 @@ namespace Duplicati.Library.Main.Operation
                                 throw;
                             }
                         }
+                    }
 
                     //Make sure we write the config
                     if (!updating)
-                        Utility.VerifyParameters(restoredb, m_options, tr);
+                        Utility.VerifyParameters(restoredb, m_options);
 
-                    using(new Logging.Timer("CommitUpdateFilesetFromRemote"))
-                        tr.Commit();
+                    restoredb.CommitTransaction("CommitUpdateFilesetFromRemote");
+                }
+                finally
+                {
                 }
             
                 if (!m_options.RepairOnlyPaths)
@@ -306,23 +313,27 @@ namespace Duplicati.Library.Main.Operation
                     var hashsize = hashalg.HashSize / 8;
 
                     //Grab all index files, and update the block table
-                    using(var tr = restoredb.BeginTransaction())
+                    try
                     {
                         var indexfiles = (
                                          from n in remotefiles
-                                          where n.FileType == RemoteVolumeType.Index
-                                          select new RemoteVolume(n.File) as IRemoteVolume).ToList();
+                                         where n.FileType == RemoteVolumeType.Index
+                                         select new RemoteVolume(n.File) as IRemoteVolume).ToList();
 
                         m_result.AddMessage(string.Format("Filelists restored, downloading {0} index files", indexfiles.Count));
 
                         var progress = 0;
-                                    
-                        foreach(var sf in new AsyncDownloader(indexfiles, backend))
+
+                        var dl = new AsyncDownloader(indexfiles, backend);
+
+                        while(dl.MoveNext())
+                        {
+                            var sf = dl.Current;
                             try
                             {
                                 if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                                 {
-                                    backend.WaitForComplete(restoredb, null);
+                                    backend.WaitForComplete();
                                     m_result.EndTime = DateTime.UtcNow;
                                     return;
                                 }
@@ -330,14 +341,14 @@ namespace Duplicati.Library.Main.Operation
                                 progress++;
                                 m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f);
 
-                                using(var tmpfile = sf.TempFile)
+                                using (var tmpfile = sf.TempFile)
                                 {
                                     if (sf.Hash != null && sf.Size > 0)
-                                        restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash, tr);
-                
-                                    using(var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize))
+                                        restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash);
+
+                                    using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize))
                                     {
-                                        foreach(var a in svr.Volumes)
+                                        foreach (var a in svr.Volumes)
                                         {
                                             var filename = a.Filename;
                                             var volumeID = restoredb.GetRemoteVolumeID(filename);
@@ -353,20 +364,20 @@ namespace Duplicati.Library.Main.Operation
                                                 if (p == null)
                                                     throw new Exception(string.Format("Unable to parse filename: {0}", filename));
                                                 m_result.AddError(string.Format("Remote file referenced as {0}, but not found in list, registering a missing remote file", filename), null);
-                                                volumeID = restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Verified, tr);
+                                                volumeID = restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Verified);
                                             }
-                                            
-                                            //Add all block/volume mappings
-                                            foreach(var b in a.Blocks)
-                                                restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr);
 
-                                            restoredb.UpdateRemoteVolume(filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr);
-                                            restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID, tr);
+                                            //Add all block/volume mappings
+                                            foreach (var b in a.Blocks)
+                                                restoredb.UpdateBlock(b.Key, b.Value, volumeID);
+
+                                            restoredb.UpdateRemoteVolume(filename, RemoteVolumeState.Verified, a.Length, a.Hash);
+                                            restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID);
                                         }
-                                
+
                                         //If there are blocklists in the index file, update the blocklists
-                                        foreach(var b in svr.BlockLists)
-                                            restoredb.UpdateBlockset(b.Hash, b.Blocklist, tr);
+                                        foreach (var b in svr.BlockLists)
+                                            restoredb.UpdateBlockset(b.Hash, b.Blocklist);
                                     }
                                 }
                             }
@@ -381,16 +392,19 @@ namespace Duplicati.Library.Main.Operation
                                 }
                             }
 
-                        using(new Logging.Timer("CommitRecreatedDb"))
-                            tr.Commit();
-                    
-                        // TODO: In some cases, we can avoid downloading all index files, 
-                        // if we are lucky and pick the right ones
+                            restoredb.CommitTransaction("CommitRecreatedDb");
+
+                            // TODO: In some cases, we can avoid downloading all index files, 
+                            // if we are lucky and pick the right ones
+                        }
+                    }
+                    finally
+                    {
                     }
 
                     // We have now grabbed as much information as possible,
                     // if we are still missing data, we must now fetch block files
-                    restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, null);
+                    restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize);
                 
                     //We do this in three passes
                     for(var i = 0; i < 3; i++)
@@ -423,47 +437,53 @@ namespace Duplicati.Library.Main.Operation
                         }
 
                         var progress = 0;
-                        foreach(var sf in new AsyncDownloader(lst, backend))
-                            using(var tmpfile = sf.TempFile)
-                            using(var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options))
-                            using(var tr = restoredb.BeginTransaction())
-                            {
-                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                        var dl = new AsyncDownloader(lst, backend);
+                        while(dl.MoveNext())
+                        {
+                            var sf = dl.Current;
+                            using (var tmpfile = sf.TempFile)
+                            using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options))
+                                try
                                 {
-                                    backend.WaitForComplete(restoredb, null);
-                                    m_result.EndTime = DateTime.UtcNow;
-                                    return;
-                                }    
-                            
-                                progress++;
-                                m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f));
+                                    if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                                    {
+                                        backend.WaitForComplete();
+                                        m_result.EndTime = DateTime.UtcNow;
+                                        return;
+                                    }
 
-                                var volumeid = restoredb.GetRemoteVolumeID(sf.Name);
+                                    progress++;
+                                    m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f));
 
-                                restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Uploaded, sf.Size, sf.Hash, tr);
-                            
-                                // Update the block table so we know about the block/volume map
-                                foreach(var h in rd.Blocks)
-                                    restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr);
-                            
-                                // Grab all known blocklists from the volume
-                                foreach(var blocklisthash in restoredb.GetBlockLists(volumeid))
-                                    restoredb.UpdateBlockset(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), tr);
-    
-                                // Update tables so we know if we are done
-                                restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, tr);
-                        
-                                using(new Logging.Timer("CommitRestoredBlocklist"))
-                                    tr.Commit();
-    
-                                //At this point we can patch files with data from the block volume
-                                if (blockprocessor != null)
-                                    blockprocessor(sf.Name, rd);
-                            }
+                                    var volumeid = restoredb.GetRemoteVolumeID(sf.Name);
+
+                                    restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Uploaded, sf.Size, sf.Hash);
+
+                                    // Update the block table so we know about the block/volume map
+                                    foreach (var h in rd.Blocks)
+                                        restoredb.UpdateBlock(h.Key, h.Value, volumeid);
+
+                                    // Grab all known blocklists from the volume
+                                    foreach (var blocklisthash in restoredb.GetBlockLists(volumeid))
+                                        restoredb.UpdateBlockset(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize));
+
+                                    // Update tables so we know if we are done
+                                    restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize);
+
+                                    restoredb.CommitTransaction("CommitRestoredBlocklist");
+
+                                    //At this point we can patch files with data from the block volume
+                                    if (blockprocessor != null)
+                                        blockprocessor(sf.Name, rd);
+                                }
+                                finally
+                                {
+                                }
+                        }
                     }
                 }
-                
-                backend.WaitForComplete(restoredb, null);
+
+                backend.WaitForEmpty();
 
                 if (m_options.RepairOnlyPaths)
                 {
@@ -480,12 +500,12 @@ namespace Duplicati.Library.Main.Operation
 
                     using (var lbfdb = new LocalListBrokenFilesDatabase(restoredb))
                     {
-                        var broken = lbfdb.GetBrokenFilesets(new DateTime(0), null, null).Count();
+                        var broken = lbfdb.GetBrokenFilesets(new DateTime(0), null).Count();
                         if (broken != 0)
                             throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"));
                     }
 
-                    restoredb.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, true);
+                    restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true);
 
                     m_result.AddMessage("Recreate completed, and consistency checks completed, marking database as complete");
 

Duplicati/Library/Main/Operation/RepairHandler.cs

@@ -99,7 +99,7 @@ namespace Duplicati.Library.Main.Operation
             m_result.OperationProgressUpdater.UpdateProgress(0);
 
             using(var db = new LocalRepairDatabase(m_options.Dbpath))
-            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result))
             {
                 m_result.SetDatabase(db);
                 Utility.UpdateOptionsFromDb(db, m_options);
@@ -150,7 +150,7 @@ namespace Duplicati.Library.Main.Operation
                                 {
                                     if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                                     {
-                                        backend.WaitForComplete(db, null);
+                                        backend.WaitForComplete();
                                         return;
                                     }
 
@@ -161,7 +161,7 @@ namespace Duplicati.Library.Main.Operation
                                     string hash;
                                     KeyValuePair<string, IEnumerable<KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>>> res;
                                    
-                                    using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
+                                using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
                                         res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, m_result, 1);
 
                                     if (res.Value.Any())
@@ -189,7 +189,7 @@ namespace Duplicati.Library.Main.Operation
                         {
                             if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                             {
-                                backend.WaitForComplete(db, null);
+                                backend.WaitForComplete();
                                 return;
                             }
 
@@ -209,18 +209,15 @@ namespace Duplicati.Library.Main.Operation
                                     {
                                         foreach(var rv in ifr.Volumes)
                                         {
-                                            string cmphash;
-                                            long cmpsize;
-                                            RemoteVolumeType cmptype;
-                                            RemoteVolumeState cmpstate;
-                                            if (!db.GetRemoteVolume(rv.Filename, out cmphash, out cmpsize, out cmptype, out cmpstate))
+                                            var entry = db.GetRemoteVolume(rv.Filename);
+                                            if (entry.ID < 0)
                                                 throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename));
                                             
-                                            if (!new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(cmpstate))
-                                                throw new Exception(string.Format("Volume {0} has local state {1}", rv.Filename, cmpstate));
+                                            if (!new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State))
+                                                throw new Exception(string.Format("Volume {0} has local state {1}", rv.Filename, entry.State));
                                         
-                                            if (cmphash != rv.Hash || cmpsize != rv.Length || ! new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(cmpstate))
-                                                throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, cmphash, cmpsize, rv.Hash, rv.Length));
+                                            if (entry.Hash != rv.Hash || entry.Size != rv.Length || ! new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State))
+                                                throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length));
 
                                             db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks);
                                         }
@@ -231,7 +228,7 @@ namespace Duplicati.Library.Main.Operation
 
                                         var selfid = db.GetRemoteVolumeID(n.File.Name);
                                         foreach(var rv in ifr.Volumes)
-                                            db.AddIndexBlockLink(selfid, db.GetRemoteVolumeID(rv.Filename), null);
+                                            db.AddIndexBlockLink(selfid, db.GetRemoteVolumeID(rv.Filename));
                                     }
                                     
                                     // All checks fine, we accept the new index file
@@ -269,7 +266,7 @@ namespace Duplicati.Library.Main.Operation
                         {  
                             if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                             {
-                                backend.WaitForComplete(db, null);
+                                backend.WaitForComplete();
                                 return;
                             }    
 
@@ -282,16 +279,16 @@ namespace Duplicati.Library.Main.Operation
                                 var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
                                 newEntry = w;
                                 w.SetRemoteFilename(n.Name);
-                                
-                                db.WriteFileset(w, null, filesetId);
-    
+
+                                db.WriteFileset(w, filesetId);
+	
                                 w.Close();
                                 if (m_options.Dryrun)
                                     m_result.AddDryrunMessage(string.Format("would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
                                 else
                                 {
-                                    db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                    backend.Put(w);
+                                    db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                                    backend.Put(w, null);
                                 }
                             }
                             else if (n.Type == RemoteVolumeType.Index)
@@ -300,6 +297,8 @@ namespace Duplicati.Library.Main.Operation
                                 newEntry = w;
                                 w.SetRemoteFilename(n.Name);
 
+                                // TODO: Most of this is actually replicated in IndexVolumeCreator
+
                                 var h = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
                                 
                                 foreach(var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
@@ -329,8 +328,8 @@ namespace Duplicati.Library.Main.Operation
                                     m_result.AddDryrunMessage(string.Format("would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
                                 else
                                 {
-                                    db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                    backend.Put(w);
+                                    db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                                    backend.Put(w, null);
                                 }
                             }
                             else if (n.Type == RemoteVolumeType.Blocks)
@@ -376,10 +375,12 @@ namespace Duplicati.Library.Main.Operation
                                             }
                                         }
                                     }
-                                    
+
                                     //Then we grab all remote volumes that have the missing blocks
-                                    foreach(var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend))
+                                    var dl = new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend);
+                                    while(dl.MoveNext())
                                     {
+                                        var vol = dl.Current;
                                         try
                                         {
                                             using(var tmpfile = vol.TempFile)
@@ -422,8 +423,8 @@ namespace Duplicati.Library.Main.Operation
                                             m_result.AddDryrunMessage(string.Format("would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
                                         else
                                         {
-                                            db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                            backend.Put(w);
+                                            db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                                            backend.Put(w, null);
                                         }
                                     }
                                 }
@@ -449,7 +450,7 @@ namespace Duplicati.Library.Main.Operation
                 }
 
                 m_result.OperationProgressUpdater.UpdateProgress(1);                
-                backend.WaitForComplete(db, null);
+                backend.WaitForEmpty();
                 db.WriteResults();
             }
         }

Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs

@@ -27,7 +27,7 @@ namespace Duplicati.Library.Main.Operation
         
             using (var tmpdb = new Library.Utility.TempFile())
             using (var db = new Database.LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result))
             {
                 m_result.SetDatabase(db);
                 
@@ -46,22 +46,17 @@ namespace Duplicati.Library.Main.Operation
                         {
                             if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                             {
-                                backend.WaitForComplete(db, null);
+                                backend.WaitForComplete();
                                 return;
                             }    
                         
                             var file = fileversion.Value.File;
-                            long size;
-                            string hash;
-                            RemoteVolumeType type;
-                            RemoteVolumeState state;
-                            if (!db.GetRemoteVolume(file.Name, out hash, out size, out type, out state))
-                                size = file.Size;
-    
+                            var entry = db.GetRemoteVolume(file.Name);
+
                             var res = new List<string>();
-                            using (var tmpfile = backend.Get(file.Name, size, hash))
-                            using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
-                                foreach (var cf in tmp.ControlFiles)
+                            using (var tmpfile = backend.Get(file.Name, entry.Size < 0 ? file.Size : entry.Size, entry.Hash))
+	                        using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
+	                            foreach (var cf in tmp.ControlFiles)
                                     if (Library.Utility.FilterExpression.Matches(filter, cf.Key))
                                     {
                                         var targetpath = System.IO.Path.Combine(m_options.Restorepath, cf.Key);
@@ -87,7 +82,7 @@ namespace Duplicati.Library.Main.Operation
                 }
                 finally
                 {
-                    backend.WaitForComplete(db, null);
+                    backend.WaitForEmpty();
                 }
 
                 db.WriteResults();

Duplicati/Library/Main/Operation/RestoreHandler.cs

@@ -337,7 +337,7 @@ namespace Duplicati.Library.Main.Operation
             //In this case, we check that the remote storage fits with the database.
             //We can then query the database and find the blocks that we need to do the restore
             using(var database = new LocalRestoreDatabase(dbparent))
-            using(var backend = new BackendManager(m_backendurl, m_options, result.BackendWriter, database))
+            using(var backend = new BackendManager(m_backendurl, m_options, result.BackendWriter, database, m_result))
             using(var metadatastorage = new RestoreHandlerMetadataStorage())
             {
                 database.SetResult(m_result);
@@ -388,7 +388,7 @@ namespace Duplicati.Library.Main.Operation
 
                 if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                 {
-                    backend.WaitForComplete(database, null);
+                    backend.WaitForComplete();
                     return;
                 }
 
@@ -402,7 +402,7 @@ namespace Duplicati.Library.Main.Operation
 
                 if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                 {
-                    backend.WaitForComplete(database, null);
+                    backend.WaitForComplete();
                     return;
                 }
                 
@@ -418,17 +418,20 @@ namespace Duplicati.Library.Main.Operation
                 }
 
                 var brokenFiles = new List<string>();
-                foreach(var blockvolume in new AsyncDownloader(volumes, backend))
+                var dl = new AsyncDownloader(volumes, backend);
+                while(dl.MoveNext())
+                {
+                    var blockvolume = dl.Current;
                     try
                     {
                         if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                         {
-                            backend.WaitForComplete(database, null);
+                            backend.WaitForComplete();
                             return;
                         }
-                    
-                        using(var tmpfile = blockvolume.TempFile)
-                        using(var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Name), tmpfile, m_options))
+
+                        using (var tmpfile = blockvolume.TempFile)
+                        using (var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Name), tmpfile, m_options))
                             PatchWithBlocklist(database, blocks, m_options, result, m_blockbuffer, metadatastorage);
                     }
                     catch (Exception ex)
@@ -438,6 +441,13 @@ namespace Duplicati.Library.Main.Operation
                         if (ex is System.Threading.ThreadAbortException)
                             throw;
                     }
+                }
+
+                // Restore empty files. They might not have any blocks so don't appear in any volume.
+                foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0)) {
+                    // Just create the file and close it right away, empty statement is intentional.
+                    using (m_systemIO.FileCreate(file.Path)) ;
+                }
 
                 // Enforcing the length of files is now already done during ScanForExistingTargetBlocks
                 // and thus not necessary anymore.
@@ -466,7 +476,7 @@ namespace Duplicati.Library.Main.Operation
                             {
                                 if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                                 {
-                                    backend.WaitForComplete(database, null);
+                                    backend.WaitForComplete();
                                     return;
                                 }
                             
@@ -502,7 +512,7 @@ namespace Duplicati.Library.Main.Operation
 
                 // Drop the temp tables
                 database.DropRestoreTable();
-                backend.WaitForComplete(database, null);
+                backend.WaitForEmpty();
             }
             
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete);
@@ -533,14 +543,6 @@ namespace Duplicati.Library.Main.Operation
                 else if (isDirTarget && !m_systemIO.DirectoryExists(targetpath))
                     m_systemIO.DirectoryCreate(targetpath);
 
-                if (metadata.TryGetValue("CoreLastWritetime", out k) && long.TryParse(k, out t))
-                {
-                    if (isDirTarget)
-                        m_systemIO.DirectorySetLastWriteTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc));
-                    else
-                        m_systemIO.FileSetLastWriteTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc));
-                }
-
                 if (metadata.TryGetValue("CoreCreatetime", out k) && long.TryParse(k, out t))
                 {
                     if (isDirTarget)
@@ -552,7 +554,25 @@ namespace Duplicati.Library.Main.Operation
                 if (metadata.TryGetValue("CoreAttributes", out k) && Enum.TryParse(k, true, out fa))
                     m_systemIO.SetFileAttributes(targetpath, fa);
 
-                m_systemIO.SetMetadata(path, metadata, restorePermissions);
+                try
+                {
+                    // Set metadata if possible
+                    m_systemIO.SetMetadata(path, metadata, restorePermissions);
+                }
+                catch (Exception ex)
+                {
+                    Logging.Log.WriteMessage(string.Format("Failed to restore metadata on {0}: {1}", path, ex.Message), Logging.LogMessageType.Warning, ex);
+                }
+
+                // Final step is to reset the LastWriteTime as the other operations
+                // may cause the filesystem to reset the timestamp
+                if (metadata.TryGetValue("CoreLastWritetime", out k) && long.TryParse(k, out t))
+                {
+                    if (isDirTarget)
+                        m_systemIO.DirectorySetLastWriteTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc));
+                    else
+                        m_systemIO.FileSetLastWriteTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc));
+                }
             }
         }
 
@@ -778,8 +798,6 @@ namespace Duplicati.Library.Main.Operation
                 {
                     // Find the largest common prefix
                     var largest_prefix = options.DontCompressRestorePaths ? "" : database.GetLargestPrefix();
-                    if (options.DontCompressRestorePaths)
-                        largest_prefix = "";
 
                     result.AddVerboseMessage("Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Library.Utility.Utility.AppendDirSeparator(options.Restorepath));
     

Duplicati/Library/Main/Operation/TestFilterHandler.cs

@@ -16,7 +16,9 @@
 //  License along with this library; if not, write to the Free Software
 //  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 using System;
+using CoCoL;
 using System.IO;
+using System.Threading.Tasks;
 using Duplicati.Library.Snapshots;
 
 namespace Duplicati.Library.Main.Operation
@@ -34,49 +36,53 @@ namespace Duplicati.Library.Main.Operation
         
         public void Run(string[] sources, Library.Utility.IFilter filter)
         {
-            var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store;
             var sourcefilter = new Library.Utility.FilterExpression(sources, true);
 
+            using(new IsolatedChannelScope(Common.Channels.LogChannel))
             using(var snapshot = BackupHandler.GetSnapshot(sources, m_options, m_result))
             {
-                foreach(var path in new BackupHandler.FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_result).EnumerateFilesAndFolders())
-                {
-                    var fa = FileAttributes.Normal;
-                    try { fa = snapshot.GetAttributes(path); }
-                    catch { }
-                    
-                    if (storeSymlinks && snapshot.IsSymlink(path, fa))
+                var enumeratorTask = Backup.FileEnumerationProcess.Run(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_options.ChangedFilelist, m_result.TaskReader);
+                var metadataTask = Backup.MetadataPreProcess.RunOnlyRecord(snapshot, m_options);
+
+                var fileCounterTask = AutomationExtensions.RunTask(
+                    new
                     {
-                        m_result.AddVerboseMessage("Storing symlink: {0}", path);
-                    }
-                    else if ((fa & FileAttributes.Directory) == FileAttributes.Directory)
+                        Input = Backup.Channels.ProcessedFiles.ForRead
+                    },
+                    async self =>
                     {
-                        m_result.AddVerboseMessage("Including folder: {0}", path);
-                    }
-                    else
-                    {
-                        m_result.FileCount++;
-                        var size = -1L;
-                    
-                        try
+                        while (await m_result.TaskReader.ProgressAsync)
                         {
-                            size = snapshot.GetFileSize(path);
-                            m_result.FileSize += size;
+                            var f = await self.Input.ReadAsync(CoCoL.Timeout.Infinite);
+                            long filestatsize = -1;
+                            try
+                            {
+                                filestatsize = snapshot.GetFileSize(f.Path);
+                            }
+                            catch
+                            {
+                            }
+
+                            var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan;
+
+                            if (tooLargeFile)
+                                m_result.AddVerboseMessage("Excluding file due to size: {0} ({1})", f.Path, filestatsize < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(filestatsize));                    
+                            else
+                                m_result.AddVerboseMessage("Including file: {0} ({1})", f.Path, filestatsize < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(filestatsize));
                         }
-                        catch
-                        {
-                        }
-                    
-                    
-                        if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan)
-                            m_result.AddVerboseMessage("Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
-                        else
-                            m_result.AddVerboseMessage("Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));                    
                     }
-                    
-                }
+                );
+
+                Task.WhenAll(
+                    enumeratorTask,
+                    metadataTask,
+                    fileCounterTask
+                )
+                .WaitForTaskOrThrow();
             }
         }
+
+
         
         #region IDisposable implementation
         public void Dispose()

Duplicati/Library/Main/Operation/TestHandler.cs

@@ -42,7 +42,7 @@ namespace Duplicati.Library.Main.Operation
                 throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath));
                                 
             using(var db = new LocalTestDatabase(m_options.Dbpath))
-            using(var backend = new BackendManager(m_backendurl, m_options, m_results.BackendWriter, db))
+            using(var backend = new BackendManager(m_backendurl, m_options, m_results.BackendWriter, db, m_results))
             {
                 db.SetResult(m_results);
                 Utility.UpdateOptionsFromDb(db, m_options);
@@ -66,13 +66,15 @@ namespace Duplicati.Library.Main.Operation
             
             if (m_options.FullRemoteVerification)
             {
-                foreach(var vol in new AsyncDownloader(files, backend))
+                var dl = new AsyncDownloader(files, backend);
+                while(dl.MoveNext())
                 {
+                    var vol = dl.Current;
                     try
                     {
                         if (m_results.TaskControlRendevouz() == TaskControlState.Stop)
                         {
-                            backend.WaitForComplete(db, null);
+                            backend.WaitForComplete();
                             m_results.EndTime = DateTime.UtcNow;
                             return;
                         }    
@@ -89,14 +91,11 @@ namespace Duplicati.Library.Main.Operation
                         {
                             if (res.Value == null || !res.Value.Any())
                             {
-                                string dbhash;
-                                long dbsize;
-                                RemoteVolumeType dbtype;
-                                RemoteVolumeState dbstate;
+                                var rv = db.GetRemoteVolume(vol.Name);
 
-                                if (db.GetRemoteVolume(vol.Name, out dbhash, out dbsize, out dbtype, out dbstate))
+                                if (rv.ID < 0)
                                 {
-                                    if (string.IsNullOrWhiteSpace(dbhash) || dbsize <= 0)
+                                    if (string.IsNullOrWhiteSpace(rv.Hash) || rv.Size <= 0)
                                     {
                                         if (m_options.Dryrun)
                                         {

Duplicati/Library/Main/Options.cs

@@ -75,6 +75,15 @@ namespace Duplicati.Library.Main
         private const string DEFAULT_LOG_RETENTION = "30D";
 
         /// <summary>
+        /// The default number of compressor instances
+        /// </summary>
+        private readonly int DEFAULT_COMPRESSORS = Math.Max(1, Environment.ProcessorCount / 2);
+
+        /// <summary>
+        /// The default number of hasher instances
+        /// </summary>
+        private readonly int DEFAULT_BLOCK_HASHERS = Math.Max(1, Environment.ProcessorCount / 2);
+
         /// The default threshold for warning about coming close to quota
         /// </summary>
         private const int DEFAULT_QUOTA_WARNING_THRESHOLD = 10;
@@ -530,6 +539,10 @@ namespace Duplicati.Library.Main
 
                     new CommandLineArgument("disable-piped-streaming", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablepipingShort, Strings.Options.DisablepipingLong, "false"),
 
+                    new CommandLineArgument("concurrency-max-threads", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencymaxthreadsShort, Strings.Options.ConcurrencymaxthreadsLong, "0"),
+                    new CommandLineArgument("concurrency-block-hashers", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencyblockhashersShort, Strings.Options.ConcurrencyblockhashersLong, DEFAULT_BLOCK_HASHERS.ToString()),
+                    new CommandLineArgument("concurrency-compressors", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencycompressorsShort, Strings.Options.ConcurrencycompressorsLong, DEFAULT_COMPRESSORS.ToString()),
+                    
                     new CommandLineArgument("auto-vacuum", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AutoVacuumShort, Strings.Options.AutoVacuumLong, "false"),
                     new CommandLineArgument("disable-file-scanner", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilescannerShort, Strings.Options.DisablefilescannerLong, "false"),
                 });
@@ -1812,6 +1825,62 @@ namespace Duplicati.Library.Main
                 return Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true);
             }
         }
+
+
+        /// <summary>
+        /// Gets the number of concurrent threads
+        /// </summary>
+        public int ConcurrencyMaxThreads
+        {
+            get
+            {
+                string value;
+                if (!m_options.TryGetValue("concurrency-max-threads", out value))
+                    value = null;
+
+                if (string.IsNullOrEmpty(value))
+                    return 0;
+                else
+                    return int.Parse(value);
+            }
+        }
+
+        /// <summary>
+        /// Gets the number of concurrent block hashers
+        /// </summary>
+        public int ConcurrencyBlockHashers
+        {
+            get
+            {
+                string value;
+                if (!m_options.TryGetValue("concurrency-block-hashers", out value))
+                    value = null;
+
+                if (string.IsNullOrEmpty(value))
+                    return DEFAULT_BLOCK_HASHERS;
+                else
+                    return Math.Max(1, int.Parse(value));
+            }
+        }
+
+        /// <summary>
+        /// Gets the number of concurrent block hashers
+        /// </summary>
+        public int ConcurrencyCompressors
+        {
+            get
+            {
+                string value;
+                if (!m_options.TryGetValue("concurrency-compressors", out value))
+                    value = null;
+
+                if (string.IsNullOrEmpty(value))
+                    return DEFAULT_COMPRESSORS;
+                else
+                    return Math.Max(1, int.Parse(value));
+            }
+        }
+
         /// <summary>
         /// Gets a lookup table with compression hints, the key is the file extension with the leading period
         /// </summary>
@@ -1821,8 +1890,7 @@ namespace Duplicati.Library.Main
             {
                 if (m_compressionHints == null)
                 {
-                    //Don't try again, if the file does not exist
-                    m_compressionHints = new Dictionary<string, CompressionHint>(Library.Utility.Utility.ClientFilenameStringComparer);
+                    var hints = new Dictionary<string, CompressionHint>(Library.Utility.Utility.ClientFilenameStringComparer);
 
                     string file;
                     if (!m_options.TryGetValue("compression-extension-file", out file))
@@ -1836,8 +1904,11 @@ namespace Duplicati.Library.Main
                             if (lix > 0)
                                 line = line.Substring(0, lix);
                             if (line.Length >= 2 && line[0] == '.')
-                                m_compressionHints[line] = CompressionHint.Noncompressible;
+                                hints[line] = CompressionHint.Noncompressible;
                         }
+
+                    //Don't try again, if the file does not exist
+                    m_compressionHints = hints;
                 }
 
                 return m_compressionHints;

Duplicati/Library/Main/ResultClasses.cs

@@ -344,7 +344,7 @@ namespace Duplicati.Library.Main
                     while (m_dbqueue.Count > 0)
                     {
                         var el = m_dbqueue.Dequeue();
-                        m_db.LogMessage(el.Type, el.Message, el.Exception, null);
+                        m_db.LogMessage(el.Type, el.Message, el.Exception);
                     }
                 }
             }
@@ -359,7 +359,7 @@ namespace Duplicati.Library.Main
             else
             {
                 FlushLog();
-                m_db.LogMessage("Message", message, ex, null);
+                m_db.LogMessage("Message", message, ex);
             }
         }
 
@@ -424,12 +424,13 @@ namespace Duplicati.Library.Main
             }
         }
 
+        private static readonly object[] NOARGS = new object[0];
+
         public void AddVerboseMessage(string message, params object[] args)
         {
             if (m_parent != null)
             {
-                if ((args != null) && (args.Length > 0)) m_parent.AddVerboseMessage(message, args);
-                else m_parent.AddMessage(message);
+                m_parent.AddVerboseMessage(message, args ?? NOARGS);
             }
             else
             {
@@ -445,8 +446,7 @@ namespace Duplicati.Library.Main
 
                         if (MessageSink != null)
                         {
-                            if ((args != null) && (args.Length > 0)) MessageSink.VerboseEvent(message, args);
-                            else MessageSink.MessageEvent(message);
+                            MessageSink.VerboseEvent(message, args ?? NOARGS);
                         }
                     }
                     finally
@@ -596,9 +596,21 @@ namespace Duplicati.Library.Main
         public IEnumerable<string> Warnings { get { return m_warnings; } }
         public IEnumerable<string> Errors { get { return m_errors; } }
 
-        public BasicResults()
-        {
-            this.BeginTime = DateTime.UtcNow;
+        protected Operation.Common.TaskControl m_taskController;
+        public Operation.Common.ITaskReader TaskReader 
+        { 
+            get 
+            {
+                if (m_parent != null)
+                    return m_parent.TaskReader;
+
+                return m_taskController;
+            } 
+        }
+
+        public BasicResults() 
+        { 
+            this.BeginTime = DateTime.UtcNow; 
             this.m_parent = null;
             m_messages = new Library.Utility.FileBackedStringList();
             m_warnings = new Library.Utility.FileBackedStringList();
@@ -609,6 +621,7 @@ namespace Duplicati.Library.Main
             m_callerThread = System.Threading.Thread.CurrentThread;
             m_backendProgressUpdater = new BackendProgressUpdater();
             m_operationProgressUpdater = new OperationProgressUpdater();
+            m_taskController = new Duplicati.Library.Main.Operation.Common.TaskControl();
         }
 
         public BasicResults(BasicResults p)

Duplicati/Library/Main/Strings.cs

@@ -225,6 +225,12 @@ namespace Duplicati.Library.Main.Strings
         public static string ForcelocaleLong { get { return LC.L(@"By default, your system locale and culture settings will be used. In some cases you may prefer to run with another locale, for example to get messages in another language. This option can be used to set the locale. Supply a blank string to choose the ""Invariant Culture""."); } }
         public static string DisablepipingShort{ get { return LC.L(@"Handle file communication with backend using threaded pipes"); } }
         public static string DisablepipingLong { get { return LC.L(@"Use this option to disable multithreaded handling of up- and downloads, that can significantly speed up backend operations depending on the hardware you're running on and the transfer rate of your backend."); } }
+        public static string ConcurrencymaxthreadsShort{ get { return LC.L(@"Limit number of concurrent threads"); } }
+        public static string ConcurrencymaxthreadsLong { get { return LC.L(@"Use this option to set the maximum number of threads used. Setting this value to zero or less will dynamically balance the number of active threads to fit the hardware."); } }
+        public static string ConcurrencyblockhashersShort{ get { return LC.L(@"Specify the number of concurrent hashing processes"); } }
+        public static string ConcurrencyblockhashersLong { get { return LC.L(@"Use this option to set the number of processes that perform hashing of data."); } }
+        public static string ConcurrencycompressorsShort{ get { return LC.L(@"Specify the number of concurrent compression processes"); } }
+        public static string ConcurrencycompressorsLong { get { return LC.L(@"Use this option to set the number of processes that perform compression of output data."); } }
         public static string HypervbackupvmShort { get { return LC.L(@"Perform backup of Hyper-V machines (Windows only)"); } }
         public static string HypervbackupvmLong { get { return LC.L(@"Use this option to specify the IDs of machines to include in the backup. Specify multiple machine IDs with a semicolon separator. (You can use this Powershell command to get ID 'Get-VM | ft VMName, ID')"); } }
         public static string DisablesyntehticfilelistLong { get { return LC.L(@"If Duplicati detects that the previous backup did not complete, it will generate a filelist that is a merge of the last completed backup and the contents that were uploaded in the incomplete backup session."); } }

Duplicati/Library/Main/Utility.cs

@@ -96,10 +96,10 @@ namespace Duplicati.Library.Main
             return new Metahash(values, options);
         }
 
-        internal static void UpdateOptionsFromDb(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null)
+        internal static void UpdateOptionsFromDb(LocalDatabase db, Options options)
         {
             string n = null;
-            var opts = db.GetDbOptions(transaction);
+            var opts = db.GetDbOptions();
             if(opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n)))
                 options.RawOptions["blocksize"] = opts["blocksize"] + "b";
 
@@ -109,13 +109,13 @@ namespace Duplicati.Library.Main
                 options.RawOptions["file-hash-algorithm"] = opts["filehash"];
         }
 
-        internal static void VerifyParameters(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null)
+        internal static void VerifyParameters(LocalDatabase db, Options options)
         {
             var newDict = new Dictionary<string, string>();
             newDict.Add("blocksize", options.Blocksize.ToString());
             newDict.Add("blockhash", options.BlockHashAlgorithm);
             newDict.Add("filehash", options.FileHashAlgorithm);
-            var opts = db.GetDbOptions(transaction);
+            var opts = db.GetDbOptions();
             
             if (options.NoEncryption)
             {
@@ -176,7 +176,7 @@ namespace Duplicati.Library.Main
                     if (!newDict.ContainsKey(k.Key))
                         newDict[k.Key] = k.Value;
                 
-                db.SetDbOptions(newDict, transaction);               
+                db.SetDbOptions(newDict);               
             }
         }
 

Duplicati/Library/Main/Volumes/VolumeWriterBase.cs

@@ -52,6 +52,9 @@ namespace Duplicati.Library.Main.Volumes
             else
                 m_localfile = new Library.Utility.TempFile();
 
+            // TODO(danstahr): This is a hack! Figure out why the file is being disposed of prematurely.
+            m_localfile.Protected = true;
+
             ResetRemoteFilename(options, timestamp);
 
             m_localFileStream = new System.IO.FileStream(m_localfile, FileMode.Create, FileAccess.Write, FileShare.Read);
@@ -78,6 +81,7 @@ namespace Duplicati.Library.Main.Volumes
                 try { m_compression.Dispose(); }
                 finally { m_compression = null; }
 
+            m_localfile.Protected = false;
             if (m_localFileStream != null)
                 try { m_localFileStream.Dispose(); }
                 finally { m_localFileStream = null; }
@@ -91,6 +95,10 @@ namespace Duplicati.Library.Main.Volumes
 
         public virtual void Close()
         {
+#if DEBUG            
+            var expectedsize = m_compression == null ? -1 : Filesize;
+#endif
+
             if (m_compression != null)
                 try { m_compression.Dispose(); }
                 finally { m_compression = null; }
@@ -98,6 +106,16 @@ namespace Duplicati.Library.Main.Volumes
             if (m_localFileStream != null)
                 try { m_localFileStream.Dispose(); }
                 finally { m_localFileStream = null; }
+
+#if DEBUG
+            if (expectedsize != -1)
+            {
+                var inf = new FileInfo(m_localfile);
+                if (inf.Exists && inf.Length != expectedsize)
+                    Console.WriteLine("Got diff in file size: {0}", Library.Utility.Utility.FormatSizeString(inf.Length - expectedsize));
+
+            }
+#endif
         }
 
         public long Filesize { get { return m_compression.Size + m_compression.FlushBufferSize; } }

Duplicati/Library/Main/packages.config

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="CoCoL" version="1.5.1" targetFramework="net45" />
   <package id="Newtonsoft.Json" version="10.0.3" targetFramework="net45" />
 </packages>

Duplicati/Library/Utility/TempFile.cs

@@ -35,7 +35,7 @@ namespace Duplicati.Library.Utility
         
         private string m_path;
         private bool m_protect;
-                
+
 #if DEBUG
         //In debug mode, we track the creation of temporary files, and encode the generating method into the name
         private static object m_lock = new object();
@@ -72,7 +72,6 @@ namespace Duplicati.Library.Utility
                 m_fileTrace.Add(s, st);
             return s;            
         }
-
 #else
         private static string GenerateUniqueName()
         {
@@ -88,7 +87,7 @@ namespace Duplicati.Library.Utility
         /// Gets all temporary files found in the current tempdir, that matches the application prefix
         /// </summary>
         /// <returns>The application temp files.</returns>
-        public static IEnumerable<string> GetApplicationTempFiles()
+        private static IEnumerable<string> GetApplicationTempFiles()
         {
             return System.IO.Directory.GetFiles(TempFolder.SystemTempPath, APPLICATION_PREFIX + "*");
         }

Duplicati/Library/Utility/Utility.cs

@@ -18,6 +18,7 @@
 // 
 using System.Text.RegularExpressions;
 using System.Linq;
+using System.Threading.Tasks;
 
 
 #endregion
@@ -478,6 +479,29 @@ namespace Duplicati.Library.Utility
             return index;
         }
 
+        /// <summary>
+        /// Some streams can return a number that is less than the requested number of bytes.
+        /// This is usually due to fragmentation, and is solved by issuing a new read.
+        /// This function wraps that functionality.
+        /// </summary>
+        /// <param name="stream">The stream to read.</param>
+        /// <param name="buf">The buffer to read into.</param>
+        /// <param name="count">The amout of bytes to read.</param>
+        /// <returns>The number of bytes read</returns>
+        public static async Task<int> ForceStreamReadAsync(this System.IO.Stream stream, byte[] buf, int count)
+        {
+            int a;
+            int index = 0;
+            do
+            {
+                a = await stream.ReadAsync(buf, index, count);
+                index += a;
+                count -= a;
+            } while (a != 0 && count > 0);
+
+            return index;
+        }
+
         /// <summary>
         /// Compares two streams to see if they are binary equals
         /// </summary>

Duplicati/UnitTest/BasicSetupHelper.cs

@@ -72,6 +72,7 @@ namespace Duplicati.UnitTest
             if (!DEBUG_OUTPUT)
                 TestContext.Progress.WriteLine(msg, args);
             Console.WriteLine(msg, args);
+            Library.Logging.Log.WriteMessage(string.Format(msg, args), Library.Logging.LogMessageType.Information);
         }
 
         [OneTimeSetUp]
@@ -90,6 +91,13 @@ namespace Duplicati.UnitTest
                 File.Delete(DBFILE);
             if (Directory.Exists(TARGETFOLDER))
                 Directory.Delete(TARGETFOLDER, true);
+            
+            var prevlog = Library.Logging.Log.CurrentLog as IDisposable;
+            Library.Logging.Log.CurrentLog = new Library.Logging.StreamLog(LOGFILE);
+            Library.Logging.Log.LogLevel = Library.Logging.LogMessageType.Information;
+
+            if (prevlog != null)
+                prevlog.Dispose();
         }
 
 
@@ -105,8 +113,6 @@ namespace Duplicati.UnitTest
 
                 opts["passphrase"] = "123456";
                 opts["debug-output"] = "";
-                opts["log-level"] = "profiling";
-                opts["log-file"] = LOGFILE;
                 opts["dblock-size"] = "10mb";
                 opts["dbpath"] = DBFILE;
                 opts["blocksize"] = "10kb";

Duplicati/UnitTest/CommandLineOperationsTests.cs

@@ -131,6 +131,18 @@ namespace Duplicati.UnitTest
                     Duplicati.CommandLine.Program.RealMain(backupargs);                
             }
 
+            ProgressWriteLine("Checking remote file sizes ...");
+            string maxsizestr;
+            TestOptions.TryGetValue("dblock-size", out maxsizestr);
+            if (string.IsNullOrWhiteSpace(maxsizestr))
+                maxsizestr = "50mb";
+
+            var maxsizeval = Library.Utility.Sizeparser.ParseSize(maxsizestr, "mb");
+            foreach (var n in System.IO.Directory.EnumerateFiles(TARGETFOLDER))
+                if (new FileInfo(n).Length > maxsizeval)
+                    Library.Logging.Log.WriteMessage(string.Format("File {0} is {1} larger than allowed volume size", n, Library.Utility.Utility.FormatSizeString(new FileInfo(n).Length - maxsizeval)), Library.Logging.LogMessageType.Warning);
+
+
             ProgressWriteLine("Compacting data ...");
             using(new Library.Logging.Timer("Compacting"))
                 Duplicati.CommandLine.Program.RealMain((new string[] { "compact", target, "--small-file-max-count=2" }.Union(opts)).ToArray());

Duplicati/UnitTest/Duplicati.UnitTest.csproj

@@ -31,7 +31,7 @@
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="nunit.framework">
-      <HintPath>..\..\packages\NUnit.3.5.0\lib\net45\nunit.framework.dll</HintPath>
+      <HintPath>..\..\packages\NUnit.3.9.0\lib\net45\nunit.framework.dll</HintPath>
     </Reference>
   </ItemGroup>
   <ItemGroup>

Duplicati/UnitTest/SVNCheckoutsTest.cs

@@ -74,6 +74,10 @@ namespace Duplicati.UnitTest
         /// <param name="target">The target destination for the backups</param>
         public static void RunTest(string[] folders, Dictionary<string, string> options, string target)
         {
+            // Don't spam the console
+            if (Log.CurrentLog != null)
+                Log.LogLevel = LogMessageType.Error;
+            
             string tempdir = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "tempdir");
             string logfilename = System.IO.Path.Combine(tempdir, string.Format("unittest-{0}.log", Library.Utility.Utility.SerializeDateTime(DateTime.Now)));
 

Duplicati/UnitTest/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="NUnit" version="3.5.0" targetFramework="net45" />
+  <package id="NUnit" version="3.9.0" targetFramework="net45" />
 </packages>