Missing commits

Duplicati/Library/Main/Database/LocalDatabase.cs

@@ -283,8 +283,9 @@ namespace Duplicati.Library.Main.Database
             return RemoteVolumeEntry.Empty;
         }
 
-        public IEnumerable<KeyValuePair<string, RemoteVolumeState>> DuplicateRemoteVolumes()
+        public IEnumerable<KeyValuePair<string, RemoteVolumeState>> DuplicateRemoteVolumes(System.Data.IDbTransaction transaction = null)
         {
+            m_selectduplicateRemoteVolumesCommand.Transaction = transaction;
             foreach(var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerable(null))
             {
                 yield return new KeyValuePair<string, RemoteVolumeState>(
@@ -455,9 +456,9 @@ namespace Duplicati.Library.Main.Database
                 cmd.ExecuteNonQuery("VACUUM");
         }
 
-        public long RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state)
+        public long RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state, System.Data.IDbTransaction transaction)
         {
-            return RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), null);
+            return RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), transaction);
         }
 
         public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, System.Data.IDbTransaction transaction)

Duplicati/Library/Main/Database/LocalDeleteDatabase.cs

@@ -20,6 +20,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.IO;
+using System.Threading.Tasks;
 
 namespace Duplicati.Library.Main.Database
 {
@@ -164,7 +165,7 @@ namespace Duplicati.Library.Main.Database
             IEnumerable<string> CompactableVolumes { get; }
             bool ShouldReclaim { get; }
             bool ShouldCompact { get; }
-            void ReportCompactData(ILogWriter log); 
+            Task ReportCompactDataAsync(bool verbose);
         }
         
         private class CompactReport : ICompactReport
@@ -203,28 +204,31 @@ namespace Duplicati.Library.Main.Database
                 m_smallspace = m_smallvolumes.Select(x => x.CompressedSize).Sum();
                 m_smallvolumecount = m_smallvolumes.Count();
             }
-            
-            public void ReportCompactData(ILogWriter log)
-            {
-                var wastepercentage = ((m_wastedspace / (float)m_fullsize) * 100);
-                if (log.VerboseOutput)
+
+			public async Task ReportCompactDataAsync(bool verbose)
+			{
+                using (var log = new Operation.Common.LogWrapper())
                 {
-                    log.AddVerboseMessage(string.Format("Found {0} fully deletable volume(s)", m_deletablevolumes));
-                    log.AddVerboseMessage(string.Format("Found {0} small volumes(s) with a total size of {1}", m_smallvolumes.Count(), Library.Utility.Utility.FormatSizeString(m_smallspace)));
-                    log.AddVerboseMessage(string.Format("Found {0} volume(s) with a total of {1:F2}% wasted space ({2} of {3})", m_wastevolumes.Count(), wastepercentage, Library.Utility.Utility.FormatSizeString(m_wastedspace), Library.Utility.Utility.FormatSizeString(m_fullsize)));
-                }
-                
-                if (m_deletablevolumes > 0)
-                    log.AddMessage(string.Format("Compacting because there are {0} fully deletable volume(s)", m_deletablevolumes));
-                else if (wastepercentage >= m_wastethreshold && m_wastevolumes.Count() >= 2)
-                    log.AddMessage(string.Format("Compacting because there is {0:F2}% wasted space and the limit is {1}%", wastepercentage, m_wastethreshold));
-                else if (m_smallspace > m_volsize)
-                    log.AddMessage(string.Format("Compacting because there are {0} in small volumes and the volume size is {1}", Library.Utility.Utility.FormatSizeString(m_smallspace), Library.Utility.Utility.FormatSizeString(m_volsize)));
-                else if (m_smallvolumecount > m_maxsmallfilecount)
-                    log.AddMessage(string.Format("Compacting because there are {0} small volumes and the maximum is {1}", m_smallvolumecount, m_maxsmallfilecount));
-                else
-                    log.AddMessage("Compacting not required");
-            }
+                    var wastepercentage = ((m_wastedspace / (float)m_fullsize) * 100);
+                    if (verbose)
+                    {
+                        await log.WriteVerboseAsync(string.Format("Found {0} fully deletable volume(s)", m_deletablevolumes));
+                        await log.WriteVerboseAsync(string.Format("Found {0} small volumes(s) with a total size of {1}", m_smallvolumes.Count(), Library.Utility.Utility.FormatSizeString(m_smallspace)));
+                        await log.WriteVerboseAsync(string.Format("Found {0} volume(s) with a total of {1:F2}% wasted space ({2} of {3})", m_wastevolumes.Count(), wastepercentage, Library.Utility.Utility.FormatSizeString(m_wastedspace), Library.Utility.Utility.FormatSizeString(m_fullsize)));
+                    }
+
+                    if (m_deletablevolumes > 0)
+                        await log.WriteInformationAsync(string.Format("Compacting because there are {0} fully deletable volume(s)", m_deletablevolumes));
+                    else if (wastepercentage >= m_wastethreshold && m_wastevolumes.Count() >= 2)
+                        await log.WriteInformationAsync(string.Format("Compacting because there is {0:F2}% wasted space and the limit is {1}%", wastepercentage, m_wastethreshold));
+                    else if (m_smallspace > m_volsize)
+                        await log.WriteInformationAsync(string.Format("Compacting because there are {0} in small volumes and the volume size is {1}", Library.Utility.Utility.FormatSizeString(m_smallspace), Library.Utility.Utility.FormatSizeString(m_volsize)));
+                    else if (m_smallvolumecount > m_maxsmallfilecount)
+                        await log.WriteInformationAsync(string.Format("Compacting because there are {0} small volumes and the maximum is {1}", m_smallvolumecount, m_maxsmallfilecount));
+                    else
+                        await log.WriteInformationAsync("Compacting not required");
+                }
+			}
             
             public bool ShouldReclaim
             {
@@ -304,7 +308,8 @@ namespace Duplicati.Library.Main.Database
         /// <summary>
         /// Builds a lookup table to enable faster response to block queries
         /// </summary>
-        /// <param name="volumename">The name of the volume to prepare for</param>
+        /// <param name="options">The options passed to the query</param>
+        /// <param name="transaction">The transaction to work within</param>
         public IBlockQuery CreateBlockQueryHelper(Options options, System.Data.IDbTransaction transaction)
         {
             return new BlockQuery(m_connection, options, transaction);

Duplicati/Library/Main/Database/LocalRecreateDatabase.cs

@@ -116,6 +116,11 @@ namespace Duplicati.Library.Main.Database
            ""FullIndex""
 ";
 
+        public LocalRecreateDatabase(string dbpath, Options options)
+            : this(new LocalDatabase(dbpath, "Recreate", true), options)
+        {
+        }
+
         public LocalRecreateDatabase(LocalDatabase parentdb, Options options)
             : base(parentdb)
         {
@@ -505,9 +510,9 @@ namespace Duplicati.Library.Main.Database
             }
         }
 
-        public IEnumerable<IRemoteVolume> GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize)
+        public IEnumerable<IRemoteVolume> GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, System.Data.IDbTransaction tr = null)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(tr))
             {
                 var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume""";
             

Duplicati/Library/Main/Database/LocalRepairDatabase.cs

@@ -18,6 +18,7 @@
 using System;
 using System.Linq;
 using System.Collections.Generic;
+using System.Data;
 
 namespace Duplicati.Library.Main.Database
 {
@@ -29,9 +30,9 @@ namespace Duplicati.Library.Main.Database
         
         }
         
-        public long GetFilesetIdFromRemotename(string filelist)
+        public long GetFilesetIdFromRemotename(string filelist, IDbTransaction transaction)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(transaction))
             {
                 var filesetid = cmd.ExecuteScalarInt64(@"SELECT ""Fileset"".""ID"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = ?", -1, filelist);
                 if (filesetid == -1)
@@ -112,9 +113,9 @@ namespace Duplicati.Library.Main.Database
             }
         }
         
-        public IEnumerable<IRemoteVolume> GetBlockVolumesFromIndexName(string name)
+        public IEnumerable<IRemoteVolume> GetBlockVolumesFromIndexName(string name, IDbTransaction transaction)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(transaction))
                 foreach(var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?))", name))
                     yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2));
         }
@@ -332,7 +333,7 @@ namespace Duplicati.Library.Main.Database
 
         }
 
-        public void FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize)
+        public bool FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize, IDbTransaction transaction)
         {
             var blockhasher = Library.Utility.HashAlgorithmHelper.Create(blockhashalgorithm);
             var hashsize = blockhasher.HashSize / 8;
@@ -344,18 +345,17 @@ namespace Duplicati.Library.Main.Database
             var sql = string.Format(@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual""", blocksize / hashsize);
             var countsql = @"SELECT COUNT(*) FROM (" + sql + @")";
 
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(transaction))
             {
                 var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0);
                 if (itemswithnoblocklisthash != 0)
                 {
                     m_result.AddMessage(string.Format("Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash));
-                    using(var c2 = m_connection.CreateCommand(tr))
-                    using(var c3 = m_connection.CreateCommand(tr))
-                    using(var c4 = m_connection.CreateCommand(tr))
-                    using(var c5 = m_connection.CreateCommand(tr))
-                    using(var c6 = m_connection.CreateCommand(tr))
+                    using(var c2 = m_connection.CreateCommand(transaction))
+                    using(var c3 = m_connection.CreateCommand(transaction))
+                    using(var c4 = m_connection.CreateCommand(transaction))
+                    using(var c5 = m_connection.CreateCommand(transaction))
+                    using(var c6 = m_connection.CreateCommand(transaction))
                     {
                         c3.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?, ?, ?) ";
                         c4.CommandText = @"SELECT COUNT(*) FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?";
@@ -432,16 +432,17 @@ namespace Duplicati.Library.Main.Database
                         throw new Duplicati.Library.Interface.UserInformationException(string.Format("Failed to repair, after repair {0} blocklisthashes were missing", itemswithnoblocklisthash));
 
                     m_result.AddMessage("Missing blocklisthashes repaired succesfully");
-                    tr.Commit();
+                    return true;
                 }
             }
 
+            return false;
+
         }
 
-        public void FixDuplicateBlocklistHashes(long blocksize, long hashsize)
+        public bool FixDuplicateBlocklistHashes(long blocksize, long hashsize, IDbTransaction transaction)
         {
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(transaction))
             {
                 var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1";
 
@@ -478,7 +479,7 @@ namespace Duplicati.Library.Main.Database
 
                     try
                     {
-                        VerifyConsistency(blocksize, hashsize, true, tr);
+                        VerifyConsistency(blocksize, hashsize, true, transaction);
                     }
                     catch(Exception ex)
                     {
@@ -486,15 +487,16 @@ namespace Duplicati.Library.Main.Database
                     }
 
                     m_result.AddMessage("Duplicate blocklisthashes repaired succesfully");
-                    tr.Commit();
+                    return true;
                 }
             }
+
+            return false;
         }
 
-        public void CheckAllBlocksAreInVolume(string filename, IEnumerable<KeyValuePair<string, long>> blocks)
+        public void CheckAllBlocksAreInVolume(string filename, IEnumerable<KeyValuePair<string, long>> blocks, IDbTransaction transaction)
         {
-            using(var tr = m_connection.BeginTransaction())
-            using(var cmd = m_connection.CreateCommand(tr))
+            using(var cmd = m_connection.CreateCommand(transaction))
             {
                 var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
 
@@ -519,9 +521,9 @@ namespace Duplicati.Library.Main.Database
             }
         }
 
-        public void CheckBlocklistCorrect(string hash, long length, IEnumerable<string> blocklist, long blocksize, long blockhashlength)
+        public void CheckBlocklistCorrect(string hash, long length, IEnumerable<string> blocklist, long blocksize, long blockhashlength, IDbTransaction transaction)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using(var cmd = m_connection.CreateCommand(transaction))
             {
                 var query = string.Format(@"
 SELECT 

Duplicati/Library/Main/Duplicati.Library.Main.csproj

@@ -146,6 +146,17 @@
     <Compile Include="Operation\ListBrokenFilesHandler.cs" />
     <Compile Include="Operation\PurgeBrokenFilesHandler.cs" />
     <Compile Include="Operation\Backup\RecreateMissingIndexFiles.cs" />
+    <Compile Include="Operation\Delete\DeleteDatabase.cs" />
+    <Compile Include="Operation\Delete\DeleteStatsCollector.cs" />
+    <Compile Include="Operation\Compact\CompactStatsCollector.cs" />
+    <Compile Include="Operation\Compact\CompactDatabase.cs" />
+    <Compile Include="Operation\Common\PrefetchDownloader.cs" />
+    <Compile Include="Operation\Recreate\RecreateDatabase.cs" />
+    <Compile Include="Operation\Recreate\RecreateStatsCollector.cs" />
+    <Compile Include="Operation\Repair\RepairDatabase.cs" />
+    <Compile Include="Operation\Repair\RepairStatsCollector.cs" />
+    <Compile Include="Operation\Test\TestDatabase.cs" />
+    <Compile Include="Operation\Test\TestStatsCollector.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Utility\Duplicati.Library.Utility.csproj">
@@ -216,5 +227,10 @@
   <ItemGroup>
     <Folder Include="Operation\Backup\" />
     <Folder Include="Operation\Common\" />
+    <Folder Include="Operation\Delete\" />
+    <Folder Include="Operation\Compact\" />
+    <Folder Include="Operation\Recreate\" />
+    <Folder Include="Operation\Repair\" />
+    <Folder Include="Operation\Test\" />
   </ItemGroup>
 </Project>

Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs

@@ -189,16 +189,6 @@ namespace Duplicati.Library.Main.Operation.Backup
             return RunOnMain(() => m_database.GetIncompleteFilesets(m_transaction).OrderBy(x => x.Value).ToArray());
         }
 
-        public Task<IEnumerable<KeyValuePair<long, DateTime>>> GetFilesetTimesAsync()
-        {
-            return RunOnMain(() => m_database.FilesetTimes);
-        }
-
-        public Task<long> CreateFilesetAsync(long volumeID, DateTime fileTime)
-        {
-            return RunOnMain(() => m_database.CreateFileset(volumeID, fileTime, m_transaction));
-        }
-
         public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid)
         {
             return RunOnMain(() => m_database.LinkFilesetToVolume(filesetid, volumeid, m_transaction));
@@ -219,11 +209,6 @@ namespace Duplicati.Library.Main.Operation.Backup
             return RunOnMain(() => m_database.UpdateChangeStatistics(result, m_transaction));
         }
 
-        public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists)
-        {
-            return RunOnMain(() => m_database.VerifyConsistency(blocksize, blockhashSize, verifyfilelists, m_transaction));
-        }
-
         public Task RemoveRemoteVolumeAsync(string remoteFilename)
         {
             return RunOnMain(() => m_database.RemoveRemoteVolume(remoteFilename, m_transaction));

Duplicati/Library/Main/Operation/BackupHandler.cs

@@ -62,31 +62,32 @@ namespace Duplicati.Library.Main.Operation
                 (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotWindows(sources, options.RawOptions);
         }
 
-        private void PreBackupVerify(BackendManager backend, string protectedfile)
+        private async Task PreBackupVerifyAsync(Common.BackendHandler backend, Backup.BackupDatabase db, Backup.BackupStatsCollector stats, string protectedfile)
         {
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupVerify);
             using(new Logging.Timer("PreBackupVerify"))
+            using(var log = new Common.LogWrapper())
             {
                 try
                 {
                     if (m_options.NoBackendverification)
                     {
-                        FilelistProcessor.VerifyLocalList(backend, m_options, m_database, m_result.BackendWriter);
+                        await FilelistProcessor.VerifyLocalListAsync(backend, m_database);
                         UpdateStorageStatsFromDatabase();
                     }
                     else
-                        FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter, protectedfile);
+                        await FilelistProcessor.VerifyRemoteListAsync(backend, m_options, db, stats, protectedfile);
                 }
                 catch (Exception ex)
                 {
                     if (m_options.AutoCleanup)
                     {
-                        m_result.AddWarning("Backend verification failed, attempting automatic cleanup", ex);
+                        await log.WriteWarningAsync("Backend verification failed, attempting automatic cleanup", ex);
                         m_result.RepairResults = new RepairResults(m_result);
-                        new RepairHandler(backend.BackendUrl, m_options, (RepairResults)m_result.RepairResults).Run();
+                        await RepairHandler.RunAsync(backend, m_options, m_result.RepairResults);
 
-                        m_result.AddMessage("Backend cleanup finished, retrying verification");
-                        FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter);
+                        await log.WriteInformationAsync("Backend cleanup finished, retrying verification");
+                        await FilelistProcessor.VerifyRemoteListAsync(backend, m_options, db, stats);
                     }
                     else
                         throw;
@@ -139,24 +140,30 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        private void CompactIfRequired(BackendManager backend, long lastVolumeSize)
+        private async Task CompactIfRequiredAsync(Common.BackendHandler backend, long lastVolumeSize)
         {
             var currentIsSmall = lastVolumeSize != -1 && lastVolumeSize <= m_options.SmallFileSize;
 
             if (m_options.KeepTime.Ticks > 0 || m_options.KeepVersions != 0)
             {
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete);
-                m_result.DeleteResults = new DeleteResults(m_result);
-                using(var db = new LocalDeleteDatabase(m_database))
-                    new DeleteHandler(backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall, backend);
+                var dr = new DeleteResults(m_result);
+                m_result.DeleteResults = dr;
+                using(var cdb = new LocalDeleteDatabase(m_database))
+                using(var db = new Delete.DeleteDatabase(cdb, m_options))
+                using(var ds = new Delete.DeleteStatsCollector(dr))
+                    await DeleteHandler.DoRunAsync(db, true, currentIsSmall, backend, m_options, dr, ds);
 
             }
             else if (currentIsSmall && !m_options.NoAutoCompact)
             {
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact);
-                m_result.CompactResults = new CompactResults(m_result);
+                var cr = new CompactResults(m_result);
+                m_result.CompactResults = cr;
                 using(var db = new LocalDeleteDatabase(m_database))
-                    new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref m_transaction, backend);
+                using(var cdb = new Compact.CompactDatabase(db, m_options))
+                using(var cstat = new Compact.CompactStatsCollector(cr))
+                    await CompactHandler.DoCompactAsync(cdb, true, backend, m_options, cstat, m_result.TaskReader);
             }
         }
 
@@ -300,7 +307,6 @@ namespace Duplicati.Library.Main.Operation
                 {
                     // Setup runners and instances here
                     using(var db = new Backup.BackupDatabase(m_database, m_options))
-                    using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
                     using(var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
                     using(var stats = new Backup.BackupStatsCollector(m_result))
                     using(var bk = new Common.BackendHandler(m_options, m_backendurl, db, stats, m_result.TaskReader))
@@ -338,7 +344,7 @@ namespace Duplicati.Library.Main.Operation
 
                                 // TODO: Rewrite to using the uploader process, or the BackendHandler interface
                                 // Do a remote verification, unless disabled
-                                PreBackupVerify(backend, lasttempfilelist);
+                                await PreBackupVerifyAsync(bk, db, stats, lasttempfilelist);
 
                                 // If the previous backup was interrupted, send a synthetic list
                                 await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskReader, lasttempfilelist, lasttempfileid);
@@ -399,7 +405,7 @@ namespace Duplicati.Library.Main.Operation
                         m_transaction = m_database.BeginTransaction();
     		                                        
                         if (await m_result.TaskReader.ProgressAsync)
-                            CompactIfRequired(backend, lastVolumeSize);
+                            await CompactIfRequiredAsync(bk, lastVolumeSize);
 
                         if (m_options.UploadVerificationFile && await m_result.TaskReader.ProgressAsync)
                         {
@@ -435,7 +441,6 @@ namespace Duplicati.Library.Main.Operation
                         {
                             m_database.Vacuum();
                         }
-                        return;
                     }
                 }
                 catch (Exception ex)

Duplicati/Library/Main/Operation/Common/BackendHandler.cs

@@ -237,17 +237,21 @@ namespace Duplicati.Library.Main.Operation.Common
                     if (createIndexFile != null)
                     {
                         var ix = await createIndexFile(fe.RemoteFilename);
-                        var indexFile = new FileEntryItem(BackendActionType.Put, ix.RemoteFilename);
-                        indexFile.SetLocalfilename(ix.LocalFilename);
+                        if (ix != null)
+                        {
+                            var indexFile = new FileEntryItem(BackendActionType.Put, ix.RemoteFilename);
+                            indexFile.SetLocalfilename(ix.LocalFilename);
 
-                        await m_database.UpdateRemoteVolumeAsync(indexFile.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                            await m_database.UpdateRemoteVolumeAsync(indexFile.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
 
-                        await DoWithRetry(indexFile, async () => {
-                            if (indexFile.IsRetry)
-                                await RenameFileAfterErrorAsync(indexFile);
+                            await DoWithRetry(indexFile, async () =>
+                            {
+                                if (indexFile.IsRetry)
+                                    await RenameFileAfterErrorAsync(indexFile);
 
-                            return await DoPut(indexFile);
-                        });
+                                return await DoPut(indexFile);
+                            });
+                        }
                     }
 
                     tcs.TrySetResult(true);
@@ -696,7 +700,33 @@ namespace Duplicati.Library.Main.Operation.Common
                 {
                 }
             }
-        }
+        }
+
+        /// <summary>
+        /// Helper method for ensuring that the queue is empty
+        /// </summary>
+        /// <returns>The async.</returns>
+        public Task ReadyAsync()
+        {
+            return RunOnMain(() => { });
+        }
+            
+
+        /// <summary>
+        /// Grabs the quota information from the backend if it supports it,
+        /// otherwise <c>null</c> is returned.
+        /// </summary>
+        /// <returns>The quota information.</returns>
+        public Task<IQuotaInfo> GetQuotaAsync()
+        {
+            return RunRetryOnMain(null, () => {                
+                var qb = m_backend as IQuotaEnabledBackend;
+                if (qb == null)
+                    return null;
+
+                return Task.FromResult(qb.Quota);
+            });
+        }
 
 		private string m_lastThrottleUploadValue = null;
 		private string m_lastThrottleDownloadValue = null;

Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs

@@ -46,6 +46,11 @@ namespace Duplicati.Library.Main.Operation.Common
             return RunOnMain(() => m_db.RegisterRemoteVolume(name, type, state, m_transaction));
         }
 
+        public Task<long> RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, long size, RemoteVolumeState state)
+        {
+            return RunOnMain(() => m_db.RegisterRemoteVolume(name, type, size, state, m_transaction));
+        }
+
         public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan))
         {
             return RunOnMain(() => m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, m_transaction));
@@ -124,10 +129,71 @@ namespace Duplicati.Library.Main.Operation.Common
         public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID)
         {
             return RunOnMain(() => m_db.AddIndexBlockLink(indexVolumeID, blockVolumeID, m_transaction));
+        }
+
+		public Task<IEnumerable<KeyValuePair<long, DateTime>>> GetFilesetTimesAsync()
+		{
+			return RunOnMain(() => m_db.FilesetTimes);
+		}
+
+        public Task UnlinkRemoteVolumeAsync(string name, RemoteVolumeState state)
+        {
+            return RunOnMain(() => m_db.UnlinkRemoteVolume(name, state, m_transaction));
+        }
+
+        public Task<IEnumerable<KeyValuePair<string, RemoteVolumeState>>> DuplicateRemoteVolumesAsync()
+        {
+			// TODO: How does the IEnumerable work with RunOnMain ?
+            return RunOnMain(() => m_db.DuplicateRemoteVolumes(m_transaction));
+        }
+
+        public Task<IEnumerable<RemoteVolumeEntry>> GetRemoteVolumesAsync()
+        {
+            return RunOnMain(() => m_db.GetRemoteVolumes(m_transaction));
+        }
+
+        public Task RemoveRemoteVolumesAsync(IEnumerable<string> names)
+        {
+            return RunOnMain(() => m_db.RemoveRemoteVolumes(names, m_transaction));
+        }
+
+		public Task WriteResultsAsync()
+		{
+			return RunOnMain(() => m_db.WriteResults());
+		}
+
+        public Task VacuumAsync()
+        {
+            return RunOnMain(() => m_db.Vacuum());
         }
 
 
-        protected override void Dispose(bool isDisposing)
+        // Shared with Backup
+
+        public Task<long> CreateFilesetAsync(long volumeID, DateTime fileTime)
+        {
+            return RunOnMain(() => m_db.CreateFileset(volumeID, fileTime, m_transaction));
+        }
+
+        public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists)
+        {
+            return RunOnMain(() => m_db.VerifyConsistency(blocksize, blockhashSize, verifyfilelists, m_transaction));
+        }
+
+        // Shared with Recreate/Repair
+
+        public Task UpdateOptionsFromDbAsync(Options options)
+        {
+            return RunOnMain(() => Utility.UpdateOptionsFromDb(m_db, options, m_transaction));
+        }
+
+        public Task VerifyParametersAsync(Options options)
+        {
+            return RunOnMain(() => Utility.VerifyParameters(m_db, options, m_transaction));
+        }
+
+
+		protected override void Dispose(bool isDisposing)
         {
             base.Dispose(isDisposing);
             if (m_transaction != null)

Duplicati/Library/Main/Operation/Common/PrefetchDownloader.cs

@@ -0,0 +1,105 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using CoCoL;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Database;
+
+namespace Duplicati.Library.Main.Operation.Common
+{
+    internal class PrefetchDownloader : IDisposable
+    {
+        private readonly IWriteChannelEnd<IAsyncDownloadedFile> m_source;
+        private readonly IReadChannelEnd<IAsyncDownloadedFile> m_result;
+
+        private class AsyncDownloadedFile : IAsyncDownloadedFile
+        {
+            public Library.Utility.TempFile TempFile { get; set;  }
+			public string Name { get; set; }
+            public string Hash { get; set; }
+            public long Size { get; set; }
+        }
+
+        public PrefetchDownloader(IEnumerable<IRemoteVolume> volumes, BackendHandler backend, int volumesahead = 1)
+        {
+            var channel = ChannelManager.CreateChannel<IAsyncDownloadedFile>(buffersize: volumesahead);
+            m_source = channel.AsWriteOnly();
+            m_result = channel.AsReadOnly();
+
+            Start(volumes, backend);
+		}
+
+        private void Start(IEnumerable<IRemoteVolume> volumes, BackendHandler backend)
+        {
+            AutomationExtensions.RunTask(
+                m_source,
+                async _ =>
+                {
+                    foreach(var n in volumes)
+                    {
+                        // Prepare to dispose it
+                        using (var tf = await backend.GetFileAsync(n.Name, n.Size, n.Hash))
+                        {
+                            await m_source.WriteAsync(new AsyncDownloadedFile()
+                            {
+                                TempFile = Library.Utility.TempFile.WrapExistingFile(tf),
+                                Name = n.Name,
+                                Hash = n.Hash,
+                                Size = n.Size
+                            });
+
+                            // If we sent it on, then do not delete it
+                            tf.Protected = true;
+                        }
+                    }
+
+                    await m_source.WriteAsync(null);
+                }
+            );
+        }
+
+        public Task<IAsyncDownloadedFile> GetNextAsync()
+        {
+            return m_result.ReadAsync();
+        }
+
+        public Task StopAsync()
+        {
+            m_source.Dispose();
+
+            return AutomationExtensions.RunTask(
+                m_result,
+                async _ =>
+                {
+                    while (true)
+                    {
+                        var tmp = await m_result.ReadAsync();
+                        if (tmp != null && tmp.TempFile != null)
+                            tmp.TempFile.Dispose();
+                    }
+				}
+            );
+            
+        }
+
+        public void Dispose()
+        {
+            m_source.Dispose();
+        }
+    }
+}

Duplicati/Library/Main/Operation/Common/StatsCollector.cs

@@ -45,7 +45,18 @@ namespace Duplicati.Library.Main.Operation.Common
         {
             if (m_bw.BackendProgressUpdater != null)
                 m_bw.BackendProgressUpdater.SetBlocking(isBlocked);
-        }
-    }
+        }
+
+        public long UnknownFileSize { set { m_bw.UnknownFileSize = value; } }
+        public long UnknownFileCount { set { m_bw.UnknownFileCount = value; } }
+		public long KnownFileCount { set { m_bw.KnownFileCount = value; } }
+		public long KnownFileSize { set { m_bw.KnownFileSize = value; } }
+		public DateTime LastBackupDate { set { m_bw.LastBackupDate = value; } }
+		public long BackupListCount { set { m_bw.BackupListCount = value; } }
+		public long TotalQuotaSpace { set { m_bw.TotalQuotaSpace = value; } }
+		public long FreeQuotaSpace { set { m_bw.FreeQuotaSpace = value; } }
+		public long AssignedQuotaSpace { set { m_bw.AssignedQuotaSpace = value; } }
+
+	}
 }
 

Duplicati/Library/Main/Operation/Compact/CompactDatabase.cs

@@ -0,0 +1,59 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Database;
+using Duplicati.Library.Main.Operation.Common;
+using static Duplicati.Library.Main.Database.LocalDeleteDatabase;
+
+namespace Duplicati.Library.Main.Operation.Compact
+{
+    internal class CompactDatabase : Delete.DeleteDatabase
+    {
+        public CompactDatabase(LocalDeleteDatabase database, Options options)
+            : base(database, options)
+        {
+            m_database = database;
+        }
+
+        public Task<ICompactReport> GetCompactReportAsync(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount)
+        {
+            return RunOnMain(() => m_database.GetCompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, m_transaction));
+        }
+
+        public Task<IEnumerable<IRemoteVolume>> GetDeletableVolumesAsync(IEnumerable<IRemoteVolume> deleteableVolumes)
+        {
+            return RunOnMain(() => m_database.GetDeletableVolumes(deleteableVolumes, m_transaction));
+        }
+
+        public Task RemoveRemoteVolumeAsync(string name)
+        {
+            return RunOnMain(() => m_db.RemoveRemoteVolume(name, m_transaction));
+        }
+
+        public Task MoveBlockToNewVolumeAsync(string hash, long size, long volumeID)
+        {
+            return RunOnMain(() => m_database.MoveBlockToNewVolume(hash, size, volumeID, m_transaction));
+        }
+
+        public Task<IBlockQuery> CreateBlockQueryHelperAsync(Options options)
+        {
+            return RunOnMain(() => m_database.CreateBlockQueryHelper(options, m_transaction));
+        }
+    }
+}

Duplicati/Library/Main/Operation/Compact/CompactStatsCollector.cs

@@ -0,0 +1,89 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Compact
+{
+    internal class CompactStatsCollector : StatsCollector
+    {
+        private CompactResults m_res;
+
+		public CompactStatsCollector(CompactResults res)
+            : base(res.BackendWriter)
+        {
+			m_res = res;
+		}
+
+		public Task SetResultAsync(long DeletedFileCount, long DownloadedFileCount, long UploadedFileCount, long DeletedFileSize, long DownloadedFileSize, long UploadedFileSize, bool Dryrun)
+		{
+			return RunOnMain(() => {
+				m_res.DeletedFileCount = DeletedFileCount;
+				m_res.DownloadedFileCount = DownloadedFileCount;
+				m_res.UploadedFileCount = UploadedFileCount;
+				m_res.DeletedFileSize = DeletedFileSize;
+				m_res.DownloadedFileSize = DownloadedFileSize;
+				m_res.UploadedFileSize = UploadedFileSize;
+				m_res.Dryrun = Dryrun;
+			});
+		}
+
+        public Task SetEndTimeAsync()
+        {
+            return RunOnMain(() =>
+            {
+                m_res.EndTime = DateTime.UtcNow;
+            });
+        }
+
+		public long DeletedFileCount
+		{
+			get { return m_res.DeletedFileCount; }
+		}
+		public long DownloadedFileCount
+		{
+			get { return m_res.DownloadedFileCount; }
+		}
+		public long UploadedFileCount
+		{
+			get { return m_res.UploadedFileCount; }
+		}
+		public long DeletedFileSize
+		{
+			get { return m_res.DeletedFileSize; }
+		}
+		public long DownloadedFileSize
+		{
+			get { return m_res.DownloadedFileSize; }
+		}
+		public long UploadedFileSize
+		{
+			get { return m_res.UploadedFileSize; }
+		}
+		public bool Dryrun
+		{
+			get { return m_res.Dryrun; }
+		}
+        public DateTime EndTime
+		{
+            get { return m_res.EndTime; }
+		}
+
+	}
+}

Duplicati/Library/Main/Operation/CompactHandler.cs

@@ -21,110 +21,95 @@ using System.Collections.Generic;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Main.Volumes;
 using System.Text;
+using System.Threading.Tasks;
+using CoCoL;
 
 namespace Duplicati.Library.Main.Operation
 {
-    internal class CompactHandler
+    internal static class CompactHandler
     {
-        protected string m_backendurl;
-        protected Options m_options;
-        protected CompactResults m_result;
-        
-        public CompactHandler(string backend, Options options, CompactResults result)
+        public static void Run(string backendurl, Options options, CompactResults result)
         {
-            m_backendurl = backend;
-            m_options = options;
-            m_result = result;
+            RunAsync(backendurl, options, result).WaitForTaskOrThrow();
         }
-        
-        public virtual void Run()
+
+        public static async Task RunAsync(string backendurl, Options options, CompactResults result)
         {
-            if (!System.IO.File.Exists(m_options.Dbpath))
-                throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
-            
-            using(var db = new LocalDeleteDatabase(m_options.Dbpath, "Compact"))
-            {
-                var tr = db.BeginTransaction();
-                try
-                {
-                    m_result.SetDatabase(db);
-                    Utility.UpdateOptionsFromDb(db, m_options);
-                    Utility.VerifyParameters(db, m_options);
-                    
-                    var changed = DoCompact(db, false, ref tr, null);
-                    
-                    if (changed && m_options.UploadVerificationFile)
-                        FilelistProcessor.UploadVerificationFile(m_backendurl, m_options, m_result.BackendWriter, db, null);
-                    
-                    if (!m_options.Dryrun)
-                    {
-                        using(new Logging.Timer("CommitCompact"))
-                            tr.Commit();
-                        if (changed)
-                        {
-                            db.WriteResults();
-                            if (m_options.AutoVacuum)
-                            {
-                                db.Vacuum();
-                            }
-                        }
-                    }
-                    else
-                        tr.Rollback();
-                    
-                    tr = null;
-                }
-                finally
-                {
-                    if (tr != null)
-                        try { tr.Rollback(); }
-                        catch { }
-                }
-            }
-        }
-        
-        internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction, BackendManager sharedBackend)
-        {
-            var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction);
-            report.ReportCompactData(m_result);
-            
-            if (report.ShouldReclaim || report.ShouldCompact)
+            if (!System.IO.File.Exists(options.Dbpath))
+                throw new Exception(string.Format("Database file does not exist: {0}", options.Dbpath));
+
+
+            using (new IsolatedChannelScope())
             {
-				// Workaround where we allow a running backendmanager to be used
-				using(var bk = sharedBackend == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
+				var lh = Common.LogHandler.Run(result);
+				using (var coredb = new LocalDeleteDatabase(options.Dbpath, "Compact"))
+                using (var db = new Compact.CompactDatabase(coredb, options))
+                using (var stats = new Compact.CompactStatsCollector(result))
+                using (var backend = new Common.BackendHandler(options, backendurl, db, stats, result.TaskReader))
+                // Keep a reference to this channel to avoid shutdown
+                using (var logtarget = ChannelManager.GetChannel(Common.Channels.LogChannel.ForWrite))
                 {
-                    var backend = bk ?? sharedBackend;
-                    if (!hasVerifiedBackend && !m_options.NoBackendverification)
-                        FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
+                    result.SetDatabase(coredb);
+                    Utility.UpdateOptionsFromDb(coredb, options);
+                    Utility.VerifyParameters(coredb, options);
+
+                    var changed = await DoCompactAsync(db, false, backend, options, stats, result.TaskReader);
+
+                    if (changed && options.UploadVerificationFile)
+                        await FilelistProcessor.UploadVerificationFileAsync(backend, options, db);
+
+                    await db.WriteResultsAsync();
+                    await db.CommitTransactionAsync("CommitCompact", false);
+                    if (changed && !options.Dryrun && options.AutoVacuum)
+                        await db.VacuumAsync();
+                }
+
+				await lh;
+			}
+
+        }
         
-                    BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
-                    newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
-    
+        internal static async Task<bool> DoCompactAsync(Compact.CompactDatabase db, bool hasVerifiedBackend, Common.BackendHandler backend, Options options, Compact.CompactStatsCollector stat, Common.ITaskReader taskreader)
+        {
+            using (var log = new Common.LogWrapper())
+            {
+                var report = await db.GetCompactReportAsync(options.VolumeSize, options.Threshold, options.SmallFileSize, options.SmallFileMaxCount);
+                await report.ReportCompactDataAsync(options.Verbose);
+
+                if (report.ShouldReclaim || report.ShouldCompact)
+                {
+                    if (!hasVerifiedBackend && !options.NoBackendverification)
+                        await FilelistProcessor.VerifyRemoteListAsync(backend, options, db, stat);
+
+                    BlockVolumeWriter newvol = new BlockVolumeWriter(options);
+                    newvol.VolumeID = await db.RegisterRemoteVolumeAsync(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
+
                     IndexVolumeWriter newvolindex = null;
-                    if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
+                    if (options.IndexfilePolicy != Options.IndexFileStrategy.None)
                     {
-                        newvolindex = new IndexVolumeWriter(m_options);
-                        newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
-                        db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
+                        newvolindex = new IndexVolumeWriter(options);
+                        newvolindex.VolumeID = await db.RegisterRemoteVolumeAsync(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);
+                        await db.AddIndexBlockLinkAsync(newvolindex.VolumeID, newvol.VolumeID);
                         newvolindex.StartVolume(newvol.RemoteFilename);
                     }
-                    
+
                     long blocksInVolume = 0;
                     long discardedBlocks = 0;
                     long discardedSize = 0;
-                    byte[] buffer = new byte[m_options.Blocksize];
-                    var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
-                    
+                    byte[] buffer = new byte[options.Blocksize];
+                    var remoteList = (await db.GetRemoteVolumesAsync()).Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
+
                     //These are for bookkeeping
                     var uploadedVolumes = new List<KeyValuePair<string, long>>();
                     var deletedVolumes = new List<KeyValuePair<string, long>>();
                     var downloadedVolumes = new List<KeyValuePair<string, long>>();
-                    
+
                     //We start by deleting unused volumes to save space before uploading new stuff
                     var fullyDeleteable = (from v in remoteList
                                            where report.DeleteableVolumes.Contains(v.Name)
-                                           select (IRemoteVolume)v).ToList();                    
-                    deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable, ref transaction));
+                                           select (IRemoteVolume)v).ToList();
+
+                    deletedVolumes.AddRange(await DoDeleteAsync(db, backend, fullyDeleteable, options));
 
                     // This list is used to pick up unused volumes,
                     // so they can be deleted once the upload of the
@@ -136,188 +121,193 @@ namespace Duplicati.Library.Main.Operation
                         var volumesToDownload = (from v in remoteList
                                                  where report.CompactableVolumes.Contains(v.Name)
                                                  select (IRemoteVolume)v).ToList();
-                        
-                        using(var q = db.CreateBlockQueryHelper(m_options, transaction))
+
+                        using (var q = await db.CreateBlockQueryHelperAsync(options))
+                        using(var pre = new Common.PrefetchDownloader(volumesToDownload, backend))
                         {
-                            foreach(var entry in new AsyncDownloader(volumesToDownload, backend))
-                            using(var tmpfile = entry.TempFile)
-                            {
-                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                            IAsyncDownloadedFile entry;
+
+                            while((entry = await pre.GetNextAsync()) != null)
+                                using (var tmpfile = entry.TempFile)
                                 {
-                                    backend.WaitForComplete(db, transaction);
-                                    return false;
-                                }
-                                    
-                                downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
-                                var inst = VolumeBase.ParseFilename(entry.Name);
-                                using(var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
-                                {
-                                    foreach(var e in f.Blocks)
+                                    if (!await taskreader.ProgressAsync)
                                     {
-                                        if (q.UseBlock(e.Key, e.Value, transaction))
+                                        await pre.StopAsync();
+                                        return false;
+                                    }
+
+                                    downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
+                                    var inst = VolumeBase.ParseFilename(entry.Name);
+                                    using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, options))
+                                    {
+                                        foreach (var e in f.Blocks)
                                         {
-                                            //TODO: How do we get the compression hint? Reverse query for filename in db?
-                                            var s = f.ReadBlock(e.Key, buffer);
-                                            if (s != e.Value)
-                                                throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
-                                                
-                                            newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
-                                            if (newvolindex != null)
-                                                newvolindex.AddBlock(e.Key, e.Value);
-                                                
-                                            db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID, transaction);
-                                            blocksInVolume++;
-                                            
-                                            if (newvol.Filesize > m_options.VolumeSize)
+                                            if (await q.UseBlockAsync(e.Key, e.Value))
                                             {
-                                                uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, newvol.Filesize));
+                                                //TODO: How do we get the compression hint? Reverse query for filename in db?
+                                                var s = f.ReadBlock(e.Key, buffer);
+                                                if (s != e.Value)
+                                                    throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
+
+                                                newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
                                                 if (newvolindex != null)
-                                                    uploadedVolumes.Add(new KeyValuePair<string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
-    
-                                                if (!m_options.Dryrun)
-                                                    backend.Put(newvol, newvolindex);
-                                                else
-                                                    m_result.AddDryrunMessage(string.Format("Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)));
-                                                
-                                                
-                                                newvol = new BlockVolumeWriter(m_options);
-                                                newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
-                
-                                                if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
+                                                    newvolindex.AddBlock(e.Key, e.Value);
+
+                                                await db.MoveBlockToNewVolumeAsync(e.Key, e.Value, newvol.VolumeID);
+                                                blocksInVolume++;
+
+                                                if (newvol.Filesize > options.VolumeSize)
                                                 {
-                                                    newvolindex = new IndexVolumeWriter(m_options);
-                                                    newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
-                                                    db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
-                                                    newvolindex.StartVolume(newvol.RemoteFilename);
+                                                    uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, newvol.Filesize));
+                                                    if (newvolindex != null)
+                                                        uploadedVolumes.Add(new KeyValuePair<string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
+
+                                                    if (!options.Dryrun)
+                                                        await backend.UploadFileAsync(newvol, (arg) => Task.FromResult(newvolindex));
+                                                    else
+                                                        await log.WriteDryRunAsync(string.Format("Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)));
+
+
+                                                    newvol = new BlockVolumeWriter(options);
+                                                    newvol.VolumeID = await db.RegisterRemoteVolumeAsync(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
+
+                                                    if (options.IndexfilePolicy != Options.IndexFileStrategy.None)
+                                                    {
+                                                        newvolindex = new IndexVolumeWriter(options);
+                                                        newvolindex.VolumeID = await db.RegisterRemoteVolumeAsync(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);
+                                                        await db.AddIndexBlockLinkAsync(newvolindex.VolumeID, newvol.VolumeID);
+                                                        newvolindex.StartVolume(newvol.RemoteFilename);
+                                                    }
+
+                                                    blocksInVolume = 0;
+
+                                                    //After we upload this volume, we can delete all previous encountered volumes
+                                                    deletedVolumes.AddRange(await DoDeleteAsync(db, backend, deleteableVolumes, options));
+                                                    deleteableVolumes = new List<IRemoteVolume>();
                                                 }
-                                                
-                                                blocksInVolume = 0;
-                                                
-                                                //After we upload this volume, we can delete all previous encountered volumes
-                                                deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
-                                                deleteableVolumes = new List<IRemoteVolume>();
+                                            }
+                                            else
+                                            {
+                                                discardedBlocks++;
+                                                discardedSize += e.Value;
                                             }
                                         }
-                                        else
-                                        {
-                                            discardedBlocks++;
-                                            discardedSize += e.Value;
-                                        }
                                     }
+
+                                    deleteableVolumes.Add(entry);
                                 }
-    
-                                deleteableVolumes.Add(entry);
-                            }
-                            
+
                             if (blocksInVolume > 0)
                             {
                                 uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, newvol.Filesize));
                                 if (newvolindex != null)
                                     uploadedVolumes.Add(new KeyValuePair<string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
-                                if (!m_options.Dryrun)
-                                    backend.Put(newvol, newvolindex);
+                                if (!options.Dryrun)
+                                    await backend.UploadFileAsync(newvol, arg => Task.FromResult(newvolindex));
                                 else
-                                    m_result.AddDryrunMessage(string.Format("Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)));
+                                    await log.WriteDryRunAsync(string.Format("Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)));
                             }
                             else
                             {
-                                db.RemoveRemoteVolume(newvol.RemoteFilename, transaction);
+                                await db.RemoveRemoteVolumeAsync(newvol.RemoteFilename);
                                 if (newvolindex != null)
                                 {
-                                    db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction);
+                                    await db.RemoveRemoteVolumeAsync(newvolindex.RemoteFilename);
                                     newvolindex.FinishVolume(null, 0);
                                 }
                             }
                         }
                     }
-                    
-                    deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
-                                        
-                    var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
-                    var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
-                    var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value);
-                    
-                    m_result.DeletedFileCount = deletedVolumes.Count;
-                    m_result.DownloadedFileCount = downloadedVolumes.Count;
-                    m_result.UploadedFileCount = uploadedVolumes.Count;
-                    m_result.DeletedFileSize = deletedSize;
-                    m_result.DownloadedFileSize = downloadSize;
-                    m_result.UploadedFileSize = uploadSize;
-                    m_result.Dryrun = m_options.Dryrun;
-                    
-                    if (m_result.Dryrun)
+
+                    deletedVolumes.AddRange(await DoDeleteAsync(db, backend, deleteableVolumes, options));
+
+                    var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
+                    var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
+                    var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
+
+                    await stat.SetResultAsync(
+                        deletedVolumes.Count,
+                        downloadedVolumes.Count,
+                        uploadedVolumes.Count,
+                        deletedSize,
+                        downloadSize,
+                        uploadSize,
+                        options.Dryrun);
+
+                    if (stat.Dryrun)
                     {
                         if (downloadedVolumes.Count == 0)
-                            m_result.AddDryrunMessage(string.Format("Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)));
+                            await log.WriteDryRunAsync(string.Format("Would delete {0} files, which would reduce storage by {1}", stat.DeletedFileCount, Library.Utility.Utility.FormatSizeString(stat.DeletedFileSize)));
                         else
-                            m_result.AddDryrunMessage(string.Format("Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}", 
-                                                                    m_result.DownloadedFileCount, 
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize), 
-                                                                    m_result.DeletedFileCount, 
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount, 
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), 
-                                                                    m_result.DeletedFileCount - m_result.UploadedFileCount, 
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)));
+                            await log.WriteDryRunAsync(string.Format("Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}",
+                                                                    stat.DownloadedFileCount,
+                                                                    Library.Utility.Utility.FormatSizeString(stat.DownloadedFileSize),
+                                                                    stat.DeletedFileCount,
+                                                                    Library.Utility.Utility.FormatSizeString(stat.DeletedFileSize), stat.UploadedFileCount,
+                                                                    Library.Utility.Utility.FormatSizeString(stat.UploadedFileSize),
+                                                                    stat.DeletedFileCount - stat.UploadedFileCount,
+                                                                    Library.Utility.Utility.FormatSizeString(stat.DeletedFileSize - stat.UploadedFileSize)));
                     }
                     else
                     {
-                        if (m_result.DownloadedFileCount == 0)
-                            m_result.AddMessage(string.Format("Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)));
+                        if (stat.DownloadedFileCount == 0)
+                            await log.WriteInformationAsync(string.Format("Deleted {0} files, which reduced storage by {1}", stat.DeletedFileCount, Library.Utility.Utility.FormatSizeString(stat.DeletedFileSize)));
                         else
-                            m_result.AddMessage(string.Format("Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}", 
-                                                              m_result.DownloadedFileCount, 
-                                                              Library.Utility.Utility.FormatSizeString(downloadSize), 
-                                                              m_result.DeletedFileCount, 
-                                                              Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), 
-                                                              m_result.UploadedFileCount, 
-                                                              Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), 
-                                                              m_result.DeletedFileCount - m_result.UploadedFileCount, 
-                                                              Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)));
+                            await log.WriteInformationAsync(string.Format("Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}",
+                                                              stat.DownloadedFileCount,
+                                                              Library.Utility.Utility.FormatSizeString(stat.DownloadedFileSize),
+                                                              stat.DeletedFileCount,
+                                                              Library.Utility.Utility.FormatSizeString(stat.DeletedFileSize),
+                                                              stat.UploadedFileCount,
+                                                              Library.Utility.Utility.FormatSizeString(stat.UploadedFileSize),
+                                                              stat.DeletedFileCount - stat.UploadedFileCount,
+                                                              Library.Utility.Utility.FormatSizeString(stat.DeletedFileSize - stat.UploadedFileSize)));
                     }
-                            
-                    backend.WaitForComplete(db, transaction);
-                }
 
-                m_result.EndTime = DateTime.UtcNow;
-                return (m_result.DeletedFileCount + m_result.UploadedFileCount) > 0;
-            }
-            else
-            {
-                m_result.EndTime = DateTime.UtcNow;
-                return false;
+                    await stat.SetEndTimeAsync();
+                    return (stat.DeletedFileCount + stat.UploadedFileCount) > 0;
+                }
+                else
+                {
+					await stat.SetEndTimeAsync();
+					return false;
+                }
             }
         }
 
-        private IEnumerable<KeyValuePair<string, long>> DoDelete(LocalDeleteDatabase db, BackendManager backend, IEnumerable<IRemoteVolume> deleteableVolumes, ref System.Data.IDbTransaction transaction)
+        private static async Task<IEnumerable<KeyValuePair<string, long>>> DoDeleteAsync(Compact.CompactDatabase db, Common.BackendHandler backend, IEnumerable<IRemoteVolume> deleteableVolumes, Options options)
         {
             // Mark all volumes as disposable
             foreach(var f in deleteableVolumes)
-                db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
+                await db.UpdateRemoteVolumeAsync(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash);
 
             // Before we commit the current state, make sure the backend has caught up
-            backend.WaitForEmpty(db, transaction);
+            await backend.ReadyAsync();
 
-            if (!m_options.Dryrun)
-            {
-                transaction.Commit();
-                transaction = db.BeginTransaction();
-            }
+            // Sync the database before we actually delete stuff
+            await db.CommitTransactionAsync("PrepareForDelete");
 
-            return PerformDelete(backend, db.GetDeletableVolumes(deleteableVolumes, transaction));
+            return await PerformDeleteAsync(backend, await db.GetDeletableVolumesAsync(deleteableVolumes), options);
         }
             
         
-        private IEnumerable<KeyValuePair<string, long>> PerformDelete(BackendManager backend, IEnumerable<IRemoteVolume> list)
+        private static async Task<IEnumerable<KeyValuePair<string, long>>> PerformDeleteAsync(Common.BackendHandler backend, IEnumerable<IRemoteVolume> list, Options options)
         {
-            foreach(var f in list)
+            var res = new List<KeyValuePair<string, long>>();
+            using (var log = new Common.LogWrapper())
             {
-                if (!m_options.Dryrun)
-                    backend.Delete(f.Name, f.Size);
-                else
-                    m_result.AddDryrunMessage(string.Format("Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)));
+                foreach (var f in list)
+                {
+                    if (!options.Dryrun)
+                        await backend.DeleteFileAsync(f.Name);
+                    else
+                        await log.WriteDryRunAsync(string.Format("Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)));
 
-                yield return new KeyValuePair<string, long>(f.Name, f.Size);
-            }               
+                    res.Add(new KeyValuePair<string, long>(f.Name, f.Size));
+                }
+            }
+
+            return res;
         }
     }
 }

Duplicati/Library/Main/Operation/Delete/DeleteDatabase.cs

@@ -0,0 +1,43 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Database;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Delete
+{
+    internal class DeleteDatabase : DatabaseCommon
+    {
+		protected LocalDeleteDatabase m_database;
+
+        public LocalDeleteDatabase BackingDatabase => m_database;
+
+		public DeleteDatabase(LocalDeleteDatabase database, Options options)
+            : base(database, options)
+        {
+			m_database = database;
+		}
+
+        public Task<IEnumerable<KeyValuePair<string, long>>> DropFilesetsFromTableAsync(DateTime[] filesets)
+		{
+			return RunOnMain(() => m_database.DropFilesetsFromTable(filesets, m_transaction));
+		}
+
+	}
+}

Duplicati/Library/Main/Operation/Delete/DeleteStatsCollector.cs

@@ -0,0 +1,46 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Delete
+{
+    internal class DeleteStatsCollector : StatsCollector
+    {
+        private DeleteResults m_res;
+
+		public DeleteStatsCollector(DeleteResults res)
+            : base(res.BackendWriter)
+        {
+			m_res = res;
+		}
+
+        public Task SetResultAsync(IEnumerable<Tuple<long, DateTime>> filesets, bool isDryRun)
+        {
+            var fs = filesets.ToList();
+            return RunOnMain(() => {
+                m_res.SetResults(
+                    fs,
+                    isDryRun
+                );
+            });
+        }		
+    }
+}

Duplicati/Library/Main/Operation/DeleteHandler.cs

@@ -20,142 +20,117 @@ using System.Linq;
 using System.Text;
 using System.Collections.Generic;
 using Duplicati.Library.Interface;
+using System.Threading.Tasks;
+using CoCoL;
 
 namespace Duplicati.Library.Main.Operation
 {
-    internal class DeleteHandler
-    {   
-        private DeleteResults m_result;
-        protected string m_backendurl;
-        protected Options m_options;
-    
-        public DeleteHandler(string backend, Options options, DeleteResults result)
+    internal static class DeleteHandler
+    {   
+		public static void Run(DeleteResults results, string backendurl, Options options)
         {
-            m_backendurl = backend;
-            m_options = options;
-            m_result = result;
+            RunAsync(results, backendurl, options).WaitForTaskOrThrow();
         }
 
-        public void Run()
+        public static async Task RunAsync(DeleteResults result, string backendurl, Options options)
         {
-            if (!System.IO.File.Exists(m_options.Dbpath))
-                throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath));
+            if (!System.IO.File.Exists(options.Dbpath))
+                throw new UserInformationException(string.Format("Database file does not exist: {0}", options.Dbpath));
 
-            using(var db = new Database.LocalDeleteDatabase(m_options.Dbpath, "Delete"))
-            {
-                var tr = db.BeginTransaction();
-                try
+            using (new IsolatedChannelScope())
+            {
+				var lh = Common.LogHandler.Run(result);
+				using (var dbcore = new Database.LocalDeleteDatabase(options.Dbpath, "Delete"))
+                using (var db = new Delete.DeleteDatabase(dbcore, options))
+                using (var stats = new Delete.DeleteStatsCollector(result))
+                using (var backend = new Common.BackendHandler(options, backendurl, db, stats, result.TaskReader))
+                // Keep a reference to this channel to avoid shutdown
+                using (var logtarget = ChannelManager.GetChannel(Common.Channels.LogChannel.ForWrite))
                 {
-                    m_result.SetDatabase(db);
-                    Utility.UpdateOptionsFromDb(db, m_options);
-                    Utility.VerifyParameters(db, m_options);
-                    
-                    DoRun(db, ref tr, false, false, null);
-                    
-                    if (!m_options.Dryrun)
-                    {
-                        using(new Logging.Timer("CommitDelete"))
-                            tr.Commit();
+                    result.SetDatabase(dbcore);
+                    Utility.UpdateOptionsFromDb(dbcore, options);
+                    Utility.VerifyParameters(dbcore, options);
 
-                        db.WriteResults();
-                    }
-                    else
-                        tr.Rollback();
+                    await DoRunAsync(db, false, false, backend, options, result, stats);
+                    await db.WriteResultsAsync();
+                    await db.CommitTransactionAsync("Finalize Delete operation", false);
+                }
+				await lh;
+			}
 
-                    tr = null;
-                }
-                finally
-                {
-                    if (tr != null)
-                        try { tr.Rollback(); }
-                        catch { }
-                }
-            }
         }
 
-        public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBacked, bool forceCompact, BackendManager sharedManager)
+        public static async Task DoRunAsync(Delete.DeleteDatabase db, bool hasVerifiedBacked, bool forceCompact, Common.BackendHandler backend, Options options, DeleteResults result, Delete.DeleteStatsCollector stats)
         {
-            // Workaround where we allow a running backendmanager to be used
-            using(var bk = sharedManager == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
+            using (var log = new Common.LogWrapper())
             {
-                var backend = bk ?? sharedManager;
+                // Workaround where we allow a running backendmanager to be used
+                if (!hasVerifiedBacked && !options.NoBackendverification)
+                    await FilelistProcessor.VerifyRemoteListAsync(backend, options, db, stats);
 
-                if (!hasVerifiedBacked && !m_options.NoBackendverification)
-                    FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter); 
-                
-                var filesetNumbers = db.FilesetTimes.Zip(Enumerable.Range(0, db.FilesetTimes.Count()), (a, b) => new Tuple<long, DateTime>(b, a.Value)).ToList();
-                var sets = db.FilesetTimes.Select(x => x.Value).ToArray();
-                var toDelete = m_options.GetFilesetsToDelete(sets);
+                var filesettimes = (await db.GetFilesetTimesAsync()).ToList();
 
-                if (!m_options.AllowFullRemoval && sets.Length == toDelete.Length)
+                var filesetNumbers = filesettimes.Zip(Enumerable.Range(0, filesettimes.Count), (a, b) => new Tuple<long, DateTime>(b, a.Value)).ToList();
+                var sets = filesettimes.Select(x => x.Value).ToArray();
+                var toDelete = options.GetFilesetsToDelete(sets);
+
+                if (!options.AllowFullRemoval && sets.Length == toDelete.Length)
                 {
-                    m_result.AddMessage(string.Format("Preventing removal of last fileset, use --{0} to allow removal ...", "allow-full-removal"));
+                    await log.WriteInformationAsync(string.Format("Preventing removal of last fileset, use --{0} to allow removal ...", "allow-full-removal"));
                     toDelete = toDelete.Skip(1).ToArray();
                 }
 
                 if (toDelete != null && toDelete.Length > 0)
-                    m_result.AddMessage(string.Format("Deleting {0} remote fileset(s) ...", toDelete.Length));
+                    await log.WriteInformationAsync(string.Format("Deleting {0} remote fileset(s) ...", toDelete.Length));
 
-                var lst = db.DropFilesetsFromTable(toDelete, transaction).ToArray();
-                foreach(var f in lst)
-                    db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction);
+                var lst = (await db.DropFilesetsFromTableAsync(toDelete)).ToArray();
+                foreach (var f in lst)
+                    await db.UpdateRemoteVolumeAsync(f.Key, RemoteVolumeState.Deleting, f.Value, null);
 
-                if (!m_options.Dryrun)
+                await db.CommitTransactionAsync("After fileset dropped");
+
+                foreach (var f in lst)
                 {
-                    transaction.Commit();
-                    transaction = db.BeginTransaction();
-                }
-
-                foreach(var f in lst)
-                {
-                    if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
-                    {
-                        backend.WaitForComplete(db, transaction);
+                    if (!await result.TaskReader.ProgressAsync)
                         return;
-                    }
 
-                    if (!m_options.Dryrun)
-                        backend.Delete(f.Key, f.Value);
+                    if (!options.Dryrun)
+                        await backend.DeleteFileAsync(f.Key);
                     else
-                        m_result.AddDryrunMessage(string.Format("Would delete remote fileset: {0}", f.Key));
+                        await log.WriteDryRunAsync(string.Format("Would delete remote fileset: {0}", f.Key));
                 }
 
-                if (sharedManager == null)
-                    backend.WaitForComplete(db, transaction);
-                else
-                    backend.WaitForEmpty(db, transaction);
-                
                 var count = lst.Length;
-                if (!m_options.Dryrun)
+                if (!options.Dryrun)
                 {
                     if (count == 0)
-                        m_result.AddMessage("No remote filesets were deleted");
+                        await log.WriteInformationAsync("No remote filesets were deleted");
                     else
-                        m_result.AddMessage(string.Format("Deleted {0} remote fileset(s)", count));
+                        await log.WriteInformationAsync(string.Format("Deleted {0} remote fileset(s)", count));
                 }
                 else
                 {
-                
                     if (count == 0)
-                        m_result.AddDryrunMessage("No remote filesets would be deleted");
+                        await log.WriteDryRunAsync("No remote filesets would be deleted");
                     else
-                        m_result.AddDryrunMessage(string.Format("{0} remote fileset(s) would be deleted", count));
+                       await log.WriteDryRunAsync(string.Format("{0} remote fileset(s) would be deleted", count));
 
-                    if (count > 0 && m_options.Dryrun)
-                        m_result.AddDryrunMessage("Remove --dry-run to actually delete files");
+                    if (count > 0 && options.Dryrun)
+                        await log.WriteDryRunAsync("Remove --dry-run to actually delete files");
                 }
-                
-                if (!m_options.NoAutoCompact && (forceCompact || (toDelete != null && toDelete.Length > 0)))
+
+                if (!options.NoAutoCompact && (forceCompact || (toDelete != null && toDelete.Length > 0)))
                 {
-                    m_result.CompactResults = new CompactResults(m_result);
-                    new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref transaction, sharedManager);
+                    var cr = new CompactResults(result);
+                    result.CompactResults = cr;
+                    using(var cs = new Compact.CompactStatsCollector(cr))
+                    using(var cdb = new Compact.CompactDatabase(db.BackingDatabase, options))
+                        await CompactHandler.DoCompactAsync(cdb, true, backend, options, cs, result.TaskReader);
                 }
-                
-                m_result.SetResults(
-                    from n in filesetNumbers
-                    where toDelete.Contains(n.Item2)
-                    select n, 
-                    m_options.Dryrun);
+
+                await stats.SetResultAsync(
+                    filesetNumbers.Where(x => toDelete.Contains(x.Item2)),
+                    options.Dryrun);
             }
         }
     }

Duplicati/Library/Main/Operation/FilelistProcessor.cs

@@ -19,11 +19,56 @@ using System;
 using Duplicati.Library.Main.Database;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace Duplicati.Library.Main.Operation
 {
     internal static class FilelistProcessor
-    {
+    {
+		/// <summary>
+		/// Helper method that verifies uploaded volumes and updates their state in the database.
+		/// Throws an error if there are issues with the remote storage
+		/// </summary>
+        /// <param name="backend">The backend instance to use</param>
+		/// <param name="database">The database to compare with</param>
+        public static async Task VerifyLocalListAsync(Common.BackendHandler backend, LocalDatabase database)
+		{
+            using (var log = new Common.LogWrapper())
+            {
+                var locallist = database.GetRemoteVolumes();
+                foreach (var i in locallist)
+                {
+                    switch (i.State)
+                    {
+                        case RemoteVolumeState.Uploaded:
+                        case RemoteVolumeState.Verified:
+                        case RemoteVolumeState.Deleted:
+                            break;
+
+                        case RemoteVolumeState.Temporary:
+                        case RemoteVolumeState.Deleting:
+                        case RemoteVolumeState.Uploading:
+                            await log.WriteInformationAsync(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
+                            try
+                            {
+                                await backend.DeleteFileAsync(i.Name, true);
+                            }
+                            catch (Exception ex)
+                            {
+                                await log.WriteWarningAsync(string.Format("Failed to erase file {0}, treating as deleted: {1}", i.Name, ex.Message), ex);
+                            }
+
+                            break;
+
+                        default:
+                            await log.WriteWarningAsync(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
+                            break;
+                    }
+
+                }
+            }
+		}
+
         /// <summary>
         /// Helper method that verifies uploaded volumes and updates their state in the database.
         /// Throws an error if there are issues with the remote storage
@@ -65,7 +110,71 @@ namespace Duplicati.Library.Main.Operation
 
                 backend.FlushDbMessages();
             }
-        }
+        }
+
+		/// <summary>
+		/// Helper method that verifies uploaded volumes and updates their state in the database.
+		/// Throws an error if there are issues with the remote storage
+		/// </summary>
+		/// <param name="backend">The backend instance to use</param>
+		/// <param name="options">The options used</param>
+		/// <param name="database">The database to compare with</param>
+		/// <param name="protectedfile">A filename that should be excempted for deletion</param>
+        public static async Task VerifyRemoteListAsync(Common.BackendHandler backend, Options options, Common.DatabaseCommon database, Common.StatsCollector stats, string protectedfile = null)
+		{
+            using (var log = new Common.LogWrapper())
+            {
+                var tp = await RemoteListAnalysisAsync(backend, options, database, stats, protectedfile);
+                long extraCount = 0;
+                long missingCount = 0;
+
+                foreach (var n in tp.ExtraVolumes)
+                {
+                    await log.WriteWarningAsync(string.Format("Extra unknown file: {0}", n.File.Name), null);
+                    extraCount++;
+                }
+
+                foreach (var n in tp.MissingVolumes)
+                {
+                    await log.WriteWarningAsync(string.Format("Missing file: {0}", n.Name), null);
+                    missingCount++;
+                }
+
+                if (extraCount > 0)
+                {
+                    var s = string.Format("Found {0} remote files that are not recorded in local storage, please run repair", extraCount);
+                    await log.WriteErrorAsync(s, null);
+                    throw new Duplicati.Library.Interface.UserInformationException(s);
+                }
+
+                var lookup = new HashSet<string>();
+                var doubles = new HashSet<string>();
+                foreach (var v in tp.ParsedVolumes)
+                {
+                    if (!lookup.Add(v.File.Name))
+                        doubles.Add(v.File.Name);
+                }
+
+                if (doubles.Count > 0)
+                {
+                    var s = string.Format("Found remote files reported as duplicates, either the backend module is broken or you need to manually remove the extra copies.\nThe following files were found multiple times: {0}", string.Join(", ", doubles));
+                    await log.WriteErrorAsync(s, null);
+                    throw new Duplicati.Library.Interface.UserInformationException(s);
+                }
+
+                if (missingCount > 0)
+                {
+                    string s;
+                    if (!tp.BackupPrefixes.Contains(options.Prefix) && tp.BackupPrefixes.Length > 0)
+                        s = string.Format("Found {0} files that are missing from the remote storage, and no files with the backup prefix {1}, but found the following backup prefixes: {2}", missingCount, options.Prefix, string.Join(", ", tp.BackupPrefixes));
+                    else
+                        s = string.Format("Found {0} files that are missing from the remote storage, please run repair", missingCount);
+
+                    await log.WriteErrorAsync(s, null);
+                    throw new Duplicati.Library.Interface.UserInformationException(s);
+                }
+            }
+		}
 
         /// <summary>
         /// Helper method that verifies uploaded volumes and updates their state in the database.
@@ -150,8 +259,20 @@ namespace Duplicati.Library.Main.Operation
         {
             var s = new Newtonsoft.Json.JsonSerializer();
             s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).Cast<IRemoteVolume>().ToArray());
-        }
-        
+        }
+
+		/// <summary>
+		/// Creates a temporary verification file.
+		/// </summary>
+		/// <returns>The verification file.</returns>
+		/// <param name="db">The database instance</param>
+		/// <param name="stream">The stream to write to</param>
+        public static async Task CreateVerificationFileAsync(Common.DatabaseCommon db, System.IO.StreamWriter stream)
+		{
+			var s = new Newtonsoft.Json.JsonSerializer();
+            s.Serialize(stream, (await db.GetRemoteVolumesAsync()).Where(x => x.State != RemoteVolumeState.Temporary).Cast<IRemoteVolume>().ToArray());
+		}
+
         /// <summary>
         /// Uploads the verification file.
         /// </summary>
@@ -179,36 +300,260 @@ namespace Duplicati.Library.Main.Operation
                     backend.WaitForComplete(db, transaction);
                 }
             }
-        }
+        }
+
+		/// <summary>
+		/// Uploads the verification file.
+		/// </summary>
+		/// <param name="backend">The backend to use</param>
+		/// <param name="options">The options to use</param>
+		/// <param name="db">The database to read from</param>
+        public static async Task UploadVerificationFileAsync(Common.BackendHandler backend, Options options, Common.DatabaseCommon db)
+		{
+			using (var tempfile = new Library.Utility.TempFile())
+			{
+				var remotename = options.Prefix + "-verification.json";
+				using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8))
+					await FilelistProcessor.CreateVerificationFileAsync(db, stream);
+
+				if (options.Dryrun)
+				{
+                    using(var log = new Common.LogWrapper())
+                        await log.WriteDryRunAsync(string.Format("Would upload verification file: {0}, size: {1}", remotename, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(tempfile).Length)));
+				}
+				else
+				{
+                    await backend.PutUnencryptedAsync(remotename, tempfile);
+				}
+			}
+		}
+
+		/// <summary>
+		/// Helper method that verifies uploaded volumes and updates their state in the database.
+		/// Throws an error if there are issues with the remote storage
+		/// </summary>
+		/// <param name="backend">The backend instance to use</param>
+		/// <param name="options">The options used</param>
+		/// <param name="database">The database to compare with</param>
+		/// <param name="protectedfile">A filename that should be excempted for deletion</param>
+        public static async Task<RemoteAnalysisResult> RemoteListAnalysisAsync(Common.BackendHandler backend, Options options, Common.DatabaseCommon database, Common.StatsCollector stats, string protectedfile)
+		{
+            using (var log = new Common.LogWrapper())
+            {
+                var rawlist = await backend.ListFilesAsync();
+                var lookup = new Dictionary<string, Volumes.IParsedVolume>();
+                protectedfile = protectedfile ?? string.Empty;
 
-        /// <summary>
-        /// Helper method that verifies uploaded volumes and updates their state in the database.
-        /// Throws an error if there are issues with the remote storage
-        /// </summary>
-        /// <param name="backend">The backend instance to use</param>
-        /// <param name="options">The options used</param>
-        /// <param name="database">The database to compare with</param>
-        /// <param name="protectedfile">A filename that should be excempted for deletion</param>
-        public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, string protectedfile)
+                var remotelist = new List<Volumes.IParsedVolume>();
+                var otherlist = new List<Volumes.IParsedVolume>();
+                var unknownlist = new List<Interface.IFileEntry>();
+
+                foreach (var n in rawlist)
+                {
+                    var p = Volumes.VolumeBase.ParseFilename(n);
+                    if (p == null)
+                    {
+                        unknownlist.Add(n);
+                    }
+                    else
+                    {
+                        if (p.Prefix != options.Prefix)
+                            otherlist.Add(p);
+                        else
+                            remotelist.Add(p);
+                    }
+                }
+
+                var filesets = (from n in remotelist
+                                where n.FileType == RemoteVolumeType.Files
+                                orderby n.Time descending
+                                select n).ToList();
+
+                stats.KnownFileCount = remotelist.Count;
+                stats.KnownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum();
+                stats.UnknownFileCount = unknownlist.Count;
+                stats.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum();
+                stats.BackupListCount = filesets.Count;
+                stats.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
+
+                // TODO: We should query through the backendmanager
+                var quota = await backend.GetQuotaAsync();
+                if (quota != null)
+                {
+                    stats.TotalQuotaSpace = quota.TotalQuotaSpace;
+                    stats.FreeQuotaSpace = quota.FreeQuotaSpace;
+                }
+
+                stats.AssignedQuotaSpace = options.QuotaSize;
+
+                foreach (var s in remotelist)
+                    lookup[s.File.Name] = s;
+
+                var missing = new List<RemoteVolumeEntry>();
+                var missingHash = new List<Tuple<long, RemoteVolumeEntry>>();
+                var cleanupRemovedRemoteVolumes = new HashSet<string>();
+
+                foreach (var e in await database.DuplicateRemoteVolumesAsync())
+                {
+                    if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary)
+                        await database.UnlinkRemoteVolumeAsync(e.Key, e.Value);
+                    else
+                        throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()));
+                }
+
+                var locallist = await database.GetRemoteVolumesAsync();
+                foreach (var i in locallist)
+                {
+                    Volumes.IParsedVolume r;
+                    var remoteFound = lookup.TryGetValue(i.Name, out r);
+                    var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
+
+                    lookup.Remove(i.Name);
+
+                    switch (i.State)
+                    {
+                        case RemoteVolumeState.Deleted:
+                            if (remoteFound)
+                                await log.WriteInformationAsync(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name));
+
+                            break;
+
+                        case RemoteVolumeState.Temporary:
+                        case RemoteVolumeState.Deleting:
+                            if (remoteFound)
+                            {
+                                await log.WriteInformationAsync(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name));
+                                await backend.DeleteFileAsync(i.Name, true);
+                            }
+                            else
+                            {
+                                if (i.DeleteGracePeriod > DateTime.UtcNow)
+                                {
+                                    await log.WriteInformationAsync(string.Format("keeping delete request for {0} until {1}", i.Name, i.DeleteGracePeriod.ToLocalTime()));
+                                }
+                                else
+                                {
+                                    if (string.Equals(i.Name, protectedfile) && i.State == RemoteVolumeState.Temporary)
+                                    {
+                                        await log.WriteInformationAsync(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
+                                    }
+                                    else
+                                    {
+                                        await log.WriteInformationAsync(string.Format("removing file listed as {0}: {1}", i.State, i.Name));
+                                        cleanupRemovedRemoteVolumes.Add(i.Name);
+                                    }
+                                }
+                            }
+                            break;
+                        case RemoteVolumeState.Uploading:
+                            if (remoteFound && correctSize && r.File.Size >= 0)
+                            {
+                                await log.WriteInformationAsync(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded));
+                                await database.UpdateRemoteVolumeAsync(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash);
+                            }
+                            else if (!remoteFound)
+                            {
+
+                                if (string.Equals(i.Name, protectedfile))
+                                {
+                                    await log.WriteInformationAsync(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
+                                    await database.UpdateRemoteVolumeAsync(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0));
+                                }
+                                else
+                                {
+                                    await log.WriteInformationAsync(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name));
+                                    cleanupRemovedRemoteVolumes.Add(i.Name);
+                                    await database.UpdateRemoteVolumeAsync(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2));
+                                }
+                            }
+                            else
+                            {
+                                if (string.Equals(i.Name, protectedfile))
+                                {
+                                    await log.WriteInformationAsync(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));
+                                }
+                                else
+                                {
+                                    await log.WriteInformationAsync(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name));
+                                    await backend.DeleteFileAsync(i.Name, true);
+                                }
+                            }
+                            break;
+
+                        case RemoteVolumeState.Uploaded:
+                            if (!remoteFound)
+                                missing.Add(i);
+                            else if (correctSize)
+                                await database.UpdateRemoteVolumeAsync(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
+                            else
+                                missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i));
+
+                            break;
+
+                        case RemoteVolumeState.Verified:
+                            if (!remoteFound)
+                                missing.Add(i);
+                            else if (!correctSize)
+                                missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i));
+
+                            break;
+
+                        default:
+                            await log.WriteWarningAsync(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null);
+                            break;
+                    }
+                }
+
+                // cleanup deleted volumes in DB en block
+                await database.RemoveRemoteVolumesAsync(cleanupRemovedRemoteVolumes);
+
+                foreach (var i in missingHash)
+                    await log.WriteWarningAsync(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null);
+
+                return new RemoteAnalysisResult()
+                {
+                    ParsedVolumes = remotelist,
+                    OtherVolumes = otherlist,
+                    ExtraVolumes = lookup.Values,
+                    MissingVolumes = missing,
+                    VerificationRequiredVolumes = missingHash.Select(x => x.Item2)
+                };
+            }
+        }
+
+		/// <summary>
+		/// Helper method that verifies uploaded volumes and updates their state in the database.
+		/// Throws an error if there are issues with the remote storage
+		/// </summary>
+		/// <param name="rawlist">The list of files returned from the backend</param>
+		/// <param name="options">The options used</param>
+		/// <param name="database">The database to compare with</param>
+		/// <param name="protectedfile">A filename that should be excempted for deletion</param>
+		private static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, string protectedfile)
         {
             var rawlist = backend.List();
             var lookup = new Dictionary<string, Volumes.IParsedVolume>();
             protectedfile = protectedfile ?? string.Empty;
 
-            var remotelist = (from n in rawlist
-                                       let p = Volumes.VolumeBase.ParseFilename(n)
-                                        where p != null && p.Prefix == options.Prefix
-                                       select p).ToList();
+            var remotelist = new List<Volumes.IParsedVolume>();
+			var otherlist = new List<Volumes.IParsedVolume>();
+			var unknownlist = new List<Interface.IFileEntry>();
 
-            var otherlist = (from n in rawlist
-                                let p = Volumes.VolumeBase.ParseFilename(n)
-                                where p != null && p.Prefix != options.Prefix
-                                select p).ToList();
-
-            var unknownlist = (from n in rawlist
-                                        let p = Volumes.VolumeBase.ParseFilename(n)
-                                        where p == null
-                                        select n).ToList();
+            foreach (var n in rawlist)
+            {
+                var p = Volumes.VolumeBase.ParseFilename(n);
+                if (p == null)
+                {
+                    unknownlist.Add(n);
+                }
+                else
+                {
+                    if (p.Prefix != options.Prefix)
+                        otherlist.Add(p);
+                    else
+                        remotelist.Add(p);
+                }
+            }
 
             var filesets = (from n in remotelist
                                      where n.FileType == RemoteVolumeType.Files orderby n.Time descending

Duplicati/Library/Main/Operation/Recreate/RecreateDatabase.cs

@@ -0,0 +1,105 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Database;
+
+namespace Duplicati.Library.Main.Operation.Recreate
+{
+    internal class RecreateDatabase : Common.DatabaseCommon
+    {
+        private readonly LocalRecreateDatabase m_database;
+        
+        public RecreateDatabase(LocalRecreateDatabase database, Options options)
+            : base(database, options)
+        {
+            m_database = database;
+        }
+
+        public Task<IEnumerable<long>> FindMatchingFilesetsAsync(DateTime restoretime, long[] versions)
+        {
+            return RunOnMain(() => m_db.FindMatchingFilesets(restoretime, versions));
+        }
+
+        public Task SetPartiallyRecreated(bool value)
+        {
+            return RunOnMain(() => m_db.PartiallyRecreated = value);
+        }
+
+        public Task SetRepairInProgressAsync(bool value)
+        {
+            return RunOnMain(() => m_db.RepairInProgress = value);
+		}
+
+        public Task<long> AddMetadatasetAsync(string hash, long size, IEnumerable<string> blocklisthashes, long expectedhashcount)
+        {
+            return RunOnMain(() => m_database.AddMetadataset(hash, size, blocklisthashes, expectedhashcount, m_transaction));
+        }
+
+        public Task AddDirectoryEntryAsync(long filesetid, string path, DateTime time, long metadataid)
+        {
+            return RunOnMain(() => m_database.AddDirectoryEntry(filesetid, path, time, metadataid, m_transaction));
+        }
+
+        public Task AddFileEntryAsync(long filesetid, string path, DateTime time, long blocksetid, long metadataid)
+        {
+            return RunOnMain(() => m_database.AddFileEntry(filesetid, path, time, blocksetid, metadataid, m_transaction));
+        }
+
+        public Task AddSymlinkEntryAsync(long filesetid, string path, DateTime time, long metadataid)
+        {
+            return RunOnMain(() => m_database.AddSymlinkEntry(filesetid, path, time, metadataid, m_transaction));
+        }
+
+        public Task<long> AddBlocksetAsync(string fullhash, long size, IEnumerable<string> blocklisthashes, long expectedblocklisthashes)
+        {
+            return RunOnMain(() => m_database.AddBlockset(fullhash, size, blocklisthashes, expectedblocklisthashes, m_transaction));
+        }
+
+        public Task AddSmallBlocksetLinkAsync(string filehash, string blockhash, long blocksize)
+        {
+            return RunOnMain(() => m_database.AddSmallBlocksetLink(filehash, blockhash, blocksize, m_transaction));
+        }
+
+        public Task<bool> UpdateBlockAsync(string hash, long size, long volumeID)
+        {
+            return RunOnMain(() => m_database.UpdateBlock(hash, size, volumeID, m_transaction));
+        }
+
+        public Task<bool> UpdateBlocksetAsync(string hash, IEnumerable<string> blocklisthashes)
+        {
+            return RunOnMain(() => m_database.UpdateBlockset(hash, blocklisthashes, m_transaction));
+        }
+
+        public Task FindMissingBlocklistHashesAsync(long hashsize, long blocksize)
+        {
+            return RunOnMain(() => m_database.FindMissingBlocklistHashes(hashsize, blocksize, m_transaction));
+        }
+
+        public Task<List<IRemoteVolume>> GetMissingBlockListVolumesAsync(int passNo, long blocksize, long hashsize)
+        {
+            return RunOnMain(() => m_database.GetMissingBlockListVolumes(passNo, blocksize, hashsize, m_transaction).ToList());
+        }
+
+        public Task<IEnumerable<string>> GetBlockListsAsync(long volumeid)
+        {
+            return RunOnMain(() => m_database.GetBlockLists(volumeid));
+        }
+    }
+}

Duplicati/Library/Main/Operation/Recreate/RecreateStatsCollector.cs

@@ -0,0 +1,53 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Recreate
+{
+    internal class RecreateStatsCollector : StatsCollector
+    {
+        private readonly RecreateDatabaseResults m_res;
+
+        public RecreateStatsCollector(RecreateDatabaseResults res)
+            : base(res.BackendWriter)
+        {
+            m_res = res;
+        }
+
+        public void UpdatePhase(OperationPhase phase)
+        {
+            m_res.OperationProgressUpdater.UpdatePhase(phase);
+        }
+
+        public void UpdateProgress(float pg)
+        {
+            m_res.OperationProgressUpdater.UpdateProgress(pg);
+        }
+
+        public Task SetEndTimeAsync()
+        {
+            return RunOnMain(() =>
+            {
+                m_res.EndTime = DateTime.UtcNow;
+            });
+        }
+    }
+}

Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs

@@ -1,28 +1,19 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
+using CoCoL;
 using Duplicati.Library.Interface;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Main.Volumes;
 
 namespace Duplicati.Library.Main.Operation
-{
-    internal class RecreateDatabaseHandler : IDisposable
+{
+    public delegate IEnumerable<KeyValuePair<long, IParsedVolume>> NumberedFilterFilelistDelegate(IEnumerable<IParsedVolume> filelist);
+    public delegate void BlockVolumePostProcessor(string volumename, BlockVolumeReader reader);
+
+    internal static class RecreateDatabaseHandler
     {
-        private string m_backendurl;
-        private Options m_options;
-        private RecreateDatabaseResults m_result;
-
-        public delegate IEnumerable<KeyValuePair<long, IParsedVolume>> NumberedFilterFilelistDelegate(IEnumerable<IParsedVolume> filelist);
-        public delegate void BlockVolumePostProcessor(string volumename,BlockVolumeReader reader);
-
-        public RecreateDatabaseHandler(string backendurl, Options options, RecreateDatabaseResults result)
-        {
-            m_options = options;
-            m_backendurl = backendurl;
-            m_result = result;
-        }
-
         /// <summary>
         /// Run the recreate procedure
         /// </summary>
@@ -30,16 +21,27 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
         /// <param name="filter">Filters the files in a filelist to prevent downloading unwanted data</param>
         /// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
-        public void Run(string path, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
+        public static async Task RunAsync(string backendurl, RecreateDatabaseResults result, Options options, string path, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
         {
             if (System.IO.File.Exists(path))
                 throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path));
 
-            using(var db = new LocalDatabase(path, "Recreate", true))
+            using (new IsolatedChannelScope())
             {
-                m_result.SetDatabase(db);
-                DoRun(db, false, filter, filelistfilter, blockprocessor);
-                db.WriteResults();
+                var lh = Common.LogHandler.Run(result);
+
+                using (var basedb = new LocalDatabase(path, "Recreate", true))
+                using (var coredb = new LocalRecreateDatabase(basedb, options))
+                using (var db = new Recreate.RecreateDatabase(coredb, options))
+                using (var stats = new Recreate.RecreateStatsCollector(result))
+                using (var backend = new Common.BackendHandler(options, backendurl, db, stats, result.TaskReader))
+                // Keep a reference to this channel to avoid shutdown
+                using (var logtarget = ChannelManager.GetChannel(Common.Channels.LogChannel.ForWrite))
+                {
+                    result.SetDatabase(coredb);
+                    await DoRunAsync(db, backend, options, false, stats, result.TaskReader, filter, filelistfilter, blockprocessor);
+                    await db.WriteResultsAsync();
+                }
             }
         }
 
@@ -49,59 +51,74 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
         /// <param name="filter">Filters the files in a filelist to prevent downloading unwanted data</param>
         /// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
-        public void RunUpdate(Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
+        public static async Task RunUpdateAsync(string backendurl, RecreateDatabaseResults result, Options options, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
         {
-            if (!m_options.RepairOnlyPaths)
+            if (!options.RepairOnlyPaths)
                 throw new UserInformationException(string.Format("Can only update with paths, try setting {0}", "--repair-only-paths"));
 
-            using(var db = new LocalDatabase(m_options.Dbpath, "Recreate", true))
+            using (new IsolatedChannelScope())
             {
-                m_result.SetDatabase(db);
+                var lh = Common.LogHandler.Run(result);
 
-                if (db.FindMatchingFilesets(m_options.Time, m_options.Version).Any())
-                    throw new UserInformationException(string.Format("The version(s) being updated to, already exists"));
+                using (var basedb = new LocalDatabase(options.Dbpath, "Recreate", true))
+                using (var coredb = new LocalRecreateDatabase(basedb, options))
+                using (var db = new Recreate.RecreateDatabase(coredb, options))
+                using (var stats = new Recreate.RecreateStatsCollector(result))
+                using (var backend = new Common.BackendHandler(options, backendurl, db, stats, result.TaskReader))
+                // Keep a reference to this channel to avoid shutdown
+                using (var logtarget = ChannelManager.GetChannel(Common.Channels.LogChannel.ForWrite))
+                {
+                    result.SetDatabase(coredb);
 
-                // Mark as incomplete
-                db.PartiallyRecreated = true;
+                    if ((await db.FindMatchingFilesetsAsync(options.Time, options.Version)).Any())
+                        throw new UserInformationException(string.Format("The version(s) being updated to, already exists"));
 
-                Utility.UpdateOptionsFromDb(db, m_options, null);
-                DoRun(db, true, filter, filelistfilter, blockprocessor);
-                db.WriteResults();
+                    // Mark as incomplete
+                    await db.SetPartiallyRecreated(true);
+
+                    Utility.UpdateOptionsFromDb(coredb, options, null);
+                    await DoRunAsync(db, backend, options, true, stats, result.TaskReader, filter, filelistfilter, blockprocessor);
+                    await db.WriteResultsAsync();
+                }
+
+                await lh;
             }
         }
 
         /// <summary>
         /// Run the recreate procedure
         /// </summary>
-        /// <param name="dbparent">The database to restore into</param>
+        /// <param name="db">The database to restore into</param>
         /// <param name="updating">True if this is an update call, false otherwise</param>
         /// <param name="filter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
         /// <param name="filelistfilter">Filters the files in a filelist to prevent downloading unwanted data</param>
         /// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
-        internal void DoRun(LocalDatabase dbparent, bool updating, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
+        internal static async Task DoRunAsync(Recreate.RecreateDatabase db, Common.BackendHandler backend, Options options, bool updating, Recreate.RecreateStatsCollector stats, Common.ITaskReader taskreader, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
         {
-            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running);
+            // Instance holder for the prefetcher
+            IAsyncDownloadedFile entry;
+
+            stats.UpdatePhase(OperationPhase.Recreate_Running);
 
             //We build a local database in steps.
-            using(var restoredb = new LocalRecreateDatabase(dbparent, m_options))
-            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb))
+            using (var log = new Common.LogWrapper())
             {
-                restoredb.RepairInProgress = true;
+                await db.SetRepairInProgressAsync(true);
 
                 var volumeIds = new Dictionary<string, long>();
 
-                var rawlist = backend.List();
-        
+                var rawlist = await backend.ListFilesAsync();
+
                 //First step is to examine the remote storage to see what
                 // kind of data we can find
                 var remotefiles =
                 (from x in rawlist
-                let n = VolumeBase.ParseFilename(x)
-                where
-                    n != null
-                        &&
-                    n.Prefix == m_options.Prefix
-                select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times
+                 let n = VolumeBase.ParseFilename(x)
+                 where
+                     n != null
+                         &&
+                     n.Prefix == options.Prefix
+                 select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times
 
                 if (remotefiles.Length == 0)
                 {
@@ -109,13 +126,13 @@ namespace Duplicati.Library.Main.Operation
                         throw new UserInformationException("No files were found at the remote location, perhaps the target url is incorrect?");
                     else
                     {
-                        var tmp = 
+                        var tmp =
                     (from x in rawlist
-                        let n = VolumeBase.ParseFilename(x)
-                    where
-                        n != null
-                    select n.Prefix).ToArray();
-                
+                     let n = VolumeBase.ParseFilename(x)
+                     where
+                         n != null
+                     select n.Prefix).ToArray();
+
                         var types = tmp.Distinct().ToArray();
                         if (tmp.Length == 0)
                             throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count));
@@ -136,7 +153,7 @@ namespace Duplicati.Library.Main.Operation
 
                 if (filelists.Count() <= 0)
                     throw new UserInformationException(string.Format("No filelists found on the remote destination"));
-                
+
                 if (filelistfilter != null)
                     filelists = filelistfilter(filelists).Select(x => x.Value).ToArray();
 
@@ -144,206 +161,208 @@ namespace Duplicati.Library.Main.Operation
                     throw new UserInformationException(string.Format("No filelists"));
 
                 // If we are updating, all files should be accounted for
-                foreach(var fl in remotefiles)
-                    volumeIds[fl.File.Name] = updating ? restoredb.GetRemoteVolumeID(fl.File.Name) : restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded);
+                foreach (var fl in remotefiles)
+                    volumeIds[fl.File.Name] = updating ? await db.GetRemoteVolumeIDAsync(fl.File.Name) : await db.RegisterRemoteVolumeAsync(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded);
 
                 var hasUpdatedOptions = false;
 
                 if (updating)
                 {
-                    Utility.UpdateOptionsFromDb(restoredb, m_options);
-                    Utility.VerifyParameters(restoredb, m_options);
+                    await db.UpdateOptionsFromDbAsync(options);
+                    await db.VerifyParametersAsync(options);
                 }
 
                 //Record all blocksets and files needed
-                using(var tr = restoredb.BeginTransaction())
+                var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList();
+                await log.WriteInformationAsync(string.Format("Rebuild database started, downloading {0} filelists", filelistWork.Count));
+
+                var progress = 0;
+
+                // Register the files we are working with, if not already updated
+                if (updating)
                 {
-                    var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList();
-                    m_result.AddMessage(string.Format("Rebuild database started, downloading {0} filelists", filelistWork.Count));
+                    foreach (var n in filelists)
+                        if (volumeIds[n.File.Name] == -1)
+                            volumeIds[n.File.Name] = await db.RegisterRemoteVolumeAsync(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Uploaded);
+                }
 
-                    var progress = 0;
+                var isFirstFilelist = true;
+                var blocksize = options.Blocksize;
+                var hashes_pr_block = blocksize / options.BlockhashSize;
 
-                    // Register the files we are working with, if not already updated
-                    if (updating)
-                    {
-                        foreach(var n in filelists)
-                            if (volumeIds[n.File.Name] == -1)
-                                volumeIds[n.File.Name] = restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), tr);
-                    }
-                                
-                    var isFirstFilelist = true;
-                    var blocksize = m_options.Blocksize;
-                    var hashes_pr_block = blocksize / m_options.BlockhashSize;
-
-                    foreach(var entry in new AsyncDownloader(filelistWork, backend))
+                using (var dn = new Common.PrefetchDownloader(filelistWork, backend))
+                    while ((entry = await dn.GetNextAsync()) != null)
                         try
                         {
-                            if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                            if (!await taskreader.ProgressAsync)
                             {
-                                backend.WaitForComplete(restoredb, null);
-                                m_result.EndTime = DateTime.UtcNow;
+                                await backend.ReadyAsync();
                                 return;
-                            }    
-                        
-                            progress++;
-                            if (filelistWork.Count == 1 && m_options.RepairOnlyPaths)
-                                m_result.OperationProgressUpdater.UpdateProgress(0.5f);
-                            else
-                                m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f));
+                            }
 
-                            using(var tmpfile = entry.TempFile)
+                            progress++;
+                            if (filelistWork.Count == 1 && options.RepairOnlyPaths)
+                                stats.UpdateProgress(0.5f);
+                            else
+                                stats.UpdateProgress(((float)progress / filelistWork.Count()) * (options.RepairOnlyPaths ? 1f : 0.2f));
+
+                            using (var tmpfile = entry.TempFile)
                             {
                                 isFirstFilelist = false;
 
                                 if (entry.Hash != null && entry.Size > 0)
-                                    restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash, tr);
+                                    await db.UpdateRemoteVolumeAsync(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash);
 
                                 var parsed = VolumeBase.ParseFilename(entry.Name);
 
-                                if (!hasUpdatedOptions && !updating) 
+                                if (!hasUpdatedOptions && !updating)
                                 {
-                                    VolumeReaderBase.UpdateOptionsFromManifest(parsed.CompressionModule, tmpfile, m_options);
+                                    VolumeReaderBase.UpdateOptionsFromManifest(parsed.CompressionModule, tmpfile, options);
                                     hasUpdatedOptions = true;
                                     // Recompute the cached sizes
-                                    blocksize = m_options.Blocksize;
-                                    hashes_pr_block = blocksize / m_options.BlockhashSize;
+                                    blocksize = options.Blocksize;
+                                    hashes_pr_block = blocksize / options.BlockhashSize;
                                 }
 
 
                                 // Create timestamped operations based on the file timestamp
-                                var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr);
-                                using(var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options))
-                                    foreach(var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path)))
+                                var filesetid = await db.CreateFilesetAsync(volumeIds[entry.Name], parsed.Time);
+                                using (var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, options))
+                                    foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path)))
                                     {
                                         try
                                         {
-                                            var expectedmetablocks = (fe.Metasize + blocksize - 1)  / blocksize;
+                                            var expectedmetablocks = (fe.Metasize + blocksize - 1) / blocksize;
                                             var expectedmetablocklisthashes = (expectedmetablocks + hashes_pr_block - 1) / hashes_pr_block;
                                             if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0;
 
                                             if (fe.Type == FilelistEntryType.Folder)
                                             {
-                                                var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
-                                                restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
+                                                var metadataid = await db.AddMetadatasetAsync(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes);
+                                                await db.AddDirectoryEntryAsync(filesetid, fe.Path, fe.Time, metadataid);
                                             }
                                             else if (fe.Type == FilelistEntryType.File)
                                             {
-                                                var expectedblocks = (fe.Size + blocksize - 1)  / blocksize;
+                                                var expectedblocks = (fe.Size + blocksize - 1) / blocksize;
                                                 var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block;
                                                 if (expectedblocks <= 1) expectedblocklisthashes = 0;
 
-                                                var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr);
-                                                var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
-                                                restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, metadataid, tr);
-                                                
+                                                var blocksetid = await db.AddBlocksetAsync(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes);
+                                                var metadataid = await db.AddMetadatasetAsync(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes);
+                                                await db.AddFileEntryAsync(filesetid, fe.Path, fe.Time, blocksetid, metadataid);
+
                                                 if (fe.Size <= blocksize)
                                                 {
                                                     if (!string.IsNullOrWhiteSpace(fe.Blockhash))
-                                                        restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, tr);
-                                                    else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
-                                                        restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, tr);
+                                                        await db.AddSmallBlocksetLinkAsync(fe.Hash, fe.Blockhash, fe.Blocksize);
+                                                    else if (options.BlockHashAlgorithm == options.FileHashAlgorithm)
+                                                        await db.AddSmallBlocksetLinkAsync(fe.Hash, fe.Hash, fe.Size);
                                                     else
-                                                        m_result.AddWarning(string.Format("No block hash found for file: {0}", fe.Path), null);
+                                                        await log.WriteWarningAsync(string.Format("No block hash found for file: {0}", fe.Path), null);
                                                 }
                                             }
                                             else if (fe.Type == FilelistEntryType.Symlink)
                                             {
-                                                var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
-                                                restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
+                                                var metadataid = await db.AddMetadatasetAsync(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes);
+                                                await db.AddSymlinkEntryAsync(filesetid, fe.Path, fe.Time, metadataid);
                                             }
                                             else
                                             {
-                                                m_result.AddWarning(string.Format("Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path), null);
+                                                await log.WriteWarningAsync(string.Format("Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path), null);
                                             }
 
                                             if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink))
                                             {
                                                 if (!string.IsNullOrWhiteSpace(fe.Metablockhash))
-                                                    restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, tr);
-                                                else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
-                                                    restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, tr);
+                                                    await db.AddSmallBlocksetLinkAsync(fe.Metahash, fe.Metablockhash, fe.Metasize);
+                                                else if (options.BlockHashAlgorithm == options.FileHashAlgorithm)
+                                                    await db.AddSmallBlocksetLinkAsync(fe.Metahash, fe.Metahash, fe.Metasize);
                                                 else
-                                                    m_result.AddWarning(string.Format("No block hash found for file metadata: {0}", fe.Path), null);                                                
+                                                    await log.WriteWarningAsync(string.Format("No block hash found for file metadata: {0}", fe.Path), null);
                                             }
                                         }
                                         catch (Exception ex)
                                         {
-                                            m_result.AddWarning(string.Format("Failed to process file-entry: {0}", fe.Path), ex);
+                                            await log.WriteWarningAsync(string.Format("Failed to process file-entry: {0}", fe.Path), ex);
                                         }
                                     }
                             }
                         }
                         catch (Exception ex)
                         {
-                            m_result.AddWarning(string.Format("Failed to process file: {0}", entry.Name), ex);
+                            await log.WriteWarningAsync(string.Format("Failed to process file: {0}", entry.Name), ex);
                             if (ex is System.Threading.ThreadAbortException)
                             {
-                                m_result.EndTime = DateTime.UtcNow;
+                                await stats.SetEndTimeAsync();
                                 throw;
                             }
 
                             if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException)
                             {
-                                m_result.EndTime = DateTime.UtcNow;
+                                await stats.SetEndTimeAsync();
                                 throw;
                             }
                         }
 
-                    //Make sure we write the config
-                    if (!updating)
-                        Utility.VerifyParameters(restoredb, m_options, tr);
+                //Make sure we write the config
+                if (!updating)
+                    await db.VerifyParametersAsync(options);
 
-                    using(new Logging.Timer("CommitUpdateFilesetFromRemote"))
-                        tr.Commit();
-                }
-            
-                if (!m_options.RepairOnlyPaths)
+                await db.CommitTransactionAsync("CommitUpdateFilesetFromRemote", true);
+
+                if (!options.RepairOnlyPaths)
                 {
-                    var hashalg = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
+                    var hashalg = Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm);
                     if (hashalg == null)
-                        throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
+                        throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm));
                     var hashsize = hashalg.HashSize / 8;
 
                     //Grab all index files, and update the block table
-                    using(var tr = restoredb.BeginTransaction())
-                    {
-                        var indexfiles = (
-                                         from n in remotefiles
-                                          where n.FileType == RemoteVolumeType.Index
-                                          select new RemoteVolume(n.File) as IRemoteVolume).ToList();
+                    var indexfiles = (
+                                     from n in remotefiles
+                                     where n.FileType == RemoteVolumeType.Index
+                                     select new RemoteVolume(n.File) as IRemoteVolume).ToList();
 
-                        m_result.AddMessage(string.Format("Filelists restored, downloading {0} index files", indexfiles.Count));
+                    await log.WriteInformationAsync(string.Format("Filelists restored, downloading {0} index files", indexfiles.Count));
 
-                        var progress = 0;
-                                    
-                        foreach(var sf in new AsyncDownloader(indexfiles, backend))
+                    progress = 0;
+
+                    using (var dn = new Common.PrefetchDownloader(indexfiles, backend))
+                        while ((entry = await dn.GetNextAsync()) != null)
                             try
                             {
-                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                                if (!await taskreader.ProgressAsync)
                                 {
-                                    backend.WaitForComplete(restoredb, null);
-                                    m_result.EndTime = DateTime.UtcNow;
+                                    await backend.ReadyAsync();
+                                    await stats.SetEndTimeAsync();
                                     return;
                                 }
 
                                 progress++;
-                                m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f);
+                                stats.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f);
 
-                                using(var tmpfile = sf.TempFile)
+                                using (var tmpfile = entry.TempFile)
                                 {
-                                    if (sf.Hash != null && sf.Size > 0)
-                                        restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash, tr);
-                
-                                    using(var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize))
+                                    if (entry.Hash != null && entry.Size > 0)
+                                        await db.UpdateRemoteVolumeAsync(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash);
+
+                                    using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(entry.Name), tmpfile, options, hashsize))
                                     {
-                                        foreach(var a in svr.Volumes)
+                                        foreach (var a in svr.Volumes)
                                         {
                                             var filename = a.Filename;
-                                            var volumeID = restoredb.GetRemoteVolumeID(filename);
+                                            var volumeID = await db.GetRemoteVolumeIDAsync(filename);
 
                                             // No such file
                                             if (volumeID < 0)
-                                                volumeID = ProbeForMatchingFilename(ref filename, restoredb);
+                                            {
+                                                var tp = await ProbeForMatchingFilenameAsync(filename, db, log);
+                                                if (tp != null)
+                                                {
+                                                    volumeID = tp.Item1;
+                                                    filename = tp.Item2;
+                                                }
+                                            }
 
                                             // Still broken, register a missing item
                                             if (volumeID < 0)
@@ -351,147 +370,144 @@ namespace Duplicati.Library.Main.Operation
                                                 var p = VolumeBase.ParseFilename(filename);
                                                 if (p == null)
                                                     throw new Exception(string.Format("Unable to parse filename: {0}", filename));
-                                                m_result.AddError(string.Format("Remote file referenced as {0}, but not found in list, registering a missing remote file", filename), null);
-                                                volumeID = restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Verified, tr);
+                                                await log.WriteWarningAsync(string.Format("Remote file referenced as {0}, but not found in list, registering a missing remote file", filename), null);
+                                                volumeID = await db.RegisterRemoteVolumeAsync(filename, p.FileType, RemoteVolumeState.Verified);
                                             }
-                                            
-                                            //Add all block/volume mappings
-                                            foreach(var b in a.Blocks)
-                                                restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr);
 
-                                            restoredb.UpdateRemoteVolume(filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr);
-                                            restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID, tr);
+                                            //Add all block/volume mappings
+                                            foreach (var b in a.Blocks)
+                                                await db.UpdateBlockAsync(b.Key, b.Value, volumeID);
+
+                                            await db.UpdateRemoteVolumeAsync(filename, RemoteVolumeState.Verified, a.Length, a.Hash);
+                                            await db.AddIndexBlockLinkAsync(await db.GetRemoteVolumeIDAsync(entry.Name), volumeID);
                                         }
-                                
+
                                         //If there are blocklists in the index file, update the blocklists
-                                        foreach(var b in svr.BlockLists)
-                                            restoredb.UpdateBlockset(b.Hash, b.Blocklist, tr);
+                                        foreach (var b in svr.BlockLists)
+                                            await db.UpdateBlocksetAsync(b.Hash, b.Blocklist);
                                     }
                                 }
                             }
                             catch (Exception ex)
                             {
                                 //Not fatal
-                                m_result.AddWarning(string.Format("Failed to process index file: {0}", sf.Name), ex);
+                                await log.WriteWarningAsync(string.Format("Failed to process index file: {0}", entry.Name), ex);
                                 if (ex is System.Threading.ThreadAbortException)
                                 {
-                                    m_result.EndTime = DateTime.UtcNow;
+                                    await stats.SetEndTimeAsync();
                                     throw;
                                 }
                             }
 
-                        using(new Logging.Timer("CommitRecreatedDb"))
-                            tr.Commit();
-                    
-                        // TODO: In some cases, we can avoid downloading all index files, 
-                        // if we are lucky and pick the right ones
-                    }
+                    await db.CommitTransactionAsync("CommitRecreatedDb", true);
+
+                    // TODO: In some cases, we can avoid downloading all index files, 
+                    // if we are lucky and pick the right ones
+
 
                     // We have now grabbed as much information as possible,
                     // if we are still missing data, we must now fetch block files
-                    restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, null);
-                
+                    await db.FindMissingBlocklistHashesAsync(hashsize, options.Blocksize);
+
                     //We do this in three passes
-                    for(var i = 0; i < 3; i++)
+                    for (var i = 0; i < 3; i++)
                     {
                         // Grab the list matching the pass type
-                        var lst = restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize).ToList();
+                        var lst = await db.GetMissingBlockListVolumesAsync(i, options.Blocksize, hashsize);
                         if (lst.Count > 0)
                         {
                             switch (i)
                             {
                                 case 0:
-                                    if (m_options.Verbose)
-                                        m_result.AddVerboseMessage("Processing required {0} blocklist volumes: {1}", lst.Count, string.Join(", ", lst.Select(x => x.Name)));
+                                    if (options.Verbose)
+                                        await log.WriteVerboseAsync("Processing required {0} blocklist volumes: {1}", lst.Count, string.Join(", ", lst.Select(x => x.Name)));
                                     else
-                                        m_result.AddMessage(string.Format("Processing required {0} blocklist volumes", lst.Count));
+                                        await log.WriteInformationAsync(string.Format("Processing required {0} blocklist volumes", lst.Count));
                                     break;
                                 case 1:
-                                    if (m_options.Verbose)
-                                        m_result.AddVerboseMessage("Probing {0} candidate blocklist volumes: {1}", lst.Count, string.Join(", ", lst.Select(x => x.Name)));
+                                    if (options.Verbose)
+                                        await log.WriteVerboseAsync("Probing {0} candidate blocklist volumes: {1}", lst.Count, string.Join(", ", lst.Select(x => x.Name)));
                                     else
-                                        m_result.AddMessage(string.Format("Probing {0} candidate blocklist volumes", lst.Count));
+                                        await log.WriteInformationAsync(string.Format("Probing {0} candidate blocklist volumes", lst.Count));
                                     break;
                                 default:
-                                    if (m_options.Verbose)
-                                        m_result.AddVerboseMessage("Processing all of the {0} volumes for blocklists: {1}", lst.Count, string.Join(", ", lst.Select(x => x.Name)));
+                                    if (options.Verbose)
+                                        await log.WriteVerboseAsync("Processing all of the {0} volumes for blocklists: {1}", lst.Count, string.Join(", ", lst.Select(x => x.Name)));
                                     else
-                                        m_result.AddMessage(string.Format("Processing all of the {0} volumes for blocklists", lst.Count));
+                                        await log.WriteInformationAsync(string.Format("Processing all of the {0} volumes for blocklists", lst.Count));
                                     break;
                             }
                         }
 
-                        var progress = 0;
-                        foreach(var sf in new AsyncDownloader(lst, backend))
-                            using(var tmpfile = sf.TempFile)
-                            using(var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options))
-                            using(var tr = restoredb.BeginTransaction())
-                            {
-                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                        progress = 0;
+                        using (var dn = new Common.PrefetchDownloader(lst, backend))
+                            while ((entry = await dn.GetNextAsync()) != null)
+                                using (var tmpfile = entry.TempFile)
+                                using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(entry.Name), tmpfile, options))
                                 {
-                                    backend.WaitForComplete(restoredb, null);
-                                    m_result.EndTime = DateTime.UtcNow;
-                                    return;
-                                }    
-                            
-                                progress++;
-                                m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f));
+                                    if (!await taskreader.ProgressAsync)
+                                    {
+                                        await backend.ReadyAsync();
+                                        await stats.SetEndTimeAsync();
+                                        return;
+                                    }
 
-                                var volumeid = restoredb.GetRemoteVolumeID(sf.Name);
+                                    progress++;
+                                    stats.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f));
 
-                                restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Uploaded, sf.Size, sf.Hash, tr);
-                            
-                                // Update the block table so we know about the block/volume map
-                                foreach(var h in rd.Blocks)
-                                    restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr);
-                            
-                                // Grab all known blocklists from the volume
-                                foreach(var blocklisthash in restoredb.GetBlockLists(volumeid))
-                                    restoredb.UpdateBlockset(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), tr);
-    
-                                // Update tables so we know if we are done
-                                restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, tr);
-                        
-                                using(new Logging.Timer("CommitRestoredBlocklist"))
-                                    tr.Commit();
-    
-                                //At this point we can patch files with data from the block volume
-                                if (blockprocessor != null)
-                                    blockprocessor(sf.Name, rd);
-                            }
+                                    var volumeid = await db.GetRemoteVolumeIDAsync(entry.Name);
+                                    await db.UpdateRemoteVolumeAsync(entry.Name, RemoteVolumeState.Uploaded, entry.Size, entry.Hash);
+
+                                    // Update the block table so we know about the block/volume map
+                                    foreach (var h in rd.Blocks)
+                                        await db.UpdateBlockAsync(h.Key, h.Value, volumeid);
+
+                                    // Grab all known blocklists from the volume
+                                    foreach (var blocklisthash in await db.GetBlockListsAsync(volumeid))
+                                        await db.UpdateBlocksetAsync(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize));
+
+                                    // Update tables so we know if we are done
+                                    await db.FindMissingBlocklistHashesAsync(hashsize, options.Blocksize);
+
+                                    await db.CommitTransactionAsync("CommitRestoredBlocklist", true);
+
+                                    //At this point we can patch files with data from the block volume
+                                    if (blockprocessor != null)
+                                        blockprocessor(entry.Name, rd);
+                                }
                     }
-                }
-                
-                backend.WaitForComplete(restoredb, null);
 
-                if (m_options.RepairOnlyPaths)
-                {
-                    m_result.AddMessage("Recreate/path-update completed, not running consistency checks");
-                }
-                else
-                {
-                    m_result.AddMessage("Recreate completed, verifying the database consistency");
+                    await backend.ReadyAsync();
 
-                    //All done, we must verify that we have all blocklist fully intact
-                    // if this fails, the db will not be deleted, so it can be used,
-                    // except to continue a backup
-                    m_result.EndTime = DateTime.UtcNow;
-
-                    using (var lbfdb = new LocalListBrokenFilesDatabase(restoredb))
+                    if (options.RepairOnlyPaths)
                     {
-                        var broken = lbfdb.GetBrokenFilesets(new DateTime(0), null, null).Count();
-                        if (broken != 0)
-                            throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"));
+                        await log.WriteInformationAsync("Recreate/path-update completed, not running consistency checks");
+                    }
+                    else
+                    {
+                        await log.WriteInformationAsync("Recreate completed, verifying the database consistency");
+
+                        //All done, we must verify that we have all blocklist fully intact
+                        // if this fails, the db will not be deleted, so it can be used,
+                        // except to continue a backup
+                        await stats.SetEndTimeAsync();
+
+                        using (var lbfdb = new LocalListBrokenFilesDatabase(db))
+                        {
+                            var broken = lbfdb.GetBrokenFilesets(new DateTime(0), null, null).Count();
+                            if (broken != 0)
+                                throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"));
+                        }
+
+                        await db.VerifyConsistencyAsync(options.Blocksize, options.BlockhashSize, true);
+
+                        await log.WriteInformationAsync("Recreate completed, and consistency checks completed, marking database as complete");
+
+                        await db.SetRepairInProgressAsync(false);
                     }
 
-                    restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null);
-
-                    m_result.AddMessage("Recreate completed, and consistency checks completed, marking database as complete");
-
-                    restoredb.RepairInProgress = false;
+                    await stats.SetEndTimeAsync();
                 }
-
-                m_result.EndTime = DateTime.UtcNow;
             }
         }
 
@@ -501,7 +517,7 @@ namespace Duplicati.Library.Main.Operation
         /// <returns>The volume id of the item</returns>
         /// <param name="filename">The filename read and written</param>
         /// <param name="restoredb">The database to query</param>
-        public long ProbeForMatchingFilename(ref string filename, LocalRestoreDatabase restoredb)
+        public static async Task<Tuple<long, string>> ProbeForMatchingFilenameAsync(string filename, Recreate.RecreateDatabase db, Common.LogWrapper log)
         {
             var p = VolumeBase.ParseFilename(filename);
             if (p != null)
@@ -510,21 +526,16 @@ namespace Duplicati.Library.Main.Operation
                     foreach(var encmodule in Library.DynamicLoader.EncryptionLoader.Keys.Union(new string[] { "" }))
                     {
                         var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule);
-                        var tvid = restoredb.GetRemoteVolumeID(testfilename);
+                        var tvid = await db.GetRemoteVolumeIDAsync(testfilename);
                         if (tvid >= 0)
                         {
-                            m_result.AddWarning(string.Format("Unable to find volume {0}, but mapping to matching file {1}", filename, testfilename), null);
-                            filename = testfilename;
-                            return tvid;
+                            await log.WriteWarningAsync(string.Format("Unable to find volume {0}, but mapping to matching file {1}", filename, testfilename), null);
+                            return new Tuple<long, string>(tvid, testfilename);
                         }
                     }
             }
 
-            return -1;
-        }
-
-        public void Dispose()
-        {
+            return null;
         }
     }
 }

Duplicati/Library/Main/Operation/Repair/RepairDatabase.cs

@@ -0,0 +1,114 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Database;
+using Duplicati.Library.Main.Volumes;
+using static Duplicati.Library.Main.Database.LocalRepairDatabase;
+
+namespace Duplicati.Library.Main.Operation.Repair
+{
+    internal class RepairDatabase : Common.DatabaseCommon
+    {
+        private readonly LocalRepairDatabase m_database;
+        
+        public RepairDatabase(LocalRepairDatabase database, Options options)
+            : base(database, options)
+        {
+            m_database = database;
+        }
+
+        public Task<bool> GetRepairInProgressAsync()
+        {
+            return RunOnMain(() => m_db.RepairInProgress);
+        }
+
+        public Task<bool> GetPartiallyRecreatedAsync()
+        {
+            return RunOnMain(() => m_db.PartiallyRecreated);
+        }
+
+        public Task FixDuplicateMetahashAsync()
+        {
+            return RunOnMain(() => m_database.FixDuplicateMetahash());
+        }
+
+        public Task FixDuplicateFileentriesAsync()
+        {
+            return RunOnMain(() => m_database.FixDuplicateFileentries());
+        }
+
+        public Task FixDuplicateBlocklistHashesAsync(int blocksize, int blockhashsize)
+        {
+            return RunOnMain(async () =>
+            {
+                if (m_database.FixDuplicateBlocklistHashes(blocksize, blockhashsize, m_transaction))
+                    await this.CommitTransactionAsync("Fixed duplicate blocklisthashes");
+            });
+        }
+
+        public Task FixMissingBlocklistHashesAsync(string blockhashalgorithm, int blockhashsize)
+        {
+            return RunOnMain(async () =>
+            {
+                if (m_database.FixMissingBlocklistHashes(blockhashalgorithm, blockhashsize, m_transaction))
+                    await this.CommitTransactionAsync("Repaired missing blocklisthashes");
+            });
+        }
+
+        public Task<RemoteVolumeEntry> GetRemoteVolumeAsync(string filename)
+        {
+            return RunOnMain(() => m_db.GetRemoteVolume(filename, m_transaction));
+        }
+
+        public Task CheckAllBlocksAreInVolumeAsync(string filename, IEnumerable<KeyValuePair<string, long>> blocks)
+        {
+            return RunOnMain(() => m_database.CheckAllBlocksAreInVolume(filename, blocks, m_transaction));
+        }
+
+        public Task CheckBlocklistCorrectAsync(string hash, long length, IEnumerable<string> blocks, int blocksize, int hashsize)
+        {
+            return RunOnMain(() => m_database.CheckBlocklistCorrect(hash, length, blocks, blocksize, hashsize, m_transaction));
+        }
+
+        public Task<IEnumerable<IRemoteVolume>> GetBlockVolumesFromIndexNameAsync(string name)
+        {
+            return RunOnMain(() => m_database.GetBlockVolumesFromIndexName(name, m_transaction));
+        }
+
+        public Task<IMissingBlockList> CreateBlockListAsync(string name)
+        {
+            return RunOnMain(() => m_database.CreateBlockList(name, m_transaction));
+        }
+
+        public Task<long> GetFilesetIdFromRemotenameAsync(string name)
+        {
+            return RunOnMain(() => m_database.GetFilesetIdFromRemotename(name, m_transaction));
+        }
+
+        public Task WriteFilesetAsync(FilesetVolumeWriter writer, long filesetid)
+        {
+            return RunOnMain(() => m_db.WriteFileset(writer, filesetid, m_transaction));
+        }
+
+        public Test.TestDatabase GetTestDatabase()
+        {
+            return new Test.TestDatabase(m_db, m_options);
+        }
+    }
+}

Duplicati/Library/Main/Operation/Repair/RepairStatsCollector.cs

@@ -0,0 +1,51 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Repair
+{
+    internal class RepairStatsCollector : StatsCollector
+    {
+        private readonly RepairResults m_res;
+
+        public RepairStatsCollector(RepairResults res)
+            : base(res.BackendWriter)
+        {
+            m_res = res;
+        }
+
+        public void UpdatePhase(OperationPhase phase)
+        {
+            m_res.OperationProgressUpdater.UpdatePhase(phase);
+        }
+
+        public void UpdateProgress(float pg)
+        {
+            m_res.OperationProgressUpdater.UpdateProgress(pg);
+        }
+
+        public Task SetEndTimeAsync()
+        {
+            return RunOnMain(() =>
+            {
+                m_res.EndTime = DateTime.UtcNow;
+            });
+        }
+    }
+}

Duplicati/Library/Main/Operation/RepairHandler.cs

@@ -2,181 +2,185 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
+using CoCoL;
 using Duplicati.Library.Interface;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Main.Volumes;
 
 namespace Duplicati.Library.Main.Operation
 {
-    internal class RepairHandler
+    internal static class RepairHandler
     {
-        private string m_backendurl;
-        private Options m_options;
-        private RepairResults m_result;
+        public static async Task RunAsync(string backendurl, Options options, RepairResults result, Library.Utility.IFilter filter = null)
+        {
+			if (options.AllowPassphraseChange)
+				throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported);
 
-        public RepairHandler(string backend, Options options, RepairResults result)
-        {
-            m_backendurl = backend;
-            m_options = options;
-            m_result = result;
-            
-            if (options.AllowPassphraseChange)
-                throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported);
+            using (new IsolatedChannelScope())
+            {
+                var lh = Common.LogHandler.Run(result);
+
+                using (var coredb = new LocalRepairDatabase(options.Dbpath))
+				using (var db = new Repair.RepairDatabase(coredb, options))
+                using(var stats = new Repair.RepairStatsCollector(result))
+                using (var backend = new Common.BackendHandler(options, backendurl, db, stats, result.TaskReader))
+				// Keep a reference to this channel to avoid shutdown
+				using (var logtarget = ChannelManager.GetChannel(Common.Channels.LogChannel.ForWrite))
+                using(var log = new Common.LogWrapper())
+				{
+
+                    if (!System.IO.File.Exists(options.Dbpath))
+                    {
+                        await RunRepairLocalAsync(backend, options, stats, result.TaskReader, filter);
+                        await RunRepairCommonAsync(options, db, stats, result.TaskReader);
+                        await stats.SetEndTimeAsync();
+                        return;
+                    }
+
+                    long knownRemotes = -1;
+                    try
+                    {
+                        using (var lrdb = new LocalRepairDatabase(options.Dbpath))
+                            knownRemotes = lrdb.GetRemoteVolumes().Count();
+                    }
+                    catch (Exception ex)
+                    {
+                        await log.WriteWarningAsync(string.Format("Failed to read local db {0}, error: {1}", options.Dbpath, ex.Message), ex);
+                    }
+
+                    if (knownRemotes <= 0)
+                    {
+                        if (options.Dryrun)
+                        {
+                            await log.WriteDryRunAsync("Performing dryrun recreate");
+                        }
+                        else
+                        {
+                            var baseName = System.IO.Path.ChangeExtension(options.Dbpath, "backup");
+                            var i = 0;
+                            while (System.IO.File.Exists(baseName) && i++ < 1000)
+                                baseName = System.IO.Path.ChangeExtension(options.Dbpath, "backup-" + i.ToString());
+
+                            await log .WriteInformationAsync(string.Format("Renaming existing db from {0} to {1}", options.Dbpath, baseName));
+                            System.IO.File.Move(options.Dbpath, baseName);
+                        }
+
+                        await RunRepairLocalAsync(backend, options, stats, result.TaskReader, filter);
+                        await RunRepairCommonAsync(options, db, stats, result.TaskReader);
+                    }
+                    else
+                    {
+                        await RunRepairCommonAsync(options, db, stats, result.TaskReader);
+                        await RunRepairRemote(backend, options, db, stats, result.TaskReader);
+                    }
+
+                    await stats.SetEndTimeAsync();
+                }
+
+                await lh;
+            }
         }
         
-        public void Run(Library.Utility.IFilter filter = null)
+        public static async Task RunRepairLocalAsync(Common.BackendHandler backend, Options options, Repair.RepairStatsCollector stats, Common.ITaskReader taskreader, Library.Utility.IFilter filter = null)
         {
-            if (!System.IO.File.Exists(m_options.Dbpath))
-            {
-                RunRepairLocal(filter);
-                RunRepairCommon();
-                m_result.EndTime = DateTime.UtcNow;
-                return;
-            }
-
-            long knownRemotes = -1;
-            try
-            {        
-                using(var db = new LocalRepairDatabase(m_options.Dbpath))
-                    knownRemotes = db.GetRemoteVolumes().Count();
-            }
-            catch (Exception ex)
-            {
-                m_result.AddWarning(string.Format("Failed to read local db {0}, error: {1}", m_options.Dbpath, ex.Message), ex);
-            }
-            
-            if (knownRemotes <= 0)
-            {                
-                if (m_options.Dryrun)
-                {
-                    m_result.AddDryrunMessage("Performing dryrun recreate");
-                }
-                else
-                {
-                    var baseName = System.IO.Path.ChangeExtension(m_options.Dbpath, "backup");
-                    var i = 0;
-                    while (System.IO.File.Exists(baseName) && i++ < 1000)
-                        baseName = System.IO.Path.ChangeExtension(m_options.Dbpath, "backup-" + i.ToString());
-                        
-                    m_result.AddMessage(string.Format("Renaming existing db from {0} to {1}", m_options.Dbpath, baseName));
-                    System.IO.File.Move(m_options.Dbpath, baseName);
-                }
-                
-                RunRepairLocal(filter);
-                RunRepairCommon();
-            }
-            else
-            {
-                RunRepairCommon();
-                RunRepairRemote();
-            }
-
-            m_result.EndTime = DateTime.UtcNow;
-
-        }
-        
-        public void RunRepairLocal(Library.Utility.IFilter filter = null)
-        {
-            m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result);
             using(new Logging.Timer("Recreate database for repair"))
-            using(var f = m_options.Dryrun ? new Library.Utility.TempFile() : null)
+            using(var f = options.Dryrun ? new Library.Utility.TempFile() : null)
             {
                 if (f != null && System.IO.File.Exists(f))
                     System.IO.File.Delete(f);
                 
-                var filelistfilter = RestoreHandler.FilterNumberedFilelist(m_options.Time, m_options.Version);
+                var filelistfilter = RestoreHandler.FilterNumberedFilelist(options.Time, options.Version);
 
-                new RecreateDatabaseHandler(m_backendurl, m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults)
-                    .Run(m_options.Dryrun ? (string)f : m_options.Dbpath, filter, filelistfilter);
+                using (var coredb = new LocalRecreateDatabase(options.Dryrun ? (string)f : options.Dbpath, options))
+                using (var db = new Recreate.RecreateDatabase(coredb, options))
+                    await RecreateDatabaseHandler.DoRunAsync(db, backend, options, false, stats, taskreader, filter, filelistfilter);
             }
         }
 
-        public void RunRepairRemote()
+        public static async Task RunRepairRemote(Common.BackendHandler backend, Options options, Repair.RepairDatabase db, Repair.RepairStatsCollector stats, Common.ITaskReader taskreader)
         {
-            if (!System.IO.File.Exists(m_options.Dbpath))
-                throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath));
+            if (!System.IO.File.Exists(options.Dbpath))
+                throw new UserInformationException(string.Format("Database file does not exist: {0}", options.Dbpath));
 
-            m_result.OperationProgressUpdater.UpdateProgress(0);
+            stats.UpdateProgress(0);
 
-            using(var db = new LocalRepairDatabase(m_options.Dbpath))
-            using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            using (var log = new Common.LogWrapper())
             {
-                m_result.SetDatabase(db);
-                Utility.UpdateOptionsFromDb(db, m_options);
-                Utility.VerifyParameters(db, m_options);
+                await db.UpdateOptionsFromDbAsync(options);
+                await db.VerifyParametersAsync(options);
 
-                if (db.PartiallyRecreated)
+                if (await db.GetPartiallyRecreatedAsync())
                     throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.");
 
-                if (db.RepairInProgress)
+                if (await db.GetRepairInProgressAsync())
                     throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.");
 
-                var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter, null);
-                var buffer = new byte[m_options.Blocksize];
-                var blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
+                var tp = await FilelistProcessor.RemoteListAnalysisAsync(backend, options, db, stats, null);
+                var buffer = new byte[options.Blocksize];
+                var blockhasher = Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm);
                 var hashsize = blockhasher.HashSize / 8;
 
                 if (blockhasher == null)
-                    throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
+                    throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm));
                 if (!blockhasher.CanReuseTransform)
-                    throw new UserInformationException(Strings.Common.InvalidCryptoSystem(m_options.BlockHashAlgorithm));
+                    throw new UserInformationException(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm));
                 
                 var progress = 0;
                 var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count();
 
-                if (m_options.Dryrun)
+                if (options.Dryrun)
                 {
-                    if (tp.ParsedVolumes.Count() == 0 && tp.OtherVolumes.Count() > 0)
+                    if (tp.ParsedVolumes.Count() == 0 && tp.OtherVolumes.Any())
                     {
                         if (tp.BackupPrefixes.Length == 1)
-                            throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", m_options.Prefix, tp.BackupPrefixes[0]));
+                            throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", options.Prefix, tp.BackupPrefixes[0]));
                         else
-                            throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)));
+                            throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", options.Prefix, string.Join(", ", tp.BackupPrefixes)));
                     }
-                    else if (tp.ParsedVolumes.Count() == 0 && tp.ExtraVolumes.Count() > 0)
+                    else if (tp.ParsedVolumes.Count() == 0 && tp.ExtraVolumes.Any())
                     {
                         throw new UserInformationException(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()));
                     }
                 }
 
-                if (tp.ExtraVolumes.Count() > 0 || tp.MissingVolumes.Count() > 0 || tp.VerificationRequiredVolumes.Count() > 0)
+                if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any())
                 {
                     if (tp.VerificationRequiredVolumes.Any())
                     {
-                        using(var testdb = new LocalTestDatabase(db))
+                        using(var testdb = db.GetTestDatabase())
                         {
                             foreach(var n in tp.VerificationRequiredVolumes)
                                 try
                                 {
-                                    if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                                    if (!await taskreader.ProgressAsync)
                                     {
-                                        backend.WaitForComplete(db, null);
+                                        await backend.ReadyAsync();
                                         return;
                                     }
 
                                     progress++;
-                                    m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
+                                    stats.UpdateProgress((float)progress / targetProgess);
 
-                                    long size;
-                                    string hash;
                                     KeyValuePair<string, IEnumerable<KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>>> res;
-                                   
-                                    using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
-                                        res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, m_result, 1);
+
+                                    var tr = await backend.GetFileWithInfoAsync(n.Name);
+                                    using(var tf = tr.Item1)
+                                        res = await TestHandler.TestVolumeInternalsAsync(testdb, n, tf, options, 1);
 
                                     if (res.Value.Any())
                                         throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First()));
 
-                                    if (!m_options.Dryrun)
+                                    if (!options.Dryrun)
                                     {
-                                        m_result.AddMessage(string.Format("Sucessfully captured hash for {0}, updating database", n.Name));
-                                        db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash);
+                                        await log.WriteInformationAsync(string.Format("Sucessfully captured hash for {0}, updating database", n.Name));
+                                        await db.UpdateRemoteVolumeAsync(n.Name, RemoteVolumeState.Verified, tr.Item2, tr.Item3);
                                     }
 
                                 }
                                 catch (Exception ex)
                                 {
-                                    m_result.AddError(string.Format("Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message), ex);
+                                    await log.WriteErrorAsync(string.Format("Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message), ex);
                                     if (ex is System.Threading.ThreadAbortException)
                                         throw;
                                 }
@@ -187,29 +191,28 @@ namespace Duplicati.Library.Main.Operation
                     foreach(var n in tp.ExtraVolumes)
                         try
                         {
-                            if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                            if (!await taskreader.ProgressAsync)
                             {
-                                backend.WaitForComplete(db, null);
+                                await backend.ReadyAsync();
                                 return;
                             }
 
                             progress++;
-                            m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
+                            stats.UpdateProgress((float)progress / targetProgess);
 
                             // If this is a new index file, we can accept it if it matches our local data
                             // This makes it possible to augment the remote store with new index data
-                            if (n.FileType == RemoteVolumeType.Index && m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
+                            if (n.FileType == RemoteVolumeType.Index && options.IndexfilePolicy != Options.IndexFileStrategy.None)
                             {
                                 try
                                 {
-                                    string hash;
-                                    long size;
-                                    using(var tf = backend.GetWithInfo(n.File.Name, out size, out hash))
-                                    using(var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize))
+                                    var tr = await backend.GetFileWithInfoAsync(n.File.Name);
+                                    using(var tf = tr.Item1)
+                                    using(var ifr = new IndexVolumeReader(n.CompressionModule, tf, options, options.BlockhashSize))
                                     {
                                         foreach(var rv in ifr.Volumes)
                                         {
-                                            var entry = db.GetRemoteVolume(rv.Filename);
+                                            var entry = await db.GetRemoteVolumeAsync(rv.Filename);
                                             if (entry.ID < 0)
                                                 throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename));
                                             
@@ -219,41 +222,41 @@ namespace Duplicati.Library.Main.Operation
                                             if (entry.Hash != rv.Hash || entry.Size != rv.Length || ! new [] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State))
                                                 throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length));
 
-                                            db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks);
+                                            await db.CheckAllBlocksAreInVolumeAsync(rv.Filename, rv.Blocks);
                                         }
 
-                                        var blocksize = m_options.Blocksize;
+                                        var blocksize = options.Blocksize;
                                         foreach(var ixb in ifr.BlockLists)
-                                            db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize);
+                                            await db.CheckBlocklistCorrectAsync(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize);
 
-                                        var selfid = db.GetRemoteVolumeID(n.File.Name);
+                                        var selfid = await db.GetRemoteVolumeIDAsync(n.File.Name);
                                         foreach(var rv in ifr.Volumes)
-                                            db.AddIndexBlockLink(selfid, db.GetRemoteVolumeID(rv.Filename), null);
+                                            await db.AddIndexBlockLinkAsync(selfid, await db.GetRemoteVolumeIDAsync(rv.Filename));
                                     }
                                     
                                     // All checks fine, we accept the new index file
-                                    m_result.AddMessage(string.Format("Accepting new index file {0}", n.File.Name));
-                                    db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, size, RemoteVolumeState.Uploading);
-                                    db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash);
+                                    await log.WriteInformationAsync(string.Format("Accepting new index file {0}", n.File.Name));
+                                    await db.RegisterRemoteVolumeAsync(n.File.Name, RemoteVolumeType.Index, tr.Item2, RemoteVolumeState.Uploading);
+                                    await db.UpdateRemoteVolumeAsync(n.File.Name, RemoteVolumeState.Verified, tr.Item2, tr.Item3);
                                     continue;
                                 }
                                 catch (Exception rex)
                                 {
-                                    m_result.AddError(string.Format("Failed to accept new index file: {0}, message: {1}", n.File.Name, rex.Message), rex);
+                                    await log.WriteErrorAsync(string.Format("Failed to accept new index file: {0}, message: {1}", n.File.Name, rex.Message), rex);
                                 }
                             }
                         
-                            if (!m_options.Dryrun)
+                            if (!options.Dryrun)
                             {
-                                db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting);
-                                backend.Delete(n.File.Name, n.File.Size);
+                                await db.RegisterRemoteVolumeAsync(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting);
+                                await backend.DeleteFileAsync(n.File.Name);
                             }
                             else
-                                m_result.AddDryrunMessage(string.Format("would delete file {0}", n.File.Name));
+                                await log.WriteDryRunAsync(string.Format("would delete file {0}", n.File.Name));
                         }
                         catch (Exception ex)
                         {
-                            m_result.AddError(string.Format("Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message), ex);
+                            await log.WriteErrorAsync(string.Format("Failed to perform cleanup for extra file: {0}, message: {1}", n.File.Name, ex.Message), ex);
                             if (ex is System.Threading.ThreadAbortException)
                                 throw;
                         }
@@ -264,53 +267,53 @@ namespace Duplicati.Library.Main.Operation
                         
                         try
                         {  
-                            if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                            if (!await taskreader.ProgressAsync)
                             {
-                                backend.WaitForComplete(db, null);
+                                await backend.ReadyAsync();
                                 return;
-                            }    
+                            }
 
                             progress++;
-                            m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
+                            stats.UpdateProgress((float)progress / targetProgess);
 
                             if (n.Type == RemoteVolumeType.Files)
                             {
-                                var filesetId = db.GetFilesetIdFromRemotename(n.Name);
-                                var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
+                                var filesetId = await db.GetFilesetIdFromRemotenameAsync(n.Name);
+                                var w = new FilesetVolumeWriter(options, DateTime.UtcNow);
                                 newEntry = w;
                                 w.SetRemoteFilename(n.Name);
 
-                                db.WriteFileset(w, filesetId, null);
+                                await db.WriteFilesetAsync(w, filesetId);
 	
                                 w.Close();
-                                if (m_options.Dryrun)
-                                    m_result.AddDryrunMessage(string.Format("would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
+                                if (options.Dryrun)
+                                    await log.WriteDryRunAsync(string.Format("would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
                                 else
                                 {
-                                    db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                    backend.Put(w);
+                                    await db.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                                    await backend.UploadFileAsync(w);
                                 }
                             }
                             else if (n.Type == RemoteVolumeType.Index)
                             {
-                                var w = new IndexVolumeWriter(m_options);
+                                var w = new IndexVolumeWriter(options);
                                 newEntry = w;
                                 w.SetRemoteFilename(n.Name);
 
-                                var h = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
+                                var h = Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm);
                                 
-                                foreach(var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
+                                foreach(var blockvolume in await db.GetBlockVolumesFromIndexNameAsync(n.Name))
                                 {                               
                                     w.StartVolume(blockvolume.Name);
-                                    var volumeid = db.GetRemoteVolumeID(blockvolume.Name);
+                                    var volumeid = await db.GetRemoteVolumeIDAsync(blockvolume.Name);
                                     
-                                    foreach(var b in db.GetBlocks(volumeid))
+                                    foreach(var b in await db.GetBlocksAsync(volumeid))
                                         w.AddBlock(b.Hash, b.Size);
                                         
                                     w.FinishVolume(blockvolume.Hash, blockvolume.Size);
                                     
-                                    if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
-                                        foreach(var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize))
+                                    if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
+                                        foreach(var b in await db.GetBlocklistsAsync(volumeid, options.Blocksize, hashsize))
                                         {
                                             var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
                                             if (bh != b.Item1)
@@ -322,24 +325,24 @@ namespace Duplicati.Library.Main.Operation
                                 
                                 w.Close();
                                 
-                                if (m_options.Dryrun)
-                                    m_result.AddDryrunMessage(string.Format("would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
+                                if (options.Dryrun)
+                                    await log.WriteDryRunAsync(string.Format("would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
                                 else
                                 {
-                                    db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                    backend.Put(w);
+                                    await db.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                                    await backend.UploadFileAsync(w);
                                 }
                             }
                             else if (n.Type == RemoteVolumeType.Blocks)
                             {
-                                var w = new BlockVolumeWriter(m_options);
+                                var w = new BlockVolumeWriter(options);
                                 newEntry = w;
                                 w.SetRemoteFilename(n.Name);
                                 
-                                using(var mbl = db.CreateBlockList(n.Name))
+                                using(var mbl = await db.CreateBlockListAsync(n.Name))
                                 {
                                     //First we grab all known blocks from local files
-                                    foreach(var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize))
+                                    foreach(var block in mbl.GetSourceFilesWithBlocks(options.Blocksize))
                                     {
                                         var hash = block.Hash;
                                         var size = (int)block.Size;
@@ -369,18 +372,20 @@ namespace Duplicati.Library.Main.Operation
                                             }
                                             catch (Exception ex)
                                             {
-                                                m_result.AddError(string.Format("Failed to access file: {0}", file), ex);
+                                                await log.WriteErrorAsync(string.Format("Failed to access file: {0}", file), ex);
                                             }
                                         }
                                     }
-                                    
+
                                     //Then we grab all remote volumes that have the missing blocks
-                                    foreach(var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend))
+                                    IAsyncDownloadedFile vol;
+                                    using(var pr = new Common.PrefetchDownloader(mbl.GetMissingBlockSources().ToList(), backend))
+                                    while((vol = await pr.GetNextAsync()) != null)
                                     {
                                         try
                                         {
                                             using(var tmpfile = vol.TempFile)
-                                            using(var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, m_options))
+                                            using(var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, options))
                                                 foreach(var b in f.Blocks)
                                                     if (mbl.SetBlockRestored(b.Key, b.Value))
                                                         if (f.ReadBlock(b.Key, buffer) == b.Value)
@@ -388,7 +393,7 @@ namespace Duplicati.Library.Main.Operation
                                         }
                                         catch (Exception ex)
                                         {
-                                            m_result.AddError(string.Format("Failed to access remote file: {0}", vol.Name), ex);
+                                            await log.WriteErrorAsync(string.Format("Failed to access remote file: {0}", vol.Name), ex);
                                         }
                                     }
                                     
@@ -396,31 +401,31 @@ namespace Duplicati.Library.Main.Operation
                                     var missingBlocks = mbl.GetMissingBlocks().Count();
                                     if (missingBlocks > 0)
                                     {                                    
-                                        m_result.AddMessage(string.Format("Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name));
+                                        await log.WriteInformationAsync(string.Format("Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name));
                                         foreach(var f in mbl.GetFilesetsUsingMissingBlocks())
-                                            m_result.AddMessage(f.Name);
+                                            await log.WriteInformationAsync(f.Name);
 
                                         var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files");
 
-                                        if (!m_options.Dryrun)
+                                        if (!options.Dryrun)
                                         {
-                                            m_result.AddMessage("This may be fixed by deleting the filesets and running repair again");
+                                            await log.WriteInformationAsync("This may be fixed by deleting the filesets and running repair again");
 
                                             throw new UserInformationException(string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks));
                                         }
                                         else
                                         {
-                                            m_result.AddMessage(recoverymsg);
+                                            await log.WriteInformationAsync(recoverymsg);
                                         }
                                     }
                                     else
                                     {
-                                        if (m_options.Dryrun)
-                                            m_result.AddDryrunMessage(string.Format("would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
+                                        if (options.Dryrun)
+                                            await log.WriteDryRunAsync(string.Format("would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)));
                                         else
                                         {
-                                            db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                            backend.Put(w);
+                                            await db.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                                            await backend.UploadFileAsync(w);
                                         }
                                     }
                                 }
@@ -433,7 +438,7 @@ namespace Duplicati.Library.Main.Operation
                                 catch { }
                                 finally { newEntry = null; }
                                 
-                            m_result.AddError(string.Format("Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message), ex);
+                            await log.WriteErrorAsync(string.Format("Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message), ex);
                             
                             if (ex is System.Threading.ThreadAbortException)
                                 throw;
@@ -442,35 +447,29 @@ namespace Duplicati.Library.Main.Operation
                 }
                 else
                 {
-                    m_result.AddMessage("Destination and database are synchronized, not making any changes");
+                    await log.WriteInformationAsync("Destination and database are synchronized, not making any changes");
                 }
 
-                m_result.OperationProgressUpdater.UpdateProgress(1);                
-                backend.WaitForComplete(db, null);
-                db.WriteResults();
+                stats.UpdateProgress(1);
             }
         }
 
-        public void RunRepairCommon()
+        public static async Task RunRepairCommonAsync(Options options, Repair.RepairDatabase db, Repair.RepairStatsCollector stats, Common.ITaskReader taskreader)
         {
-            if (!System.IO.File.Exists(m_options.Dbpath))
-                throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath));
+            if (!System.IO.File.Exists(options.Dbpath))
+                throw new UserInformationException(string.Format("Database file does not exist: {0}", options.Dbpath));
 
-            m_result.OperationProgressUpdater.UpdateProgress(0);
-
-            using(var db = new LocalRepairDatabase(m_options.Dbpath))
+            stats.UpdateProgress(0);
+            await db.UpdateOptionsFromDbAsync(options);
+            using (var log = new Common.LogWrapper())
             {
-                db.SetResult(m_result);
+                if (await db.GetRepairInProgressAsync() || await db.GetPartiallyRecreatedAsync())
+                    await log.WriteWarningAsync("The database is marked as \"in-progress\" and may be incomplete.", null);
 
-                Utility.UpdateOptionsFromDb(db, m_options);
-
-                if (db.RepairInProgress || db.PartiallyRecreated)
-                    m_result.AddWarning("The database is marked as \"in-progress\" and may be incomplete.", null);
-
-                db.FixDuplicateMetahash();
-                db.FixDuplicateFileentries();
-                db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize);
-                db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize);
+                await db.FixDuplicateMetahashAsync();
+                await db.FixDuplicateFileentriesAsync();
+                await db.FixDuplicateBlocklistHashesAsync(options.Blocksize, options.BlockhashSize);
+                await db.FixMissingBlocklistHashesAsync(options.BlockHashAlgorithm, options.Blocksize);
             }
         }
     }

Duplicati/Library/Main/Operation/Test/TestDatabase.cs

@@ -0,0 +1,56 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Database;
+using static Duplicati.Library.Main.Database.LocalTestDatabase;
+
+namespace Duplicati.Library.Main.Operation.Test
+{
+    internal class TestDatabase : Common.DatabaseCommon
+    {
+        private readonly LocalTestDatabase m_database;
+
+        public TestDatabase(LocalTestDatabase db, Options options)
+            : base(db, options)
+        {
+            m_database = db;
+        }
+
+        public Task<IFilelist> CreateFilelistAsync(string name)
+        {
+            return RunOnMain(() => m_database.CreateFilelist(name, m_transaction));
+        }
+
+        public Task<IBlocklist> CreateBlocklistAsync(string name)
+        {
+            return RunOnMain(() => m_database.CreateBlocklist(name, m_transaction));
+        }
+
+        public Task<IIndexlist> CreateIndexlistAsync(string name)
+        {
+            return RunOnMain(() => m_database.CreateIndexlist(name, m_transaction));
+        }
+
+        public Task<IEnumerable<IRemoteVolume>> SelectTestTargetsAsync(long samples, Options options)
+        {
+            return RunOnMain(() => m_database.SelectTestTargets(samples, options, m_transaction));
+        }
+
+    }
+}

Duplicati/Library/Main/Operation/Test/TestStatsCollector.cs

@@ -0,0 +1,51 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Threading.Tasks;
+using Duplicati.Library.Main.Operation.Common;
+
+namespace Duplicati.Library.Main.Operation.Test
+{
+    internal class TestStatsCollector : StatsCollector
+    {
+        private readonly TestResults m_res;
+
+        public TestStatsCollector(TestResults res)
+            : base(res.BackendWriter)
+        {
+            m_res = res;
+        }
+
+        public void UpdatePhase(OperationPhase phase)
+        {
+            m_res.OperationProgressUpdater.UpdatePhase(phase);
+        }
+
+        public void UpdateProgress(float pg)
+        {
+            m_res.OperationProgressUpdater.UpdateProgress(pg);
+        }
+
+        public Task SetEndTimeAsync()
+        {
+            return RunOnMain(() =>
+            {
+                m_res.EndTime = DateTime.UtcNow;
+            });
+        }
+    }
+}

Duplicati/Library/Main/Operation/TestHandler.cs

@@ -20,96 +20,88 @@ using Duplicati.Library.Main.Database;
 using System.Linq;
 using System.Collections.Generic;
 using Duplicati.Library.Interface;
+using System.Threading.Tasks;
 
 namespace Duplicati.Library.Main.Operation
 {
-    internal class TestHandler
+    internal static class TestHandler
     {
-        private readonly Options m_options;
-        private string m_backendurl;
-        private TestResults m_results;
-        
-        public TestHandler(string backendurl, Options options, TestResults results)
+        public static async Task Run(long samples, string backendurl, Options options, TestResults results)
         {
-            m_options = options;
-            m_backendurl = backendurl;
-            m_results = results;
-        }
-        
-        public void Run(long samples)
-        {
-            if (!System.IO.File.Exists(m_options.Dbpath))
-                throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath));
+            if (!System.IO.File.Exists(options.Dbpath))
+                throw new UserInformationException(string.Format("Database file does not exist: {0}", options.Dbpath));
                                 
-            using(var db = new LocalTestDatabase(m_options.Dbpath))
-            using(var backend = new BackendManager(m_backendurl, m_options, m_results.BackendWriter, db))
+            using (var coredb = new LocalTestDatabase(options.Dbpath))
+            using (var db = new Test.TestDatabase(coredb, options))
+            using (var backend = new Common.BackendHandler(options, backendurl, db, stats, reader))
             {
-                db.SetResult(m_results);
-                Utility.UpdateOptionsFromDb(db, m_options);
-                Utility.VerifyParameters(db, m_options);
-                
-                if (!m_options.NoBackendverification)
-                    FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_results.BackendWriter);
+                db.SetResult(results);
+                await db.UpdateOptionsFromDbAsync(options);
+                await db.VerifyParametersAsync(options);
+
+                if (!options.NoBackendverification)
+                    await FilelistProcessor.VerifyRemoteListAsync(backend, options, db, results.BackendWriter);
                     
-                DoRun(samples, db, backend);
+                await DoRunAsync(samples, options, db, stats, backend, reader);
                 db.WriteResults();
             }
         }
         
-        public void DoRun(long samples, LocalTestDatabase db, BackendManager backend)
+        public static async Task DoRunAsync(long samples, Options options, Test.TestDatabase db, Test.TestStatsCollector stats, Common.BackendHandler backend, Common.ITaskReader taskreader)
         {
-            var files = db.SelectTestTargets(samples, m_options).ToList();
+            var files = (await db.SelectTestTargetsAsync(samples, options)).ToList();
 
-            m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running);
-            m_results.OperationProgressUpdater.UpdateProgress(0);
+            stats.UpdatePhase(OperationPhase.Verify_Running);
+            stats.UpdateProgress(0);
             var progress = 0L;
             
-            if (m_options.FullRemoteVerification)
+            if (options.FullRemoteVerification)
             {
-                foreach(var vol in new AsyncDownloader(files, backend))
+                IAsyncDownloadedFile vol;
+                using(var n = new Common.PrefetchDownloader(files, backend))
+                while((vol = await n.GetNextAsync()) != null)
                 {
                     try
                     {
-                        if (m_results.TaskControlRendevouz() == TaskControlState.Stop)
+                        if (!await taskreader.ProgressAsync)
                         {
-                            backend.WaitForComplete(db, null);
-                            m_results.EndTime = DateTime.UtcNow;
+                            await backend.ReadyAsync();
                             return;
-                        }    
+                        }
 
                         progress++;
-                        m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
+                        stats.UpdateProgress((float)progress / files.Count);
 
                         KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
                         using(var tf = vol.TempFile)
-                            res = TestVolumeInternals(db, vol, tf, m_options, m_results, m_options.FullBlockVerification ? 1.0 : 0.2);
+                            res = await TestVolumeInternalsAsync(db, vol, tf, options, options.FullBlockVerification ? 1.0 : 0.2);
                         m_results.AddResult(res.Key, res.Value);
 
                         if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0)
                         {
                             if (res.Value == null || !res.Value.Any())
                             {
-                                var rv = db.GetRemoteVolume(vol.Name, null);
+                                var rv = await db.GetRemoteVolumeAsync(vol.Name);
 
                                 if (rv.ID < 0)
                                 {
                                     if (string.IsNullOrWhiteSpace(rv.Hash) || rv.Size <= 0)
                                     {
-                                        if (m_options.Dryrun)
+                                        if (options.Dryrun)
                                         {
                                             m_results.AddDryrunMessage(string.Format("Sucessfully captured hash and size for {0}, would update database", vol.Name));
                                         }
                                         else
                                         {
                                             m_results.AddMessage(string.Format("Sucessfully captured hash and size for {0}, updating database", vol.Name));
-                                            db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash);
+                                            await db.UpdateRemoteVolumeAsync(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash);
                                         }
                                     }
                                 }
                             }
                         }
                         
-                        db.UpdateVerificationCount(vol.Name);
+                        await db.UpdateVerificationCountAsync(vol.Name);
                     }
                     catch (Exception ex)
                     {
@@ -117,7 +109,7 @@ namespace Duplicati.Library.Main.Operation
                         m_results.AddError(string.Format("Failed to process file {0}", vol.Name), ex);
                         if (ex is System.Threading.ThreadAbortException)
                         {
-                            m_results.EndTime = DateTime.UtcNow;
+                            await stats.SetEndTimeAsync();
                             throw;
                         }
                     }
@@ -129,14 +121,14 @@ namespace Duplicati.Library.Main.Operation
                 {
                     try
                     {
-                        if (m_results.TaskControlRendevouz() == TaskControlState.Stop)
+                        if (!await taskreader.ProgressAsync)
                         {
-                            m_results.EndTime = DateTime.UtcNow;
+                            await backend.ReadyAsync();
                             return;
                         }
 
                         progress++;
-                        m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
+                        stats.UpdateProgress((float)progress / files.Count);
 
                         if (f.Size <= 0 || string.IsNullOrWhiteSpace(f.Hash))
                         {
@@ -145,30 +137,33 @@ namespace Duplicati.Library.Main.Operation
                             string hash;
                             long size;
 
-                            using (var tf = backend.GetWithInfo(f.Name, out size, out hash))
-                                res = TestVolumeInternals(db, f, tf, m_options, m_results, 1);
+                            var rf = await backend.GetFileWithInfoAsync(f.Name);
+                            using (var tf = rf.Item1)
+                                res = await TestVolumeInternalsAsync(db, f, tf, options, 1);
                             m_results.AddResult(res.Key, res.Value);
 
-                            if (!string.IsNullOrWhiteSpace(hash) && size > 0)
+                            if (!string.IsNullOrWhiteSpace(rf.Item3) && rf.Item2 > 0)
                             {
                                 if (res.Value == null || !res.Value.Any())
                                 {
-                                    if (m_options.Dryrun)
+                                    if (options.Dryrun)
                                     {
                                         m_results.AddDryrunMessage(string.Format("Sucessfully captured hash and size for {0}, would update database", f.Name));
                                     }
                                     else
                                     {
                                         m_results.AddMessage(string.Format("Sucessfully captured hash and size for {0}, updating database", f.Name));
-                                        db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash);
+                                        await db.UpdateRemoteVolumeAsync(f.Name, RemoteVolumeState.Verified, rf.Item2, rf.Item3);
                                     }
                                 }
                             }
                         }
                         else
-                            backend.GetForTesting(f.Name, f.Size, f.Hash);
+                        {
+                            await backend.GetFileForTestingAsync(f.Name, f.Size, f.Hash);
+                        }
                         
-                        db.UpdateVerificationCount(f.Name);
+                        await db.UpdateVerificationCountAsync(f.Name);
                         m_results.AddResult(f.Name, new KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>[0]);
                     }
                     catch (Exception ex)
@@ -177,14 +172,14 @@ namespace Duplicati.Library.Main.Operation
                         m_results.AddError(string.Format("Failed to process file {0}", f.Name), ex);
                         if (ex is System.Threading.ThreadAbortException)
                         {
-                            m_results.EndTime = DateTime.UtcNow;
+                            await stats.SetEndTimeAsync();
                             throw;
                         }
                     }
                 }
             }
 
-            m_results.EndTime = DateTime.UtcNow;
+            await stats.SetEndTimeAsync();
         }
 
         /// <summary>
@@ -193,7 +188,7 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="vol">The remote volume being examined</param>
         /// <param name="tf">The path to the downloaded copy of the file</param>
         /// <param name="sample_percent">A value between 0 and 1 that indicates how many blocks are tested in a dblock file</param>
-        public static KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, ILogWriter log, double sample_percent)
+        public static async Task<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>> TestVolumeInternalsAsync(Test.TestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent)
         {
             var blockhasher = Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm);
  
@@ -210,7 +205,7 @@ namespace Duplicati.Library.Main.Operation
             {
                 //Compare with db and see if all files are accounted for 
                 // with correct file hashes and blocklist hashes
-                using(var fl = db.CreateFilelist(vol.Name))
+                using(var fl = await db.CreateFilelistAsync(vol.Name))
                 {
                     using(var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options))
                         foreach(var f in rd.Files)
@@ -229,7 +224,7 @@ namespace Duplicati.Library.Main.Operation
                     foreach(var v in rd.Volumes)
                     {
                         blocklinks.Add(new Tuple<string, string, long>(v.Filename, v.Hash, v.Length));
-                        using(var bl = db.CreateBlocklist(v.Filename))
+                        using(var bl = await db.CreateBlocklistAsync(v.Filename))
                         {
                             foreach(var h in v.Blocks)
                                 bl.AddBlock(h.Key, h.Value);
@@ -238,7 +233,7 @@ namespace Duplicati.Library.Main.Operation
                         }
                     }
                                 
-                using(var il = db.CreateIndexlist(vol.Name))
+                using(var il = await db.CreateIndexlistAsync(vol.Name))
                 {
                     foreach(var t in blocklinks)
                         il.AddBlockLink(t.Item1, t.Item2, t.Item3);
@@ -250,7 +245,7 @@ namespace Duplicati.Library.Main.Operation
             }
             else if (parsedInfo.FileType == RemoteVolumeType.Blocks)
             {
-                using(var bl = db.CreateBlocklist(vol.Name))
+                using(var bl = await db.CreateBlocklistAsync(vol.Name))
                 using(var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options))
                 {                                    
                     //Verify that all blocks are in the file
@@ -284,7 +279,9 @@ namespace Duplicati.Library.Main.Operation
                 }                                
             }
 
-            log.AddWarning(string.Format("Unexpected file type {0} for {1}", parsedInfo.FileType, vol.Name), null);
+            using(var log = new Common.LogWrapper())
+                await log.WriteWarningAsync(string.Format("Unexpected file type {0} for {1}", parsedInfo.FileType, vol.Name), null);
+            
             return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, null);
         }
     }