Merge branch 'master' into webapi/dropbox

.travis.yml

@@ -5,6 +5,7 @@ install:
   - nuget restore Duplicati.sln
   - nuget install NUnit.Runners -Version 3.5.0 -OutputDirectory testrunner
   - sudo pip install selenium
+  - sudo pip install --upgrade urllib3
   - if [ ! -d "${TRAVIS_BUILD_DIR}"/packages/SharpCompress.0.18.2 ]; then ln -s "${TRAVIS_BUILD_DIR}"/packages/sharpcompress.0.18.2 "${TRAVIS_BUILD_DIR}"/packages/SharpCompress.0.18.2; fi
 addons:
   coverity_scan:
@@ -68,4 +69,4 @@ before_install:
       - echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-
 
 script:
-  - ./build.sh ${TRAVIS_BUILD_DIR} ${CATEGORY}
+  - ./build.sh ${CATEGORY} ${TRAVIS_BUILD_DIR}

Duplicati/CommandLine/BackendTester/Program.cs

@@ -31,7 +31,9 @@ namespace Duplicati.CommandLine.BackendTester
         /// <summary>
         /// Used to maintain a reference to initialized system settings.
         /// </summary>
+        #pragma warning disable CS0414 // The private field `Duplicati.CommandLine.BackendTester.Program.SystemSettings' is assigned but its value is never used
         private static IDisposable SystemSettings;
+        #pragma warning restore CS0414 // The private field `Duplicati.CommandLine.BackendTester.Program.SystemSettings' is assigned but its value is never used
 
         class TempFile
         {
@@ -71,7 +73,7 @@ namespace Duplicati.CommandLine.BackendTester
                 {
                     try
                     {
-                        var p = Library.Utility.Utility.ExpandEnvironmentVariables(_args[0]);
+                        var p = Environment.ExpandEnvironmentVariables(_args[0]);
                         if (System.IO.File.Exists(p))
                             _args = (from x in System.IO.File.ReadLines(p)
                                 where !string.IsNullOrWhiteSpace(x) && !x.Trim().StartsWith("#", StringComparison.Ordinal)
@@ -86,7 +88,7 @@ namespace Duplicati.CommandLine.BackendTester
                 List<string> args = new List<string>(_args);
                 Dictionary<string, string> options = Library.Utility.CommandLineParser.ExtractOptions(args);
 
-                if (args.Count != 1 || args[0].ToLower() == "help" || args[0] == "?")
+                if (args.Count != 1 || String.Equals(args[0], "help", StringComparison.OrdinalIgnoreCase) || args[0] == "?")
                 {
                     Console.WriteLine("Usage: <protocol>://<username>:<password>@<path>");
                     Console.WriteLine("Example: ftp://user:pass@server/folder");
@@ -135,14 +137,6 @@ namespace Duplicati.CommandLine.BackendTester
 
         static bool Run(List<string> args, Dictionary<string, string> options, bool first)
         {
-            string allowedChars = ValidFilenameChars;
-            if (options.ContainsKey("extended-chars"))
-                allowedChars += options["extended-chars"];
-            else
-                allowedChars += ExtendedChars;
-
-            bool autoCreateFolders = Library.Utility.Utility.ParseBoolOption(options, "auto-create-folder");
-
             Library.Interface.IBackend backend = Library.DynamicLoader.BackendLoader.GetBackend(args[0], options);
             if (backend == null)
             {
@@ -152,6 +146,14 @@ namespace Duplicati.CommandLine.BackendTester
                 return false;
             }
 
+            string allowedChars = ValidFilenameChars;
+            if (options.ContainsKey("extended-chars"))
+            {
+                allowedChars += String.IsNullOrEmpty(options["extended-chars"]) ? ExtendedChars : options["extended-chars"];
+            }
+
+            bool autoCreateFolders = Library.Utility.Utility.ParseBoolOption(options, "auto-create-folder");
+
             string disabledModulesValue;
             string enabledModulesValue;
             options.TryGetValue("enable-module", out enabledModulesValue);
@@ -161,7 +163,7 @@ namespace Duplicati.CommandLine.BackendTester
 
             List<Library.Interface.IGenericModule> loadedModules = new List<IGenericModule>();
             foreach (Library.Interface.IGenericModule m in Library.DynamicLoader.GenericLoader.Modules)
-                if (Array.IndexOf<string>(disabledModules, m.Key.ToLower()) < 0 && (m.LoadAsDefault || Array.IndexOf<string>(enabledModules, m.Key.ToLower()) >= 0))
+                if (!disabledModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase) && (m.LoadAsDefault || enabledModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase)))
                 {
                     m.Configure(options);
                     loadedModules.Add(m);

Duplicati/CommandLine/BackendTool/Program.cs

@@ -73,7 +73,7 @@ namespace Duplicati.CommandLine.BackendTool
                 }
 
 
-                if (args.Count < 2 || args[0].ToLower() == "help" || args[0] == "?" || command == null)
+                if (args.Count < 2 || String.Equals(args[0], "help", StringComparison.OrdinalIgnoreCase) || args[0] == "?" || command == null)
                 {
                     if (command == null && args.Count >= 2)
                     {

Duplicati/CommandLine/Commands.cs

@@ -174,7 +174,7 @@ namespace Duplicati.CommandLine
                 setup(i);
                 i.ListAffected(args, res => 
                 { 
-                    if (res.Filesets != null && res.Filesets.Count() != 0)
+                    if (res.Filesets != null && res.Filesets.Any())
                     {
                         outwriter.WriteLine("The following filesets are affected:");
                         foreach (var e in res.Filesets)
@@ -314,8 +314,8 @@ namespace Duplicati.CommandLine
                 // try again with all-versions set
                 var compareFilter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(args), filter);
                 var isRequestForFiles = 
-                    !controlFiles && res.Filesets.Count() != 0 && 
-                    (res.Files == null || res.Files.Count() == 0) && 
+                    !controlFiles && res.Filesets.Any() && 
+                    (res.Files == null || !res.Files.Any()) && 
                     !compareFilter.Empty;
                 
                 if (isRequestForFiles && !Library.Utility.Utility.ParseBoolOption(options, "all-versions"))
@@ -327,7 +327,7 @@ namespace Duplicati.CommandLine
                     res = i.List(args, filter);
                 }
 
-                if (res.Filesets.Count() != 0 && (res.Files == null || res.Files.Count() == 0) && compareFilter.Empty)
+                if (res.Filesets.Any() && (res.Files == null || !res.Files.Any()) && compareFilter.Empty)
                 {
                     outwriter.WriteLine("Listing filesets:");
                     
@@ -345,11 +345,11 @@ namespace Duplicati.CommandLine
                 }
                 else
                 {
-                    if (res.Filesets.Count() == 0)
+                    if (!res.Filesets.Any())
                     {
                         outwriter.WriteLine("No time or version matched a fileset");
                     }
-                    else if (res.Files == null || res.Files.Count() == 0)
+                    else if (res.Files == null || !res.Files.Any())
                     {
                         outwriter.WriteLine("Found {0} filesets, but no files matched", res.Filesets.Count());
                     }
@@ -383,7 +383,7 @@ namespace Duplicati.CommandLine
         {
             var requiredOptions = new string[] { "keep-time", "keep-versions", "version" };
             
-            if (!options.Keys.Where(x => requiredOptions.Contains(x, StringComparer.OrdinalIgnoreCase)).Any())
+            if (!options.Keys.Any(x => requiredOptions.Contains(x, StringComparer.OrdinalIgnoreCase)))
             {
                 outwriter.WriteLine(Strings.Program.DeleteCommandNeedsOptions("delete", requiredOptions)); 
                 return 200;
@@ -396,7 +396,7 @@ namespace Duplicati.CommandLine
                 args.RemoveAt(0);
                 var res = i.Delete();
                 
-                if (res.DeletedSets.Count() == 0)
+                if (!res.DeletedSets.Any())
                 {
                     outwriter.WriteLine(Strings.Program.NoFilesetsMatching);
                 }
@@ -519,7 +519,7 @@ namespace Duplicati.CommandLine
                         if (output.FullResults)
                             Library.Utility.Utility.PrintSerializeObject(res, outwriter);
 
-                        if (res.Warnings.Count() > 0)
+                        if (res.Warnings.Any())
                             return 2;
                     }
                 }
@@ -623,7 +623,7 @@ namespace Duplicati.CommandLine
                 output.MessageEvent(string.Format("  Data uploaded: {0}", Library.Utility.Utility.FormatSizeString(result.BackendStatistics.BytesUploaded)));
                 output.MessageEvent(string.Format("  Data downloaded: {0}", Library.Utility.Utility.FormatSizeString(result.BackendStatistics.BytesDownloaded)));
 
-                if (result.ExaminedFiles == 0 && (filter != null || !filter.Empty))
+                if (result.ExaminedFiles == 0 && (filter != null && !filter.Empty))
                     output.MessageEvent("No files were processed. If this was not intentional you may want to use the \"test-filters\" command");
 
                 output.MessageEvent("Backup completed successfully!");
@@ -694,8 +694,8 @@ namespace Duplicati.CommandLine
             }
             else
             {
-                var filtered = from n in result.Verifications where n.Value.Count() != 0 select n;
-                if (filtered.Count() == 0)
+                var filtered = from n in result.Verifications where n.Value.Any() select n;
+                if (!filtered.Any())
                 {
                     outwriter.WriteLine("Examined {0} files and found no errors", totalFiles);
                     return 0;
@@ -708,24 +708,24 @@ namespace Duplicati.CommandLine
                         if (changecount == 0)
                         {
                             if (fullResults)
-                                Console.WriteLine("{0}: No errors", n.Key);
+                                outwriter.WriteLine("{0}: No errors", n.Key);
                         }
                         else
                         {
-                            Console.WriteLine("{0}: {1} errors", n.Key, changecount);
+                            outwriter.WriteLine("{0}: {1} errors", n.Key, changecount);
                             var count = 0;
                             foreach (var c in n.Value)
                             {
                                 count++;
-                                Console.WriteLine("\t{0}: {1}", c.Key, c.Value);
+                                outwriter.WriteLine("\t{0}: {1}", c.Key, c.Value);
                                 if (!fullResults && count == 10 && changecount > 10)
                                 {
-                                    Console.WriteLine("\t... and {0} more", changecount - count);
+                                    outwriter.WriteLine("\t... and {0} more", changecount - count);
                                     break;
                                 }
                             }
 
-                            Console.WriteLine();
+                            outwriter.WriteLine();
                         }
                     }
 
@@ -740,7 +740,7 @@ namespace Duplicati.CommandLine
             return 200;
         }
 
-        public static int PrintInvalidCommand(TextWriter outwriter, string command, List<string> args)
+        public static int PrintInvalidCommand(TextWriter outwriter, string command)
         {
             outwriter.WriteLine(Strings.Program.InvalidCommandError(command));
             return 200;
@@ -996,7 +996,7 @@ namespace Duplicati.CommandLine
 
 
             using (var console = new ConsoleOutput(outwriter, options))
-            using (var i = new Library.Main.Controller(args[0], options, console))
+            using (var i = new Library.Main.Controller(backend, options, console))
             {
                 setup(i);
                 i.PurgeFiles(filter);
@@ -1052,7 +1052,7 @@ namespace Duplicati.CommandLine
             using (var i = new Library.Main.Controller(args[0], options, console))
             {
                 setup(i);
-                var res = i.PurgeBrokenFiles(filter);
+                i.PurgeBrokenFiles(filter);
             }
 
             return 0;

Duplicati/CommandLine/Duplicati.CommandLine.csproj

@@ -43,6 +43,9 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
+  <PropertyGroup>
+    <ApplicationManifest>app.manifest</ApplicationManifest>
+  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />
@@ -207,6 +210,9 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <None Include="app.manifest">
+      <SubType>Designer</SubType>
+    </None>
     <None Include="Duplicati.snk" />
   </ItemGroup>
   <ItemGroup>

Duplicati/CommandLine/Help.cs

@@ -374,19 +374,6 @@ namespace Duplicati.CommandLine
             lines.Add("");
         }
 
-        private static string PrintArguments(IEnumerable<Duplicati.Library.Interface.ICommandLineArgument> args)
-        {
-            if (args == null)
-                return "";
-
-            List<string> lines = new List<string>();
-            foreach (Library.Interface.ICommandLineArgument arg in args)
-                Library.Interface.CommandLineArgument.PrintArgument(lines, arg, "  ");
-
-            return string.Join(Environment.NewLine, lines.ToArray());
-
-        }
-
         private static void PrintFormatted(TextWriter outwriter, IEnumerable<string> lines)
         {
             int windowWidth = 80;

Duplicati/CommandLine/Program.cs

@@ -203,6 +203,21 @@ namespace Duplicati.CommandLine
             string command = cargs[0];
             cargs.RemoveAt(0);
 
+            if (verboseErrors)
+            {
+                outwriter.WriteLine("Input command: {0}", command);
+                outwriter.WriteLine("Input arguments: ");
+                foreach (var a in cargs)
+                    outwriter.WriteLine("\t{0}", a);
+                outwriter.WriteLine();
+
+                outwriter.WriteLine("Input options: ");
+                foreach (var n in options)
+                    outwriter.WriteLine("{0}: {1}", n.Key, n.Value);
+                outwriter.WriteLine();
+            }
+
+
             if (CommandMap.ContainsKey(command))
             {
                 var autoupdate = Library.Utility.Utility.ParseBoolOption(options, "auto-update");
@@ -219,7 +234,7 @@ namespace Duplicati.CommandLine
             }
             else
             {
-                Commands.PrintInvalidCommand(outwriter, command, cargs);
+                Commands.PrintInvalidCommand(outwriter, command);
                 return 200;
             }
         }
@@ -241,6 +256,7 @@ namespace Duplicati.CommandLine
                 if (ex is Duplicati.Library.Interface.UserInformationException && !verboseErrors)
                 {
                     errwriter.WriteLine();
+                    errwriter.WriteLine("ErrorID: {0}", ((Duplicati.Library.Interface.UserInformationException)ex).HelpID);
                     errwriter.WriteLine(ex.Message);
                 }
                 else if (!(ex is Library.Interface.CancelException))
@@ -283,7 +299,7 @@ namespace Duplicati.CommandLine
         {
             try
             {
-                List<string> fargs = new List<string>(Library.Utility.Utility.ReadFileWithDefaultEncoding(Library.Utility.Utility.ExpandEnvironmentVariables(filename)).Replace("\r\n", "\n").Replace("\r", "\n").Split(new String[] { "\n" }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()));
+                List<string> fargs = new List<string>(Library.Utility.Utility.ReadFileWithDefaultEncoding(Environment.ExpandEnvironmentVariables(filename)).Replace("\r\n", "\n").Replace("\r", "\n").Split(new String[] { "\n" }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()));
                 var newsource = new List<string>();
                 string newtarget = null;
                 string prependfilter = null;

Duplicati/CommandLine/RecoveryTool/Program.cs

@@ -106,7 +106,7 @@ namespace Duplicati.CommandLine.RecoveryTool
         {
             try
             {
-                List<string> fargs = new List<string>(Library.Utility.Utility.ReadFileWithDefaultEncoding(Library.Utility.Utility.ExpandEnvironmentVariables(filename)).Replace("\r\n", "\n").Replace("\r", "\n").Split(new String[] { "\n" }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()));
+                List<string> fargs = new List<string>(Library.Utility.Utility.ReadFileWithDefaultEncoding(Environment.ExpandEnvironmentVariables(filename)).Replace("\r\n", "\n").Replace("\r", "\n").Split(new String[] { "\n" }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()));
                 var tmpparsed = Library.Utility.FilterCollector.ExtractOptions(fargs);
 
                 var opt = tmpparsed.Item1;

Duplicati/CommandLine/app.manifest

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
+  <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
+    <application>
+      <!-- A list of the Windows versions that this application has been tested on
+           and is designed to work with. Uncomment the appropriate elements
+           and Windows will automatically select the most compatible environment. -->
+
+      <!-- Windows 7 -->
+      <!--<supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}" />-->
+
+      <!-- Windows 8 -->
+      <!--<supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}" />-->
+
+      <!-- Windows 8.1 -->
+      <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}" />
+
+      <!-- Windows 10 -->
+      <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}" />
+
+    </application>
+  </compatibility>
+</assembly>

Duplicati/GUI/Duplicati.GUI.TrayIcon/CocoaRunner.cs

@@ -33,7 +33,7 @@ namespace Duplicati.GUI.TrayIcon
             
             public NSMenuItem MenuItem { get { return m_item; } }
             
-            public MenuItemWrapper(string text, Duplicati.GUI.TrayIcon.MenuIcons icon, Action callback, IList<Duplicati.GUI.TrayIcon.IMenuItem> subitems)
+            public MenuItemWrapper(string text, Action callback, IList<Duplicati.GUI.TrayIcon.IMenuItem> subitems)
             {
                 if (text == "-")
                     m_item = NSMenuItem.SeparatorItem;
@@ -199,7 +199,7 @@ namespace Duplicati.GUI.TrayIcon
 
         protected override Duplicati.GUI.TrayIcon.IMenuItem CreateMenuItem (string text, Duplicati.GUI.TrayIcon.MenuIcons icon, Action callback, System.Collections.Generic.IList<Duplicati.GUI.TrayIcon.IMenuItem> subitems)
         {
-            return new MenuItemWrapper(text, icon, callback, subitems);
+            return new MenuItemWrapper(text, callback, subitems);
         }
 
         protected override void Exit()

Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj

@@ -63,6 +63,9 @@
   <PropertyGroup>
     <AssemblyOriginatorKeyFile>Duplicati.snk</AssemblyOriginatorKeyFile>
   </PropertyGroup>
+  <PropertyGroup>
+    <ApplicationManifest>app.manifest</ApplicationManifest>
+  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />
@@ -103,6 +106,7 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="HttpServerConnection.cs" />
     <None Include="app.config" />
+    <None Include="app.manifest" />
     <None Include="Duplicati.snk" />
     <None Include="Properties\Settings.settings">
       <Generator>SettingsSingleFileGenerator</Generator>

Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs

@@ -76,9 +76,7 @@ namespace Duplicati.GUI.TrayIcon
 
         public HttpServerConnection(Uri server, string password, bool saltedpassword, Program.PasswordSource passwordSource, bool disableTrayIconLogin, Dictionary<string, string> options)
         {
-            m_baseUri = server.ToString();
-            if (!m_baseUri.EndsWith("/", StringComparison.Ordinal))
-                m_baseUri += "/";
+            m_baseUri = Duplicati.Library.Utility.Utility.AppendDirSeparator(server.ToString(), "/");
 
             m_apiUri = m_baseUri + "api/v1";
 

Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs

@@ -36,11 +36,11 @@ namespace Duplicati.GUI.TrayIcon
         public static string BrowserCommand { get { return _browser_command; } }
         public static Server.Database.Connection databaseConnection = null;
 
-        private static string GetDefaultToolKit(bool printwarnings)
+        private static string GetDefaultToolKit()
         {
             // No longer using Cocoa directly as it fails on 32bit as well            
             if (Duplicati.Library.Utility.Utility.IsClientOSX)
-                    return TOOLKIT_RUMPS;
+                return TOOLKIT_RUMPS;
 
 #if __MonoCS__ || __WindowsGTK__ || ENABLE_GTK
             if (Duplicati.Library.Utility.Utility.IsClientLinux)
@@ -114,7 +114,7 @@ namespace Duplicati.GUI.TrayIcon
                 if (Library.Utility.Utility.IsClientLinux && !Library.Utility.Utility.IsClientOSX)
                     Console.WriteLine("Warning: this build does not support GTK, rebuild with ENABLE_GTK defined");
 #endif
-                toolkit = GetDefaultToolKit(true);
+                toolkit = GetDefaultToolKit();
             }
             else 
             {
@@ -131,7 +131,7 @@ namespace Duplicati.GUI.TrayIcon
                 else if (TOOLKIT_RUMPS.Equals(toolkit, StringComparison.OrdinalIgnoreCase))
                     toolkit = TOOLKIT_RUMPS;
                 else
-                    toolkit = GetDefaultToolKit(true);
+                    toolkit = GetDefaultToolKit();
             }
 
             HostedInstanceKeeper hosted = null;
@@ -437,7 +437,7 @@ namespace Duplicati.GUI.TrayIcon
                 
                 var args = new List<Duplicati.Library.Interface.ICommandLineArgument>()
                 {
-                    new Duplicati.Library.Interface.CommandLineArgument(TOOLKIT_OPTION, CommandLineArgument.ArgumentType.Enumeration, "Selects the toolkit to use", "Choose the toolkit used to generate the TrayIcon, note that it will fail if the selected toolkit is not supported on this machine", GetDefaultToolKit(false), null, toolkits.ToArray()),
+                    new Duplicati.Library.Interface.CommandLineArgument(TOOLKIT_OPTION, CommandLineArgument.ArgumentType.Enumeration, "Selects the toolkit to use", "Choose the toolkit used to generate the TrayIcon, note that it will fail if the selected toolkit is not supported on this machine", GetDefaultToolKit(), null, toolkits.ToArray()),
                     new Duplicati.Library.Interface.CommandLineArgument(HOSTURL_OPTION, CommandLineArgument.ArgumentType.String, "Selects the url to connect to", "Supply the url that the TrayIcon will connect to and show status for", DEFAULT_HOSTURL),
                     new Duplicati.Library.Interface.CommandLineArgument(NOHOSTEDSERVER_OPTION, CommandLineArgument.ArgumentType.String, "Disables local server", "Set this option to not spawn a local service, use if the TrayIcon should connect to a running service"),
                     new Duplicati.Library.Interface.CommandLineArgument(READCONFIGFROMDB_OPTION, CommandLineArgument.ArgumentType.String, "Read server connection info from DB", $"Set this option to read server connection info for running service from its database (only together with {NOHOSTEDSERVER_OPTION})"),               

Duplicati/GUI/Duplicati.GUI.TrayIcon/RumpsRunner.cs

@@ -36,7 +36,7 @@ namespace Duplicati.GUI.TrayIcon
             private bool m_enabled;
             private bool m_default;
 
-            public MenuItemWrapper(RumpsRunner parent, string text, Duplicati.GUI.TrayIcon.MenuIcons icon, Action callback, IList<Duplicati.GUI.TrayIcon.IMenuItem> subitems)
+            public MenuItemWrapper(RumpsRunner parent, string text, Action callback, IList<Duplicati.GUI.TrayIcon.IMenuItem> subitems)
             {
                 m_parent = parent;
                 Key = Guid.NewGuid().ToString("N");
@@ -176,9 +176,11 @@ namespace Duplicati.GUI.TrayIcon
             var ch = ChannelManager.CreateChannel<string>();
             m_toRumps = ch.AsWriteOnly();
 
-            WriteChannel(m_rumpsProcess.StandardInput, ch.AsReadOnly());
-            var standardOutputTask = ReadChannel(m_rumpsProcess.StandardOutput);
-            var standardErrorTask = ReadChannel(m_rumpsProcess.StandardError);
+            WriteChannel(m_rumpsProcess.StandardInput, ch.AsReadOnly());
+            #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
+            ReadChannel(m_rumpsProcess.StandardOutput);
+            ReadChannel(m_rumpsProcess.StandardError);
+            #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
 
             m_toRumps.WriteNoWait(JsonConvert.SerializeObject(new {Action = "background"}));
             //m_toRumps.WriteNoWait(JsonConvert.SerializeObject(new {Action = "setappicon", Image = GetIcon(m_lastIcon)}));
@@ -213,7 +215,7 @@ namespace Duplicati.GUI.TrayIcon
                         while(true)
                         {
                             var line = await self.Input.ReadAsync();
-                            await stream.WriteLineAsync(line);
+                            await stream.WriteLineAsync(line).ConfigureAwait(false);
                             //Console.WriteLine("Wrote {0}", line);
                         }
                     }
@@ -225,7 +227,7 @@ namespace Duplicati.GUI.TrayIcon
         {
             string line;
             using(stream)
-                while ((line = await stream.ReadLineAsync()) != null)
+                while ((line = await stream.ReadLineAsync().ConfigureAwait(false)) != null)
                 {
                     //Console.WriteLine("Got message: {0}", line);
 
@@ -233,7 +235,7 @@ namespace Duplicati.GUI.TrayIcon
                     if (line.StartsWith("click:", StringComparison.OrdinalIgnoreCase))
                     {
                         var key = line.Substring("click:".Length);
-                        var menu = m_menus.Where(x => string.Equals(x.Key, key)).FirstOrDefault();
+                        var menu = m_menus.FirstOrDefault(x => string.Equals(x.Key, key));
                         if (menu == null)
                         {
                             Console.WriteLine("Menu not found: {0}", key);
@@ -323,7 +325,7 @@ namespace Duplicati.GUI.TrayIcon
 
         protected override Duplicati.GUI.TrayIcon.IMenuItem CreateMenuItem (string text, Duplicati.GUI.TrayIcon.MenuIcons icon, Action callback, System.Collections.Generic.IList<Duplicati.GUI.TrayIcon.IMenuItem> subitems)
         {
-            return new MenuItemWrapper(this, text, icon, callback, subitems);
+            return new MenuItemWrapper(this, text, callback, subitems);
         }
 
         protected override void Exit()
@@ -331,9 +333,9 @@ namespace Duplicati.GUI.TrayIcon
             m_isQuitting = true;
             if (m_rumpsProcess != null && !m_rumpsProcess.HasExited)
             {
-                m_toRumps.WriteNoWait(JsonConvert.SerializeObject(new {Action = "shutdown"}));
                 if (m_toRumps != null)
-                {
+                {
+                    m_toRumps.WriteNoWait(JsonConvert.SerializeObject(new { Action = "shutdown" }));
                     m_toRumps.Dispose();
                     m_toRumps = null;
                 }

Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs

@@ -165,27 +165,11 @@ namespace Duplicati.GUI.TrayIcon
             ShowStatusWindow();
         }
 
-        protected void OnWizardClicked()
-        {
-        }
-        
-        protected void OnOptionsClicked()
-        {
-        }
-        
-        protected void OnStopClicked()
-        {
-        }
-
         protected void OnQuitClicked()
         {
             Exit();
         }
 
-        protected void OnThrottleClicked()
-        {
-        }
-
         protected void OnPauseClicked()
         {
             if (m_stateIsPaused)

Duplicati/GUI/Duplicati.GUI.TrayIcon/app.manifest

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
+  <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
+    <application>
+      <!-- A list of the Windows versions that this application has been tested on
+           and is designed to work with. Uncomment the appropriate elements
+           and Windows will automatically select the most compatible environment. -->
+
+      <!-- Windows 7 -->
+      <!--<supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}" />-->
+
+      <!-- Windows 8 -->
+      <!--<supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}" />-->
+
+      <!-- Windows 8.1 -->
+      <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}" />
+
+      <!-- Windows 10 -->
+      <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}" />
+
+    </application>
+  </compatibility>
+</assembly>

Duplicati/Library/AutoUpdater/AutoUpdateSettings.cs

@@ -106,7 +106,7 @@ namespace Duplicati.Library.AutoUpdater
                 if (UsesAlternateURLs)
                     return Environment.GetEnvironmentVariable(string.Format(UPDATEURL_ENVNAME_TEMPLATE, AppName)).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                 else
-                    return ReadResourceText(UPDATE_URL, OEM_UPDATE_URL).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);; 
+                    return ReadResourceText(UPDATE_URL, OEM_UPDATE_URL).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
             }
         }
 

Duplicati/Library/AutoUpdater/SignatureReadingStream.cs

@@ -174,7 +174,9 @@ namespace Duplicati.Library.AutoUpdater
             }
         }
 
-        public override long Position
+        // Since the constructor sets the Position, we seal the implementation here to prevent subclasses
+        // from potentially referencing uninitialized members.
+        public sealed override long Position
         {
             get
             {

Duplicati/Library/AutoUpdater/UpdaterManager.cs

@@ -21,7 +21,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
 using Duplicati.Library.Interface;
-
+
 namespace Duplicati.Library.AutoUpdater
 {
     public enum AutoUpdateStrategy
@@ -51,7 +51,7 @@ namespace Duplicati.Library.AutoUpdater
         private static readonly string INSTALLED_BASE_DIR =
             string.IsNullOrWhiteSpace(System.Environment.GetEnvironmentVariable(string.Format(BASEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)))
             ? System.IO.Path.GetDirectoryName(Duplicati.Library.Utility.Utility.getEntryAssembly().Location)
-            : Library.Utility.Utility.ExpandEnvironmentVariables(System.Environment.GetEnvironmentVariable(string.Format(BASEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)));
+            : Environment.ExpandEnvironmentVariables(System.Environment.GetEnvironmentVariable(string.Format(BASEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)));
 
         private static readonly bool DISABLE_UPDATE_DOMAIN = !string.IsNullOrWhiteSpace(System.Environment.GetEnvironmentVariable(string.Format(SKIPUPDATE_ENVNAME_TEMPLATE, APPNAME)));
 
@@ -169,7 +169,7 @@ namespace Duplicati.Library.AutoUpdater
 
                 if (string.IsNullOrWhiteSpace(installdir))
                     foreach (var p in legacypaths)
-                        if (!string.IsNullOrWhiteSpace(p) && System.IO.Directory.Exists(p) && System.IO.Directory.EnumerateFiles(p, "*", System.IO.SearchOption.TopDirectoryOnly).Count() > 0 && TestDirectoryIsWriteable(p))
+                        if (!string.IsNullOrWhiteSpace(p) && System.IO.Directory.Exists(p) && System.IO.Directory.EnumerateFiles(p, "*", System.IO.SearchOption.TopDirectoryOnly).Any() && TestDirectoryIsWriteable(p))
                         {
                             installdir = p;
                             break;
@@ -187,7 +187,7 @@ namespace Duplicati.Library.AutoUpdater
             }
             else
             {
-                INSTALLDIR = Library.Utility.Utility.ExpandEnvironmentVariables(System.Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)));
+                INSTALLDIR = Environment.ExpandEnvironmentVariables(System.Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)));
             }
 
 
@@ -460,7 +460,7 @@ namespace Duplicati.Library.AutoUpdater
                             var areq = new Duplicati.Library.Utility.AsyncHttpRequest(wreq);
                             using (var resp = areq.GetResponse())
                             using (var rss = areq.GetResponseStream())
-                            using (var pgs = new Duplicati.Library.Utility.ProgressReportingStream(rss, version.CompressedSize, cb))
+                            using (var pgs = new Duplicati.Library.Utility.ProgressReportingStream(rss, cb))
                             {
                                 Duplicati.Library.Utility.Utility.CopyStream(pgs, tempfile);
                             }
@@ -1081,6 +1081,10 @@ namespace Duplicati.Library.AutoUpdater
             // If we are not the primary entry, just execute
             if (IsRunningInUpdateEnvironment)
             {
+                // For some reason this does not work
+                //if (Library.Utility.Utility.IsClientWindows)
+                    //Duplicati.Library.Utility.Win32.AttachConsole(Duplicati.Library.Utility.Win32.ATTACH_PARENT_PROCESS);
+
                 int r = 0;
                 WrapWithUpdater(defaultstrategy, () => {
                     r = RunMethod(method, cmdargs);
@@ -1106,7 +1110,7 @@ namespace Duplicati.Library.AutoUpdater
                 {
                     CreateNoWindow = true,
                     UseShellExecute = false,
-                    ErrorDialog = false
+                    ErrorDialog = false,
                 };
                 pi.EnvironmentVariables.Clear();
 
@@ -1118,8 +1122,36 @@ namespace Duplicati.Library.AutoUpdater
                 pi.EnvironmentVariables[string.Format(BASEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)] = InstalledBaseDir;
                 pi.EnvironmentVariables["LOCALIZATION_FOLDER"] = InstalledBaseDir;
 
+                // On Windows, we manually redirect the streams
+                if (Library.Utility.Utility.IsClientWindows)
+                {
+                    pi.RedirectStandardError = true;
+                    pi.RedirectStandardInput = true;
+                    pi.RedirectStandardOutput = true;
+                }
+
                 var proc = System.Diagnostics.Process.Start(pi);
+                Task tasks = null;
+                if (Library.Utility.Utility.IsClientWindows)
+                {
+                    // On Windows, we manually redirect the streams
+                    tasks = Task.WhenAll(
+                        // This does some unwanted buffering that breaks things
+                        //Console.OpenStandardInput().CopyToAsync(proc.StandardInput.BaseStream),
+                        Task.Run(async () => {
+                            var stdin = new StreamReader(Console.OpenStandardInput());
+                            var line = string.Empty;
+                            while ((line = await stdin.ReadLineAsync().ConfigureAwait(false)) != null)
+                                await proc.StandardInput.WriteLineAsync(line);
+                        }),
+                        proc.StandardOutput.BaseStream.CopyToAsync(Console.OpenStandardOutput()),
+                        proc.StandardError.BaseStream.CopyToAsync(Console.OpenStandardError())
+                    );
+                }
+
                 proc.WaitForExit();
+                if (tasks != null)
+                    tasks.Wait(1000);
 
                 if (proc.ExitCode != MAGIC_EXIT_CODE)
                     return proc.ExitCode;

Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs

@@ -147,11 +147,7 @@ namespace Duplicati.Library.Backend.AlternativeFTP
                 _userInfo.Domain = "";
 
             _url = u.SetScheme("ftp").SetQuery(null).SetCredentials(null, null).ToString();
-            if (!_url.EndsWith("/", StringComparison.Ordinal))
-            {
-                _url += "/";
-            }
-
+            _url = Duplicati.Library.Utility.Utility.AppendDirSeparator(_url, "/");
             _listVerify = !CoreUtility.ParseBoolOption(options, "disable-upload-verify");
 
             // Process the aftp-data-connection-type option

Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs

@@ -31,7 +31,7 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
         private const string DELAY_OPTION = "amzcd-consistency-delay";
 
         private const string DEFAULT_LABELS = "duplicati,backup";
-        private const string DEFAULT_DELAY = "15s";
+        private const string DEFAULT_DELAY = "30s";
 
         private const int PAGE_SIZE = 200;
 
@@ -48,23 +48,22 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
         private readonly string m_path;
         private readonly string[] m_labels;
 
+        private readonly string m_authid;
         private readonly OAuthHelper m_oauth;
         private Dictionary<string, string> m_filecache;
         private readonly string m_userid;
-        private DateTime m_waitUntil;
         private readonly TimeSpan m_delayTimeSpan;
 
-        private enum RemoteOperation
-        {
-            First,
-            List,
-            Get,
-            Put,
-            Delete,
-            Rename
-        }
+        private static readonly object m_waitUntilLock;
+        private static Dictionary<string, DateTime> m_waitUntilAuthId;
+        private static Dictionary<string, DateTime> m_waitUntilRemotename;
 
-        private RemoteOperation m_lastOperation = RemoteOperation.First;
+        static AmzCD()
+        {
+            m_waitUntilLock = new object();
+            m_waitUntilAuthId = new Dictionary<string, DateTime>();
+            m_waitUntilRemotename = new Dictionary<string, DateTime>();
+        }
 
         public AmzCD()
         {
@@ -73,14 +72,10 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
         public AmzCD(string url, Dictionary<string, string> options)
         {
             var uri = new Utility.Uri(url);
+            m_path = Duplicati.Library.Utility.Utility.AppendDirSeparator(uri.HostAndPath, "/");
 
-            m_path = uri.HostAndPath;
-            if (!m_path.EndsWith("/", StringComparison.Ordinal))
-                m_path += "/";
-
-            string authid = null;
             if (options.ContainsKey(AUTHID_OPTION))
-                authid = options[AUTHID_OPTION];
+                m_authid = options[AUTHID_OPTION];
 
             string labels = DEFAULT_LABELS;
             if (options.ContainsKey(LABELS_OPTION))
@@ -97,23 +92,64 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
                 m_delayTimeSpan = new TimeSpan(0);
             else
                 m_delayTimeSpan = Library.Utility.Timeparser.ParseTimeSpan(delay);
-            m_waitUntil = DateTime.Now + m_delayTimeSpan;
 
-            m_oauth = new OAuthHelper(authid, this.ProtocolKey) { AutoAuthHeader = true };
-            m_userid = authid.Split(new string[] {":"}, StringSplitOptions.RemoveEmptyEntries).First();
+            m_oauth = new OAuthHelper(m_authid, this.ProtocolKey) { AutoAuthHeader = true };
+            m_userid = m_authid.Split(new [] {":"}, StringSplitOptions.RemoveEmptyEntries).First();
         }
 
-        private void EnforceConsistencyDelay(RemoteOperation lastop)
+        private static void CleanWaitUntil()
         {
-            if (m_lastOperation == RemoteOperation.First)
-                m_lastOperation = lastop;
-                 
-            if (lastop == m_lastOperation)
-                return;
+            lock (m_waitUntilLock)
+            {
+                DateTime now = DateTime.Now;
+                m_waitUntilRemotename = m_waitUntilRemotename.Where(pair => pair.Value > now)
+                                 .ToDictionary(pair => pair.Key, pair => pair.Value);
+            }
+        }
 
-            m_lastOperation = lastop;
+        private DateTime GetWaitUntil(string remotename)
+        {
+            lock (m_waitUntilLock)
+            {
+                CleanWaitUntil();
+
+                DateTime result;
+                if (string.IsNullOrEmpty(remotename))
+                {
+                    if (m_waitUntilAuthId.TryGetValue(m_authid, out result))
+                        return result;
+                }
+                else
+                {
+                    if (m_waitUntilRemotename.TryGetValue(remotename, out result))
+                        return result;
+                }
+
+                return DateTime.MinValue;
+            }
+        }
+
+        private void SetWaitUntil(string remotename, DateTime value)
+        {
+            lock (m_waitUntilLock)
+            {
+                DateTime oldValue;
+
+                if (!string.IsNullOrEmpty(remotename))
+                {
+                    if (!m_waitUntilRemotename.TryGetValue(remotename, out oldValue) || value > oldValue)
+                        m_waitUntilRemotename[remotename] = value;
+                }
+
+                if (!m_waitUntilAuthId.TryGetValue(m_authid, out oldValue) || value > oldValue)
+                    m_waitUntilAuthId[m_authid] = value;
+            }
+        }
+
+        private void EnforceConsistencyDelay(string remotename)
+        {
+            var wait = GetWaitUntil(remotename) - DateTime.Now;
 
-            var wait = m_waitUntil - DateTime.Now;
             if (wait.Ticks > 0)
                 System.Threading.Thread.Sleep(wait);
         }
@@ -221,7 +257,7 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
                                 rs.Write(data, 0, data.Length);
                         }
                     );
-                    m_waitUntil = DateTime.Now + m_delayTimeSpan;
+                    SetWaitUntil(null, DateTime.Now + m_delayTimeSpan);
                 }
                 else if (self != null && self.Count > 1)
                     throw new UserInformationException(Strings.AmzCD.MultipleEntries(p, "/" + string.Join("/", curpath)), "AmzCDMultipleEntries");
@@ -275,9 +311,10 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
         }
 
         #region IStreamingBackend implementation
+
         public void Put(string remotename, System.IO.Stream stream)
         {
-            EnforceConsistencyDelay(RemoteOperation.Put);
+            EnforceConsistencyDelay(remotename);
 
             var overwrite = FileCache.ContainsKey(remotename);
             var fileid = overwrite ? m_filecache[remotename] : null;
@@ -301,8 +338,8 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
                         req.Method = overwrite ? "PUT" : "POST";
                     },
 
-                    new MultipartItem(createreq, name: "metadata"),
-                    new MultipartItem(stream, name: "content", filename: remotename)
+                    new MultipartItem(createreq, "metadata"),
+                    new MultipartItem(stream, "content", remotename)
 
                 );
 
@@ -322,23 +359,26 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
             }
             finally
             {
-                m_waitUntil = DateTime.Now + m_delayTimeSpan;
+                SetWaitUntil(remotename, DateTime.Now + m_delayTimeSpan);
             }
         }
+
         public void Get(string remotename, System.IO.Stream stream)
         {
-            EnforceConsistencyDelay(RemoteOperation.Get);
+            EnforceConsistencyDelay(remotename);
 
             using (var resp = m_oauth.GetResponse(string.Format("{0}/nodes/{1}/content", ContentUrl, GetFileID(remotename))))
             using(var rs = Library.Utility.AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream()))
                 Utility.Utility.CopyStream(rs, stream);
         }
+
         #endregion
 
         #region IBackend implementation
+
         public IEnumerable<IFileEntry> List()
         {
-            EnforceConsistencyDelay(RemoteOperation.List);
+            EnforceConsistencyDelay(null);
 
             var query = string.Format("{0}/nodes?filters=parents:{1}&limit={2}", MetadataUrl, Utility.Uri.UrlEncode(CurrentDirectory.ID), PAGE_SIZE);
             var res = new List<IFileEntry>();
@@ -386,23 +426,26 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
             return res;
 
         }
+
         public void Put(string remotename, string filename)
         {
             using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
                 Put(remotename, fs);
         }
+
         public void Get(string remotename, string filename)
         {
             using (System.IO.FileStream fs = System.IO.File.Create(filename))
                 Get(remotename, fs);
         }
+
         public void Delete(string remotename)
         {
-            EnforceConsistencyDelay(RemoteOperation.Delete);
+            EnforceConsistencyDelay(remotename);
 
             try
             {
-                using(m_oauth.GetResponse(string.Format("{0}/trash/{1}", MetadataUrl, GetFileID(remotename)), null, "PUT"))
+                using (m_oauth.GetResponse(string.Format("{0}/trash/{1}", MetadataUrl, GetFileID(remotename)), null, "PUT"))
                 {
                 }
 
@@ -412,17 +455,20 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
             {
                 m_filecache = null;
             }
-            m_waitUntil = DateTime.Now + m_delayTimeSpan;
+            SetWaitUntil(remotename, DateTime.Now + m_delayTimeSpan);
         }
+
         public void Test()
         {
             this.TestList();
         }
+
         public void CreateFolder()
         {
-            EnforceConsistencyDelay(RemoteOperation.List);
+            EnforceConsistencyDelay(null);
             GetCurrentDirectory(true);            
         }
+
         public string DisplayName
         {
             get
@@ -482,16 +528,20 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
         }
 
         #endregion
+
         #region IDisposable implementation
+
         public void Dispose()
         {
         }
+
         #endregion
 
         #region IRenameEnabledBackend
+
         public void Rename(string oldname, string newname)
         {
-            EnforceConsistencyDelay(RemoteOperation.Rename);
+            EnforceConsistencyDelay(oldname);
 
             var id = GetFileID(oldname);
 
@@ -512,7 +562,7 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
 
                     req =>
                     {
-                        using(var rs = req.GetRequestStream())
+                        using (var rs = req.GetRequestStream())
                             rs.Write(data, 0, data.Length);
                     }
                 );
@@ -527,12 +577,15 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
             }
             finally
             {
-                m_waitUntil = DateTime.Now + m_delayTimeSpan;
+                SetWaitUntil(oldname, DateTime.Now + m_delayTimeSpan);
+                SetWaitUntil(newname, DateTime.Now + m_delayTimeSpan);
             }
         }
+
         #endregion
 
         #region JSON Classes
+
         private class ListResponse
         {
             [JsonProperty("count")]
@@ -611,9 +664,8 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive
             [JsonProperty("parents")]
             public string[] Parents { get; set; } 
         }
+
         #endregion
-
-
     }
 }
 

Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs

@@ -119,7 +119,6 @@ namespace Duplicati.Library.Backend.AzureBlob
                         if (x is CloudBlockBlob)
                         {
                             var cb = (CloudBlockBlob)x;
-                            var modified = cb.Properties.LastModified;
                             var lastModified = new System.DateTime();
                             if (cb.Properties.LastModified != null)
                                 lastModified = new System.DateTime(cb.Properties.LastModified.Value.Ticks, System.DateTimeKind.Utc);

Duplicati/Library/Backend/Backblaze/B2.cs

@@ -56,9 +56,7 @@ namespace Duplicati.Library.Backend.Backblaze
             var uri = new Utility.Uri(url);
 
             m_bucketname = uri.Host;
-            m_prefix = "/" + uri.Path;
-            if (!m_prefix.EndsWith("/", StringComparison.Ordinal))
-                m_prefix += "/";
+            m_prefix = Duplicati.Library.Utility.Utility.AppendDirSeparator("/" + uri.Path, "/");
 
             // For B2 we do not use a leading slash
             while(m_prefix.StartsWith("/", StringComparison.Ordinal))
@@ -118,7 +116,7 @@ namespace Duplicati.Library.Backend.Backblaze
                     );
 
                     if (buckets != null && buckets.Buckets != null)
-                        m_bucket = buckets.Buckets.Where(x => string.Equals(x.BucketName, m_bucketname, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
+                        m_bucket = buckets.Buckets.FirstOrDefault(x => string.Equals(x.BucketName, m_bucketname, StringComparison.OrdinalIgnoreCase));
 
                     if (m_bucket == null)
                         throw new FolderMissingException();

Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs

@@ -101,8 +101,6 @@ namespace Duplicati.Library.Backend.Backblaze
                         }
                         catch (Exception ex)
                         {
-                            
-                            var msg = ex.Message;
                             var clienterror = false;
 
                             try

Duplicati/Library/Backend/Box/BoxBackend.cs

@@ -94,10 +94,8 @@ namespace Duplicati.Library.Backend.Box
         {
             var uri = new Utility.Uri(url);
 
-            m_path = uri.HostAndPath;
-            if (!m_path.EndsWith("/", StringComparison.Ordinal))
-                m_path += "/";
-            
+            m_path = Duplicati.Library.Utility.Utility.AppendDirSeparator(uri.HostAndPath, "/");
+
             string authid = null;
             if (options.ContainsKey(AUTHID_OPTION))
                 authid = options[AUTHID_OPTION];
@@ -124,7 +122,7 @@ namespace Duplicati.Library.Backend.Box
 
             foreach(var p in m_path.Split(new string[] {"/"}, StringSplitOptions.RemoveEmptyEntries))
             {
-                var el = (MiniFolder)PagedFileListResponse(parentid, true).Where(x => x.Name == p).FirstOrDefault();
+                var el = (MiniFolder)PagedFileListResponse(parentid, true).FirstOrDefault(x => x.Name == p);
                 if (el == null)
                 {
                     if (!create)
@@ -218,7 +216,7 @@ namespace Duplicati.Library.Backend.Box
                 {
                     res = m_oauth.PostMultipartAndGetJSONData<FileList>(
                         string.Format("{0}/{1}/content", BOX_UPLOAD_URL, m_filecache[remotename]),
-                        new MultipartItem(stream, name: "file", filename: remotename)
+                        new MultipartItem(stream, "file", remotename)
                     ).Entries.First();
                 }
                 else
@@ -226,8 +224,8 @@ namespace Duplicati.Library.Backend.Box
 
                     res = m_oauth.PostMultipartAndGetJSONData<FileList>(
                         string.Format("{0}/content", BOX_UPLOAD_URL),
-                        new MultipartItem(createreq, name: "attributes"),
-                        new MultipartItem(stream, name: "file", filename: remotename)
+                        new MultipartItem(createreq, "attributes"),
+                        new MultipartItem(stream, "file", remotename)
                     ).Entries.First();
                 }
 
@@ -405,14 +403,6 @@ namespace Duplicati.Library.Backend.Box
             public long Limit { get; set; }
         }
 
-        private class SharePermissions
-        {
-            [JsonProperty("can_download")]
-            public bool CanDownload { get; set; }
-            [JsonProperty("can_preview")]
-            public bool CanPreview { get; set; }
-        }
-
         private class UploadEmail
         {
             [JsonProperty("access")]
@@ -421,28 +411,6 @@ namespace Duplicati.Library.Backend.Box
             public string Email { get; set; }
         }
 
-        private class SharedLink
-        {
-            [JsonProperty("url")]
-            public string Url { get; set; }
-            [JsonProperty("download_url")]
-            public string DownloadUrl { get; set; }
-            [JsonProperty("vanity_url")]
-            public string VanityUrl { get; set; }
-            [JsonProperty("is_password_enabled")]
-            public bool IsPasswordEnabled { get; set; }
-            [JsonProperty("unshared_at")]
-            public DateTime? UnsharedAt { get; set; }
-            [JsonProperty("download_count")]
-            public long DownloadCount { get; set; }
-            [JsonProperty("preview_count")]
-            public long PreviewCount { get; set; }
-            [JsonProperty("access")]
-            public string Access { get; set; }
-            [JsonProperty("permissions")]
-            public SharePermissions Permissions { get; set; }
-         }
-
         private class ListFolderResponse : MiniFolder
         {
             [JsonProperty("created_at")]
@@ -518,9 +486,6 @@ namespace Duplicati.Library.Backend.Box
             public int Status { get; set; }
             [JsonProperty("code")]
             public string Code { get; set; }
-            // Not working exactly his way ...
-            //[JsonProperty("context_info")]
-            //public ErrorItem[] ContextInfo { get; set; }
             [JsonProperty("help_url")]
             public string HelpUrl { get; set; }
             [JsonProperty("message")]
@@ -529,17 +494,6 @@ namespace Duplicati.Library.Backend.Box
             public string RequestId { get; set; }
 
         }
-
-        private class ErrorItem
-        {
-            [JsonProperty("reason")]
-            public string Reason { get; set; }
-            [JsonProperty("name")]
-            public string Name { get; set; }
-            [JsonProperty("message")]
-            public string Message { get; set; }
-        }
-
     }
 }
 

Duplicati/Library/Backend/CloudFiles/CloudFiles.cs

@@ -308,7 +308,7 @@ namespace Duplicati.Library.Backend
                 string md5Hash = resp.Headers["ETag"];
                 Utility.Utility.CopyStream(mds, stream, true, m_copybuffer);
 
-                if (mds.GetFinalHashString().ToLower() != md5Hash.ToLower())
+                if (!String.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase))
                     throw new Exception(Strings.CloudFiles.ETagVerificationError);
             }
         }
@@ -384,7 +384,7 @@ namespace Duplicati.Library.Backend
                 }
 
 
-                if (md5Hash == null || md5Hash.ToLower() != fileHash.ToLower())
+                if (md5Hash == null || !String.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase))
                 {
                     //Remove the broken file
                     try { Delete(remotename); }

Duplicati/Library/Backend/Dropbox/Dropbox.cs

@@ -8,7 +8,6 @@ namespace Duplicati.Library.Backend
     public class Dropbox : IBackend, IStreamingBackend
     {
         private const string AUTHID_OPTION = "authid";
-        private const int MAX_FILE_LIST = 10000;
 
         private readonly string m_accesToken;
         private readonly string m_path;

Duplicati/Library/Backend/FTP/FTPBackend.cs

@@ -19,7 +19,6 @@
 #endregion
 using System;
 using System.Collections.Generic;
-using System.Text;
 using System.Text.RegularExpressions;
 using Duplicati.Library.Interface;
 using System.Linq;
@@ -32,8 +31,7 @@ namespace Duplicati.Library.Backend
         private readonly string m_url;
 
         private readonly bool m_useSSL = false;
-        private readonly bool m_defaultPassive = true;
-        private readonly bool m_passive = false;
+        private readonly bool m_passiveMode = false;
         private readonly bool m_listVerify = true;
 
         private readonly byte[] m_copybuffer = new byte[Duplicati.Library.Utility.Utility.DEFAULT_BUFFER_SIZE];
@@ -57,49 +55,46 @@ namespace Duplicati.Library.Backend
 
             var u = new Utility.Uri(url);
             u.RequireHost();
+            string username = null;
+            string password = null;
 
             if (!string.IsNullOrEmpty(u.Username))
             {
-                m_userInfo = new System.Net.NetworkCredential();
-                m_userInfo.UserName = u.Username;
-                if (!string.IsNullOrEmpty(u.Password))
-                    m_userInfo.Password = u.Password;
-                else if (options.ContainsKey("auth-password"))
-                    m_userInfo.Password = options["auth-password"];
+                username = u.Username;
             }
-            else
+            else if (options.ContainsKey("auth-username"))
+            {
+                username = options["auth-username"];
+            }
+
+            if (!string.IsNullOrEmpty(u.Username) && !string.IsNullOrEmpty(u.Password)) {
+                        password = u.Password;
+            }
+            else if (options.ContainsKey("auth-password")) {
+                password = options["auth-password"];
+            }
+
+            m_userInfo = new System.Net.NetworkCredential
             {
-                if (options.ContainsKey("auth-username"))
-                {
-                    m_userInfo = new System.Net.NetworkCredential();
-                    m_userInfo.UserName = options["auth-username"];
-                    if (options.ContainsKey("auth-password"))
-                        m_userInfo.Password = options["auth-password"];
-                }
-            }
+                UserName = username,
+                Password = password
+            };
 
             //Bugfix, see http://connect.microsoft.com/VisualStudio/feedback/details/695227/networkcredential-default-constructor-leaves-domain-null-leading-to-null-object-reference-exceptions-in-framework-code
             if (m_userInfo != null)
                 m_userInfo.Domain = "";
 
             m_url = u.SetScheme("ftp").SetQuery(null).SetCredentials(null, null).ToString();
-            if (!m_url.EndsWith("/", StringComparison.Ordinal))
-                m_url += "/";
-
+            m_url = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_url, "/");
+
             m_useSSL = Utility.Utility.ParseBoolOption(options, "use-ssl");
 
             m_listVerify = !Utility.Utility.ParseBoolOption(options, "disable-upload-verify");
 
             if (Utility.Utility.ParseBoolOption(options, "ftp-passive"))
             {
-                m_defaultPassive = false;
-                m_passive = true;
-            }
-            if (Utility.Utility.ParseBoolOption(options, "ftp-regular"))
-            {
-                m_defaultPassive = false;
-                m_passive = false;
-            }
+                m_passiveMode = true;
+            } else m_passiveMode = !Utility.Utility.ParseBoolOption(options, "ftp-regular");
         }
 
         #region Regular expression to parse list lines
@@ -138,18 +133,20 @@ namespace Duplicati.Library.Backend
             string time = m.Groups["timestamp"].Value;
             string dir = m.Groups["dir"].Value;
 
-            //Unused
-            //string permission = m.Groups["permission"].Value;
-
-            if (dir != "" && dir != "-")
-                f.IsFolder = true;
-            else
-                f.Size = long.Parse(m.Groups["size"].Value);
-
-            DateTime t;
-            if (DateTime.TryParse(time, out t))
-                f.LastAccess = f.LastModification = t;
-
+            if (dir != "" && dir != "-")
+            {
+                f.IsFolder = true;
+            }
+            else
+            {
+                f.Size = long.Parse(m.Groups["size"].Value);
+            }
+
+            if (DateTime.TryParse(time, out DateTime t))
+            {
+                f.LastAccess = f.LastModification = t;
+            }
+
             return f;
         }
 
@@ -221,7 +218,7 @@ namespace Duplicati.Library.Backend
                     req);
                 
                 string line;
-                while ((line = HandleListExceptions(() => sr.ReadLine(), req)) != null)
+                while ((line = HandleListExceptions(sr.ReadLine, req)) != null)
                 {
                     FileEntry f = ParseLine(line);
                     if (f != null)
@@ -380,9 +377,8 @@ namespace Duplicati.Library.Backend
         #region IDisposable Members
 
         public void Dispose()
-        {
-            if (m_userInfo != null)
-                m_userInfo = null;
+        {
+            m_userInfo = null;
         }
 
         #endregion
@@ -398,14 +394,15 @@ namespace Duplicati.Library.Backend
             if (createFolder && url.EndsWith("/", StringComparison.Ordinal))
                 url = url.Substring(0, url.Length - 1);
             
-            System.Net.FtpWebRequest req = (System.Net.FtpWebRequest)System.Net.FtpWebRequest.Create(url + remotename);
+            System.Net.FtpWebRequest req = (System.Net.FtpWebRequest)System.Net.WebRequest.Create(url + remotename);
 
-            if (m_userInfo != null)
-                req.Credentials = m_userInfo;
+            if (m_userInfo != null)
+            {
+                req.Credentials = m_userInfo;
+            }
+
             req.KeepAlive = false;
-
-            if (!m_defaultPassive)
-                req.UsePassive = m_passive;
+            req.UsePassive = m_passiveMode;
 
             if (m_useSSL)
                 req.EnableSsl = m_useSSL;

Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs

@@ -54,9 +54,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
             var uri = new Utility.Uri(url);
 
             m_bucket = uri.Host;
-            m_prefix = "/" + uri.Path;
-            if (!m_prefix.EndsWith("/", StringComparison.Ordinal))
-                m_prefix += "/";
+            m_prefix = Duplicati.Library.Utility.Utility.AppendDirSeparator("/" + uri.Path, "/");
 
             // For GCS we do not use a leading slash
             if (m_prefix.StartsWith("/", StringComparison.Ordinal))
@@ -147,7 +145,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
                 if (string.IsNullOrWhiteSpace(token))
                     break;
                 url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix), token);
-            };
+            }
         }
 
         public void Put(string remotename, string filename)

Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs

@@ -161,7 +161,7 @@ namespace Duplicati.Library.Backend.GoogleServices
                     var chunkSize = Math.Min(UPLOAD_CHUNK_SIZE, stream.Length - offset);
 
                     req.ContentLength = chunkSize;
-                    req.Headers["Content-Range"] = string.Format("bytes {0}-{1}/{2}", offset, offset + chunkSize - 1, stream.Length);;
+                    req.Headers["Content-Range"] = string.Format("bytes {0}-{1}/{2}", offset, offset + chunkSize - 1, stream.Length);
 
                     // Upload the remaining data
                     var areq = new AsyncHttpRequest(req);

Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs

@@ -49,9 +49,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
         {
             var uri = new Utility.Uri(url);
 
-            m_path = uri.HostAndPath;
-            if (!m_path.EndsWith("/", StringComparison.Ordinal))
-                m_path += "/";
+            m_path = Duplicati.Library.Utility.Utility.AppendDirSeparator(uri.HostAndPath, "/");
 
             string authid = null;
             if (options.ContainsKey(AUTHID_OPTION))

Duplicati/Library/Backend/GoogleServices/Strings.cs

@@ -46,7 +46,7 @@ namespace Duplicati.Library.Backend.Strings
         public static string MissingAuthID(string url) { return LC.L(@"You need an AuthID, you can get it from: {0}", url); }
         public static string MultipleEntries(string folder, string parent) { return LC.L(@"There is more than one item named ""{0}"" in the folder ""{1}""", folder, parent); }
         public static string DisableTeamDriveShort { get { return LC.L("Hide team drives"); } }
-        public static string DisableTeamDriveLong { get { return LC.L("This option disables the team drives, showing only files and folders accesible with the account itself"); } }
+        public static string DisableTeamDriveLong { get { return LC.L("This option disables the team drives, showing only files and folders accessible with the account itself"); } }
     }
 }
 

Duplicati/Library/Backend/GoogleServices/WebApi.cs

@@ -100,7 +100,7 @@ namespace Duplicati.Library.Backend.WebApi
             {
                 { QueryParam.UploadType, QueryValue.Resumable }
             };
-            var path = UrlPath.Create(Path.Bucket).Append(bucketId).ToString();
+            var path = UrlPath.Create(Path.Bucket).Append(bucketId).Append(Path.Object).ToString();
             return Uri.UriBuilder(Url.UPLOAD, path, queryParams);
         }
 

Duplicati/Library/Backend/Jottacloud/Jottacloud.cs

@@ -36,7 +36,6 @@ namespace Duplicati.Library.Backend
         private const string JFS_DEVICE_OPTION = "jottacloud-device";
         private const string JFS_MOUNT_POINT_OPTION = "jottacloud-mountpoint";
         private const string JFS_DATE_FORMAT = "yyyy'-'MM'-'dd-'T'HH':'mm':'ssK";
-        private const bool ALLOW_USER_DEFINED_MOUNT_POINTS = false;
         private readonly string m_device;
         private readonly bool m_device_builtin;
         private readonly string m_mountPoint;
@@ -115,8 +114,7 @@ namespace Duplicati.Library.Backend
             m_path = u.HostAndPath; // Host and path of "jottacloud://folder/subfolder" is "folder/subfolder", so the actual folder path within the mount point.
             if (string.IsNullOrEmpty(m_path)) // Require a folder. Actually it is possible to store files directly on the root level of the mount point, but that does not seem to be a good option.
                 throw new UserInformationException(Strings.Jottacloud.NoPathError, "JottaNoPath");
-            if (!m_path.EndsWith("/", StringComparison.Ordinal))
-                m_path += "/";
+            m_path = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_path, "/");
             if (!string.IsNullOrEmpty(u.Username))
             {
                 m_userInfo = new System.Net.NetworkCredential();

Duplicati/Library/Backend/Mega/Duplicati.Library.Backend.Mega.csproj

@@ -32,13 +32,13 @@
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Net.Http" />
-    <Reference Include="MegaApiClient">
-      <HintPath>..\..\..\..\packages\MegaApiClient.1.6.0\lib\net45\MegaApiClient.dll</HintPath>
-    </Reference>
     <Reference Include="System.Web" />
     <Reference Include="Newtonsoft.Json">
       <HintPath>..\..\..\..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
+    <Reference Include="MegaApiClient">
+      <HintPath>..\..\..\..\packages\MegaApiClient.1.6.3\lib\net45\MegaApiClient.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Properties\AssemblyInfo.cs" />

Duplicati/Library/Backend/Mega/MegaBackend.cs

@@ -77,11 +77,11 @@ namespace Duplicati.Library.Backend.Mega
         {
             var parts = m_prefix.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries);
             var nodes = Client.GetNodes();
-            INode parent = nodes.Where(x => x.Type == NodeType.Root).First();
+            INode parent = nodes.First(x => x.Type == NodeType.Root);
 
             foreach(var n in parts)
             {
-                var item = nodes.Where(x => x.Name == n && x.Type == NodeType.Directory && x.ParentId == parent.Id).FirstOrDefault();
+                var item = nodes.FirstOrDefault(x => x.Name == n && x.Type == NodeType.Directory && x.ParentId == parent.Id);
                 if (item == null)
                 {
                     if (!autocreate)

Duplicati/Library/Backend/Mega/packages.config

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="MegaApiClient" version="1.6.0" targetFramework="net45" />
+  <package id="MegaApiClient" version="1.6.3" targetFramework="net45" />
   <package id="Newtonsoft.Json" version="10.0.3" targetFramework="net45" />
   <package id="System.Net.Http" version="4.3.3" targetFramework="net45" />
 </packages>

Duplicati/Library/Backend/OAuthHelper/MultipartItem.cs

@@ -28,28 +28,34 @@ namespace Duplicati.Library
             this.Headers = new Dictionary<string, string>();
         }
 
-        public MultipartItem(string contenttype, string name = null, string filename = null)
+        public MultipartItem(string contenttype, string name, string filename)
             : this()
         {
             ContentType = contenttype;
             SetContentDisposition(name, filename);
         }
 
-        public MultipartItem(object content, string contenttype = "application/json; charset=utf-8", string name = null, string filename = null)
-            : this(JsonConvert.SerializeObject(content), contenttype, name, filename)
+        public MultipartItem(object content, string name)
+            : this(JsonConvert.SerializeObject(content), "application/json; charset=utf-8", name, null)
         {
         }
 
-        public MultipartItem(string content, string contenttype = null, string name = null, string filename = null)
+        public MultipartItem(string content, string contenttype, string name, string filename)
             : this(System.Text.Encoding.UTF8.GetBytes(content), contenttype, name, filename)
         {
         }
 
-        public MultipartItem(byte[] content, string contenttype = "application/octet-stream", string name = null, string filename = null)
+        public MultipartItem(byte[] content, string contenttype, string name, string filename)
             : this(new MemoryStream(content), contenttype, name, filename)
         {
         }
-        public MultipartItem(Stream content, string contenttype = "application/octet-stream", string name = null, string filename = null)
+
+        public MultipartItem(Stream content, string name, string filename)
+            : this(content, "application/octet-stream", name, filename)
+        {
+        }
+
+        public MultipartItem(Stream content, string contenttype, string name, string filename)
             : this(contenttype, name, filename)
         {
             ContentData = content;

Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs

@@ -67,7 +67,6 @@ namespace Duplicati.Library
         private DateTime m_tokenExpires = DateTime.UtcNow;
 
         public const string DUPLICATI_OAUTH_SERVICE = "https://duplicati-oauth-handler.appspot.com/refresh";
-        private const string OAUTH_LOGIN_URL_TEMPLATE = "https://duplicati-oauth-handler.appspot.com/?type={0}";
 
         public static string OAUTH_LOGIN_URL(string modulename) 
         {

Duplicati/Library/Backend/OAuthHelper/OAuthHttpClient.cs

@@ -67,7 +67,7 @@ namespace Duplicati.Library
                 this.PreventAuthentication(request);
             }
 
-            return await this.SendAsync(request);
+            return await this.SendAsync(request).ConfigureAwait(false);
         }
 
         /// <summary>

Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs

@@ -74,21 +74,23 @@ namespace Duplicati.Library.Backend
 
         private readonly JsonSerializer m_serializer = new JsonSerializer();
         private readonly OAuthHttpClient m_client;
-        private readonly string m_path;
         private readonly int fragmentSize;
         private readonly int fragmentRetryCount;
         private readonly int fragmentRetryDelay; // In milliseconds
 
         private string[] dnsNames = null;
 
+        private readonly Lazy<string> rootPathFromURL;
+        private string RootPath => this.rootPathFromURL.Value;
+
         protected MicrosoftGraphBackend() { } // Constructor needed for dynamic loading to find it
 
-        protected MicrosoftGraphBackend(string url, Dictionary<string, string> options)
+        protected MicrosoftGraphBackend(string url, string protocolKey, Dictionary<string, string> options)
         {
             string authid;
             options.TryGetValue(AUTHID_OPTION, out authid);
             if (string.IsNullOrEmpty(authid))
-                throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey)), "MicrosoftGraphBackendMissingAuthId");
+                throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(protocolKey)), "MicrosoftGraphBackendMissingAuthId");
 
             string fragmentSizeStr;
             if (options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize))
@@ -117,11 +119,12 @@ namespace Duplicati.Library.Backend
                 this.fragmentRetryDelay = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY;
             }
 
-            this.m_client = new OAuthHttpClient(authid, this.ProtocolKey);
+            this.m_client = new OAuthHttpClient(authid, protocolKey);
             this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS);
 
-            // Extract out the path to the backup root folder from the given URI
-            this.m_path = NormalizeSlashes(this.GetRootPathFromUrl(url));
+            // Extract out the path to the backup root folder from the given URI.  Since this can be an expensive operation, 
+            // we will cache the value using a lazy initializer.
+            this.rootPathFromURL = new Lazy<string>(() => this.GetRootPathFromUrl(url));
         }
 
         public abstract string ProtocolKey { get; }
@@ -167,7 +170,7 @@ namespace Duplicati.Library.Backend
                     // To get the upload session endpoint, we can start an upload session and then immediately cancel it.
                     // We pick a random file name (using a guid) to make sure we don't conflict with an existing file
                     string dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid());
-                    UploadSession uploadSession = this.Post<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.m_path, NormalizeSlashes(dnsTestFile)), null);
+                    UploadSession uploadSession = this.Post<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), null);
 
                     // Canceling an upload session is done by sending a DELETE to the upload URL
                     var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl);
@@ -240,7 +243,7 @@ namespace Duplicati.Library.Backend
         {
             string parentFolder = "root";
             string parentFolderPath = string.Empty;
-            foreach (string folder in this.m_path.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries))
+            foreach (string folder in this.RootPath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries))
             {
                 string nextPath = parentFolderPath + "/" + folder;
                 DriveItem folderItem;
@@ -268,7 +271,7 @@ namespace Duplicati.Library.Backend
         {
             try
             {
-                return this.Enumerate<DriveItem>(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.m_path))
+                return this.Enumerate<DriveItem>(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath))
                     .Where(item => item.IsFile && !item.IsDeleted) // Exclude non-files and deleted items (not sure if they show up in this listing, but make sure anyway)
                     .Select(item =>
                         new FileEntry(
@@ -296,7 +299,7 @@ namespace Duplicati.Library.Backend
         {
             try
             {
-                var response = this.m_client.GetAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename))).Await();
+                var response = this.m_client.GetAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename))).Await();
                 this.CheckResponse(response);
                 using (Stream responseStream = response.Content.ReadAsStreamAsync().Await())
                 {
@@ -314,7 +317,7 @@ namespace Duplicati.Library.Backend
         {
             try
             {
-                this.Patch(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.m_path, NormalizeSlashes(oldname)), new DriveItem() { Name = newname });
+                this.Patch(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(oldname)), new DriveItem() { Name = newname });
             }
             catch (DriveItemNotFoundException ex)
             {
@@ -338,10 +341,10 @@ namespace Duplicati.Library.Backend
             {
                 StreamContent streamContent = new StreamContent(stream);
                 streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = this.m_client.PutAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename)), streamContent).Await();
+                var response = this.m_client.PutAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename)), streamContent).Await();
                 
                 // Make sure this response is a valid drive item, though we don't actually use it for anything currently.
-                var result = this.ParseResponse<DriveItem>(response);
+                this.ParseResponse<DriveItem>(response);
             }
             else
             {
@@ -350,22 +353,22 @@ namespace Duplicati.Library.Backend
                 // The documentation seems somewhat contradictory - it states that uploads must be done sequentially,
                 // but also states that the nextExpectedRanges value returned may indicate multiple ranges...
                 // For now, this plays it safe and does a sequential upload.
-                HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename)));
+                HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename)));
 
                 // Indicate that we want to replace any existing content with this new data we're uploading
-                StringContent createSessionContent = this.PrepareContent(new UploadSession() { Item = new DriveItem() { ConflictBehavior = ConflictBehavior.Replace } });
+                this.PrepareContent(new UploadSession() { Item = new DriveItem() { ConflictBehavior = ConflictBehavior.Replace } });
 
                 HttpResponseMessage createSessionResponse = this.m_client.SendAsync(createSessionRequest).Await();
                 UploadSession uploadSession = this.ParseResponse<UploadSession>(createSessionResponse);
 
                 // If the stream's total length is less than the chosen fragment size, then we should make the buffer only as large as the stream.
-                int fragmentSize = (int)Math.Min(this.fragmentSize, stream.Length);
+                int bufferSize = (int)Math.Min(this.fragmentSize, stream.Length);
 
-                byte[] fragmentBuffer = new byte[fragmentSize];
+                byte[] fragmentBuffer = new byte[bufferSize];
                 int read = 0;
                 for (int offset = 0; offset < stream.Length; offset += read)
                 {
-                    read = stream.Read(fragmentBuffer, 0, fragmentSize);
+                    read = stream.Read(fragmentBuffer, 0, bufferSize);
 
                     int retryCount = this.fragmentRetryCount;
                     for (int attempt = 0; attempt < retryCount; attempt++)
@@ -384,7 +387,7 @@ namespace Duplicati.Library.Backend
                             response = this.m_client.SendAsync(request, false).Await();
 
                             // Note: On the last request, the json result includes the default properties of the item that was uploaded
-                            var result = this.ParseResponse<UploadSession>(response);
+                            this.ParseResponse<UploadSession>(response);
                         }
                         catch (MicrosoftGraphException ex)
                         {
@@ -393,7 +396,7 @@ namespace Duplicati.Library.Backend
                             if (attempt >= retryCount - 1)
                             {
                                 // We've used up all our retry attempts
-                                throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex);
+                                throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);
                             }
                             else if ((int)ex.Response.StatusCode >= 500 && (int)ex.Response.StatusCode < 600)
                             {
@@ -406,7 +409,7 @@ namespace Duplicati.Library.Backend
                             {
                                 // 404 is a special case indicating the upload session no longer exists, so the fragment shouldn't be retried.
                                 // Instead we'll let the caller re-attempt the whole file.
-                                throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex);
+                                throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);
                             }
                             else if ((int)ex.Response.StatusCode >= 400 && (int)ex.Response.StatusCode < 500)
                             {
@@ -416,7 +419,7 @@ namespace Duplicati.Library.Backend
                             else
                             {
                                 // Other errors should be rethrown
-                                throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex);
+                                throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);
                             }
                         }
 
@@ -429,7 +432,7 @@ namespace Duplicati.Library.Backend
 
         public void Delete(string remotename)
         {
-            var response = this.m_client.DeleteAsync(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename))).Await();
+            var response = this.m_client.DeleteAsync(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename))).Await();
             try
             {
                 this.CheckResponse(response);
@@ -445,8 +448,8 @@ namespace Duplicati.Library.Backend
         {
             try
             {
-                string rootPath = string.Format("{0}/root:{1}", this.DrivePrefix, this.m_path);
-                DriveItem rootFolder = this.Get<DriveItem>(rootPath);
+                string rootPath = string.Format("{0}/root:{1}", this.DrivePrefix, this.RootPath);
+                this.Get<DriveItem>(rootPath);
             }
             catch (DriveItemNotFoundException ex)
             {
@@ -476,12 +479,12 @@ namespace Duplicati.Library.Backend
             return this.SendRequest<T>(HttpMethod.Get, url);
         }
 
-        protected T Post<T>(string url, T body)
+        protected T Post<T>(string url, T body) where T : class
         {
             return this.SendRequest(HttpMethod.Post, url, body);
         }
 
-        protected T Patch<T>(string url, T body)
+        protected T Patch<T>(string url, T body) where T : class
         {
             return this.SendRequest(PatchMethod, url, body);
         }
@@ -492,7 +495,7 @@ namespace Duplicati.Library.Backend
             return this.SendRequest<T>(request);
         }
 
-        private T SendRequest<T>(HttpMethod method, string url, T body)
+        private T SendRequest<T>(HttpMethod method, string url, T body) where T : class
         {
             var request = new HttpRequestMessage(method, url);
             if (body != null)

Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs

@@ -4,14 +4,13 @@ using System.Linq;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 
-/// <summary>
-/// Types are based on definitions from:
-/// https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/onedrive
-/// 
-/// Note that some classes don't have the full set of properties defined, particularly if they don't seem like they are needed.
-/// </summary>
 namespace Duplicati.Library.Backend.MicrosoftGraph
 {
+    /// Types are based on definitions from:
+    /// https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/onedrive
+    ///
+    /// Note that some classes don't have the full set of properties defined, particularly if they don't seem like they are needed.
+
     public class Identity
     {
         [JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)]
@@ -64,7 +63,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
 
         /// <summary>
         /// Note: OneDrive and OneDrive for Business don't allow the following characters in file names:
-        ///   " * : < > ? / \ |
+        ///   &quot; * : &lt; &gt; ? / \ |
         /// https://support.office.com/en-us/article/Invalid-file-names-and-file-types-in-OneDrive-OneDrive-for-Business-and-SharePoint-64883a5d-228e-48f5-b3d2-eb39e07630fa
         /// If appears it also follows the Windows conventions for handling leading and trailing spaces,
         /// meaning the ASCII space character is trimmed off of both the front and back of the file name:

Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs

@@ -10,13 +10,14 @@ namespace Duplicati.Library.Backend
     {
         private const string GROUP_EMAIL_OPTION = "group-email";
         private const string GROUP_ID_OPTION = "group-id";
+        private const string PROTOCOL_KEY = "msgroup";
 
         private readonly string drivePath;
 
         public MicrosoftGroup() { } // Constructor needed for dynamic loading to find it
 
         public MicrosoftGroup(string url, Dictionary<string, string> options)
-            : base(url, options)
+            : base(url, MicrosoftGroup.PROTOCOL_KEY, options)
         {
             string groupId = null;
             string groupEmail;
@@ -46,7 +47,7 @@ namespace Duplicati.Library.Backend
 
         public override string ProtocolKey
         {
-            get { return "msgroup"; }
+            get { return MicrosoftGroup.PROTOCOL_KEY; }
         }
 
         public override string DisplayName

Duplicati/Library/Backend/OneDrive/OneDrive.cs

@@ -46,9 +46,7 @@ namespace Duplicati.Library.Backend
             var uri = new Utility.Uri(url);
 
             m_rootfolder = uri.Host;
-            m_prefix = "/" + uri.Path;
-            if (!m_prefix.EndsWith("/", StringComparison.Ordinal))
-                m_prefix += "/";
+            m_prefix = Duplicati.Library.Utility.Utility.AppendDirSeparator("/" + uri.Path, "/");
 
             string authid = null;
             if (options.ContainsKey(AUTHID_OPTION))
@@ -57,13 +55,6 @@ namespace Duplicati.Library.Backend
             m_oauth = new OAuthHelper(authid, this.ProtocolKey);
         }
 
-        private class WLID_Service_Response
-        {
-            public string access_token { get; set; }
-            [Newtonsoft.Json.JsonProperty(NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-            public int expires { get; set; }
-        }
-
         private class WLID_DataItem
         {
             public WLID_FolderItem[] data { get; set; }
@@ -91,16 +82,6 @@ namespace Duplicati.Library.Backend
             public string description;
         }
 
-        private class WLID_ContinuationResponse
-        {
-            [Newtonsoft.Json.JsonProperty("uploadUrl")]
-            public string UploadUrl { get; set; }
-            [Newtonsoft.Json.JsonProperty("expirationDateTime", NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-            public DateTime Expires { get; set; } 
-            [Newtonsoft.Json.JsonProperty("nextExpectedRanges")]
-            public string[] NextRanges { get; set; }
-        }
-
         private class WLID_UserInfo
         {
             public string id { get; set; }

Duplicati/Library/Backend/OneDrive/OneDriveV2.cs

@@ -7,15 +7,15 @@ namespace Duplicati.Library.Backend
     public class OneDriveV2 : MicrosoftGraphBackend
     {
         private const string DRIVE_ID_OPTION = "drive-id";
-
         private const string DEFAULT_DRIVE_PATH = "/me/drive";
+        private const string PROTOCOL_KEY = "onedrivev2";
 
         private readonly string drivePath;
 
         public OneDriveV2() { } // Constructor needed for dynamic loading to find it
 
         public OneDriveV2(string url, Dictionary<string, string> options)
-            : base(url, options)
+            : base(url, OneDriveV2.PROTOCOL_KEY, options)
         {
             string driveId;
             if (options.TryGetValue(DRIVE_ID_OPTION, out driveId))
@@ -30,7 +30,7 @@ namespace Duplicati.Library.Backend
 
         public override string ProtocolKey
         {
-            get { return "onedrivev2"; }
+            get { return OneDriveV2.PROTOCOL_KEY; }
         }
 
         public override string DisplayName

Duplicati/Library/Backend/OneDrive/SharePointV2.cs

@@ -11,6 +11,7 @@ namespace Duplicati.Library.Backend
     public class SharePointV2 : MicrosoftGraphBackend
     {
         private const string SITE_ID_OPTION = "site-id";
+        private const string PROTOCOL_KEY = "sharepoint";
 
         private readonly string drivePath;
         private string siteId = null;
@@ -18,7 +19,7 @@ namespace Duplicati.Library.Backend
         public SharePointV2() { } // Constructor needed for dynamic loading to find it
 
         public SharePointV2(string url, Dictionary<string, string> options)
-            : base(url, options)
+            : base(url, SharePointV2.PROTOCOL_KEY, options)
         {
             // Check to see if a site ID was explicitly provided
             string siteIdOption;
@@ -42,7 +43,7 @@ namespace Duplicati.Library.Backend
 
         public override string ProtocolKey
         {
-            get { return "sharepoint"; }
+            get { return SharePointV2.PROTOCOL_KEY; }
         }
 
         public override string DisplayName

Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs

@@ -319,9 +319,7 @@ namespace Duplicati.Library.Backend.OpenStack
             var uri = new Utility.Uri(url);
 
             m_container = uri.Host;
-            m_prefix = "/" + uri.Path;
-            if (!m_prefix.EndsWith("/", StringComparison.Ordinal))
-                m_prefix += "/";
+            m_prefix = Duplicati.Library.Utility.Utility.AppendDirSeparator("/" + uri.Path, "/");
 
             // For OpenStack we do not use a leading slash
             if (m_prefix.StartsWith("/", StringComparison.Ordinal))
@@ -461,11 +459,11 @@ namespace Duplicati.Library.Backend.OpenStack
             m_accessToken = resp.access.token;
 
             // Grab the endpoint now that we have received it anyway
-            var fileservice = resp.access.serviceCatalog.Where(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
+            var fileservice = resp.access.serviceCatalog.FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase));
             if (fileservice == null)
                 throw new Exception("No object-store service found, is this service supported by the provider?");
 
-            var endpoint = fileservice.endpoints.Where(x => string.Equals(m_region, x.region)).FirstOrDefault() ?? fileservice.endpoints.First();
+            var endpoint = fileservice.endpoints.FirstOrDefault(x => string.Equals(m_region, x.region)) ?? fileservice.endpoints.First();
 
             m_simplestorageendpoint = endpoint.publicURL;
 

Duplicati/Library/Backend/Rclone/Rclone.cs

@@ -135,7 +135,7 @@ namespace Duplicati.Library.Backend
 #endif
                         // append the new data to the data already read-in
                         outputBuilder.Append(e.Data);
-                    };
+                    }
                 }
             );  
 

Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj

@@ -68,6 +68,10 @@
       <Project>{B68F2214-951F-4F78-8488-66E1ED3F50BF}</Project>
       <Name>Duplicati.Library.Localization</Name>
     </ProjectReference>
+    <ProjectReference Include="..\..\Logging\Duplicati.Library.Logging.csproj">
+      <Project>{D10A5FC0-11B4-4E70-86AA-8AEA52BD9798}</Project>
+      <Name>Duplicati.Library.Logging</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
     <None Include="app.config" />

Duplicati/Library/Backend/S3/S3Backend.cs

@@ -28,6 +28,8 @@ namespace Duplicati.Library.Backend
 {
     public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend
     {
+        private static string LOGTAG = Logging.Log.LogTagFromType<S3>();
+
         public const string RRS_OPTION = "s3-use-rrs";
         public const string STORAGECLASS_OPTION = "s3-storage-class";
         public const string EU_BUCKETS_OPTION = "s3-european-buckets";
@@ -44,6 +46,7 @@ namespace Duplicati.Library.Backend
             new KeyValuePair<string, string>("dinCloud - Los Angeles", "d3-lax.dincloud.com"),
             new KeyValuePair<string, string>("IBM COS (S3) Public US", "s3-api.us-geo.objectstorage.softlayer.net"),
             new KeyValuePair<string, string>("Wasabi Hot Storage", "s3.wasabisys.com"),
+            new KeyValuePair<string, string>("Wasabi Hot Storage (US West)", "s3.us-west-1.wasabisys.com"),
         };
 
         //Updated list: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
@@ -214,7 +217,7 @@ namespace Duplicati.Library.Backend
                 host = u.Host;
                 m_prefix = "";
 
-                if (host.ToLower() == s3host)
+                if (String.Equals(host, s3host, StringComparison.OrdinalIgnoreCase))
                 {
                     m_bucket = Library.Utility.Uri.UrlDecode(u.PathAndQuery);
 
@@ -230,7 +233,7 @@ namespace Duplicati.Library.Backend
                 else
                 {
                     //Subdomain type lookup
-                    if (host.ToLower().EndsWith("." + s3host, StringComparison.Ordinal))
+                    if (host.EndsWith("." + s3host, StringComparison.OrdinalIgnoreCase))
                     {
                         m_bucket = host.Substring(0, host.Length - ("." + s3host).Length);
                         host = s3host;
@@ -243,8 +246,7 @@ namespace Duplicati.Library.Backend
                         throw new UserInformationException(Strings.S3Backend.UnableToDecodeBucketnameError(url), "S3CannotDecodeBucketName");
                 }
 
-                try { Console.Error.WriteLine(Strings.S3Backend.DeprecatedUrlFormat("s3://" + m_bucket + "/" + m_prefix)); }
-                catch { }
+                Logging.Log.WriteWarningMessage(LOGTAG, "DeprecatedS3Format", null, Strings.S3Backend.DeprecatedUrlFormat("s3://" + m_bucket + "/" + m_prefix)); 
             }
             else
             {
@@ -255,8 +257,10 @@ namespace Duplicati.Library.Backend
 
             m_options = options;
             m_prefix = m_prefix.Trim();
-            if (m_prefix.Length != 0 && !m_prefix.EndsWith("/", StringComparison.Ordinal))
-                m_prefix += "/";
+            if (m_prefix.Length != 0)
+            {
+                m_prefix = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_prefix, "/");
+            }
 
             // Auto-disable dns lookup for non AWS configurations
             var hasForcePathStyle = options.ContainsKey("s3-ext-forcepathstyle");

Duplicati/Library/Backend/S3/S3Wrapper.cs

@@ -32,6 +32,7 @@ namespace Duplicati.Library.Backend
     /// </summary>
     public class S3Wrapper : IDisposable
     {
+        private static string LOGTAG = Logging.Log.LogTagFromType<S3Wrapper>();
         private const int ITEM_LIST_LIMIT = 1000;
 
         protected string m_locationConstraint;
@@ -51,7 +52,7 @@ namespace Duplicati.Library.Backend
 
             foreach(var opt in options.Keys.Where(x => x.StartsWith("s3-ext-", StringComparison.OrdinalIgnoreCase)))
             {
-                var prop = cfg.GetType().GetProperties().Where(x => string.Equals(x.Name, opt.Substring("s3-ext-".Length), StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
+                var prop = cfg.GetType().GetProperties().FirstOrDefault(x => string.Equals(x.Name, opt.Substring("s3-ext-".Length), StringComparison.OrdinalIgnoreCase));
                 if (prop != null && prop.CanWrite)
                 {
                     if (prop.PropertyType == typeof(bool))
@@ -67,8 +68,7 @@ namespace Duplicati.Library.Backend
                 }
 
                 if (prop == null)
-                    try { Console.Error.WriteLine("Unsupported option: {0}", opt); }
-                    catch { }
+                    Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedOption", null, "Unsupported option: {0}", opt);
             }
 
             m_client = new Amazon.S3.AmazonS3Client(awsID, awsKey, cfg);

Duplicati/Library/Backend/SSHv2/KeyUploader.cs

@@ -78,7 +78,7 @@ namespace Duplicati.Library.Backend
                     client.ChangeDirectory(SSH_FOLDER);
                 }
                 
-                var sshfolder = client.ListDirectory(".").Where(x => x.Name == ".").First();                    
+                var sshfolder = client.ListDirectory(".").First(x => x.Name == ".");                    
                 client.ChangeDirectory("..");
                 
                 if (!sshfolder.OwnerCanRead || !sshfolder.OwnerCanWrite)
@@ -87,7 +87,7 @@ namespace Duplicati.Library.Backend
                 string authorized_keys = "";
                 byte[] authorized_keys_bytes = null;
                 
-                var existing_authorized_keys = client.ListDirectory(SSH_FOLDER).Where(x => x.Name == AUTHORIZED_KEYS_FILE).Any();
+                var existing_authorized_keys = client.ListDirectory(SSH_FOLDER).Any(x => x.Name == AUTHORIZED_KEYS_FILE);
                 if (existing_authorized_keys)
                 {
                     using(var ms = new System.IO.MemoryStream())
@@ -102,11 +102,11 @@ namespace Duplicati.Library.Backend
                 var cleaned_keys = keys.Select(x => x.Trim()).Where(x => x.Length > 0 && !x.StartsWith("#", StringComparison.Ordinal));
                 
                 // Does the key already exist?
-                if (cleaned_keys.Where(x =>
+                if (cleaned_keys.Any(x =>
                 {
                     var els = x.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).Select(y => y.Trim()).Where(y => y.Length > 0).ToArray();
                     return els.Length == 3 && els[0] == pubkey[0] && els[1] == pubkey[1];
-                }).Any())
+                }))
                 {
                     res["status"] = "Key already existed";
                 }

Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs

@@ -78,8 +78,10 @@ namespace Duplicati.Library.Backend
 
             m_path = uri.Path;
 
-            if (!string.IsNullOrWhiteSpace(m_path) && !m_path.EndsWith("/", StringComparison.Ordinal))
-                m_path += "/";
+            if (!string.IsNullOrWhiteSpace(m_path))
+            {
+                m_path = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_path, "/");
+            }
 
             if (!m_path.StartsWith("/", StringComparison.Ordinal))
                 m_path = "/" + m_path;
@@ -270,7 +272,7 @@ namespace Duplicati.Library.Backend
                 if (string.IsNullOrEmpty(m_fingerprint))
                     throw new Library.Utility.HostKeyException(Strings.SSHv2Backend.FingerprintNotSpecifiedManagedError(hostFingerprint.ToLower(), SSH_FINGERPRINT_OPTION, SSH_FINGERPRINT_ACCEPT_ANY_OPTION), hostFingerprint, m_fingerprint);
 
-                if (hostFingerprint.ToLower() != m_fingerprint.ToLower())
+                if (!String.Equals(hostFingerprint, m_fingerprint, StringComparison.OrdinalIgnoreCase))
                     throw new Library.Utility.HostKeyException(Strings.SSHv2Backend.FingerprintNotMatchManagedError(hostFingerprint.ToLower()), hostFingerprint, m_fingerprint);
                 else
                     e.CanTrust = true;
@@ -291,11 +293,7 @@ namespace Duplicati.Library.Backend
             if (string.IsNullOrEmpty(path))
                 return;
 
-            string working_dir = m_con.WorkingDirectory;
-
-            if (!working_dir.EndsWith("/", StringComparison.Ordinal))
-                working_dir += "/";
-
+            string working_dir = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_con.WorkingDirectory, "/");
             if (working_dir == path)
                 return;
 

Duplicati/Library/Backend/SharePoint/SharePointBackend.cs

@@ -169,8 +169,7 @@ namespace Duplicati.Library.Backend
             m_serverRelPath = u.Path;
             if (!m_serverRelPath.StartsWith("/", StringComparison.Ordinal))
                 m_serverRelPath = "/" + m_serverRelPath;
-            if (!m_serverRelPath.EndsWith("/", StringComparison.Ordinal))
-                m_serverRelPath += "/";
+            m_serverRelPath = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_serverRelPath, "/");
             // remove marker for SP-Web
             m_serverRelPath = m_serverRelPath.Replace("//", "/");
 

Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs

@@ -105,8 +105,7 @@ namespace Duplicati.Library.Backend
             m_useSSL = Utility.Utility.ParseBoolOption(options, "use-ssl");
 
             m_url = u.SetScheme(m_useSSL ? "https" : "http").SetQuery(null).SetCredentials(null, null).ToString();
-            if (!m_url.EndsWith("/", StringComparison.Ordinal))
-                m_url += "/";
+            m_url = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_url, "/");
         }
 
         private System.Net.HttpWebRequest CreateRequest(string remotename, string queryparams)

Duplicati/Library/Backend/WEBDAV/WEBDAV.cs

@@ -93,14 +93,12 @@ namespace Duplicati.Library.Backend
             m_useSSL = Utility.Utility.ParseBoolOption(options, "use-ssl");
 
             m_url = u.SetScheme(m_useSSL ? "https" : "http").SetCredentials(null, null).SetQuery(null).ToString();
-            if (!m_url.EndsWith("/", StringComparison.Ordinal))
-                m_url += "/";
+            m_url = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_url, "/");
 
             m_path = u.Path;
             if (!m_path.StartsWith("/", StringComparison.Ordinal))
                 m_path = "/" + m_path;
-            if (!m_path.EndsWith("/", StringComparison.Ordinal))
-                m_path += "/";
+            m_path = Duplicati.Library.Utility.Utility.AppendDirSeparator(m_path, "/");
 
             m_path = Library.Utility.Uri.UrlDecode(m_path);
             m_rawurl = new Utility.Uri(m_useSSL ? "https" : "http", u.Host, m_path).ToString();

Duplicati/Library/DynamicLoader/BackendLoader.cs

@@ -83,10 +83,10 @@ namespace Duplicati.Library.DynamicLoader
                             if (m_interfaces.ContainsKey(tmpscheme))
                             {
                                 var commands = m_interfaces[tmpscheme].SupportedCommands;
-                                if (commands != null && (commands.Where(x =>
+                                if (commands != null && (commands.Any(x =>
                                 x.Name.Equals("use-ssl", StringComparison.OrdinalIgnoreCase) ||
-                                (x.Aliases != null && x.Aliases.Where(y => y.Equals("use-ssl", StringComparison.OrdinalIgnoreCase)).Any())
-                                ).Any()))
+                                (x.Aliases != null && x.Aliases.Any(y => y.Equals("use-ssl", StringComparison.OrdinalIgnoreCase)))
+                                )))
                                 {
                                     newOpts["use-ssl"] = "true";
                                     return (IBackend)Activator.CreateInstance(m_interfaces[tmpscheme].GetType(), url, newOpts);

Duplicati/Library/Encryption/GPGEncryption.cs

@@ -178,7 +178,7 @@ namespace Duplicati.Library.Encryption
                 m_decryption_args += " " + GPG_DECRYPTION_COMMAND;
 
             if (options.ContainsKey(COMMANDLINE_OPTIONS_PATH))
-                m_programpath = Library.Utility.Utility.ExpandEnvironmentVariables(options[COMMANDLINE_OPTIONS_PATH]);
+                m_programpath = Environment.ExpandEnvironmentVariables(options[COMMANDLINE_OPTIONS_PATH]);
 
         }
 

Duplicati/Library/Interface/CustomExceptions.cs

@@ -139,5 +139,48 @@ namespace Duplicati.Library.Interface
         public CancelException(string message, Exception innerException)
             : base(message, "Cancelled", innerException)
         { }
-}
+    }
+
+    /// <summary>
+    /// The reason why an operation is aborted
+    /// </summary>
+    public enum OperationAbortReason
+    {
+        /// <summary>
+        /// The operation is aborted, but this is considered a normal operation
+        /// </summary>
+        Normal,
+        /// <summary>
+        /// The operation is aborted and this should give a warning
+        /// </summary>
+        Warning,
+        /// <summary>
+        /// The operation is aborted and this is an error
+        /// </summary>
+        Error
+    }
+
+    /// <summary>
+    /// A class that signals the operation should be aborted
+    /// </summary>
+    [Serializable]
+    public class OperationAbortException : UserInformationException
+    {
+        /// <summary>
+        /// The reason for the abort operation
+        /// </summary>
+        public readonly OperationAbortReason AbortReason;
+
+        public OperationAbortException(OperationAbortReason reason, string message)
+            : base(message, "OperationAborted")
+        {
+            AbortReason = reason;
+        }
+
+        public OperationAbortException(OperationAbortReason reason, string message, Exception innerException)
+            : base(message, "OperationAborted", innerException)
+        {
+            AbortReason = reason;
+        }
+    }
 }

Duplicati/Library/Localization/MoLocalizationService.cs

@@ -72,7 +72,7 @@ namespace Duplicati.Library.Localization
         {
             var filenames = new string[] { 
                 // Load the specialized version first
-                string.Format("localization-{0}.mo", ci.Name), 
+                string.Format("localization-{0}.mo", ci.Name.Replace('-', '_')), 
                 // Then try the generic language version
                 string.Format("localization-{0}.mo", ci.TwoLetterISOLanguageName) 
             };

Duplicati/Library/Logging/Log.cs

@@ -93,7 +93,7 @@ namespace Duplicati.Library.Logging
         public static object Lock { get { return m_lock; } }
 
         /// <summary>
-        /// Gets a log tag taht reflects the type
+        /// Gets a log tag that reflects the type
         /// </summary>
         /// <returns>The log-tag for the type.</returns>
         /// <typeparam name="T">The type to get the tag for.</typeparam>
@@ -104,7 +104,7 @@ namespace Duplicati.Library.Logging
 
 
         /// <summary>
-        /// Gets a log tag taht reflects the type
+        /// Gets a log tag that reflects the type
         /// </summary>
         /// <returns>The log-tag for the type.</returns>
         /// <param name="t">The type to get the tag for.</param>
@@ -341,10 +341,32 @@ namespace Duplicati.Library.Logging
         /// <summary>
         /// Starts a new scope, that can be closed by disposing the returned instance
         /// </summary>
+        /// <param name="detached">Flag indicating if the scope should be detached from the parent</param>
         /// <returns>The new scope.</returns>
-        public static IDisposable StartIsolatingScope()
+        public static IDisposable StartIsolatingScope(bool detached)
         {
-            return StartScope((ILogDestination)null, null, true);
+            lock (m_lock)
+            {
+                var scope = StartScope((ILogDestination)null, null, true);
+                if (detached)
+                    DetachCurrentScope(scope);
+                return scope;
+            }
+        }
+
+        /// <summary>
+        /// Detaches the current scope, such that new scopes do not chain onto this
+        /// </summary>
+        /// <param name="scope">The current scope.</param>
+        public static IDisposable DetachCurrentScope(IDisposable scope)
+        {
+            lock (m_lock)
+            {
+                if (CurrentScope == scope && scope != null && CurrentScope.Parent != null)
+                    CurrentScope = CurrentScope.Parent;
+            }
+
+            return scope;
         }
 
         /// <summary>
@@ -445,7 +467,6 @@ namespace Duplicati.Library.Logging
                     {
                         System.Runtime.Remoting.Messaging.CallContext.LogicalSetData(LOGICAL_CONTEXT_KEY, null);
                     }
-                        
                 }
             }
         }

Duplicati/Library/Logging/RepeatingLogScope.cs

@@ -66,7 +66,7 @@ namespace Duplicati.Library.Logging
             var remainingTime = m_maxIdleTime;
             while (!m_completed)
             {
-                await Task.Delay(new TimeSpan(Math.Max(TimeSpan.FromMilliseconds(500).Ticks, remainingTime)));
+                await Task.Delay(new TimeSpan(Math.Max(TimeSpan.FromMilliseconds(500).Ticks, remainingTime))).ConfigureAwait(false);
                 if (m_completed)
                     return;
                 if (m_lastEntry == null)

Duplicati/Library/Logging/Timer.cs

@@ -18,8 +18,6 @@
 // 
 #endregion
 using System;
-using System.Collections.Generic;
-using System.Text;
 
 namespace Duplicati.Library.Logging
 {

Duplicati/Library/Main/BackendManager.cs

@@ -248,7 +248,6 @@ namespace Duplicati.Library.Main
             private readonly LocalDatabase m_database;
             private readonly System.Threading.Thread m_callerThread;
             private List<IDbEntry> m_dbqueue;
-            private readonly IBackendWriter m_stats;
 
             private interface IDbEntry { }
 
@@ -273,10 +272,9 @@ namespace Duplicati.Library.Main
                 public string Newname;
             }
 
-            public DatabaseCollector(LocalDatabase database, IBackendWriter stats)
+            public DatabaseCollector(LocalDatabase database)
             {
                 m_database = database;
-                m_stats = stats;
                 m_dbqueue = new List<IDbEntry>();
                 if (m_database != null)
                     m_callerThread = System.Threading.Thread.CurrentThread;
@@ -378,7 +376,7 @@ namespace Duplicati.Library.Main
             m_numberofretries = options.NumberOfRetries;
             m_retrydelay = options.RetryDelay;
 
-            m_db = new DatabaseCollector(database, statwriter);
+            m_db = new DatabaseCollector(database);
 
             m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions);
             if (m_backend == null)
@@ -633,7 +631,7 @@ namespace Duplicati.Library.Main
         private void RenameFileAfterError(FileEntryItem item)
         {
             var p = VolumeBase.ParseFilename(item.RemoteFilename);
-            var guid = VolumeWriterBase.GenerateGuid(m_options);
+            var guid = VolumeWriterBase.GenerateGuid();
             var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1);
             var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule);
             var oldname = item.RemoteFilename;
@@ -735,7 +733,7 @@ namespace Duplicati.Library.Main
             {
                 using (var fs = System.IO.File.OpenRead(item.LocalFilename))
                 using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
-                using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
+                using (var pgs = new Library.Utility.ProgressReportingStream(ts, pg => HandleProgress(ts, pg)))
                     ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs);
             }
             else
@@ -751,7 +749,7 @@ namespace Duplicati.Library.Main
 
             if (m_options.ListVerifyUploads)
             {
-                var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
+                var f = m_backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase));
                 if (f == null)
                     throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
                 else if (f.Size != item.Size && f.Size >= 0)
@@ -829,7 +827,7 @@ namespace Duplicati.Library.Main
                         using (var ss = new ShaderStream(nextTierWriter, false))
                         {
                             using (var ts = new ThrottledStream(ss, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond))
-                            using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
+                            using (var pgs = new Library.Utility.ProgressReportingStream(ts, pg => HandleProgress(ts, pg)))
                             {
                                 taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed. 
                                 if (taskDecrypter != null) taskDecrypter.Start();
@@ -858,7 +856,7 @@ namespace Duplicati.Library.Main
                     // are properly ended and tidied up. For what is thrown: If exceptions in main thread occured (download) it is thrown,
                     // then hasher task is checked and last decryption. This resembles old logic.
                     try { retHashcode = taskHasher.Result; }
-                    catch (AggregateException ex) { if (!hadException) { hadException = true; throw ex.InnerExceptions[0]; } }
+                    catch (AggregateException ex) { if (!hadException) { hadException = true; throw ex.Flatten().InnerException; } }
                     finally
                     {
                         if (taskDecrypter != null)
@@ -869,10 +867,11 @@ namespace Duplicati.Library.Main
                                 if (!hadException)
                                 {
                                     hadException = true;
-                                    if (ex.InnerExceptions[0] is System.Security.Cryptography.CryptographicException)
-                                        throw ex.InnerExceptions[0];
+                                    AggregateException flattenedException = ex.Flatten();
+                                    if (flattenedException.InnerException is System.Security.Cryptography.CryptographicException)
+                                        throw flattenedException.InnerException;
                                     else
-                                        throw new System.Security.Cryptography.CryptographicException(ex.InnerExceptions[0].Message, ex.InnerExceptions[0]);
+                                        throw new System.Security.Cryptography.CryptographicException(flattenedException.InnerException.Message, flattenedException.InnerException);
                                 }
                             }
                         }
@@ -914,7 +913,7 @@ namespace Duplicati.Library.Main
                     using (var ss = new ShaderStream(hs, true))
                     {
                         using (var ts = new ThrottledStream(ss, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond))
-                        using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
+                        using (var pgs = new Library.Utility.ProgressReportingStream(ts, pg => HandleProgress(ts, pg)))
                         { ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); }
                         ss.Flush();
                         retDownloadSize = ss.TotalBytesWritten;

Duplicati/Library/Main/Controller.cs

@@ -22,7 +22,6 @@ using System.Linq;
 #endregion
 using System;
 using System.Collections.Generic;
-using System.Text;
 using Duplicati.Library.Utility;
 
 namespace Duplicati.Library.Main
@@ -56,11 +55,6 @@ namespace Duplicati.Library.Main
         /// </summary>
         private System.Threading.Thread m_currentTaskThread = null;
 
-        /// <summary>
-        /// Holds various keys that need to be reset after running the task
-        /// </summary>
-        private readonly Dictionary<string, string> m_resetKeys = new Dictionary<string, string>();
-
         /// <summary>
         /// The thread priority to reset to
         /// </summary>
@@ -164,7 +158,7 @@ namespace Duplicati.Library.Main
             return List((IEnumerable<string>)null, filter);
         }
 
-        public Duplicati.Library.Interface.IListResults List (string filterstring, Library.Utility.IFilter filter = null)
+        public Duplicati.Library.Interface.IListResults List(string filterstring)
         {
             return List(filterstring == null ? null : new string[] { filterstring }, null);
         }
@@ -436,14 +430,29 @@ namespace Duplicati.Library.Main
                 {
                     result.EndTime = DateTime.UtcNow;
 
-                    try { (result as BasicResults).OperationProgressUpdater.UpdatePhase(OperationPhase.Error); }
-                    catch { }
+                    if (ex is Library.Interface.OperationAbortException oae)
+                    {
+                        // Perform the module shutdown
+                        OnOperationComplete(ex);
 
-                    OnOperationComplete(ex);
+                        // Log this as a normal operation, as the script rasing the exception,
+                        // has already populated either warning or log messages as required
+                        Logging.Log.WriteInformationMessage(LOGTAG, "AbortOperation", "Aborting operation by request, requested result: {0}", oae.AbortReason);
 
-                    Logging.Log.WriteErrorMessage(LOGTAG, "FailedOperation", ex, Strings.Controller.FailedOperationMessage(m_options.MainAction, ex.Message));
+                        return result;
+                    }
+                    else
+                    {
+                        try { (result as BasicResults).OperationProgressUpdater.UpdatePhase(OperationPhase.Error); }
+                        catch { }
+
+                        OnOperationComplete(ex);
+
+                        Logging.Log.WriteErrorMessage(LOGTAG, "FailedOperation", ex, Strings.Controller.FailedOperationMessage(m_options.MainAction, ex.Message));
+
+                        throw;
+                    }
 
-                    throw;
                 }
                 finally
                 {
@@ -506,21 +515,6 @@ namespace Duplicati.Library.Main
                 m_resetLocaleUI = null;
             }
 
-            if (m_resetKeys != null)
-            {
-                var keys = m_resetKeys.Keys.ToArray();
-                foreach(var k in keys)
-                {
-                    try
-                    {
-                        Environment.SetEnvironmentVariable(k, m_resetKeys[k]);
-                    }
-                    catch { }
-
-                    m_resetKeys.Remove(k);
-                }
-            }
-
             if (m_logTarget != null)
             {
                 m_logTarget.Dispose();
@@ -548,7 +542,7 @@ namespace Duplicati.Library.Main
             m_options.LoadedModules.Clear();
 
             foreach (Library.Interface.IGenericModule m in DynamicLoader.GenericLoader.Modules)
-                m_options.LoadedModules.Add(new KeyValuePair<bool, Library.Interface.IGenericModule>(Array.IndexOf<string>(m_options.DisableModules, m.Key.ToLower()) < 0 && (m.LoadAsDefault || Array.IndexOf<string>(m_options.EnableModules, m.Key.ToLower()) >= 0), m));
+                m_options.LoadedModules.Add(new KeyValuePair<bool, Library.Interface.IGenericModule>(!m_options.DisableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase) && (m.LoadAsDefault || m_options.EnableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase)), m));
 
             // Make the filter read-n-write able in the generic modules
             var pristinefilter = string.Join(System.IO.Path.PathSeparator.ToString(), FilterExpression.Serialize(filter));
@@ -563,7 +557,7 @@ namespace Duplicati.Library.Main
             //// Since Configure in RunScript can alter the RawOptions, make sure it is first in the list for Configure
             var LoadedModules = new List<KeyValuePair<bool, Interface.IGenericModule>>();
             foreach (var mx in m_options.LoadedModules)
-                if (mx.Value.ToString().ToLower().Contains("runscript"))
+                if (mx.Value.ToString().IndexOf("runscript", StringComparison.OrdinalIgnoreCase) >= 0)
                 {
                     LoadedModules.Insert(0, mx);
                 }
@@ -622,18 +616,6 @@ namespace Duplicati.Library.Main
             if (m_options.HasTempDir)
             {
                 Library.Utility.TempFolder.SystemTempPath = m_options.TempDir;
-                if (Library.Utility.Utility.IsClientLinux)
-                {
-                    m_resetKeys["TMPDIR"] = Environment.GetEnvironmentVariable("TMPDIR");
-                    Environment.SetEnvironmentVariable("TMPDIR", m_options.TempDir);
-                }
-                else
-                {
-                    m_resetKeys["TMP"] = Environment.GetEnvironmentVariable("TMP");
-                    m_resetKeys["TEMP"] = Environment.GetEnvironmentVariable("TEMP");
-                    Environment.SetEnvironmentVariable("TMP", m_options.TempDir);
-                    Environment.SetEnvironmentVariable("TEMP", m_options.TempDir);
-                }
             }
 
             if (m_options.HasForcedLocale)
@@ -685,7 +667,7 @@ namespace Duplicati.Library.Main
                 selectedRetentionOptions.Add("keep-versions");
             }
 
-            if (m_options.RetentionPolicy.Count() > 0)
+            if (m_options.RetentionPolicy.Any())
             {
                 selectedRetentionOptions.Add("retention-policy");
             }
@@ -762,7 +744,7 @@ namespace Duplicati.Library.Main
                 if (l != null)
                     foreach (Library.Interface.ICommandLineArgument a in l)
                     {
-                        if (supportedOptions.ContainsKey(a.Name) && Array.IndexOf(Options.KnownDuplicates, a.Name.ToLower()) < 0)
+                        if (supportedOptions.ContainsKey(a.Name) && !Options.KnownDuplicates.Contains(a.Name, StringComparer.OrdinalIgnoreCase))
                             Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(a.Name));
 
                         supportedOptions[a.Name] = a;
@@ -770,7 +752,7 @@ namespace Duplicati.Library.Main
                         if (a.Aliases != null)
                             foreach (string s in a.Aliases)
                             {
-                                if (supportedOptions.ContainsKey(s) && Array.IndexOf(Options.KnownDuplicates, s.ToLower()) < 0)
+                                if (supportedOptions.ContainsKey(s) && !Options.KnownDuplicates.Contains(s, StringComparer.OrdinalIgnoreCase))
                                     Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(s));
 
                                 supportedOptions[s] = a;
@@ -900,6 +882,7 @@ namespace Duplicati.Library.Main
                     string source;
                     try
                     {
+                        // TODO: This expands "C:" to CWD, but not C:\
                         source = System.IO.Path.GetFullPath(expandedSource);
                     }
                     catch (Exception ex)
@@ -1059,6 +1042,9 @@ namespace Duplicati.Library.Main
                 {
                     return Strings.Controller.UnsupportedSizeValue(optionname, value);
                 }
+
+                if (!string.IsNullOrWhiteSpace(value) && char.IsDigit(value.Last()))
+                    return Strings.Controller.NonQualifiedSizeValue(optionname, value);
             }
             else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan)
             {

Duplicati/Library/Main/Database/ExtensionMethods.cs

@@ -41,7 +41,7 @@ namespace Duplicati.Library.Main.Database
         public static string GetPrintableCommandText(this System.Data.IDbCommand self)
         {
             var txt = self.CommandText;
-#if DEBUG
+
             foreach(var p in self.Parameters.Cast<System.Data.IDbDataParameter>())
             {
                 var ix = txt.IndexOf('?');
@@ -58,11 +58,21 @@ namespace Duplicati.Library.Main.Database
                     txt = txt.Substring(0, ix) + v + txt.Substring(ix + 1);
                 }
             }
-#endif
+
             return txt;
         }
 
+        public static int ExecuteNonQuery(this System.Data.IDbCommand self, bool writeLog)
+        {
+            return ExecuteNonQuery(self, writeLog, null, null);
+        }
+
         public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values)
+        {
+            return ExecuteNonQuery(self, true, cmd, values);
+        }
+
+        public static int ExecuteNonQuery(this System.Data.IDbCommand self, bool writeLog, string cmd, params object[] values)
         {
             if (cmd != null)
                 self.CommandText = cmd;
@@ -74,11 +84,16 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())))
+            using(writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null)
                 return self.ExecuteNonQuery();
         }
 
         public static object ExecuteScalar(this System.Data.IDbCommand self, string cmd, params object[] values)
+        {
+            return ExecuteScalar(self, true, cmd, values);
+        }
+
+        public static object ExecuteScalar(this System.Data.IDbCommand self, bool writeLog, string cmd, params object[] values)
         {
             if (cmd != null)
                 self.CommandText = cmd;
@@ -90,21 +105,36 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(LOGTAG, "ExecuteScalar", string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())))
+            using(writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalar", string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())) : null)
                 return self.ExecuteScalar();
         }
 
+        public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, long defaultvalue = -1)
+        {
+            return ExecuteScalarInt64(self, writeLog, null, defaultvalue);
+        }
+
         public static long ExecuteScalarInt64(this System.Data.IDbCommand self, long defaultvalue = -1)
         {
-            return ExecuteScalarInt64(self, null, defaultvalue, null);
+            return ExecuteScalarInt64(self, true, null, defaultvalue);
+        }
+
+        public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, string cmd, long defaultvalue = -1)
+        {
+            return ExecuteScalarInt64(self, writeLog, cmd, defaultvalue, null);
         }
 
         public static long ExecuteScalarInt64(this System.Data.IDbCommand self, string cmd, long defaultvalue = -1)
         {
-            return ExecuteScalarInt64(self, cmd, defaultvalue, null);
+            return ExecuteScalarInt64(self, true, cmd, defaultvalue, null);
         }
 
         public static long ExecuteScalarInt64(this System.Data.IDbCommand self, string cmd, long defaultvalue, params object[] values)
+        {
+            return ExecuteScalarInt64(self, true, cmd, defaultvalue, values);
+        }
+
+        public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, string cmd, long defaultvalue, params object[] values)
         {
             if (cmd != null)
                 self.CommandText = cmd;
@@ -116,7 +146,7 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())))
+            using(writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())) : null)
                 using(var rd = self.ExecuteReader())
                     if (rd.Read())
                         return ConvertValueToInt64(rd, 0, defaultvalue);
@@ -125,6 +155,11 @@ namespace Duplicati.Library.Main.Database
         }
 
         public static System.Data.IDataReader ExecuteReader(this System.Data.IDbCommand self, string cmd, params object[] values)
+        {
+            return ExecuteReader(self, true, cmd, values);
+        }
+
+        public static System.Data.IDataReader ExecuteReader(this System.Data.IDbCommand self, bool writeLog, string cmd, params object[] values)
         {
             if (cmd != null)
                 self.CommandText = cmd;
@@ -136,7 +171,7 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(LOGTAG, "ExcuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())))
+            using(writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null)
                 return self.ExecuteReader();
         }
 

Duplicati/Library/Main/Database/LocalBackupDatabase.cs

@@ -86,16 +86,20 @@ namespace Duplicati.Library.Main.Database
 
         private readonly System.Data.IDbCommand m_findfileCommand;
         private readonly System.Data.IDbCommand m_selectfilelastmodifiedCommand;
+        private readonly System.Data.IDbCommand m_selectfilelastmodifiedWithSizeCommand;
         private readonly System.Data.IDbCommand m_selectfileHashCommand;
         private readonly System.Data.IDbCommand m_selectblocklistHashesCommand;
 
         private readonly System.Data.IDbCommand m_insertfileOperationCommand;
-        
+        private readonly System.Data.IDbCommand m_selectfilemetadatahashandsizeCommand;
+
         private PathLookupHelper<PathEntryKeeper> m_pathLookup;
         private Dictionary<string, long> m_blockCache;
         
         private long m_filesetId;
 
+        private readonly bool m_logQueries;
+
         public LocalBackupDatabase(string path, Options options)
             : this(new LocalDatabase(path, "Backup", false), options)
         {
@@ -105,6 +109,8 @@ namespace Duplicati.Library.Main.Database
         public LocalBackupDatabase(LocalDatabase db, Options options)
             : base(db)
         {
+            m_logQueries = options.ProfileAllDatabaseQueries;
+
             m_findblockCommand = m_connection.CreateCommand();
             m_insertblockCommand = m_connection.CreateCommand();
             m_insertfileCommand = m_connection.CreateCommand();
@@ -119,8 +125,10 @@ namespace Duplicati.Library.Main.Database
             m_insertfileOperationCommand = m_connection.CreateCommand();
             m_findfileCommand = m_connection.CreateCommand();
             m_selectfilelastmodifiedCommand = m_connection.CreateCommand();
+            m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand();
             m_selectfileHashCommand = m_connection.CreateCommand();
             m_insertblocksetentryFastCommand = m_connection.CreateCommand();
+            m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand();
                 
             m_findblockCommand.CommandText = @"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?";
             m_findblockCommand.AddParameters(2);
@@ -161,17 +169,97 @@ namespace Duplicati.Library.Main.Database
             m_selectfilelastmodifiedCommand.CommandText = @"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?";
             m_selectfilelastmodifiedCommand.AddParameters(2);
 
-            //Need a temporary table with path/lastmodified lookups
-            m_findfileCommand.CommandText =
-                @" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
-                @"   FROM ""File"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
-                @"  WHERE ""File"".""Path"" = ? " +
-                @"    AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
-                @"    AND ""FileBlockset"".""ID"" = ""File"".""BlocksetID"" " +
-                @"    AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
-                @"  ORDER BY ""Fileset"".""Timestamp"" DESC " +
-                @"  LIMIT 1 ";
-            m_findfileCommand.AddParameters(1);
+            m_selectfilelastmodifiedWithSizeCommand.CommandText = @"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" ";
+            m_selectfilelastmodifiedWithSizeCommand.AddParameters(2);
+
+            m_selectfilemetadatahashandsizeCommand.CommandText = @"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = ? AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" ";
+            m_selectfilemetadatahashandsizeCommand.AddParameters(1);
+
+            // Allow users to test on real-world data
+            // to get feedback on potential performance
+            int.TryParse(Environment.GetEnvironmentVariable("TEST_QUERY_VERSION"), out var testqueryversion);
+
+            if (testqueryversion != 0)
+                Logging.Log.WriteWarningMessage(LOGTAG, "TestFileQuery", null, "Using performance test query version {0} as the TEST_QUERY_VERSION environment variable is set", testqueryversion);
+
+            // The original query (v==1) finds the most recent entry of the file in question, 
+            // but it requires some large joins to extract the required information.
+            // To speed it up, we use a slightly simpler approach that only looks at the
+            // previous fileset, and uses information here.
+            // If there is a case where a file is sometimes there and sometimes not
+            // (i.e. filter file, remove filter) we will not find the file.
+            // We currently use this faster version, 
+            // but allow users to switch back via an environment variable
+            // such that we can get performance feedback
+
+            switch (testqueryversion)
+            {
+                // The query used in Duplicati until 2.0.3.9
+                case 1:
+                    m_findfileCommand.CommandText =
+                        @" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
+                        @"   FROM ""File"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
+                        @"  WHERE ""File"".""Path"" = ? " +
+                        @"    AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
+                        @"    AND ""FileBlockset"".""ID"" = ""File"".""BlocksetID"" " +
+                        @"    AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
+                        @"    AND ? IS NOT NULL" +
+                        @"  ORDER BY ""Fileset"".""Timestamp"" DESC " +
+                        @"  LIMIT 1 ";
+                    break;
+
+                // The fastest reported query in Duplicati 2.0.3.10, but with "LIMIT 1" added
+                default:
+                case 2:
+                    var getLastFileEntryForPath =
+                        @"SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"", ""A"".""MetadataID"" " +
+                        @"  FROM (SELECT ""ID"", ""BlocksetID"", ""MetadataID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" " +
+                        @"  CROSS JOIN ""FilesetEntry"" ""B"" " +
+                        @"  WHERE ""A"".""ID"" = ""B"".""FileID"" " +
+                        @"    AND ""B"".""FilesetID"" = ? ";
+
+                    m_findfileCommand.CommandText = string.Format(
+                        @"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" " +
+                        @"  FROM " +
+                        @"  ({0}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" " +
+                        @" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" AND ""C"".""MetadataID"" == ""F"".""ID"" AND ""F"".""BlocksetID"" = ""E"".""ID"" " +
+                        @" LIMIT 1",
+                        getLastFileEntryForPath
+                    );
+                    break;
+
+                // Potentially faster query: https://forum.duplicati.com/t/release-2-0-3-10-canary-2018-08-30/4497/25
+                case 3:
+                    m_findfileCommand.CommandText =
+                        @"    SELECT File.ID as FileID, FilesetEntry.Lastmodified, FileBlockset.Length,  " +
+                        @"           MetaBlockset.FullHash AS Metahash, MetaBlockset.Length as Metasize " +
+                        @"      FROM FilesetEntry " +
+                        @"INNER JOIN Fileset ON (FileSet.ID = FilesetEntry.FilesetID) " +
+                        @"INNER JOIN File ON (File.ID = FilesetEntry.FileID) " +
+                        @"INNER JOIN Metadataset ON (Metadataset.ID = File.MetadataID) " +
+                        @"INNER JOIN Blockset AS MetaBlockset ON (MetaBlockset.ID = Metadataset.BlocksetID) " +
+                        @" LEFT JOIN Blockset AS FileBlockset ON (FileBlockset.ID = File.BlocksetID) " +
+                        @"     WHERE File.Path = ? AND FilesetID = ? " +
+                        @"     LIMIT 1 ";
+                    break;
+
+                // The slow query used in Duplicati 2.0.3.10, but with "LIMIT 1" added
+                case 4:
+                    m_findfileCommand.CommandText =
+                        @" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
+                        @"   FROM ""File"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
+                        @"  WHERE ""File"".""Path"" = ? " +
+                        @"    AND ""Fileset"".""ID"" = ? " +
+                        @"    AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
+                        @"    AND ""FileBlockset"".""ID"" = ""File"".""BlocksetID"" " +
+                        @"    AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
+                        @"  LIMIT 1 ";
+                    break;
+
+            }
+
+            m_findfileCommand.AddParameters(2);
+
 
             m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""File"" WHERE ""Blockset"".""ID"" = ""File"".""BlocksetID"" AND ""File"".""ID"" = ?  ";
             m_selectfileHashCommand.AddParameters(1);
@@ -289,7 +377,7 @@ namespace Duplicati.Library.Main.Database
             m_findblockCommand.Transaction = transaction;
             m_findblockCommand.SetParameterValue(0, key);
             m_findblockCommand.SetParameterValue(1, size);
-            return m_findblockCommand.ExecuteScalarInt64(-1);
+            return m_findblockCommand.ExecuteScalarInt64(m_logQueries, -1);
         }
 
 		/// <summary>
@@ -314,7 +402,7 @@ namespace Duplicati.Library.Main.Database
             m_findblockCommand.Transaction = transaction;
             m_findblockCommand.SetParameterValue(0, key);
             m_findblockCommand.SetParameterValue(1, size);
-            var r = m_findblockCommand.ExecuteScalarInt64(-1);
+            var r = m_findblockCommand.ExecuteScalarInt64(m_logQueries, -1);
 
             if (r == -1L)
             {
@@ -322,7 +410,7 @@ namespace Duplicati.Library.Main.Database
                 m_insertblockCommand.SetParameterValue(0, key);
                 m_insertblockCommand.SetParameterValue(1, volumeid);
                 m_insertblockCommand.SetParameterValue(2, size);
-                r = m_insertblockCommand.ExecuteScalarInt64();
+                m_insertblockCommand.ExecuteScalarInt64(m_logQueries);
                 if (m_blockCache != null)
                     m_blockCache.Add(key, size);
                 return true;
@@ -346,7 +434,7 @@ namespace Duplicati.Library.Main.Database
         public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes, out long blocksetid, System.Data.IDbTransaction transaction = null)
         {
             m_findblocksetCommand.Transaction = transaction;
-            blocksetid = m_findblocksetCommand.ExecuteScalarInt64(null, -1, filehash, size);
+            blocksetid = m_findblocksetCommand.ExecuteScalarInt64(m_logQueries, null, -1, filehash, size);
             if (blocksetid != -1)
                 return false; //Found it
 
@@ -355,7 +443,7 @@ namespace Duplicati.Library.Main.Database
                 m_insertblocksetCommand.Transaction = tr.Parent;
                 m_insertblocksetCommand.SetParameterValue(0, size);
                 m_insertblocksetCommand.SetParameterValue(1, filehash);
-                blocksetid = m_insertblocksetCommand.ExecuteScalarInt64();
+                blocksetid = m_insertblocksetCommand.ExecuteScalarInt64(m_logQueries);
 
                 long ix = 0;
                 if (blocklistHashes != null)
@@ -366,7 +454,7 @@ namespace Duplicati.Library.Main.Database
                     {
                         m_insertblocklistHashesCommand.SetParameterValue(1, ix);
                         m_insertblocklistHashesCommand.SetParameterValue(2, bh);
-                        m_insertblocklistHashesCommand.ExecuteNonQuery();
+                        m_insertblocklistHashesCommand.ExecuteNonQuery(m_logQueries);
                         ix++;
                     }
                 }
@@ -385,7 +473,7 @@ namespace Duplicati.Library.Main.Database
                     m_insertblocksetentryCommand.SetParameterValue(1, ix);
                     m_insertblocksetentryCommand.SetParameterValue(2, h);
                     m_insertblocksetentryCommand.SetParameterValue(3, exsize);
-                    var c = m_insertblocksetentryCommand.ExecuteNonQuery();
+                    var c = m_insertblocksetentryCommand.ExecuteNonQuery(m_logQueries);
                     if (c != 1)
                     {
                         Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size);
@@ -414,7 +502,7 @@ namespace Duplicati.Library.Main.Database
         /// <summary>
         /// Gets the metadataset ID from the filehash
         /// </summary>
-        /// <returns><c>true</c>, if metadataset should be recorded, false if it already exists.</returns>
+        /// <returns><c>true</c>, if metadataset found, false if does not exist.</returns>
         /// <param name="filehash">The metadata hash.</param>
         /// <param name="size">The size of the metadata.</param>
         /// <param name="metadataid">The ID of the metadataset.</param>
@@ -424,7 +512,7 @@ namespace Duplicati.Library.Main.Database
             if (size > 0)
             {
                 m_findmetadatasetCommand.Transaction = transaction;
-                metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size);
+                metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(m_logQueries, null, -1, filehash, size);
                 return metadataid != -1;
             }
 
@@ -435,7 +523,10 @@ namespace Duplicati.Library.Main.Database
         /// <summary>
         /// Adds a metadata set to the database, and returns a value indicating if the record was new
         /// </summary>
-        /// <param name="hash">The metadata hash</param>
+        /// <param name="filehash">The metadata hash</param>
+        /// <param name="size">The size of the metadata</param>
+        /// <param name="transaction">The transaction to execute under</param>
+        /// <param name="blocksetid">The id of the blockset to add</param>
         /// <param name="metadataid">The id of the metadata set</param>
         /// <returns>True if the set was added to the database, false otherwise</returns>
         public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, System.Data.IDbTransaction transaction = null)
@@ -447,7 +538,7 @@ namespace Duplicati.Library.Main.Database
             {
                 m_insertmetadatasetCommand.Transaction = tr.Parent;
                 m_insertmetadatasetCommand.SetParameterValue(0, blocksetid);
-                metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64();
+                metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64(m_logQueries);
                 tr.Commit();
                 return true;
             }
@@ -482,7 +573,7 @@ namespace Duplicati.Library.Main.Database
                 m_findfilesetCommand.SetParameterValue(0, blocksetID);
                 m_findfilesetCommand.SetParameterValue(1, metadataID);
                 m_findfilesetCommand.SetParameterValue(2, filename);
-                fileidobj = m_findfilesetCommand.ExecuteScalarInt64();
+                fileidobj = m_findfilesetCommand.ExecuteScalarInt64(m_logQueries);
             }
             
             if (fileidobj == -1)
@@ -493,7 +584,7 @@ namespace Duplicati.Library.Main.Database
                     m_insertfileCommand.SetParameterValue(0, filename);
                     m_insertfileCommand.SetParameterValue(1, blocksetID);
                     m_insertfileCommand.SetParameterValue(2, metadataID);
-                    fileidobj = m_insertfileCommand.ExecuteScalarInt64();
+                    fileidobj = m_insertfileCommand.ExecuteScalarInt64(m_logQueries);
                     tr.Commit();                    
 
                     // We do not need to update this, because we will not ask for the same file twice
@@ -515,7 +606,7 @@ namespace Duplicati.Library.Main.Database
             m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
             m_insertfileOperationCommand.SetParameterValue(1, fileidobj);
             m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
-            m_insertfileOperationCommand.ExecuteNonQuery();
+            m_insertfileOperationCommand.ExecuteNonQuery(m_logQueries);
 
         }
 
@@ -525,7 +616,7 @@ namespace Duplicati.Library.Main.Database
             m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
             m_insertfileOperationCommand.SetParameterValue(1, fileid);
             m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
-            m_insertfileOperationCommand.ExecuteNonQuery();
+            m_insertfileOperationCommand.ExecuteNonQuery(m_logQueries);
         }
 
         public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, System.Data.IDbTransaction transaction = null)
@@ -538,23 +629,53 @@ namespace Duplicati.Library.Main.Database
             AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction);
         }
 
-        public long GetFileLastModified(string path, long filesetid, out DateTime oldModified, System.Data.IDbTransaction transaction = null)
+        public long GetFileLastModified(string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, System.Data.IDbTransaction transaction = null)
         {
-			m_selectfileHashCommand.Transaction = transaction;
-			m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
-            m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid);
-            using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader())
-                if (rd.Read())
-                {
-                    oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
-                    return rd.ConvertValueToInt64(0);    
-                }
+            if (includeLength)
+            {
+                m_selectfilelastmodifiedWithSizeCommand.Transaction = transaction;
+                m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(0, path);
+                m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(1, filesetid);
+                using (var rd = m_selectfilelastmodifiedWithSizeCommand.ExecuteReader(m_logQueries, null))
+                    if (rd.Read())
+                    {
+                        oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
+                        length = rd.ConvertValueToInt64(2);
+                        return rd.ConvertValueToInt64(0);
+                    }
 
+            }
+            else
+            {
+                m_selectfilelastmodifiedCommand.Transaction = transaction;
+                m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
+                m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid);
+                using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null))
+                    if (rd.Read())
+                    {
+                        length = -1;
+                        oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
+                        return rd.ConvertValueToInt64(0);
+                    }
+
+            }
             oldModified = new DateTime(0, DateTimeKind.Utc);
+            length = -1;
             return -1;
         }
 
-        public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize)
+        public Tuple<long, string> GetMetadataHashAndSizeForFile(long fileid, System.Data.IDbTransaction transaction)
+        {
+            m_selectfilemetadatahashandsizeCommand.Transaction = transaction;
+            m_selectfilemetadatahashandsizeCommand.SetParameterValue(0, fileid);
+            using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader(m_logQueries, null))
+                if (rd.Read())
+                    return new Tuple<long, string>(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1));
+
+            return null;
+        }
+
+        public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize, System.Data.IDbTransaction transaction)
         {
             if (m_pathLookup != null)
             {            
@@ -579,8 +700,9 @@ namespace Duplicati.Library.Main.Database
             else
             {
                 m_findfileCommand.SetParameterValue(0, path);
-
-                using(var rd = m_findfileCommand.ExecuteReader())
+                m_findfileCommand.SetParameterValue(1, filesetid);
+                m_findfileCommand.Transaction = transaction;
+                using(var rd = m_findfileCommand.ExecuteReader(m_logQueries, null))
                     if (rd.Read())
                     {
                         oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
@@ -603,7 +725,7 @@ namespace Duplicati.Library.Main.Database
         public string GetFileHash(long fileid)
         {
             m_selectfileHashCommand.SetParameterValue(0, fileid);
-            var r = m_selectfileHashCommand.ExecuteScalar();
+            var r = m_selectfileHashCommand.ExecuteScalar(m_logQueries, null);
             if (r == null || r == DBNull.Value)
                 return null;
                 

Duplicati/Library/Main/Database/LocalDatabase.cs

@@ -625,7 +625,7 @@ namespace Duplicati.Library.Main.Database
             {
                 get
                 {
-                    return m_reader.ConvertValueToInt64(1);;
+                    return m_reader.ConvertValueToInt64(1);
                 }
             }
 
@@ -807,13 +807,19 @@ ON
                     using(var cmd2 = m_connection.CreateCommand(transaction))
                     foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1)))
                     {
-                            var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS);
+                        var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS);
                         var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid);
                         //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
                         var storedlist = cmd2.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?", 0, filesetid);
 
                         if (expandedlist != storedlist)
-                            throw new Exception(string.Format("Unexpected difference in fileset {0}, found {1} entries, but expected {2}", filesetid, expandedlist, storedlist));
+                        {
+                            var filesetname = filesetid.ToString();
+                            var fileset = FilesetTimes.Zip(Enumerable.Range(0, FilesetTimes.Count()), (a, b) => new Tuple<long, long, DateTime>(b, a.Key, a.Value)).FirstOrDefault(x => x.Item2 == filesetid);
+                            if (fileset != null)
+                                filesetname = string.Format("version {0}: {1} (database id: {2})", fileset.Item1, fileset.Item3, fileset.Item2);
+                            throw new Interface.UserInformationException(string.Format("Unexpected difference in fileset {0}, found {1} entries, but expected {2}", filesetname, expandedlist, storedlist), "FilesetDifferences");
+                        }
                     }
                 }
             }
@@ -1301,7 +1307,6 @@ ORDER BY
                         {
                             yield return new Tuple<string, byte[], int>(curHash, buffer, index);
                             buffer = new byte[blocksize];
-                            curHash = null;
                             index = 0;
                         }
 

Duplicati/Library/Main/Database/LocalDeleteDatabase.cs

@@ -51,12 +51,7 @@ namespace Duplicati.Library.Main.Database
             m_moveBlockToNewVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?";
             m_moveBlockToNewVolumeCommand.AddParameters(3);
         }
-        
-        private long GetLastFilesetID(System.Data.IDbCommand cmd)
-        {
-            return cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1", -1);
-        }
-        
+
         /// <summary>
         /// Drops all entries related to operations listed in the table.
         /// </summary>
@@ -272,7 +267,7 @@ namespace Duplicati.Library.Main.Database
         {
             private System.Data.IDbCommand m_command;
 
-            public BlockQuery(System.Data.IDbConnection con, Options options, System.Data.IDbTransaction transaction)
+            public BlockQuery(System.Data.IDbConnection con, System.Data.IDbTransaction transaction)
             {
                 m_command = con.CreateCommand();
                 m_command.Transaction = transaction;
@@ -302,9 +297,9 @@ namespace Duplicati.Library.Main.Database
         /// <summary>
         /// Builds a lookup table to enable faster response to block queries
         /// </summary>
-        public IBlockQuery CreateBlockQueryHelper(Options options, System.Data.IDbTransaction transaction)
+        public IBlockQuery CreateBlockQueryHelper(System.Data.IDbTransaction transaction)
         {
-            return new BlockQuery(m_connection, options, transaction);
+            return new BlockQuery(m_connection, transaction);
         }
 
         public void MoveBlockToNewVolume(string hash, long size, long volumeID, System.Data.IDbTransaction tr)

Duplicati/Library/Main/Database/LocalRecreateDatabase.cs

@@ -6,7 +6,7 @@ using System.Text;
 
 namespace Duplicati.Library.Main.Database
 {
-    internal partial class LocalRecreateDatabase : LocalRestoreDatabase
+    internal class LocalRecreateDatabase : LocalRestoreDatabase
     {
         /// <summary>
         /// The tag used for logging
@@ -294,20 +294,20 @@ namespace Duplicati.Library.Main.Database
         
         public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
         {
-            AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction);
+            AddEntry(filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction);
         }
 
         public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
         {
-            AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
+            AddEntry(filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
         }
         
         public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
         {
-            AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metadataid, transaction);
+            AddEntry(filesetid, path, time, blocksetid, metadataid, transaction);
         }
         
-        private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
+        private void AddEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
         {
             var fileid = -1L;
                         

Duplicati/Library/Main/Database/LocalRepairDatabase.cs

@@ -426,7 +426,6 @@ namespace Duplicati.Library.Main.Database
 
                                 // Add to table
                                 c3.ExecuteNonQuery(null, blocksetid, ix, blkeyfinal);
-                                ix++;
                             }
                         }
                     }

Duplicati/Library/Main/Database/LocalRestoreDatabase.cs

@@ -6,7 +6,7 @@ using Duplicati.Library.Main.Volumes;
 
 namespace Duplicati.Library.Main.Database
 {
-    internal partial class LocalRestoreDatabase : LocalDatabase
+    internal class LocalRestoreDatabase : LocalDatabase
     {
         /// <summary>
         /// The tag used for logging
@@ -100,7 +100,7 @@ namespace Duplicati.Library.Main.Database
                         , m_fileprogtable, m_tempfiletable, m_tempblocktable);
 
                     // Will be one row per file.
-                    int fileCnt = cmd.ExecuteNonQuery(sql);
+                    cmd.ExecuteNonQuery(sql);
 
                     sql = string.Format(
                           @"INSERT INTO ""{0}"" ("
@@ -115,7 +115,7 @@ namespace Duplicati.Library.Main.Database
                         , m_totalprogtable, m_fileprogtable);
 
                     // Will result in a single line (no support to also track metadata)
-                    int totalStatRowCount = cmd.ExecuteNonQuery(sql);
+                    cmd.ExecuteNonQuery(sql);
 
                     // Finally we create TRIGGERs to keep all our statistics up to date.
                     // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic.

Duplicati/Library/Main/Database/LocalTestDatabase.cs

@@ -169,29 +169,25 @@ namespace Duplicati.Library.Main.Database
               protected string m_tablename;
               protected System.Data.IDbTransaction m_transaction;
               protected System.Data.IDbCommand m_insertCommand;
-              protected abstract string TABLE_PREFIX { get; }
-              protected abstract string TABLEFORMAT { get; }
-              protected abstract string INSERTCOMMAND { get; }
-              protected abstract int INSERTARGUMENTS { get; }
-              
-              protected Basiclist(System.Data.IDbConnection connection, string volumename)
+
+              protected Basiclist(System.Data.IDbConnection connection, string volumename, string tablePrefix, string tableFormat, string insertCommand, int insertArguments)
               {
                 m_connection = connection;
                 m_volumename = volumename;
                 m_transaction = m_connection.BeginTransaction();
-                var tablename = TABLE_PREFIX + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
+                var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
 
                 using(var cmd = m_connection.CreateCommand())
                 {
                     cmd.Transaction = m_transaction;
-                    cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" {1}", tablename, TABLEFORMAT));
+                    cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" {1}", tablename, tableFormat));
                     m_tablename = tablename;
                 }
                 
                 m_insertCommand = m_connection.CreateCommand();
                 m_insertCommand.Transaction = m_transaction;
-                m_insertCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" {1}", m_tablename, INSERTCOMMAND);
-                m_insertCommand.AddParameters(INSERTARGUMENTS);
+                m_insertCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" {1}", m_tablename, insertCommand);
+                m_insertCommand.AddParameters(insertArguments);
             }
 
             public virtual void Dispose()
@@ -228,13 +224,13 @@ namespace Duplicati.Library.Main.Database
         
         private class Filelist : Basiclist, IFilelist
         {
-            protected override string TABLE_PREFIX { get { return "Filelist"; } }
-            protected override string TABLEFORMAT { get { return @"(""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL)"; } }
-            protected override string INSERTCOMMAND { get { return @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (?,?,?,?,?)"; } }
-            protected override int INSERTARGUMENTS { get { return 5; } }
+            private const string TABLE_PREFIX = "Filelist";
+            private const string TABLE_FORMAT = @"(""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL)";
+            private const string INSERT_COMMAND = @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (?,?,?,?,?)";
+            private const int INSERT_ARGUMENTS = 5;
 
             public Filelist(System.Data.IDbConnection connection, string volumename)
-                : base(connection, volumename)
+                : base(connection, volumename, Filelist.TABLE_PREFIX, Filelist.TABLE_FORMAT, Filelist.INSERT_COMMAND, Filelist.INSERT_ARGUMENTS)
             { 
             }
             
@@ -287,13 +283,13 @@ namespace Duplicati.Library.Main.Database
         
         private class Indexlist : Basiclist, IIndexlist
         {
-            protected override string TABLE_PREFIX { get { return "Indexlist"; } }
-            protected override string TABLEFORMAT { get { return @"(""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; } }
-            protected override string INSERTCOMMAND { get { return @"(""Name"", ""Hash"", ""Size"") VALUES (?,?,?)"; } }
-            protected override int INSERTARGUMENTS { get { return 3; } }
+            private const string TABLE_PREFIX = "Indexlist";
+            private const string TABLE_FORMAT = @"(""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)";
+            private const string INSERT_COMMAND = @"(""Name"", ""Hash"", ""Size"") VALUES (?,?,?)";
+            private const int INSERT_ARGUMENTS = 3;
             
             public Indexlist(System.Data.IDbConnection connection, string volumename)
-                : base(connection, volumename)
+                : base(connection, volumename, Indexlist.TABLE_PREFIX, Indexlist.TABLE_FORMAT, Indexlist.INSERT_COMMAND, Indexlist.INSERT_ARGUMENTS)
             {
             }
                     
@@ -343,13 +339,13 @@ namespace Duplicati.Library.Main.Database
 
        private class Blocklist : Basiclist, IBlocklist
        {
-            protected override string TABLE_PREFIX { get { return "Blocklist"; } }
-            protected override string TABLEFORMAT { get { return @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; } }
-            protected override string INSERTCOMMAND { get { return @"(""Hash"", ""Size"") VALUES (?,?)"; } }
-            protected override int INSERTARGUMENTS { get { return 2; } }
+            private const string TABLE_PREFIX = "Blocklist";
+            private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)";
+            private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (?,?)";
+            private const int INSERT_ARGUMENTS = 2;
             
             public Blocklist(System.Data.IDbConnection connection, string volumename)
-                : base(connection, volumename)
+                : base(connection, volumename, Blocklist.TABLE_PREFIX, Blocklist.TABLE_FORMAT, Blocklist.INSERT_COMMAND, Blocklist.INSERT_ARGUMENTS)
             { }            
         
             public void AddBlock(string hash, long size)

Duplicati/Library/Main/DatabaseLocator.cs

@@ -88,10 +88,9 @@ namespace Duplicati.Library.Main
             string type = uri.Scheme;
             int port = uri.Port;
             string username = uri.Username;
-            string password = uri.Password;
             string prefix = options.Prefix;
             
-            if (username == null || password == null)
+            if (username == null || uri.Password == null)
             {
                 var sopts = DynamicLoader.BackendLoader.GetSupportedCommands(backend);
                 var ropts = new Dictionary<string, string>(options.RawOptions);
@@ -104,23 +103,16 @@ namespace Duplicati.Library.Main
                     {
                         if (username == null && o.Aliases != null && o.Aliases.Contains("auth-username", StringComparer.OrdinalIgnoreCase) && ropts.ContainsKey(o.Name))
                             username = ropts[o.Name];
-                        if (password == null && o.Aliases != null && o.Aliases.Contains("auth-password", StringComparer.OrdinalIgnoreCase) && ropts.ContainsKey(o.Name))
-                            password = ropts[o.Name];
                     }
                 
                     foreach(var o in sopts)
                     {
                         if (username == null && o.Name.Equals("auth-username", StringComparison.OrdinalIgnoreCase) && ropts.ContainsKey("auth-username"))
                             username = ropts["auth-username"];
-                        if (password == null && o.Name.Equals("auth-password", StringComparison.OrdinalIgnoreCase) && ropts.ContainsKey("auth-password"))
-                            password = ropts["auth-password"];
                     }
                 }
             }
             
-            if (password != null)
-                password = Library.Utility.Utility.ByteArrayAsHexString(System.Security.Cryptography.SHA256.Create().ComputeHash(System.Text.Encoding.UTF8.GetBytes(password + "!" + uri.Scheme + "!" + uri.HostAndPath)));
-                
             //Now find the one that matches :)
             var matches = (from n in configs
                 where 

Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs

@@ -94,15 +94,19 @@ namespace Duplicati.Library.Main.Operation.Backup
         {
             return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction));
         }
+
+        public Task<Tuple<long, string>> GetMetadataHashAndSizeForFileAsync(long fileid)
+        {
+            return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, m_transaction));
+        }
 
-        public Task<KeyValuePair<long, DateTime>> GetFileLastModifiedAsync(string path, long lastfilesetid)
+        public Task<Tuple<long, DateTime, long>> GetFileLastModifiedAsync(string path, long lastfilesetid, bool includeLength)
         {
             return RunOnMain(() =>
             {
-                DateTime lastModified;
-                var id = m_database.GetFileLastModified(path, lastfilesetid, out lastModified, m_transaction);
+                var id = m_database.GetFileLastModified(path, lastfilesetid, includeLength, out var lastModified, out var length, m_transaction);
 
-                return new KeyValuePair<long, DateTime>(id, lastModified);
+                return new Tuple<long, DateTime, long>(id, lastModified, length);
             });
 		}
 
@@ -114,7 +118,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                 string oldMetahash;
                 long oldMetasize;
 
-                var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize);
+                var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize, m_transaction);
                 return
                     id < 0 ?
                     null :

Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs

@@ -26,12 +26,12 @@ namespace Duplicati.Library.Main.Operation.Backup
 {
     internal static class CountFilesHandler
     {
-        public static Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, Common.ITaskReader taskreader, System.Threading.CancellationToken token)
+        public static async Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, Common.ITaskReader taskreader, System.Threading.CancellationToken token)
         {
             // Make sure we create the enumeration process in a seperate scope,
             // but keep the log channel from the parent scope
-            using(Logging.Log.StartIsolatingScope())
-            using(new IsolatedChannelScope())
+            using(Logging.Log.StartIsolatingScope(true))
+            using (new IsolatedChannelScope())
             {
                 var enumeratorTask = Backup.FileEnumerationProcess.Run(sources, snapshot, null, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, options.ChangedFilelist, taskreader);
                 var counterTask = AutomationExtensions.RunTask(new 
@@ -60,7 +60,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                             {
                             }
 
-                            result.OperationProgressUpdater.UpdatefileCount(count, size, false);                    
+                            result.OperationProgressUpdater.UpdatefileCount(count, size, false);
                         }
                     }
                     finally
@@ -69,7 +69,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                     }
                 });
 
-                return Task.WhenAll(enumeratorTask, counterTask);
+                await Task.WhenAll(enumeratorTask, counterTask);
             }
         }
     }

Duplicati/Library/Main/Operation/Backup/DataBlock.cs

@@ -48,7 +48,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                 TaskCompletion = tcs
             });
 
-            var r = await tcs.Task;
+            var r = await tcs.Task.ConfigureAwait(false);
             return r;
         }
     }

Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs

@@ -47,8 +47,6 @@ namespace Duplicati.Library.Main.Operation.Backup
 
             async self =>
             {
-                var blocksize = options.Blocksize;
-
                 while (await taskreader.ProgressAsync)
                 {
                     var e = await self.Input.ReadAsync();
@@ -68,10 +66,10 @@ namespace Duplicati.Library.Main.Operation.Backup
                                 if (!e.MetadataChanged)
                                 {
                                     var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length);
-                                    if (!res.Item1)
+                                    if (res.Item1)
                                         return res.Item2;
 
-                                    Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpextedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2);
+                                    Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpextedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Path);
                                     e.MetadataChanged = true;
                                 }
 
@@ -99,14 +97,14 @@ namespace Duplicati.Library.Main.Operation.Backup
                                 await stats.AddAddedFile(filesize);
 
                                 if (options.Dryrun)
-                                    Logging.Log.WriteVerboseMessage(FILELOGTAG, "WoudlAddNewFile", "Would add new file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
+                                    Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddNewFile", "Would add new file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
                             }
                             else
                             {
                                 await stats.AddModifiedFile(filesize);
 
                                 if (options.Dryrun)
-                                    Logging.Log.WriteVerboseMessage(FILELOGTAG, "WoudlAddChangedFile", "Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
+                                    Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
                             }
 
                             await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid);
@@ -120,7 +118,15 @@ namespace Duplicati.Library.Main.Operation.Backup
                         {
                             // When we write the file to output, update the last modified time
                             Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Path);
-                            await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+
+                            try
+                            {
+                                await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                            }
+                            catch (Exception ex)
+                            {
+                                Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
+                            }
                         }
                     }
                     catch(Exception ex)

Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs

@@ -38,7 +38,7 @@ namespace Duplicati.Library.Main.Operation.Backup
         /// </summary>
         private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(typeof(FileEnumerationProcess));
 
-        public static Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, string[] changedfilelist, ITaskReader taskreader)
+        public static Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, string[] changedfilelist, ITaskReader taskreader)
         {
             return AutomationExtensions.RunTask(
             new
@@ -77,21 +77,21 @@ namespace Duplicati.Library.Main.Operation.Backup
                         {
                         }
 
-                        return AttributeFilterAsync(null, x, fa, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, attributeFilter, enumeratefilter, ignorenames, mixinqueue).WaitForTask().Result;
+                        return AttributeFilter(x, fa, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
                     });
                 }
                 else
                 {
-					Library.Utility.Utility.EnumerationFilterDelegate AttributeFilter = (root, path, attr) =>
-					    AttributeFilterAsync(root, path, attr, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, attributeFilter, enumeratefilter, ignorenames, mixinqueue).WaitForTask().Result;
+                    Library.Utility.Utility.EnumerationFilterDelegate attributeFilter = (root, path, attr) =>
+                        AttributeFilter(path, attr, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
 
                     if (journalService != null)
                     {
                         // filter sources using USN journal, to obtain a sub-set of files / folders that may have been modified
-                        sources = journalService.GetModifiedSources(AttributeFilter);
+                        sources = journalService.GetModifiedSources(attributeFilter);
                     }
 
-                    worklist = snapshot.EnumerateFilesAndFolders(sources, AttributeFilter, (rootpath, errorpath, ex) =>
+                    worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) =>
                     {
                         Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath);
                     });
@@ -219,10 +219,9 @@ namespace Duplicati.Library.Main.Operation.Backup
         /// Plugin filter for enumerating a list of files.
         /// </summary>
         /// <returns>True if the path should be returned, false otherwise.</returns>
-        /// <param name="rootpath">The root path that initiated this enumeration.</param>
         /// <param name="path">The current path.</param>
         /// <param name="attributes">The file or folder attributes.</param>
-        private static async Task<bool> AttributeFilterAsync(string rootpath, string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary<string, string> hardlinkmap, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue<string> mixinqueue)
+        private static bool AttributeFilter(string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary<string, string> hardlinkmap, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue<string> mixinqueue)
         {
 			// Step 1, exclude block devices
 			try
@@ -304,7 +303,7 @@ namespace Duplicati.Library.Main.Operation.Backup
             }
 
             // If we exclude files based on attributes, filter that
-            if ((attributeFilter & attributes) != 0)
+            if ((fileAttributes & attributes) != 0)
             {
                 Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path);
                 return false;

Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs

@@ -47,7 +47,18 @@ namespace Duplicati.Library.Main.Operation.Backup
             async self =>
             {
                 var EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);
-                var blocksize = options.Blocksize;
+
+                // Pre-cache the option variables here to simplify and
+                // speed up repeated option access below
+
+                var SKIPFILESLARGERTHAN = options.SkipFilesLargerThan;
+                // Zero and max both indicate no size limit
+                if (SKIPFILESLARGERTHAN == long.MaxValue)
+                    SKIPFILESLARGERTHAN = 0;
+
+                var DISABLEFILETIMECHECK = options.DisableFiletimeCheck;
+                var CHECKFILETIMEONLY = options.CheckFiletimeOnly;
+                var SKIPMETADATA = options.SkipMetadata;
 
                 while (true)
                 {
@@ -60,31 +71,78 @@ namespace Duplicati.Library.Main.Operation.Backup
                     }
                     catch(Exception ex)
                     {
-                        Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed tp read size of file: {0}", e.Path);
+                        Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path);
                     }
 
                     await stats.AddExaminedFile(filestatsize);
 
-                    e.MetaHashAndSize = options.StoreMetadata ? Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot), options) : EMPTY_METADATA;
-
-                    var timestampChanged = e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
-                    var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
-                    var tooLargeFile = options.SkipFilesLargerThan != long.MaxValue && options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > options.SkipFilesLargerThan;
-                    e.MetadataChanged = !options.CheckFiletimeOnly && !options.SkipMetadata && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
-
-                    if ((e.OldId < 0 || options.DisableFiletimeCheck || timestampChanged || filesizeChanged || e.MetadataChanged) && !tooLargeFile)
+                    // Stop now if the file is too large
+                    var tooLargeFile = SKIPFILESLARGERTHAN != 0 && filestatsize >= 0 && filestatsize > SKIPFILESLARGERTHAN;
+                    if (tooLargeFile)
                     {
-                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, e.OldId <= 0, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
+                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);
+                        continue;
+                    }
+
+                    // Invalid ID indicates a new file
+                    var isNewFile = e.OldId < 0;
+
+                    // If we disable the filetime check, we always assume that the file has changed
+                    // Otherwise we check that the timestamps are different or if any of them are empty
+                    var timestampChanged = DISABLEFILETIMECHECK || e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
+
+                    // Avoid generating a new matadata blob if timestamp has not changed
+                    // and we only check for timestamp changes
+                    if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile)
+                    {
+                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);                                                
+                        try
+                        {
+                            await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                        }
+                        catch (Exception ex)
+                        {
+                            if (ex.IsRetiredException())
+                                throw;
+                            Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
+                        }
+                        continue;
+                    }
+
+                    // If we have have disabled the filetime check, we do not have the metadata info
+                    // but we want to know if the metadata is potentially changed
+                    if (!isNewFile && DISABLEFILETIMECHECK)
+                    {
+                        var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId);
+                        if (tp != null)
+                        {
+                            e.OldMetaSize = tp.Item1;
+                            e.OldMetaHash = tp.Item2;
+                        }
+                    }
+
+                    // Compute current metadata
+                    e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Path, e.Attributes, options, snapshot), options);
+                    e.MetadataChanged = !SKIPMETADATA && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
+
+                    // Check if the file is new, or something indicates a change
+                    var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
+                    if (isNewFile || timestampChanged || filesizeChanged || e.MetadataChanged)
+                    {
+                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
                         await self.Output.WriteAsync(e);
                     }
                     else
                     {
-                        if (tooLargeFile)
-                            Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);                        
-                        else
-                            Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);                        
-                        
-                        await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Path);
+                        try
+                        {
+                            await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                        }
+                        catch (Exception ex)
+                        {
+                            Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
+                        }
                     }
                 }
             });

Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs

@@ -30,13 +30,13 @@ namespace Duplicati.Library.Main.Operation.Backup
     {
         private static readonly string METALOGTAG = Logging.Log.LogTagFromType(typeof(MetadataGenerator)) + ".Metadata";
 
-        public static async Task<Dictionary<string, string>> GenerateMetadataAsync(string path, System.IO.FileAttributes attributes, Options options, Snapshots.ISnapshotService snapshot)
+        public static Dictionary<string, string> GenerateMetadata(string path, System.IO.FileAttributes attributes, Options options, Snapshots.ISnapshotService snapshot)
         {
             try
             {
                 Dictionary<string, string> metadata;
 
-                if (options.StoreMetadata)
+                if (!options.SkipMetadata)
                 {
                     metadata = snapshot.GetMetadata(path, snapshot.IsSymlink(path, attributes), options.SymlinkPolicy == Options.SymlinkStrategy.Follow);
                     if (metadata == null)

Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs

@@ -67,6 +67,9 @@ namespace Duplicati.Library.Main.Operation.Backup
             {
                 var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), options);
 
+                var CHECKFILETIMEONLY = options.CheckFiletimeOnly;
+                var DISABLEFILETIMECHECK = options.DisableFiletimeCheck;
+
                 while (true)
                 {
                     var path = await self.Input.ReadAsync();
@@ -92,20 +95,20 @@ namespace Duplicati.Library.Main.Operation.Backup
                     }
 
                     // If we only have metadata, stop here
-                    if (await ProcessMetadata(path, attributes, lastwrite, options, snapshot, emptymetadata, database, self.StreamBlockChannel))
+                    if (await ProcessMetadata(path, attributes, lastwrite, options, snapshot, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false))
                     {
                         try
                         {
-                            if (options.CheckFiletimeOnly || options.DisableFiletimeCheck)
+                            if (CHECKFILETIMEONLY || DISABLEFILETIMECHECK)
                             {
-                                var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid);
+                                var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid, false);
                                 await self.Output.WriteAsync(new FileEntry() {
-                                    OldId = tmp.Key < 0 ? -1 : tmp.Key,
+                                    OldId = tmp.Item1,
                                     Path = path,
                                     Attributes = attributes,
                                     LastWrite = lastwrite,
-                                    OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value,
-                                    LastFileSize = -1 ,
+                                    OldModified = tmp.Item2,
+                                    LastFileSize = tmp.Item3 ,
                                     OldMetaHash = null,
                                     OldMetaSize = -1
                                 });
@@ -159,13 +162,13 @@ namespace Duplicati.Library.Main.Operation.Backup
 
                     if (options.SymlinkPolicy == Options.SymlinkStrategy.Store)
                     {
-                        var metadata = await MetadataGenerator.GenerateMetadataAsync(path, attributes, options, snapshot);
+                        var metadata = MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot);
 
                         if (!metadata.ContainsKey("CoreSymlinkTarget"))
                             metadata["CoreSymlinkTarget"] = symlinkTarget;
 
                         var metahash = Utility.WrapMetadata(metadata, options);
-                        await AddSymlinkToOutputAsync(path, DateTime.UtcNow, metahash, database, streamblockchannel);
+                        await AddSymlinkToOutputAsync(path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false);
 
                         Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", path);
                         // Don't process further
@@ -182,9 +185,9 @@ namespace Duplicati.Library.Main.Operation.Backup
             {
                 IMetahash metahash;
 
-                if (options.StoreMetadata)
+                if (!options.SkipMetadata)
                 {
-                    metahash = Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(path, attributes, options, snapshot), options);
+                    metahash = Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot), options);
                 }
                 else
                 {
@@ -192,7 +195,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                 }
 
                 Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", path);
-                await AddFolderToOutputAsync(path, lastwrite, metahash, database, streamblockchannel);
+                await AddFolderToOutputAsync(path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false);
                 return false;
             }
 
@@ -224,7 +227,7 @@ namespace Duplicati.Library.Main.Operation.Backup
         /// <param name="lastModified">The value of the lastModified timestamp</param>
         private static async Task AddFolderToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
         {
-            var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel);
+            var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel).ConfigureAwait(false);
             await database.AddDirectoryEntryAsync(filename, metadataid.Item2, lastModified);
         }
 
@@ -238,7 +241,7 @@ namespace Duplicati.Library.Main.Operation.Backup
         /// <param name="meta">The metadata ti record</param>
         private static async Task AddSymlinkToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
         {
-            var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel);
+            var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel).ConfigureAwait(false);
             await database.AddSymlinkEntryAsync(filename, metadataid.Item2, lastModified);
         }
 

Duplicati/Library/Main/Operation/Backup/ProgressHandler.cs

@@ -39,6 +39,8 @@ namespace Duplicati.Library.Main.Operation.Backup
             {
                 var filesStarted = new Dictionary<string, long>();
                 var fileProgress = new Dictionary<string, long>();
+                long processedFileCount = 0;
+                long processedFileSize = 0;
                 string current = null;
 
                 while(true)
@@ -63,6 +65,11 @@ namespace Duplicati.Library.Main.Operation.Backup
                                 stat.OperationProgressUpdater.UpdateFileProgress(t.Length);
                                 current = null;
                             }
+
+                            processedFileCount += 1;
+                            processedFileSize += t.Length;
+
+                            stat.OperationProgressUpdater.UpdatefilesProcessed(processedFileCount, processedFileSize);
                             filesStarted.Remove(t.Filepath);
                             fileProgress.Remove(t.Filepath);
                             break;

Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs

@@ -28,7 +28,7 @@ namespace Duplicati.Library.Main.Operation.Backup
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecreateMissingIndexFiles));
 
-        public static Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader)
+        public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader)
         {
             return AutomationExtensions.RunTask(new
             {

Duplicati/Library/Main/Operation/Backup/StreamBlock.cs

@@ -49,7 +49,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                 Result = tcs
             });
 
-            return await tcs.Task;
+            return await tcs.Task.ConfigureAwait(false);
         }
     }
 }

Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs

@@ -67,7 +67,6 @@ namespace Duplicati.Library.Main.Operation.Backup
                 {
                     var send_close = false;
                     var filesize = 0L;
-                    var filename = string.Empty;
 
                     var e = await self.Input.ReadAsync();
                     var cur = e.Result;
@@ -145,7 +144,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                                 }
 
                                 // Make sure the filehasher is done with the buf instance before we pass it on
-                                await pftask;
+                                await pftask.ConfigureAwait(false);
                                 await DataBlock.AddBlockToOutputAsync(self.BlockOutput, hashkey, buf, 0, lastread, e.Hint, false);
                                 buf = new byte[blocksize];
                             }

Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs

@@ -93,7 +93,6 @@ namespace Duplicati.Library.Main.Operation.Backup
                     {
                         var s = 1;
                         var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);
-                        var oldFilesetID = incompleteSet.Key;
 
                         // Probe for an unused filename
                         while (s < 60)

Duplicati/Library/Main/Operation/BackupHandler.cs

@@ -142,7 +142,7 @@ namespace Duplicati.Library.Main.Operation
                 {
                     if (m_options.NoBackendverification)
                     {
-                        FilelistProcessor.VerifyLocalList(backend, m_options, m_database, m_result.BackendWriter);
+                        FilelistProcessor.VerifyLocalList(backend, m_database);
                         UpdateStorageStatsFromDatabase();
                     }
                     else
@@ -201,7 +201,7 @@ namespace Duplicati.Library.Main.Operation
                     );
                 }
 
-                await all;
+                await all.ConfigureAwait(false);
 
                 if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1)
                 {
@@ -269,7 +269,7 @@ namespace Duplicati.Library.Main.Operation
                 backend.WaitForComplete(m_database, null);
             }
 
-            if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Count() > 0)
+            if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Any())
             {
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest);
                 m_result.TestResults = new TestResults(m_result);
@@ -354,7 +354,7 @@ namespace Duplicati.Library.Main.Operation
 
             // In case the uploader crashes, we grab the exception here
             if (await Task.WhenAny(uploader, flushReq.LastWriteSizeAync) == uploader)
-                await uploader;
+                await uploader.ConfigureAwait(false);
 
             // Grab the size of the last uploaded volume
             return await flushReq.LastWriteSizeAync;
@@ -421,7 +421,7 @@ namespace Duplicati.Library.Main.Operation
                                 }
 
                                 // Make sure the database is sane
-                                await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, true);
+                                await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks);
 
                                 // Start the uploader process
                                 uploader = Backup.BackendUploader.Run(bk, m_options, db, m_result, m_result.TaskReader, stats);
@@ -454,7 +454,7 @@ namespace Duplicati.Library.Main.Operation
                                 var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key;
 
                                 // Rebuild any index files that are missing
-                                await Backup.RecreateMissingIndexFiles.Run(db, m_options, m_result, m_result.TaskReader);
+                                await Backup.RecreateMissingIndexFiles.Run(db, m_options, m_result.TaskReader);
 
                                 // This should be removed as the lookups are no longer used
                                 m_database.BuildLookupTable(m_options);
@@ -477,7 +477,7 @@ namespace Duplicati.Library.Main.Operation
 
                                 // Run the backup operation
                                 if (await m_result.TaskReader.ProgressAsync)
-                                    await RunMainOperation(sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskReader, lastfilesetid);
+                                    await RunMainOperation(sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskReader, lastfilesetid).ConfigureAwait(false);
                             }
                             finally
                             {
@@ -496,7 +496,7 @@ namespace Duplicati.Library.Main.Operation
 
                         // Wait for upload completion
                         m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
-                        var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploader);
+                        var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploader).ConfigureAwait(false);
 
                         // Make sure we have the database up-to-date
                         await db.CommitTransactionAsync("CommitAfterUpload", false);

Duplicati/Library/Main/Operation/Common/BackendHandler.cs

@@ -108,16 +108,16 @@ namespace Duplicati.Library.Main.Operation.Common
                 this.LocalTempfile = Library.Utility.TempFile.WrapExistingFile(name);
                 this.LocalTempfile.Protected = true;
             }
-                
-            public async Task Encrypt(Options options)
+
+            public void Encrypt(Options options)
             {
                 if (!this.Encrypted && !options.NoEncryption)
                 {
                     var tempfile = new Library.Utility.TempFile();
-                    using(var enc = DynamicLoader.EncryptionLoader.GetModule(options.EncryptionModule, options.Passphrase, options.RawOptions))
+                    using (var enc = DynamicLoader.EncryptionLoader.GetModule(options.EncryptionModule, options.Passphrase, options.RawOptions))
                         enc.Encrypt(this.LocalFilename, tempfile);
 
-                    await this.DeleteLocalFile();
+                    this.DeleteLocalFile();
 
                     this.LocalTempfile = tempfile;
                     this.Hash = null;
@@ -146,12 +146,18 @@ namespace Duplicati.Library.Main.Operation.Common
                 return false;
             }
 
-            public async Task DeleteLocalFile()
+            public void DeleteLocalFile()
             {
                 if (this.LocalTempfile != null)
-                    try { this.LocalTempfile.Dispose(); }
-                catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DeleteTemporaryFileError", ex, "Failed to dispose temporary file: {0}", this.LocalTempfile); }
-                finally { this.LocalTempfile = null; }
+                {
+                    try
+                    {
+                        this.LocalTempfile.Protected = false;
+                        this.LocalTempfile.Dispose();
+                    }
+                    catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DeleteTemporaryFileError", ex, "Failed to dispose temporary file: {0}", this.LocalTempfile); }
+                    finally { this.LocalTempfile = null; }
+                }
             }
         }
 
@@ -173,18 +179,18 @@ namespace Duplicati.Library.Main.Operation.Common
             m_backendurl = backendUrl;
             m_stats = stats;
             m_taskreader = taskreader;
-            m_backend = DynamicLoader.BackendLoader.GetBackend(backendUrl, options.RawOptions);
-			
-            var shortname = m_backendurl;
-
-			// Try not to leak hostnames or other information in the error messages
-			try { shortname = new Library.Utility.Uri(shortname).Scheme; }
-			catch { }
-
-			if (m_backend == null)
-                throw new Duplicati.Library.Interface.UserInformationException(string.Format("Backend not supported: {0}", shortname), "BackendNotSupported");
-		}
-            
+            m_backend = DynamicLoader.BackendLoader.GetBackend(backendUrl, options.RawOptions);
+
+            var shortname = m_backendurl;
+
+            // Try not to leak hostnames or other information in the error messages
+            try { shortname = new Library.Utility.Uri(shortname).Scheme; }
+            catch { }
+
+            if (m_backend == null)
+                throw new Duplicati.Library.Interface.UserInformationException(string.Format("Backend not supported: {0}", shortname), "BackendNotSupported");
+        }
+
         protected Task<T> RunRetryOnMain<T>(FileEntryItem fe, Func<Task<T>> method)
         {
             return RunOnMain<T>(() =>
@@ -201,13 +207,13 @@ namespace Duplicati.Library.Main.Operation.Common
 
             return RunRetryOnMain<bool>(fe, async () =>
             {
-                await DoPut(fe);
+                await DoPut(fe).ConfigureAwait(false);
                 m_uploadSuccess = true;
                 return true;
             });
 
         }
-            
+
         public async Task UploadFileAsync(VolumeWriterBase item, Func<string, Task<IndexVolumeWriter>> createIndexFile = null)
         {
             var fe = new FileEntryItem(BackendActionType.Put, item.RemoteFilename);
@@ -215,9 +221,9 @@ namespace Duplicati.Library.Main.Operation.Common
 
             var tcs = new TaskCompletionSource<bool>();
 
-            var backgroundhashAndEncrypt = Task.Run(async () =>
+            var backgroundhashAndEncrypt = Task.Run(() =>
             {
-                await fe.Encrypt(m_options).ConfigureAwait(false);
+                fe.Encrypt(m_options);
                 return fe.UpdateHashAndSize(m_options);
             });
 
@@ -225,44 +231,46 @@ namespace Duplicati.Library.Main.Operation.Common
             {
                 try
                 {
-                    await DoWithRetry(fe, async () => {
+                    await DoWithRetry(fe, async () =>
+                    {
                         if (fe.IsRetry)
-                            await RenameFileAfterErrorAsync(fe);
+                            await RenameFileAfterErrorAsync(fe).ConfigureAwait(false);
 
                         // Make sure the encryption and hashing has completed
-                        await backgroundhashAndEncrypt;
+                        await backgroundhashAndEncrypt.ConfigureAwait(false);
 
-                        return await DoPut(fe);
-                    });
+                        return await DoPut(fe).ConfigureAwait(false);
+                    }).ConfigureAwait(false);
 
                     if (createIndexFile != null)
                     {
-                        var ix = await createIndexFile(fe.RemoteFilename);
+                        var ix = await createIndexFile(fe.RemoteFilename).ConfigureAwait(false);
                         var indexFile = new FileEntryItem(BackendActionType.Put, ix.RemoteFilename);
                         indexFile.SetLocalfilename(ix.LocalFilename);
 
                         await m_database.UpdateRemoteVolumeAsync(indexFile.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
 
-                        await DoWithRetry(indexFile, async () => {
+                        await DoWithRetry(indexFile, async () =>
+                        {
                             if (indexFile.IsRetry)
-                                await RenameFileAfterErrorAsync(indexFile);
+                                await RenameFileAfterErrorAsync(indexFile).ConfigureAwait(false);
 
-                            var res = await DoPut(indexFile);
+                            var res = await DoPut(indexFile).ConfigureAwait(false);
 
                             // Register that the index file is tracking the block file
                             await m_database.AddIndexBlockLinkAsync(
                                 ix.VolumeID,
                                 await m_database.GetRemoteVolumeIDAsync(fe.RemoteFilename)
-                            );
+                            ).ConfigureAwait(false);
 
 
                             return res;
-                        });
+                        }).ConfigureAwait(false);
                     }
 
                     tcs.TrySetResult(true);
                 }
-                catch(Exception ex)
+                catch (Exception ex)
                 {
                     if (ex is System.Threading.ThreadAbortException)
                         tcs.TrySetCanceled();
@@ -271,7 +279,7 @@ namespace Duplicati.Library.Main.Operation.Common
                 }
             });
 
-            await tcs.Task;
+            await tcs.Task.ConfigureAwait(false);
         }
 
         public Task DeleteFileAsync(string remotename, bool suppressCleanup = false)
@@ -294,22 +302,23 @@ namespace Duplicati.Library.Main.Operation.Common
         public Task<IList<Library.Interface.IFileEntry>> ListFilesAsync()
         {
             var fe = new FileEntryItem(BackendActionType.List, null);
-            return RunRetryOnMain(fe, () => 
-                DoList(fe)
+            return RunRetryOnMain(fe, () =>
+                DoList()
             );
         }
 
         public Task<Library.Utility.TempFile> GetFileAsync(string remotename, long size, string remotehash)
         {
             var fe = new FileEntryItem(BackendActionType.Get, remotename, size, remotehash);
-            return RunRetryOnMain(fe, () => DoGet(fe) );
+            return RunRetryOnMain(fe, () => DoGet(fe));
         }
 
         public Task<Tuple<Library.Utility.TempFile, long, string>> GetFileWithInfoAsync(string remotename)
         {
             var fe = new FileEntryItem(BackendActionType.Get, remotename);
-            return RunRetryOnMain(fe, async () => {
-                var res = await DoGet(fe);
+            return RunRetryOnMain(fe, async () =>
+            {
+                var res = await DoGet(fe).ConfigureAwait(false);
                 return new Tuple<Library.Utility.TempFile, long, string>(
                     res,
                     fe.Size,
@@ -318,26 +327,25 @@ namespace Duplicati.Library.Main.Operation.Common
             });
         }
 
-        public Task<Library.Utility.TempFile> GetFileForTestingAsync(string remotename, long size, string remotehash)
+        public Task<Library.Utility.TempFile> GetFileForTestingAsync(string remotename)
         {
             var fe = new FileEntryItem(BackendActionType.Get, remotename);
             fe.VerifyHashOnly = true;
             return RunRetryOnMain(fe, () => DoGet(fe));
         }
 
-        private async Task ResetBackendAsync(Exception ex)
+        private void ResetBackend(Exception ex)
         {
             try
             {
                 if (m_backend != null)
                     m_backend.Dispose();
             }
-            catch (Exception dex) 
-            { 
-                Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", dex, "Failed to dispose backend instance: {0}", ex.Message); 
+            catch (Exception dex)
+            {
+                Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", dex, "Failed to dispose backend instance: {0}", ex.Message);
             }
             m_backend = null;
-
         }
 
         private async Task<T> DoWithRetry<T>(FileEntryItem item, Func<Task<T>> method)
@@ -350,11 +358,11 @@ namespace Duplicati.Library.Main.Operation.Common
 
             if (m_workerSource.IsCancellationRequested)
                 throw new OperationCanceledException();
-            
-            for(var i = 0; i < m_options.NumberOfRetries; i++)
+
+            for (var i = 0; i < m_options.NumberOfRetries; i++)
             {
                 if (m_options.RetryDelay.Ticks != 0 && i != 0)
-                    await Task.Delay(m_options.RetryDelay);
+                    await Task.Delay(m_options.RetryDelay).ConfigureAwait(false);
 
                 if (!await m_taskreader.TransferProgressAsync)
                     throw new OperationCanceledException();
@@ -368,8 +376,8 @@ namespace Duplicati.Library.Main.Operation.Common
                         m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions);
                     if (m_backend == null)
                         throw new Exception("Backend failed to re-load");
-                    
-                    var r = await method();
+
+                    var r = await method().ConfigureAwait(false);
                     return r;
                 }
                 catch (Exception ex)
@@ -387,24 +395,24 @@ namespace Duplicati.Library.Main.Operation.Common
                     if (!m_uploadSuccess && ex is Duplicati.Library.Interface.FolderMissingException && m_options.AutocreateFolders)
                     {
                         try
-                        { 
+                        {
                             // If we successfully create the folder, we can re-use the connection
-                            m_backend.CreateFolder(); 
+                            m_backend.CreateFolder();
                             recovered = true;
                         }
                         catch (Exception dex)
-                        { 
+                        {
                             Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateError", dex, "Failed to create folder: {0}", ex.Message);
                         }
                     }
-                        
+
                     if (!recovered)
-                        await ResetBackendAsync(ex);
+                        ResetBackend(ex);
                 }
                 finally
                 {
                     if (m_options.NoConnectionReuse)
-                        await ResetBackendAsync(null);
+                        ResetBackend(null);
                 }
             }
 
@@ -414,7 +422,7 @@ namespace Duplicati.Library.Main.Operation.Common
         private async Task RenameFileAfterErrorAsync(FileEntryItem item)
         {
             var p = VolumeBase.ParseFilename(item.RemoteFilename);
-            var guid = VolumeWriterBase.GenerateGuid(m_options);
+            var guid = VolumeWriterBase.GenerateGuid();
             var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1);
             var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule);
             var oldname = item.RemoteFilename;
@@ -429,7 +437,7 @@ namespace Duplicati.Library.Main.Operation.Common
         private async Task<bool> DoPut(FileEntryItem item, bool updatedHash = false)
         {
             // If this is not already encrypted, do it now
-            await item.Encrypt(m_options);
+            item.Encrypt(m_options);
 
             updatedHash |= item.UpdateHashAndSize(m_options);
 
@@ -439,10 +447,10 @@ namespace Duplicati.Library.Main.Operation.Common
             if (m_options.Dryrun)
             {
                 Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length));
-                await item.DeleteLocalFile();
+                item.DeleteLocalFile();
                 return true;
             }
-            
+
             await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
             await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size);
 
@@ -452,7 +460,7 @@ namespace Duplicati.Library.Main.Operation.Common
             {
                 using (var fs = System.IO.File.OpenRead(item.LocalFilename))
                 using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
-                using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
+                using (var pgs = new Library.Utility.ProgressReportingStream(ts, pg => HandleProgress(ts, pg)))
                     ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs);
             }
             else
@@ -468,20 +476,20 @@ namespace Duplicati.Library.Main.Operation.Common
 
             if (m_options.ListVerifyUploads)
             {
-                var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
+                var f = m_backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase));
                 if (f == null)
                     throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
                 else if (f.Size != item.Size && f.Size >= 0)
                     throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
             }
-                
-            await item.DeleteLocalFile();
+
+            item.DeleteLocalFile();
             await m_database.CommitTransactionAsync("CommitAfterUpload");
 
             return true;
         }
 
-        private async Task<IList<Library.Interface.IFileEntry>> DoList(FileEntryItem item)
+        private async Task<IList<Library.Interface.IFileEntry>> DoList()
         {
             await m_stats.SendEventAsync(BackendActionType.List, BackendEventType.Started, null, -1);
 
@@ -570,7 +578,7 @@ namespace Duplicati.Library.Main.Operation.Common
             try
             {
                 m_backend.CreateFolder();
-            } 
+            }
             catch (Exception ex)
             {
                 result = ex.ToString();
@@ -599,7 +607,7 @@ namespace Duplicati.Library.Main.Operation.Common
                 {
                     using (var fs = System.IO.File.OpenWrite(tmpfile))
                     using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
-                    using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
+                    using (var pgs = new Library.Utility.ProgressReportingStream(ts, pg => HandleProgress(ts, pg)))
                         ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
                 }
                 else
@@ -636,14 +644,14 @@ namespace Duplicati.Library.Main.Operation.Common
                 // Fast exit
                 if (item.VerifyHashOnly)
                     return null;
-                
+
                 // Decrypt before returning
                 if (!m_options.NoEncryption)
                 {
                     try
                     {
-                        using(var tmpfile2 = tmpfile)
-                        { 
+                        using (var tmpfile2 = tmpfile)
+                        {
                             tmpfile = new Library.Utility.TempFile();
 
                             // Auto-guess the encryption module
@@ -653,11 +661,11 @@ namespace Duplicati.Library.Main.Operation.Common
                                 // Check if the file is encrypted with something else
                                 if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase))
                                 {
-                                    using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(ext, m_options.Passphrase, m_options.RawOptions))
+                                    using (var encmodule = DynamicLoader.EncryptionLoader.GetModule(ext, m_options.Passphrase, m_options.RawOptions))
                                         if (encmodule != null)
                                         {
-                                        Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", using matching encryption module", ext, m_options.EncryptionModule);
-                                        encmodule.Decrypt(tmpfile2, tmpfile);
+                                            Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", using matching encryption module", ext, m_options.EncryptionModule);
+                                            encmodule.Decrypt(tmpfile2, tmpfile);
                                         }
                                 }
                                 // Check if the file is not encrypted
@@ -669,13 +677,13 @@ namespace Duplicati.Library.Main.Operation.Common
                                 else
                                 {
                                     Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, m_options.EncryptionModule);
-                                    using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions))
+                                    using (var encmodule = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions))
                                         encmodule.Decrypt(tmpfile2, tmpfile);
                                 }
                             }
                             else
                             {
-                                using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions))
+                                using (var encmodule = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions))
                                     encmodule.Decrypt(tmpfile2, tmpfile);
                             }
                         }
@@ -696,42 +704,42 @@ namespace Duplicati.Library.Main.Operation.Common
             }
             finally
             {
-                try 
-                { 
-                    if (tmpfile != null) 
+                try
+                {
+                    if (tmpfile != null)
                         tmpfile.Dispose();
                 }
                 catch
                 {
                 }
             }
-        }
-
-		private string m_lastThrottleUploadValue = null;
-		private string m_lastThrottleDownloadValue = null;
-
-		private void HandleProgress(ThrottledStream ts, long pg)
-		{
+        }
+
+        private string m_lastThrottleUploadValue = null;
+        private string m_lastThrottleDownloadValue = null;
+
+        private void HandleProgress(ThrottledStream ts, long pg)
+        {
             if (!m_taskreader.TransferProgressAsync.WaitForTask().Result)
-                throw new OperationCanceledException();
-
-			// Update the throttle speeds if they have changed
-			string tmp;
-			m_options.RawOptions.TryGetValue("throttle-upload", out tmp);
-			if (tmp != m_lastThrottleUploadValue)
-			{
-				ts.WriteSpeed = m_options.MaxUploadPrSecond;
-				m_lastThrottleUploadValue = tmp;
-			}
-
-			m_options.RawOptions.TryGetValue("throttle-download", out tmp);
-			if (tmp != m_lastThrottleDownloadValue)
-			{
-				ts.ReadSpeed = m_options.MaxDownloadPrSecond;
-				m_lastThrottleDownloadValue = tmp;
-			}
-
-			m_stats.UpdateBackendProgress(pg);
+                throw new OperationCanceledException();
+
+            // Update the throttle speeds if they have changed
+            string tmp;
+            m_options.RawOptions.TryGetValue("throttle-upload", out tmp);
+            if (tmp != m_lastThrottleUploadValue)
+            {
+                ts.WriteSpeed = m_options.MaxUploadPrSecond;
+                m_lastThrottleUploadValue = tmp;
+            }
+
+            m_options.RawOptions.TryGetValue("throttle-download", out tmp);
+            if (tmp != m_lastThrottleDownloadValue)
+            {
+                ts.ReadSpeed = m_options.MaxDownloadPrSecond;
+                m_lastThrottleDownloadValue = tmp;
+            }
+
+            m_stats.UpdateBackendProgress(pg);
         }
 
         protected override void Dispose(bool disposing)
@@ -740,7 +748,7 @@ namespace Duplicati.Library.Main.Operation.Common
 
             if (m_backend != null)
                 try { m_backend.Dispose(); }
-                catch {}
+                catch { }
                 finally { m_backend = null; }
         }
     }

Duplicati/Library/Main/Operation/Common/SingleRunner.cs

@@ -27,79 +27,20 @@ namespace Duplicati.Library.Main.Operation.Common
     /// </summary>
     internal abstract class SingleRunner : IDisposable
     {
-        protected IChannel<Func<Task>> m_channel;
-        protected readonly Task m_worker;
-        protected CancellationTokenSource m_workerSource;
+        protected AsyncLock m_lock = new AsyncLock();
+        protected CancellationTokenSource m_workerSource = new CancellationTokenSource();
 
-        public SingleRunner()
+        protected async Task<T> DoRunOnMain<T>(Func<Task<T>> method)
         {
-            AutomationExtensions.AutoWireChannels(this, null);
-            m_channel = ChannelManager.CreateChannel<Func<Task>>();
-            m_workerSource = new System.Threading.CancellationTokenSource();
-            m_worker = AutomationExtensions.RunProtected(this, Start);
-        }
+            m_workerSource.Token.ThrowIfCancellationRequested();
 
-        private async Task Start()
-        {            
-            var ct = m_workerSource.Token;
-            while(!ct.IsCancellationRequested)
+            using (await m_lock.LockAsync())
             {
-                // Grab next task
-                var nextTask = await m_channel.ReadAsync();
-
-                // Execute it
-                await nextTask();
+                m_workerSource.Token.ThrowIfCancellationRequested();
+                return await method().ConfigureAwait(false);
             }
         }
 
-        protected Task<T> DoRunOnMain<T>(Func<Task<T>> method)
-        {
-            var res = new TaskCompletionSource<T>();
-
-            Task.Run(async () =>
-            {
-                try
-                {
-                    if (m_workerSource.IsCancellationRequested)
-                    {
-                        res.TrySetCanceled();
-                        return;
-                    }
-
-                    await m_channel.WriteAsync(async () =>
-                    {
-                        if (m_workerSource.IsCancellationRequested)
-                        {
-                            res.TrySetCanceled();
-                            return;
-                        }
-
-                        try
-                        {
-                            var r = await method().ConfigureAwait(false);
-                            res.SetResult(r);
-                        }
-                        catch (Exception ex)
-                        {
-                            if (ex is System.Threading.ThreadAbortException)
-                                res.TrySetCanceled();
-                            else
-                                res.TrySetException(ex);
-                        }
-                    }).ConfigureAwait(false);
-                }
-                catch (Exception ex)
-                {
-                    if (ex is System.Threading.ThreadAbortException)
-                        res.TrySetCanceled();
-                    else
-                        res.TrySetException(ex);
-                }
-            });
-
-            return res.Task;
-        }
-
         protected Task RunOnMain(Action method)
         {
             return DoRunOnMain<bool>(() =>
@@ -138,13 +79,6 @@ namespace Duplicati.Library.Main.Operation.Common
         protected virtual void Dispose(bool isDisposing)
         {
             m_workerSource.Cancel();
-
-            if (m_channel != null)
-                try { m_channel.Retire(); }
-                catch { }
-                finally { }
-
-            AutomationExtensions.RetireAllChannels(this);
         }
     }
 }

Duplicati/Library/Main/Operation/CompactHandler.cs

@@ -141,7 +141,7 @@ namespace Duplicati.Library.Main.Operation
                                                  where report.CompactableVolumes.Contains(v.Name)
                                                  select (IRemoteVolume)v).ToList();
                         
-                        using(var q = db.CreateBlockQueryHelper(m_options, transaction))
+                        using(var q = db.CreateBlockQueryHelper(transaction))
                         {
                             foreach (var entry in new AsyncDownloader(volumesToDownload, backend))
                             {

Duplicati/Library/Main/Operation/FilelistProcessor.cs

@@ -33,10 +33,8 @@ namespace Duplicati.Library.Main.Operation
         /// Helper method that verifies uploaded volumes and updates their state in the database.
         /// Throws an error if there are issues with the remote storage
         /// </summary>
-        /// <param name="options">The options used</param>
         /// <param name="database">The database to compare with</param>
-        /// <param name="log">The log instance to use</param>
-        public static void VerifyLocalList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)
+        public static void VerifyLocalList(BackendManager backend, LocalDatabase database)
         {
             var locallist = database.GetRemoteVolumes();
             foreach(var i in locallist)

Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs

@@ -105,7 +105,14 @@ namespace Duplicati.Library.Main.Operation
             if (brokensets.Length == 0)
             {
                 m_result.BrokenFiles = new Tuple<long, DateTime, IEnumerable<Tuple<string, long>>>[0];
-                Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "No broken filesets found");
+
+                if (missing == null)
+                    Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets");
+                else if (missing.Count == 0)
+                    Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files");
+                else
+                    Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Run purge-broken-files.", missing.Count));
+
                 return;
             }
 

Duplicati/Library/Main/Operation/ListChangesHandler.cs

@@ -66,13 +66,13 @@ namespace Duplicati.Library.Main.Operation
             
             var useLocalDb = !m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath);
             baseVersion = string.IsNullOrEmpty(baseVersion) ? "1" : baseVersion;
-            compareVersion = string.IsNullOrEmpty(compareVersion) ? "0" : compareVersion;
-            
-            long baseVersionIndex = -1;
-            long compareVersionIndex = -1;
-            
-            DateTime baseVersionTime = new DateTime(0);
-            DateTime compareVersionTime = new DateTime(0);
+            compareVersion = string.IsNullOrEmpty(compareVersion) ? "0" : compareVersion;
+
+            long baseVersionIndex;
+            long compareVersionIndex;
+
+            DateTime baseVersionTime;
+            DateTime compareVersionTime;
             
             using(var tmpdb = useLocalDb ? null : new Library.Utility.TempFile())
             using(var db = new Database.LocalListChangesDatabase(useLocalDb ? m_options.Dbpath : (string)tmpdb))

Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs

@@ -65,103 +65,105 @@ namespace Duplicati.Library.Main.Operation
                     else if (missing.Count == 0)
                         Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files");
                     else
-                        throw new UserInformationException(string.Format("Found no broken filesets, but {0} missing remote files", sets.Length), "NoBrokenSetsButMissingRemoteFiles");
+                        Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count));
                 }
+                else
+                { 
+                    Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.Item3));
 
-                Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.Item3));
+                    var pgoffset = 0.0f;
+                    var pgspan = 0.95f / sets.Length;
 
-                var pgoffset = 0.0f;
-                var pgspan = 0.95f / sets.Length;
+                    var filesets = db.FilesetTimes.ToList();
 
-                var filesets = db.FilesetTimes.ToList();
-
-                var compare_list = sets.Select(x => new
-                {
-                    FilesetID = x.Item2,
-                    Timestamp = x.Item1,
-                    RemoveCount = x.Item3,
-                    Version = filesets.FindIndex(y => y.Key == x.Item2),
-                    SetCount = db.GetFilesetFileCount(x.Item2, tr)
-                }).ToArray();
-
-                var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
-                var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
-
-                if (fully_emptied.Length != 0)
-                {
-                    if (fully_emptied.Length == 1)
-                        Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount);
-                    else
-                        Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())));
-
-                    m_result.DeleteResults = new DeleteResults(m_result);
-                    using (var rmdb = new Database.LocalDeleteDatabase(db))
+                    var compare_list = sets.Select(x => new
                     {
-                        var deltr = rmdb.BeginTransaction();
-                        try
+                        FilesetID = x.Item2,
+                        Timestamp = x.Item1,
+                        RemoveCount = x.Item3,
+                        Version = filesets.FindIndex(y => y.Key == x.Item2),
+                        SetCount = db.GetFilesetFileCount(x.Item2, tr)
+                    }).ToArray();
+
+                    var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
+                    var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
+
+                    if (fully_emptied.Length != 0)
+                    {
+                        if (fully_emptied.Length == 1)
+                            Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount);
+                        else
+                            Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())));
+
+                        m_result.DeleteResults = new DeleteResults(m_result);
+                        using (var rmdb = new Database.LocalDeleteDatabase(db))
                         {
-                            var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
-                            opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
-                            opts.RawOptions.Remove("time");
-                            opts.RawOptions["no-auto-compact"] = "true";
-
-                            new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults)
-                                .DoRun(rmdb, ref deltr, true, false, null);
-
-                            if (!m_options.Dryrun)
+                            var deltr = rmdb.BeginTransaction();
+                            try
                             {
-                                using (new Logging.Timer(LOGTAG, "CommitDelete", "CommitDelete"))
-                                    deltr.Commit();
+                                var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
+                                opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
+                                opts.RawOptions.Remove("time");
+                                opts.RawOptions["no-auto-compact"] = "true";
 
-                                rmdb.WriteResults();
+                                new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults)
+                                    .DoRun(rmdb, ref deltr, true, false, null);
+
+                                if (!m_options.Dryrun)
+                                {
+                                    using (new Logging.Timer(LOGTAG, "CommitDelete", "CommitDelete"))
+                                        deltr.Commit();
+
+                                    rmdb.WriteResults();
+                                }
+                                else
+                                    deltr.Rollback();
                             }
-                            else
-                                deltr.Rollback();
-                        }
-                        finally
-                        {
-                            if (deltr != null)
-                                try { deltr.Rollback(); }
-                                catch { }
+                            finally
+                            {
+                                if (deltr != null)
+                                    try { deltr.Rollback(); }
+                                    catch { }
+                            }
+
                         }
 
+                        pgoffset += (pgspan * fully_emptied.Length);
+                        m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
                     }
 
-                    pgoffset += (pgspan * fully_emptied.Length);
-                    m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
-                }
-
-                if (to_purge.Length > 0)
-                {
-                    m_result.PurgeResults = new PurgeFilesResults(m_result);
-
-                    foreach (var bs in to_purge)
+                    if (to_purge.Length > 0)
                     {
-                        Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime());
-                        var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
+                        m_result.PurgeResults = new PurgeFilesResults(m_result);
 
-                        using (var pgdb = new Database.LocalPurgeDatabase(db))
+                        foreach (var bs in to_purge)
                         {
-                            // Recompute the version number after we deleted the versions before
-                            filesets = pgdb.FilesetTimes.ToList();
-                            var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID);
-                            if (thisversion < 0)
-                                throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString()))));
+                            Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime());
+                            var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
 
-                            opts.RawOptions["version"] = thisversion.ToString();
-                            opts.RawOptions.Remove("time");
-                            opts.RawOptions["no-auto-compact"] = "true";
-
-                            new PurgeFilesHandler(m_backendurl, opts, (PurgeFilesResults)m_result.PurgeResults).Run(pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) =>
+                            using (var pgdb = new Database.LocalPurgeDatabase(db))
                             {
-                                if (filesetid != bs.FilesetID)
-                                    throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
-                                db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction);
-                            });
-                        }
+                                // Recompute the version number after we deleted the versions before
+                                filesets = pgdb.FilesetTimes.ToList();
+                                var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID);
+                                if (thisversion < 0)
+                                    throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString()))));
 
-                        pgoffset += pgspan;
-                        m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
+                                opts.RawOptions["version"] = thisversion.ToString();
+                                opts.RawOptions.Remove("time");
+                                opts.RawOptions["no-auto-compact"] = "true";
+
+                                new PurgeFilesHandler(m_backendurl, opts, (PurgeFilesResults)m_result.PurgeResults).Run(pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) =>
+                                {
+                                    if (filesetid != bs.FilesetID)
+                                        throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
+                                    db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction);
+                                });
+                            }
+
+                            pgoffset += pgspan;
+                            m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
+                        }
                     }
                 }
 
@@ -172,18 +174,6 @@ namespace Duplicati.Library.Main.Operation
 
                 m_result.OperationProgressUpdater.UpdateProgress(0.95f);
 
-                if (missing != null && missing.Count > 0)
-                {
-                    using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
-                    {
-                        foreach (var f in missing)
-                            if (m_options.Dryrun)
-                                Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size));
-                            else
-                                backend.Delete(f.Name, f.Size);
-                    }
-                }
-
                 if (!m_options.Dryrun && db.RepairInProgress)
                 {                    
                     Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files");

Duplicati/Library/Main/Operation/PurgeFilesHandler.cs

@@ -85,7 +85,7 @@ namespace Duplicati.Library.Main.Operation
                     db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null);
 
                     if (m_options.NoBackendverification)
-                        FilelistProcessor.VerifyLocalList(backend, m_options, db, m_result.BackendWriter);
+                        FilelistProcessor.VerifyLocalList(backend, db);
                     else
                         FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter, null);
                 }

Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs

@@ -92,7 +92,7 @@ namespace Duplicati.Library.Main.Operation
             using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb))
             {
                 restoredb.RepairInProgress = true;
-
+                var autoDetectBlockSize = !(m_options.HasBlocksize && restoredb.GetDbOptions().ContainsKey("blocksize"));                    
                 var volumeIds = new Dictionary<string, long>();
 
                 var rawlist = backend.List();
@@ -139,13 +139,13 @@ namespace Duplicati.Library.Main.Operation
                     orderby n.Time descending
                     select n;
 
-                if (filelists.Count() <= 0)
+                if (!filelists.Any())
                     throw new UserInformationException("No filelists found on the remote destination", "EmptyRemoteLocation");
                 
                 if (filelistfilter != null)
                     filelists = filelistfilter(filelists).Select(x => x.Value).ToArray();
 
-                if (filelists.Count() <= 0)
+                if (!filelists.Any())
                     throw new UserInformationException("No filelists", "NoMatchingRemoteFilelists");
 
                 // If we are updating, all files should be accounted for
@@ -205,7 +205,7 @@ namespace Duplicati.Library.Main.Operation
 
                                 var parsed = VolumeBase.ParseFilename(entry.Name);
 
-                                if (!hasUpdatedOptions && !updating) 
+                                if (!hasUpdatedOptions && (!updating || autoDetectBlockSize)) 
                                 {
                                     VolumeReaderBase.UpdateOptionsFromManifest(parsed.CompressionModule, tmpfile, m_options);
                                     hasUpdatedOptions = true;

Duplicati/Library/Main/Operation/RepairHandler.cs

@@ -130,20 +130,20 @@ namespace Duplicati.Library.Main.Operation
 
                 if (m_options.Dryrun)
                 {
-                    if (tp.ParsedVolumes.Count() == 0 && tp.OtherVolumes.Count() > 0)
+                    if (!tp.ParsedVolumes.Any() && tp.OtherVolumes.Any())
                     {
                         if (tp.BackupPrefixes.Length == 1)
                             throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefix {1}, did you forget to set the backup prefix?", m_options.Prefix, tp.BackupPrefixes[0]), "RemoteFolderEmptyWithPrefix");
                         else
                             throw new UserInformationException(string.Format("Found no backup files with prefix {0}, but files with prefixes {1}, did you forget to set the backup prefix?", m_options.Prefix, string.Join(", ", tp.BackupPrefixes)), "RemoteFolderEmptyWithPrefix");
                     }
-                    else if (tp.ParsedVolumes.Count() == 0 && tp.ExtraVolumes.Count() > 0)
+                    else if (!tp.ParsedVolumes.Any() && tp.ExtraVolumes.Any())
                     {
                         throw new UserInformationException(string.Format("No files were missing, but {0} remote files were, found, did you mean to run recreate-database?", tp.ExtraVolumes.Count()), "NoRemoteFilesMissing");
                     }
                 }
 
-                if (tp.ExtraVolumes.Count() > 0 || tp.MissingVolumes.Count() > 0 || tp.VerificationRequiredVolumes.Count() > 0)
+                if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any())
                 {
                     if (tp.VerificationRequiredVolumes.Any())
                     {
@@ -261,6 +261,13 @@ namespace Duplicati.Library.Main.Operation
                             if (ex is System.Threading.ThreadAbortException)
                                 throw;
                         }
+
+                    if (!m_options.RebuildMissingDblockFiles)
+                    {
+                        var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks).ToArray();
+                        if (missingDblocks.Length > 0)
+                            throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles");
+                    }
                             
                     foreach(var n in tp.MissingVolumes)
                     {

Duplicati/Library/Main/Operation/RestoreHandler.cs

@@ -114,74 +114,17 @@ namespace Duplicati.Library.Main.Operation
                 {
                     using(var metadatastorage = new RestoreHandlerMetadataStorage())
                     {
-                        System.Security.Cryptography.HashAlgorithm blockhasher = null;
-                        System.Security.Cryptography.HashAlgorithm filehasher = null;
-
-                        bool first = true;
-                        RecreateDatabaseHandler.BlockVolumePostProcessor localpatcher =
-                            (key, rd) =>
-                            {
-                                if (first)
-                                {
-                                    Utility.UpdateOptionsFromDb(database, m_options);
-                                    m_blockbuffer = new byte[m_options.Blocksize];
-
-                                    //Figure out what files are to be patched, and what blocks are needed
-                                    PrepareBlockAndFileList(database, m_options, filter, m_result);
-
-                                    blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
-                                    filehasher = Library.Utility.HashAlgorithmHelper.Create(m_options.FileHashAlgorithm);
-                                    if (blockhasher == null)
-                                        throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm), "BlockHashAlgorithmNotSupported");
-                                    if (!blockhasher.CanReuseTransform)
-                                        throw new UserInformationException(Strings.Common.InvalidCryptoSystem(m_options.BlockHashAlgorithm), "BlockHashAlgorithmNotSupported");
-
-                                    if (filehasher == null)
-                                        throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.FileHashAlgorithm), "FileHashAlgorithmNotSupported");
-                                    if (!filehasher.CanReuseTransform)
-                                        throw new UserInformationException(Strings.Common.InvalidCryptoSystem(m_options.FileHashAlgorithm), "FileHashAlgorithmNotSupported");
-
-                                    // Don't run this again
-                                    first = false;
-                                }
-                                else
-                                {
-                                    // Patch the missing blocks list to include the newly discovered blocklists
-                                    //UpdateMissingBlocksTable(key);
-                                }
-
-                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
-                                    return;
-
-                                CreateDirectoryStructure(database, m_options, m_result);
-
-                                //If we are patching an existing target folder, do not touch stuff that is already updated
-                                ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, m_result);
-
-                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
-                                    return;
-
-                                // If other local files already have the blocks we want, we use them instead of downloading
-                                if (!m_options.NoLocalBlocks)
-                                    ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result, metadatastorage);
-                                
-                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
-                                    return;
-                            
-                                //Update files with data
-                                PatchWithBlocklist(database, rd, m_options, m_result, m_blockbuffer, metadatastorage);
-                            };
-
-                        // TODO: When UpdateMissingBlocksTable is implemented, the localpatcher can be activated
+                        // TODO: When UpdateMissingBlocksTable is implemented, the localpatcher
+                        // (removed in revision 9ce1e807 ("Remove unused variables and fields") can be activated
                         // and this will reduce the need for multiple downloads of the same volume
                         // TODO: This will need some work to preserve the missing block list for use with --fh-dryrun
                         m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result);
                         using(new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore"))
                             new RecreateDatabaseHandler(m_backendurl, m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults)
-                                .DoRun(database, false, filter, filelistfilter, /*localpatcher*/null);
+                                .DoRun(database, false, filter, filelistfilter, null);
 
                         if (!m_options.SkipMetadata)
-                            ApplyStoredMetadata(database, m_options, m_result, metadatastorage);
+                            ApplyStoredMetadata(m_options, metadatastorage);
                     }
 
                     //If we have --version set, we need to adjust, as the db has only the required versions
@@ -310,7 +253,7 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        private static void ApplyStoredMetadata(LocalRestoreDatabase database, Options options, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage)
+        private static void ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage)
         {
             foreach(var metainfo in metadatastorage.Records)
             {
@@ -464,7 +407,7 @@ namespace Duplicati.Library.Main.Operation
 
                 // Apply metadata
                 if (!m_options.SkipMetadata)
-                    ApplyStoredMetadata(database, m_options, m_result, metadatastorage);
+                    ApplyStoredMetadata(m_options, metadatastorage);
                 
                 // Reset the filehasher if it was used to verify existing files
                 filehasher.Initialize();
@@ -904,7 +847,7 @@ namespace Duplicati.Library.Main.Operation
                                 if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) // clear readonly attribute
                                 {
                                     if (options.Dryrun) 
-                                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribyte", "Would reset read-only attribute on file: {0}", targetpath);
+                                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribute", "Would reset read-only attribute on file: {0}", targetpath);
                                     else m_systemIO.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly);
                                 }
                                 if (options.Dryrun)

Duplicati/Library/Main/Operation/SystemInfoHandler.cs

@@ -49,10 +49,7 @@ namespace Duplicati.Library.Main.Operation
             yield return string.Format("Locale: {0}, {1}, {2}", System.Threading.Thread.CurrentThread.CurrentCulture, System.Threading.Thread.CurrentThread.CurrentUICulture, System.Globalization.CultureInfo.InstalledUICulture);
             yield return string.Format("Date/time strings: {0} - {1}", System.Threading.Thread.CurrentThread.CurrentCulture.DateTimeFormat.LongDatePattern, System.Threading.Thread.CurrentThread.CurrentCulture.DateTimeFormat.LongTimePattern);
             yield return string.Format("Tempdir: {0}", Library.Utility.TempFolder.SystemTempPath);
-            foreach(var e in new string[] {"TEMP", "TMP", "TMPDIR"})
-                if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(e)))
-                    yield return string.Format("Environment variable: {0} = {1}", e, Environment.GetEnvironmentVariable(e));
-            
+
             Type sqlite = null;
             string sqliteversion = "";