Merge branch 'master' into netstandard

.travis.yml

@@ -5,6 +5,7 @@ install:
   - nuget restore Duplicati.sln
   - nuget install NUnit.Runners -Version 3.5.0 -OutputDirectory testrunner
   - sudo pip install selenium
+  - sudo pip install --upgrade urllib3
   - if [ ! -d "${TRAVIS_BUILD_DIR}"/packages/SharpCompress.0.18.2 ]; then ln -s "${TRAVIS_BUILD_DIR}"/packages/sharpcompress.0.18.2 "${TRAVIS_BUILD_DIR}"/packages/SharpCompress.0.18.2; fi
 addons:
   coverity_scan:

Duplicati/CommandLine/BackendTester/Program.cs

@@ -135,14 +135,6 @@ namespace Duplicati.CommandLine.BackendTester
 
         static bool Run(List<string> args, Dictionary<string, string> options, bool first)
         {
-            string allowedChars = ValidFilenameChars;
-            if (options.ContainsKey("extended-chars"))
-                allowedChars += options["extended-chars"];
-            else
-                allowedChars += ExtendedChars;
-
-            bool autoCreateFolders = Library.Utility.Utility.ParseBoolOption(options, "auto-create-folder");
-
             Library.Interface.IBackend backend = Library.DynamicLoader.BackendLoader.GetBackend(args[0], options);
             if (backend == null)
             {
@@ -152,6 +144,14 @@ namespace Duplicati.CommandLine.BackendTester
                 return false;
             }
 
+            string allowedChars = ValidFilenameChars;
+            if (options.ContainsKey("extended-chars"))
+            {
+                allowedChars += String.IsNullOrEmpty(options["extended-chars"]) ? ExtendedChars : options["extended-chars"];
+            }
+
+            bool autoCreateFolders = Library.Utility.Utility.ParseBoolOption(options, "auto-create-folder");
+
             string disabledModulesValue;
             string enabledModulesValue;
             options.TryGetValue("enable-module", out enabledModulesValue);

Duplicati/CommandLine/Program.cs

@@ -203,6 +203,21 @@ namespace Duplicati.CommandLine
             string command = cargs[0];
             cargs.RemoveAt(0);
 
+            if (verboseErrors)
+            {
+                outwriter.WriteLine("Input command: {0}", command);
+                outwriter.WriteLine("Input arguments: ");
+                foreach (var a in cargs)
+                    outwriter.WriteLine("\t{0}", a);
+                outwriter.WriteLine();
+
+                outwriter.WriteLine("Input options: ");
+                foreach (var n in options)
+                    outwriter.WriteLine("{0}: {1}", n.Key, n.Value);
+                outwriter.WriteLine();
+            }
+
+
             if (CommandMap.ContainsKey(command))
             {
                 var autoupdate = Library.Utility.Utility.ParseBoolOption(options, "auto-update");

Duplicati/Library/Backend/FTP/FTPBackend.cs

@@ -19,7 +19,6 @@
 #endregion
 using System;
 using System.Collections.Generic;
-using System.Text;
 using System.Text.RegularExpressions;
 using Duplicati.Library.Interface;
 using System.Linq;
@@ -32,8 +31,7 @@ namespace Duplicati.Library.Backend
         private readonly string m_url;
 
         private readonly bool m_useSSL = false;
-        private readonly bool m_defaultPassive = true;
-        private readonly bool m_passive = false;
+        private readonly bool m_passiveMode = false;
         private readonly bool m_listVerify = true;
 
         private readonly byte[] m_copybuffer = new byte[Duplicati.Library.Utility.Utility.DEFAULT_BUFFER_SIZE];
@@ -57,49 +55,49 @@ namespace Duplicati.Library.Backend
 
             var u = new Utility.Uri(url);
             u.RequireHost();
+            string username = null;
+            string password = null;
 
             if (!string.IsNullOrEmpty(u.Username))
             {
-                m_userInfo = new System.Net.NetworkCredential();
-                m_userInfo.UserName = u.Username;
-                if (!string.IsNullOrEmpty(u.Password))
-                    m_userInfo.Password = u.Password;
-                else if (options.ContainsKey("auth-password"))
-                    m_userInfo.Password = options["auth-password"];
+                username = u.Username;
             }
-            else
+            else if (options.ContainsKey("auth-username"))
+            {
+                username = options["auth-username"];
+            }
+
+            if (!string.IsNullOrEmpty(u.Username) && !string.IsNullOrEmpty(u.Password)) {
+                        password = u.Password;
+            }
+            else if (options.ContainsKey("auth-password")) {
+                password = options["auth-password"];
+            }
+
+            m_userInfo = new System.Net.NetworkCredential
             {
-                if (options.ContainsKey("auth-username"))
-                {
-                    m_userInfo = new System.Net.NetworkCredential();
-                    m_userInfo.UserName = options["auth-username"];
-                    if (options.ContainsKey("auth-password"))
-                        m_userInfo.Password = options["auth-password"];
-                }
-            }
+                UserName = username,
+                Password = password
+            };
 
             //Bugfix, see http://connect.microsoft.com/VisualStudio/feedback/details/695227/networkcredential-default-constructor-leaves-domain-null-leading-to-null-object-reference-exceptions-in-framework-code
             if (m_userInfo != null)
                 m_userInfo.Domain = "";
 
             m_url = u.SetScheme("ftp").SetQuery(null).SetCredentials(null, null).ToString();
-            if (!m_url.EndsWith("/", StringComparison.Ordinal))
-                m_url += "/";
-
+            if (!m_url.EndsWith("/", StringComparison.Ordinal))
+            {
+                m_url += "/";
+            }
+
             m_useSSL = Utility.Utility.ParseBoolOption(options, "use-ssl");
 
             m_listVerify = !Utility.Utility.ParseBoolOption(options, "disable-upload-verify");
 
             if (Utility.Utility.ParseBoolOption(options, "ftp-passive"))
             {
-                m_defaultPassive = false;
-                m_passive = true;
-            }
-            if (Utility.Utility.ParseBoolOption(options, "ftp-regular"))
-            {
-                m_defaultPassive = false;
-                m_passive = false;
-            }
+                m_passiveMode = true;
+            } else m_passiveMode = !Utility.Utility.ParseBoolOption(options, "ftp-regular");
         }
 
         #region Regular expression to parse list lines
@@ -138,18 +136,20 @@ namespace Duplicati.Library.Backend
             string time = m.Groups["timestamp"].Value;
             string dir = m.Groups["dir"].Value;
 
-            //Unused
-            //string permission = m.Groups["permission"].Value;
-
-            if (dir != "" && dir != "-")
-                f.IsFolder = true;
-            else
-                f.Size = long.Parse(m.Groups["size"].Value);
-
-            DateTime t;
-            if (DateTime.TryParse(time, out t))
-                f.LastAccess = f.LastModification = t;
-
+            if (dir != "" && dir != "-")
+            {
+                f.IsFolder = true;
+            }
+            else
+            {
+                f.Size = long.Parse(m.Groups["size"].Value);
+            }
+
+            if (DateTime.TryParse(time, out DateTime t))
+            {
+                f.LastAccess = f.LastModification = t;
+            }
+
             return f;
         }
 
@@ -221,7 +221,7 @@ namespace Duplicati.Library.Backend
                     req);
                 
                 string line;
-                while ((line = HandleListExceptions(() => sr.ReadLine(), req)) != null)
+                while ((line = HandleListExceptions(sr.ReadLine, req)) != null)
                 {
                     FileEntry f = ParseLine(line);
                     if (f != null)
@@ -380,9 +380,8 @@ namespace Duplicati.Library.Backend
         #region IDisposable Members
 
         public void Dispose()
-        {
-            if (m_userInfo != null)
-                m_userInfo = null;
+        {
+            m_userInfo = null;
         }
 
         #endregion
@@ -398,14 +397,15 @@ namespace Duplicati.Library.Backend
             if (createFolder && url.EndsWith("/", StringComparison.Ordinal))
                 url = url.Substring(0, url.Length - 1);
             
-            System.Net.FtpWebRequest req = (System.Net.FtpWebRequest)System.Net.FtpWebRequest.Create(url + remotename);
+            System.Net.FtpWebRequest req = (System.Net.FtpWebRequest)System.Net.WebRequest.Create(url + remotename);
 
-            if (m_userInfo != null)
-                req.Credentials = m_userInfo;
+            if (m_userInfo != null)
+            {
+                req.Credentials = m_userInfo;
+            }
+
             req.KeepAlive = false;
-
-            if (!m_defaultPassive)
-                req.UsePassive = m_passive;
+            req.UsePassive = m_passiveMode;
 
             if (m_useSSL)
                 req.EnableSsl = m_useSSL;

Duplicati/Library/Backend/GoogleServices/Strings.cs

@@ -46,7 +46,7 @@ namespace Duplicati.Library.Backend.Strings
         public static string MissingAuthID(string url) { return LC.L(@"You need an AuthID, you can get it from: {0}", url); }
         public static string MultipleEntries(string folder, string parent) { return LC.L(@"There is more than one item named ""{0}"" in the folder ""{1}""", folder, parent); }
         public static string DisableTeamDriveShort { get { return LC.L("Hide team drives"); } }
-        public static string DisableTeamDriveLong { get { return LC.L("This option disables the team drives, showing only files and folders accesible with the account itself"); } }
+        public static string DisableTeamDriveLong { get { return LC.L("This option disables the team drives, showing only files and folders accessible with the account itself"); } }
     }
 }
 

Duplicati/Library/Backend/GoogleServices/WebApi.cs

@@ -100,7 +100,7 @@ namespace Duplicati.Library.Backend.WebApi
             {
                 { QueryParam.UploadType, QueryValue.Resumable }
             };
-            var path = UrlPath.Create(Path.Bucket).Append(bucketId).ToString();
+            var path = UrlPath.Create(Path.Bucket).Append(bucketId).Append(Path.Object).ToString();
             return Uri.UriBuilder(Url.UPLOAD, path, queryParams);
         }
 

Duplicati/Library/Interface/CustomExceptions.cs

@@ -139,5 +139,48 @@ namespace Duplicati.Library.Interface
         public CancelException(string message, Exception innerException)
             : base(message, "Cancelled", innerException)
         { }
-}
+    }
+
+    /// <summary>
+    /// The reason why an operation is aborted
+    /// </summary>
+    public enum OperationAbortReason
+    {
+        /// <summary>
+        /// The operation is aborted, but this is considered a normal operation
+        /// </summary>
+        Normal,
+        /// <summary>
+        /// The operation is aborted and this should give a warning
+        /// </summary>
+        Warning,
+        /// <summary>
+        /// The operation is aborted and this is an error
+        /// </summary>
+        Error
+    }
+
+    /// <summary>
+    /// A class that signals the operation should be aborted
+    /// </summary>
+    [Serializable]
+    public class OperationAbortException : UserInformationException
+    {
+        /// <summary>
+        /// The reason for the abort operation
+        /// </summary>
+        public readonly OperationAbortReason AbortReason;
+
+        public OperationAbortException(OperationAbortReason reason, string message)
+            : base(message, "OperationAborted")
+        {
+            AbortReason = reason;
+        }
+
+        public OperationAbortException(OperationAbortReason reason, string message, Exception innerException)
+            : base(message, "OperationAborted", innerException)
+        {
+            AbortReason = reason;
+        }
+    }
 }

Duplicati/Library/Localization/MoLocalizationService.cs

@@ -72,7 +72,7 @@ namespace Duplicati.Library.Localization
         {
             var filenames = new string[] { 
                 // Load the specialized version first
-                string.Format("localization-{0}.mo", ci.Name), 
+                string.Format("localization-{0}.mo", ci.Name.Replace('-', '_')), 
                 // Then try the generic language version
                 string.Format("localization-{0}.mo", ci.TwoLetterISOLanguageName) 
             };

Duplicati/Library/Logging/Log.cs

@@ -93,7 +93,7 @@ namespace Duplicati.Library.Logging
         public static object Lock { get { return m_lock; } }
 
         /// <summary>
-        /// Gets a log tag taht reflects the type
+        /// Gets a log tag that reflects the type
         /// </summary>
         /// <returns>The log-tag for the type.</returns>
         /// <typeparam name="T">The type to get the tag for.</typeparam>
@@ -104,7 +104,7 @@ namespace Duplicati.Library.Logging
 
 
         /// <summary>
-        /// Gets a log tag taht reflects the type
+        /// Gets a log tag that reflects the type
         /// </summary>
         /// <returns>The log-tag for the type.</returns>
         /// <param name="t">The type to get the tag for.</param>
@@ -341,10 +341,32 @@ namespace Duplicati.Library.Logging
         /// <summary>
         /// Starts a new scope, that can be closed by disposing the returned instance
         /// </summary>
+        /// <param name="detached">Flag indicating if the scope should be detached from the parent</param>
         /// <returns>The new scope.</returns>
-        public static IDisposable StartIsolatingScope()
+        public static IDisposable StartIsolatingScope(bool detached)
         {
-            return StartScope((ILogDestination)null, null, true);
+            lock (m_lock)
+            {
+                var scope = StartScope((ILogDestination)null, null, true);
+                if (detached)
+                    DetachCurrentScope(scope);
+                return scope;
+            }
+        }
+
+        /// <summary>
+        /// Detaches the current scope, such that new scopes do not chain onto this
+        /// </summary>
+        /// <param name="scope">The current scope.</param>
+        public static IDisposable DetachCurrentScope(IDisposable scope)
+        {
+            lock (m_lock)
+            {
+                if (CurrentScope == scope && scope != null && CurrentScope.Parent != null)
+                    CurrentScope = CurrentScope.Parent;
+            }
+
+            return scope;
         }
 
         /// <summary>
@@ -451,7 +473,6 @@ namespace Duplicati.Library.Logging
                     {
                         System.Runtime.Remoting.Messaging.CallContext.LogicalSetData(LOGICAL_CONTEXT_KEY, null);
                     }
-                        
                 }
                 */
             }

Duplicati/Library/Main/Controller.cs

@@ -430,14 +430,29 @@ namespace Duplicati.Library.Main
                 {
                     result.EndTime = DateTime.UtcNow;
 
-                    try { (result as BasicResults).OperationProgressUpdater.UpdatePhase(OperationPhase.Error); }
-                    catch { }
+                    if (ex is Library.Interface.OperationAbortException oae)
+                    {
+                        // Perform the module shutdown
+                        OnOperationComplete(ex);
 
-                    OnOperationComplete(ex);
+                        // Log this as a normal operation, as the script rasing the exception,
+                        // has already populated either warning or log messages as required
+                        Logging.Log.WriteInformationMessage(LOGTAG, "AbortOperation", "Aborting operation by request, requested result: {0}", oae.AbortReason);
 
-                    Logging.Log.WriteErrorMessage(LOGTAG, "FailedOperation", ex, Strings.Controller.FailedOperationMessage(m_options.MainAction, ex.Message));
+                        return result;
+                    }
+                    else
+                    {
+                        try { (result as BasicResults).OperationProgressUpdater.UpdatePhase(OperationPhase.Error); }
+                        catch { }
+
+                        OnOperationComplete(ex);
+
+                        Logging.Log.WriteErrorMessage(LOGTAG, "FailedOperation", ex, Strings.Controller.FailedOperationMessage(m_options.MainAction, ex.Message));
+
+                        throw;
+                    }
 
-                    throw;
                 }
                 finally
                 {
@@ -867,6 +882,7 @@ namespace Duplicati.Library.Main
                     string source;
                     try
                     {
+                        // TODO: This expands "C:" to CWD, but not C:\
                         source = System.IO.Path.GetFullPath(expandedSource);
                     }
                     catch (Exception ex)
@@ -1026,6 +1042,9 @@ namespace Duplicati.Library.Main
                 {
                     return Strings.Controller.UnsupportedSizeValue(optionname, value);
                 }
+
+                if (!string.IsNullOrWhiteSpace(value) && char.IsDigit(value.Last()))
+                    return Strings.Controller.NonQualifiedSizeValue(optionname, value);
             }
             else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan)
             {

Duplicati/Library/Main/Database/ExtensionMethods.cs

@@ -41,7 +41,7 @@ namespace Duplicati.Library.Main.Database
         public static string GetPrintableCommandText(this System.Data.IDbCommand self)
         {
             var txt = self.CommandText;
-#if DEBUG
+
             foreach(var p in self.Parameters.Cast<System.Data.IDbDataParameter>())
             {
                 var ix = txt.IndexOf('?');
@@ -58,11 +58,21 @@ namespace Duplicati.Library.Main.Database
                     txt = txt.Substring(0, ix) + v + txt.Substring(ix + 1);
                 }
             }
-#endif
+
             return txt;
         }
 
+        public static int ExecuteNonQuery(this System.Data.IDbCommand self, bool writeLog)
+        {
+            return ExecuteNonQuery(self, writeLog, null, null);
+        }
+
         public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values)
+        {
+            return ExecuteNonQuery(self, true, cmd, values);
+        }
+
+        public static int ExecuteNonQuery(this System.Data.IDbCommand self, bool writeLog, string cmd, params object[] values)
         {
             if (cmd != null)
                 self.CommandText = cmd;
@@ -74,11 +84,16 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())))
+            using(writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQuery", string.Format("ExecuteNonQuery: {0}", self.GetPrintableCommandText())) : null)
                 return self.ExecuteNonQuery();
         }
 
         public static object ExecuteScalar(this System.Data.IDbCommand self, string cmd, params object[] values)
+        {
+            return ExecuteScalar(self, true, cmd, values);
+        }
+
+        public static object ExecuteScalar(this System.Data.IDbCommand self, bool writeLog, string cmd, params object[] values)
         {
             if (cmd != null)
                 self.CommandText = cmd;
@@ -90,21 +105,36 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(LOGTAG, "ExecuteScalar", string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())))
+            using(writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalar", string.Format("ExecuteScalar: {0}", self.GetPrintableCommandText())) : null)
                 return self.ExecuteScalar();
         }
 
+        public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, long defaultvalue = -1)
+        {
+            return ExecuteScalarInt64(self, writeLog, null, defaultvalue);
+        }
+
         public static long ExecuteScalarInt64(this System.Data.IDbCommand self, long defaultvalue = -1)
         {
-            return ExecuteScalarInt64(self, null, defaultvalue, null);
+            return ExecuteScalarInt64(self, true, null, defaultvalue);
+        }
+
+        public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, string cmd, long defaultvalue = -1)
+        {
+            return ExecuteScalarInt64(self, writeLog, cmd, defaultvalue, null);
         }
 
         public static long ExecuteScalarInt64(this System.Data.IDbCommand self, string cmd, long defaultvalue = -1)
         {
-            return ExecuteScalarInt64(self, cmd, defaultvalue, null);
+            return ExecuteScalarInt64(self, true, cmd, defaultvalue, null);
         }
 
         public static long ExecuteScalarInt64(this System.Data.IDbCommand self, string cmd, long defaultvalue, params object[] values)
+        {
+            return ExecuteScalarInt64(self, true, cmd, defaultvalue, values);
+        }
+
+        public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, string cmd, long defaultvalue, params object[] values)
         {
             if (cmd != null)
                 self.CommandText = cmd;
@@ -116,7 +146,7 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())))
+            using(writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64", string.Format("ExecuteScalarInt64: {0}", self.GetPrintableCommandText())) : null)
                 using(var rd = self.ExecuteReader())
                     if (rd.Read())
                         return ConvertValueToInt64(rd, 0, defaultvalue);
@@ -125,6 +155,11 @@ namespace Duplicati.Library.Main.Database
         }
 
         public static System.Data.IDataReader ExecuteReader(this System.Data.IDbCommand self, string cmd, params object[] values)
+        {
+            return ExecuteReader(self, true, cmd, values);
+        }
+
+        public static System.Data.IDataReader ExecuteReader(this System.Data.IDbCommand self, bool writeLog, string cmd, params object[] values)
         {
             if (cmd != null)
                 self.CommandText = cmd;
@@ -136,7 +171,7 @@ namespace Duplicati.Library.Main.Database
                     self.AddParameter(n);
             }
 
-            using(new Logging.Timer(LOGTAG, "ExcuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())))
+            using(writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null)
                 return self.ExecuteReader();
         }
 

Duplicati/Library/Main/Database/LocalBackupDatabase.cs

@@ -86,16 +86,20 @@ namespace Duplicati.Library.Main.Database
 
         private readonly System.Data.IDbCommand m_findfileCommand;
         private readonly System.Data.IDbCommand m_selectfilelastmodifiedCommand;
+        private readonly System.Data.IDbCommand m_selectfilelastmodifiedWithSizeCommand;
         private readonly System.Data.IDbCommand m_selectfileHashCommand;
         private readonly System.Data.IDbCommand m_selectblocklistHashesCommand;
 
         private readonly System.Data.IDbCommand m_insertfileOperationCommand;
-        
+        private readonly System.Data.IDbCommand m_selectfilemetadatahashandsizeCommand;
+
         private PathLookupHelper<PathEntryKeeper> m_pathLookup;
         private Dictionary<string, long> m_blockCache;
         
         private long m_filesetId;
 
+        private readonly bool m_logQueries;
+
         public LocalBackupDatabase(string path, Options options)
             : this(new LocalDatabase(path, "Backup", false), options)
         {
@@ -105,6 +109,8 @@ namespace Duplicati.Library.Main.Database
         public LocalBackupDatabase(LocalDatabase db, Options options)
             : base(db)
         {
+            m_logQueries = options.ProfileAllDatabaseQueries;
+
             m_findblockCommand = m_connection.CreateCommand();
             m_insertblockCommand = m_connection.CreateCommand();
             m_insertfileCommand = m_connection.CreateCommand();
@@ -119,8 +125,10 @@ namespace Duplicati.Library.Main.Database
             m_insertfileOperationCommand = m_connection.CreateCommand();
             m_findfileCommand = m_connection.CreateCommand();
             m_selectfilelastmodifiedCommand = m_connection.CreateCommand();
+            m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand();
             m_selectfileHashCommand = m_connection.CreateCommand();
             m_insertblocksetentryFastCommand = m_connection.CreateCommand();
+            m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand();
                 
             m_findblockCommand.CommandText = @"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?";
             m_findblockCommand.AddParameters(2);
@@ -161,17 +169,97 @@ namespace Duplicati.Library.Main.Database
             m_selectfilelastmodifiedCommand.CommandText = @"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?";
             m_selectfilelastmodifiedCommand.AddParameters(2);
 
-            //Need a temporary table with path/lastmodified lookups
-            m_findfileCommand.CommandText =
-                @" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
-                @"   FROM ""File"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
-                @"  WHERE ""File"".""Path"" = ? " +
-                @"    AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
-                @"    AND ""FileBlockset"".""ID"" = ""File"".""BlocksetID"" " +
-                @"    AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
-                @"  ORDER BY ""Fileset"".""Timestamp"" DESC " +
-                @"  LIMIT 1 ";
-            m_findfileCommand.AddParameters(1);
+            m_selectfilelastmodifiedWithSizeCommand.CommandText = @"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" ";
+            m_selectfilelastmodifiedWithSizeCommand.AddParameters(2);
+
+            m_selectfilemetadatahashandsizeCommand.CommandText = @"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = ? AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" ";
+            m_selectfilemetadatahashandsizeCommand.AddParameters(1);
+
+            // Allow users to test on real-world data
+            // to get feedback on potential performance
+            int.TryParse(Environment.GetEnvironmentVariable("TEST_QUERY_VERSION"), out var testqueryversion);
+
+            if (testqueryversion != 0)
+                Logging.Log.WriteWarningMessage(LOGTAG, "TestFileQuery", null, "Using performance test query version {0} as the TEST_QUERY_VERSION environment variable is set", testqueryversion);
+
+            // The original query (v==1) finds the most recent entry of the file in question, 
+            // but it requires some large joins to extract the required information.
+            // To speed it up, we use a slightly simpler approach that only looks at the
+            // previous fileset, and uses information here.
+            // If there is a case where a file is sometimes there and sometimes not
+            // (i.e. filter file, remove filter) we will not find the file.
+            // We currently use this faster version, 
+            // but allow users to switch back via an environment variable
+            // such that we can get performance feedback
+
+            switch (testqueryversion)
+            {
+                // The query used in Duplicati until 2.0.3.9
+                case 1:
+                    m_findfileCommand.CommandText =
+                        @" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
+                        @"   FROM ""File"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
+                        @"  WHERE ""File"".""Path"" = ? " +
+                        @"    AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
+                        @"    AND ""FileBlockset"".""ID"" = ""File"".""BlocksetID"" " +
+                        @"    AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
+                        @"    AND ? IS NOT NULL" +
+                        @"  ORDER BY ""Fileset"".""Timestamp"" DESC " +
+                        @"  LIMIT 1 ";
+                    break;
+
+                // The fastest reported query in Duplicati 2.0.3.10, but with "LIMIT 1" added
+                default:
+                case 2:
+                    var getLastFileEntryForPath =
+                        @"SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"", ""A"".""MetadataID"" " +
+                        @"  FROM (SELECT ""ID"", ""BlocksetID"", ""MetadataID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" " +
+                        @"  CROSS JOIN ""FilesetEntry"" ""B"" " +
+                        @"  WHERE ""A"".""ID"" = ""B"".""FileID"" " +
+                        @"    AND ""B"".""FilesetID"" = ? ";
+
+                    m_findfileCommand.CommandText = string.Format(
+                        @"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" " +
+                        @"  FROM " +
+                        @"  ({0}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" " +
+                        @" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" AND ""C"".""MetadataID"" == ""F"".""ID"" AND ""F"".""BlocksetID"" = ""E"".""ID"" " +
+                        @" LIMIT 1",
+                        getLastFileEntryForPath
+                    );
+                    break;
+
+                // Potentially faster query: https://forum.duplicati.com/t/release-2-0-3-10-canary-2018-08-30/4497/25
+                case 3:
+                    m_findfileCommand.CommandText =
+                        @"    SELECT File.ID as FileID, FilesetEntry.Lastmodified, FileBlockset.Length,  " +
+                        @"           MetaBlockset.FullHash AS Metahash, MetaBlockset.Length as Metasize " +
+                        @"      FROM FilesetEntry " +
+                        @"INNER JOIN Fileset ON (FileSet.ID = FilesetEntry.FilesetID) " +
+                        @"INNER JOIN File ON (File.ID = FilesetEntry.FileID) " +
+                        @"INNER JOIN Metadataset ON (Metadataset.ID = File.MetadataID) " +
+                        @"INNER JOIN Blockset AS MetaBlockset ON (MetaBlockset.ID = Metadataset.BlocksetID) " +
+                        @" LEFT JOIN Blockset AS FileBlockset ON (FileBlockset.ID = File.BlocksetID) " +
+                        @"     WHERE File.Path = ? AND FilesetID = ? " +
+                        @"     LIMIT 1 ";
+                    break;
+
+                // The slow query used in Duplicati 2.0.3.10, but with "LIMIT 1" added
+                case 4:
+                    m_findfileCommand.CommandText =
+                        @" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
+                        @"   FROM ""File"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
+                        @"  WHERE ""File"".""Path"" = ? " +
+                        @"    AND ""Fileset"".""ID"" = ? " +
+                        @"    AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
+                        @"    AND ""FileBlockset"".""ID"" = ""File"".""BlocksetID"" " +
+                        @"    AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
+                        @"  LIMIT 1 ";
+                    break;
+
+            }
+
+            m_findfileCommand.AddParameters(2);
+
 
             m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""File"" WHERE ""Blockset"".""ID"" = ""File"".""BlocksetID"" AND ""File"".""ID"" = ?  ";
             m_selectfileHashCommand.AddParameters(1);
@@ -289,7 +377,7 @@ namespace Duplicati.Library.Main.Database
             m_findblockCommand.Transaction = transaction;
             m_findblockCommand.SetParameterValue(0, key);
             m_findblockCommand.SetParameterValue(1, size);
-            return m_findblockCommand.ExecuteScalarInt64(-1);
+            return m_findblockCommand.ExecuteScalarInt64(m_logQueries, -1);
         }
 
 		/// <summary>
@@ -314,7 +402,7 @@ namespace Duplicati.Library.Main.Database
             m_findblockCommand.Transaction = transaction;
             m_findblockCommand.SetParameterValue(0, key);
             m_findblockCommand.SetParameterValue(1, size);
-            var r = m_findblockCommand.ExecuteScalarInt64(-1);
+            var r = m_findblockCommand.ExecuteScalarInt64(m_logQueries, -1);
 
             if (r == -1L)
             {
@@ -322,7 +410,7 @@ namespace Duplicati.Library.Main.Database
                 m_insertblockCommand.SetParameterValue(0, key);
                 m_insertblockCommand.SetParameterValue(1, volumeid);
                 m_insertblockCommand.SetParameterValue(2, size);
-                r = m_insertblockCommand.ExecuteScalarInt64();
+                r = m_insertblockCommand.ExecuteScalarInt64(m_logQueries);
                 if (m_blockCache != null)
                     m_blockCache.Add(key, size);
                 return true;
@@ -346,7 +434,7 @@ namespace Duplicati.Library.Main.Database
         public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes, out long blocksetid, System.Data.IDbTransaction transaction = null)
         {
             m_findblocksetCommand.Transaction = transaction;
-            blocksetid = m_findblocksetCommand.ExecuteScalarInt64(null, -1, filehash, size);
+            blocksetid = m_findblocksetCommand.ExecuteScalarInt64(m_logQueries, null, -1, filehash, size);
             if (blocksetid != -1)
                 return false; //Found it
 
@@ -355,7 +443,7 @@ namespace Duplicati.Library.Main.Database
                 m_insertblocksetCommand.Transaction = tr.Parent;
                 m_insertblocksetCommand.SetParameterValue(0, size);
                 m_insertblocksetCommand.SetParameterValue(1, filehash);
-                blocksetid = m_insertblocksetCommand.ExecuteScalarInt64();
+                blocksetid = m_insertblocksetCommand.ExecuteScalarInt64(m_logQueries);
 
                 long ix = 0;
                 if (blocklistHashes != null)
@@ -366,7 +454,7 @@ namespace Duplicati.Library.Main.Database
                     {
                         m_insertblocklistHashesCommand.SetParameterValue(1, ix);
                         m_insertblocklistHashesCommand.SetParameterValue(2, bh);
-                        m_insertblocklistHashesCommand.ExecuteNonQuery();
+                        m_insertblocklistHashesCommand.ExecuteNonQuery(m_logQueries);
                         ix++;
                     }
                 }
@@ -385,7 +473,7 @@ namespace Duplicati.Library.Main.Database
                     m_insertblocksetentryCommand.SetParameterValue(1, ix);
                     m_insertblocksetentryCommand.SetParameterValue(2, h);
                     m_insertblocksetentryCommand.SetParameterValue(3, exsize);
-                    var c = m_insertblocksetentryCommand.ExecuteNonQuery();
+                    var c = m_insertblocksetentryCommand.ExecuteNonQuery(m_logQueries);
                     if (c != 1)
                     {
                         Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size);
@@ -414,7 +502,7 @@ namespace Duplicati.Library.Main.Database
         /// <summary>
         /// Gets the metadataset ID from the filehash
         /// </summary>
-        /// <returns><c>true</c>, if metadataset should be recorded, false if it already exists.</returns>
+        /// <returns><c>true</c>, if metadataset found, false if does not exist.</returns>
         /// <param name="filehash">The metadata hash.</param>
         /// <param name="size">The size of the metadata.</param>
         /// <param name="metadataid">The ID of the metadataset.</param>
@@ -424,7 +512,7 @@ namespace Duplicati.Library.Main.Database
             if (size > 0)
             {
                 m_findmetadatasetCommand.Transaction = transaction;
-                metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size);
+                metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(m_logQueries, null, -1, filehash, size);
                 return metadataid != -1;
             }
 
@@ -435,7 +523,10 @@ namespace Duplicati.Library.Main.Database
         /// <summary>
         /// Adds a metadata set to the database, and returns a value indicating if the record was new
         /// </summary>
-        /// <param name="hash">The metadata hash</param>
+        /// <param name="filehash">The metadata hash</param>
+        /// <param name="size">The size of the metadata</param>
+        /// <param name="transaction">The transaction to execute under</param>
+        /// <param name="blocksetid">The id of the blockset to add</param>
         /// <param name="metadataid">The id of the metadata set</param>
         /// <returns>True if the set was added to the database, false otherwise</returns>
         public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, System.Data.IDbTransaction transaction = null)
@@ -447,7 +538,7 @@ namespace Duplicati.Library.Main.Database
             {
                 m_insertmetadatasetCommand.Transaction = tr.Parent;
                 m_insertmetadatasetCommand.SetParameterValue(0, blocksetid);
-                metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64();
+                metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64(m_logQueries);
                 tr.Commit();
                 return true;
             }
@@ -482,7 +573,7 @@ namespace Duplicati.Library.Main.Database
                 m_findfilesetCommand.SetParameterValue(0, blocksetID);
                 m_findfilesetCommand.SetParameterValue(1, metadataID);
                 m_findfilesetCommand.SetParameterValue(2, filename);
-                fileidobj = m_findfilesetCommand.ExecuteScalarInt64();
+                fileidobj = m_findfilesetCommand.ExecuteScalarInt64(m_logQueries);
             }
             
             if (fileidobj == -1)
@@ -493,7 +584,7 @@ namespace Duplicati.Library.Main.Database
                     m_insertfileCommand.SetParameterValue(0, filename);
                     m_insertfileCommand.SetParameterValue(1, blocksetID);
                     m_insertfileCommand.SetParameterValue(2, metadataID);
-                    fileidobj = m_insertfileCommand.ExecuteScalarInt64();
+                    fileidobj = m_insertfileCommand.ExecuteScalarInt64(m_logQueries);
                     tr.Commit();                    
 
                     // We do not need to update this, because we will not ask for the same file twice
@@ -515,7 +606,7 @@ namespace Duplicati.Library.Main.Database
             m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
             m_insertfileOperationCommand.SetParameterValue(1, fileidobj);
             m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
-            m_insertfileOperationCommand.ExecuteNonQuery();
+            m_insertfileOperationCommand.ExecuteNonQuery(m_logQueries);
 
         }
 
@@ -525,7 +616,7 @@ namespace Duplicati.Library.Main.Database
             m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
             m_insertfileOperationCommand.SetParameterValue(1, fileid);
             m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
-            m_insertfileOperationCommand.ExecuteNonQuery();
+            m_insertfileOperationCommand.ExecuteNonQuery(m_logQueries);
         }
 
         public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, System.Data.IDbTransaction transaction = null)
@@ -538,23 +629,53 @@ namespace Duplicati.Library.Main.Database
             AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction);
         }
 
-        public long GetFileLastModified(string path, long filesetid, out DateTime oldModified, System.Data.IDbTransaction transaction = null)
+        public long GetFileLastModified(string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, System.Data.IDbTransaction transaction = null)
         {
-			m_selectfileHashCommand.Transaction = transaction;
-			m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
-            m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid);
-            using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader())
-                if (rd.Read())
-                {
-                    oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
-                    return rd.ConvertValueToInt64(0);    
-                }
+            if (includeLength)
+            {
+                m_selectfilelastmodifiedWithSizeCommand.Transaction = transaction;
+                m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(0, path);
+                m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(1, filesetid);
+                using (var rd = m_selectfilelastmodifiedWithSizeCommand.ExecuteReader(m_logQueries, null))
+                    if (rd.Read())
+                    {
+                        oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
+                        length = rd.ConvertValueToInt64(2);
+                        return rd.ConvertValueToInt64(0);
+                    }
 
+            }
+            else
+            {
+                m_selectfilelastmodifiedCommand.Transaction = transaction;
+                m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
+                m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid);
+                using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null))
+                    if (rd.Read())
+                    {
+                        length = -1;
+                        oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
+                        return rd.ConvertValueToInt64(0);
+                    }
+
+            }
             oldModified = new DateTime(0, DateTimeKind.Utc);
+            length = -1;
             return -1;
         }
 
-        public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize)
+        public Tuple<long, string> GetMetadataHashAndSizeForFile(long fileid, System.Data.IDbTransaction transaction)
+        {
+            m_selectfilemetadatahashandsizeCommand.Transaction = transaction;
+            m_selectfilemetadatahashandsizeCommand.SetParameterValue(0, fileid);
+            using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader(m_logQueries, null))
+                if (rd.Read())
+                    return new Tuple<long, string>(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1));
+
+            return null;
+        }
+
+        public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize, System.Data.IDbTransaction transaction)
         {
             if (m_pathLookup != null)
             {            
@@ -579,8 +700,9 @@ namespace Duplicati.Library.Main.Database
             else
             {
                 m_findfileCommand.SetParameterValue(0, path);
-
-                using(var rd = m_findfileCommand.ExecuteReader())
+                m_findfileCommand.SetParameterValue(1, filesetid);
+                m_findfileCommand.Transaction = transaction;
+                using(var rd = m_findfileCommand.ExecuteReader(m_logQueries, null))
                     if (rd.Read())
                     {
                         oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
@@ -603,7 +725,7 @@ namespace Duplicati.Library.Main.Database
         public string GetFileHash(long fileid)
         {
             m_selectfileHashCommand.SetParameterValue(0, fileid);
-            var r = m_selectfileHashCommand.ExecuteScalar();
+            var r = m_selectfileHashCommand.ExecuteScalar(m_logQueries, null);
             if (r == null || r == DBNull.Value)
                 return null;
                 

Duplicati/Library/Main/Database/LocalDatabase.cs

@@ -807,7 +807,7 @@ ON
                     using(var cmd2 = m_connection.CreateCommand(transaction))
                     foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1)))
                     {
-                            var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS);
+                        var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS);
                         var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid);
                         //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
                         var storedlist = cmd2.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?", 0, filesetid);

Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs

@@ -94,15 +94,19 @@ namespace Duplicati.Library.Main.Operation.Backup
         {
             return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction));
         }
+
+        public Task<Tuple<long, string>> GetMetadataHashAndSizeForFileAsync(long fileid)
+        {
+            return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, m_transaction));
+        }
 
-        public Task<KeyValuePair<long, DateTime>> GetFileLastModifiedAsync(string path, long lastfilesetid)
+        public Task<Tuple<long, DateTime, long>> GetFileLastModifiedAsync(string path, long lastfilesetid, bool includeLength)
         {
             return RunOnMain(() =>
             {
-                DateTime lastModified;
-                var id = m_database.GetFileLastModified(path, lastfilesetid, out lastModified, m_transaction);
+                var id = m_database.GetFileLastModified(path, lastfilesetid, includeLength, out var lastModified, out var length, m_transaction);
 
-                return new KeyValuePair<long, DateTime>(id, lastModified);
+                return new Tuple<long, DateTime, long>(id, lastModified, length);
             });
 		}
 
@@ -114,7 +118,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                 string oldMetahash;
                 long oldMetasize;
 
-                var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize);
+                var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize, m_transaction);
                 return
                     id < 0 ?
                     null :

Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs

@@ -26,12 +26,12 @@ namespace Duplicati.Library.Main.Operation.Backup
 {
     internal static class CountFilesHandler
     {
-        public static Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, Common.ITaskReader taskreader, System.Threading.CancellationToken token)
+        public static async Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, Common.ITaskReader taskreader, System.Threading.CancellationToken token)
         {
             // Make sure we create the enumeration process in a seperate scope,
             // but keep the log channel from the parent scope
-            using(Logging.Log.StartIsolatingScope())
-            using(new IsolatedChannelScope())
+            using(Logging.Log.StartIsolatingScope(true))
+            using (new IsolatedChannelScope())
             {
                 var enumeratorTask = Backup.FileEnumerationProcess.Run(sources, snapshot, null, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, options.ChangedFilelist, taskreader);
                 var counterTask = AutomationExtensions.RunTask(new 
@@ -60,7 +60,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                             {
                             }
 
-                            result.OperationProgressUpdater.UpdatefileCount(count, size, false);                    
+                            result.OperationProgressUpdater.UpdatefileCount(count, size, false);
                         }
                     }
                     finally
@@ -69,7 +69,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                     }
                 });
 
-                return Task.WhenAll(enumeratorTask, counterTask);
+                await Task.WhenAll(enumeratorTask, counterTask);
             }
         }
     }

Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs

@@ -68,10 +68,10 @@ namespace Duplicati.Library.Main.Operation.Backup
                                 if (!e.MetadataChanged)
                                 {
                                     var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length);
-                                    if (!res.Item1)
+                                    if (res.Item1)
                                         return res.Item2;
 
-                                    Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpextedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2);
+                                    Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpextedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Path);
                                     e.MetadataChanged = true;
                                 }
 
@@ -120,7 +120,15 @@ namespace Duplicati.Library.Main.Operation.Backup
                         {
                             // When we write the file to output, update the last modified time
                             Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Path);
-                            await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+
+                            try
+                            {
+                                await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                            }
+                            catch (Exception ex)
+                            {
+                                Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
+                            }
                         }
                     }
                     catch(Exception ex)

Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs

@@ -38,7 +38,7 @@ namespace Duplicati.Library.Main.Operation.Backup
         /// </summary>
         private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(typeof(FileEnumerationProcess));
 
-        public static Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, string[] changedfilelist, ITaskReader taskreader)
+        public static Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, string[] changedfilelist, ITaskReader taskreader)
         {
             return AutomationExtensions.RunTask(
             new
@@ -77,21 +77,21 @@ namespace Duplicati.Library.Main.Operation.Backup
                         {
                         }
 
-                        return AttributeFilterAsync(null, x, fa, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, attributeFilter, enumeratefilter, ignorenames, mixinqueue).WaitForTask().Result;
+                        return AttributeFilter(null, x, fa, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
                     });
                 }
                 else
                 {
-					Library.Utility.Utility.EnumerationFilterDelegate AttributeFilter = (root, path, attr) =>
-					    AttributeFilterAsync(root, path, attr, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, attributeFilter, enumeratefilter, ignorenames, mixinqueue).WaitForTask().Result;
+                    Library.Utility.Utility.EnumerationFilterDelegate attributeFilter = (root, path, attr) =>
+                        AttributeFilter(root, path, attr, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
 
                     if (journalService != null)
                     {
                         // filter sources using USN journal, to obtain a sub-set of files / folders that may have been modified
-                        sources = journalService.GetModifiedSources(AttributeFilter);
+                        sources = journalService.GetModifiedSources(attributeFilter);
                     }
 
-                    worklist = snapshot.EnumerateFilesAndFolders(sources, AttributeFilter, (rootpath, errorpath, ex) =>
+                    worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) =>
                     {
                         Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath);
                     });
@@ -222,7 +222,7 @@ namespace Duplicati.Library.Main.Operation.Backup
         /// <param name="rootpath">The root path that initiated this enumeration.</param>
         /// <param name="path">The current path.</param>
         /// <param name="attributes">The file or folder attributes.</param>
-        private static async Task<bool> AttributeFilterAsync(string rootpath, string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary<string, string> hardlinkmap, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue<string> mixinqueue)
+        private static bool AttributeFilter(string rootpath, string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary<string, string> hardlinkmap, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue<string> mixinqueue)
         {
 			// Step 1, exclude block devices
 			try
@@ -304,7 +304,7 @@ namespace Duplicati.Library.Main.Operation.Backup
             }
 
             // If we exclude files based on attributes, filter that
-            if ((attributeFilter & attributes) != 0)
+            if ((fileAttributes & attributes) != 0)
             {
                 Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path);
                 return false;

Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs

@@ -49,6 +49,19 @@ namespace Duplicati.Library.Main.Operation.Backup
                 var EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);
                 var blocksize = options.Blocksize;
 
+
+                // Pre-cache the option variables here to simplify and
+                // speed up repeated option access below
+
+                var SKIPFILESLARGERTHAN = options.SkipFilesLargerThan;
+                // Zero and max both indicate no size limit
+                if (SKIPFILESLARGERTHAN == long.MaxValue)
+                    SKIPFILESLARGERTHAN = 0;
+
+                var DISABLEFILETIMECHECK = options.DisableFiletimeCheck;
+                var CHECKFILETIMEONLY = options.CheckFiletimeOnly;
+                var SKIPMETADATA = options.SkipMetadata;
+
                 while (true)
                 {
                     var e = await self.Input.ReadAsync();
@@ -60,31 +73,78 @@ namespace Duplicati.Library.Main.Operation.Backup
                     }
                     catch(Exception ex)
                     {
-                        Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed tp read size of file: {0}", e.Path);
+                        Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path);
                     }
 
                     await stats.AddExaminedFile(filestatsize);
 
-                    e.MetaHashAndSize = options.StoreMetadata ? Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot), options) : EMPTY_METADATA;
-
-                    var timestampChanged = e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
-                    var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
-                    var tooLargeFile = options.SkipFilesLargerThan != long.MaxValue && options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > options.SkipFilesLargerThan;
-                    e.MetadataChanged = !options.CheckFiletimeOnly && !options.SkipMetadata && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
-
-                    if ((e.OldId < 0 || options.DisableFiletimeCheck || timestampChanged || filesizeChanged || e.MetadataChanged) && !tooLargeFile)
+                    // Stop now if the file is too large
+                    var tooLargeFile = SKIPFILESLARGERTHAN != 0 && filestatsize >= 0 && filestatsize > SKIPFILESLARGERTHAN;
+                    if (tooLargeFile)
                     {
-                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, e.OldId <= 0, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
+                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);
+                        continue;
+                    }
+
+                    // Invalid ID indicates a new file
+                    var isNewFile = e.OldId < 0;
+
+                    // If we disable the filetime check, we always assume that the file has changed
+                    // Otherwise we check that the timestamps are different or if any of them are empty
+                    var timestampChanged = DISABLEFILETIMECHECK || e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
+
+                    // Avoid generating a new matadata blob if timestamp has not changed
+                    // and we only check for timestamp changes
+                    if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile)
+                    {
+                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);                                                
+                        try
+                        {
+                            await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                        }
+                        catch (Exception ex)
+                        {
+                            if (ex.IsRetiredException())
+                                throw;
+                            Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
+                        }
+                        continue;
+                    }
+
+                    // If we have have disabled the filetime check, we do not have the metadata info
+                    // but we want to know if the metadata is potentially changed
+                    if (!isNewFile && DISABLEFILETIMECHECK)
+                    {
+                        var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId);
+                        if (tp != null)
+                        {
+                            e.OldMetaSize = tp.Item1;
+                            e.OldMetaHash = tp.Item2;
+                        }
+                    }
+
+                    // Compute current metadata
+                    e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Path, e.Attributes, options, snapshot), options);
+                    e.MetadataChanged = !SKIPMETADATA && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
+
+                    // Check if the file is new, or something indicates a change
+                    var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
+                    if (isNewFile || timestampChanged || filesizeChanged || e.MetadataChanged)
+                    {
+                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
                         await self.Output.WriteAsync(e);
                     }
                     else
                     {
-                        if (tooLargeFile)
-                            Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);                        
-                        else
-                            Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);                        
-                        
-                        await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                        Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Path);
+                        try
+                        {
+                            await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
+                        }
+                        catch (Exception ex)
+                        {
+                            Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
+                        }
                     }
                 }
             });

Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs

@@ -30,13 +30,13 @@ namespace Duplicati.Library.Main.Operation.Backup
     {
         private static readonly string METALOGTAG = Logging.Log.LogTagFromType(typeof(MetadataGenerator)) + ".Metadata";
 
-        public static async Task<Dictionary<string, string>> GenerateMetadataAsync(string path, System.IO.FileAttributes attributes, Options options, Snapshots.ISnapshotService snapshot)
+        public static Dictionary<string, string> GenerateMetadata(string path, System.IO.FileAttributes attributes, Options options, Snapshots.ISnapshotService snapshot)
         {
             try
             {
                 Dictionary<string, string> metadata;
 
-                if (options.StoreMetadata)
+                if (!options.SkipMetadata)
                 {
                     metadata = snapshot.GetMetadata(path, snapshot.IsSymlink(path, attributes), options.SymlinkPolicy == Options.SymlinkStrategy.Follow);
                     if (metadata == null)

Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs

@@ -67,6 +67,9 @@ namespace Duplicati.Library.Main.Operation.Backup
             {
                 var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), options);
 
+                var CHECKFILETIMEONLY = options.CheckFiletimeOnly;
+                var DISABLEFILETIMECHECK = options.DisableFiletimeCheck;
+
                 while (true)
                 {
                     var path = await self.Input.ReadAsync();
@@ -96,16 +99,16 @@ namespace Duplicati.Library.Main.Operation.Backup
                     {
                         try
                         {
-                            if (options.CheckFiletimeOnly || options.DisableFiletimeCheck)
+                            if (CHECKFILETIMEONLY || DISABLEFILETIMECHECK)
                             {
-                                var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid);
+                                var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid, false);
                                 await self.Output.WriteAsync(new FileEntry() {
-                                    OldId = tmp.Key < 0 ? -1 : tmp.Key,
+                                    OldId = tmp.Item1,
                                     Path = path,
                                     Attributes = attributes,
                                     LastWrite = lastwrite,
-                                    OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value,
-                                    LastFileSize = -1 ,
+                                    OldModified = tmp.Item2,
+                                    LastFileSize = tmp.Item3 ,
                                     OldMetaHash = null,
                                     OldMetaSize = -1
                                 });
@@ -159,7 +162,7 @@ namespace Duplicati.Library.Main.Operation.Backup
 
                     if (options.SymlinkPolicy == Options.SymlinkStrategy.Store)
                     {
-                        var metadata = await MetadataGenerator.GenerateMetadataAsync(path, attributes, options, snapshot);
+                        var metadata = MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot);
 
                         if (!metadata.ContainsKey("CoreSymlinkTarget"))
                             metadata["CoreSymlinkTarget"] = symlinkTarget;
@@ -182,9 +185,9 @@ namespace Duplicati.Library.Main.Operation.Backup
             {
                 IMetahash metahash;
 
-                if (options.StoreMetadata)
+                if (!options.SkipMetadata)
                 {
-                    metahash = Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(path, attributes, options, snapshot), options);
+                    metahash = Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot), options);
                 }
                 else
                 {

Duplicati/Library/Main/Operation/BackupHandler.cs

@@ -421,7 +421,7 @@ namespace Duplicati.Library.Main.Operation
                                 }
 
                                 // Make sure the database is sane
-                                await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, true);
+                                await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks);
 
                                 // Start the uploader process
                                 uploader = Backup.BackendUploader.Run(bk, m_options, db, m_result, m_result.TaskReader, stats);

Duplicati/Library/Main/Operation/Common/BackendHandler.cs

@@ -109,7 +109,7 @@ namespace Duplicati.Library.Main.Operation.Common
                 this.LocalTempfile.Protected = true;
             }
                 
-            public async Task Encrypt(Options options)
+            public void Encrypt(Options options)
             {
                 if (!this.Encrypted && !options.NoEncryption)
                 {
@@ -221,9 +221,9 @@ namespace Duplicati.Library.Main.Operation.Common
 
             var tcs = new TaskCompletionSource<bool>();
 
-            var backgroundhashAndEncrypt = Task.Run(async () =>
+            var backgroundhashAndEncrypt = Task.Run(() =>
             {
-                await fe.Encrypt(m_options).ConfigureAwait(false);
+                fe.Encrypt(m_options);
                 return fe.UpdateHashAndSize(m_options);
             });
 
@@ -331,7 +331,7 @@ namespace Duplicati.Library.Main.Operation.Common
             return RunRetryOnMain(fe, () => DoGet(fe));
         }
 
-        private async Task ResetBackendAsync(Exception ex)
+        private void ResetBackend(Exception ex)
         {
             try
             {
@@ -343,7 +343,6 @@ namespace Duplicati.Library.Main.Operation.Common
                 Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", dex, "Failed to dispose backend instance: {0}", ex.Message); 
             }
             m_backend = null;
-
         }
 
         private async Task<T> DoWithRetry<T>(FileEntryItem item, Func<Task<T>> method)
@@ -405,12 +404,12 @@ namespace Duplicati.Library.Main.Operation.Common
                     }
 
                     if (!recovered)
-                        await ResetBackendAsync(ex).ConfigureAwait(false);
+                        ResetBackend(ex);
                 }
                 finally
                 {
                     if (m_options.NoConnectionReuse)
-                        await ResetBackendAsync(null).ConfigureAwait(false);
+                        ResetBackend(null);
                 }
             }
 
@@ -435,7 +434,7 @@ namespace Duplicati.Library.Main.Operation.Common
         private async Task<bool> DoPut(FileEntryItem item, bool updatedHash = false)
         {
             // If this is not already encrypted, do it now
-            await item.Encrypt(m_options).ConfigureAwait(false);
+            item.Encrypt(m_options);
 
             updatedHash |= item.UpdateHashAndSize(m_options);
 

Duplicati/Library/Main/Operation/Common/SingleRunner.cs

@@ -27,79 +27,20 @@ namespace Duplicati.Library.Main.Operation.Common
     /// </summary>
     internal abstract class SingleRunner : IDisposable
     {
-        protected IChannel<Func<Task>> m_channel;
-        protected readonly Task m_worker;
-        protected CancellationTokenSource m_workerSource;
+        protected AsyncLock m_lock = new AsyncLock();
+        protected CancellationTokenSource m_workerSource = new CancellationTokenSource();
 
-        public SingleRunner()
+        protected async Task<T> DoRunOnMain<T>(Func<Task<T>> method)
         {
-            AutomationExtensions.AutoWireChannels(this, null);
-            m_channel = ChannelManager.CreateChannel<Func<Task>>();
-            m_workerSource = new System.Threading.CancellationTokenSource();
-            m_worker = AutomationExtensions.RunProtected(this, Start);
-        }
+            m_workerSource.Token.ThrowIfCancellationRequested();
 
-        private async Task Start()
-        {            
-            var ct = m_workerSource.Token;
-            while(!ct.IsCancellationRequested)
+            using (await m_lock.LockAsync())
             {
-                // Grab next task
-                var nextTask = await m_channel.ReadAsync();
-
-                // Execute it
-                await nextTask();
+                m_workerSource.Token.ThrowIfCancellationRequested();
+                return await method().ConfigureAwait(false);
             }
         }
 
-        protected Task<T> DoRunOnMain<T>(Func<Task<T>> method)
-        {
-            var res = new TaskCompletionSource<T>();
-
-            Task.Run(async () =>
-            {
-                try
-                {
-                    if (m_workerSource.IsCancellationRequested)
-                    {
-                        res.TrySetCanceled();
-                        return;
-                    }
-
-                    await m_channel.WriteAsync(async () =>
-                    {
-                        if (m_workerSource.IsCancellationRequested)
-                        {
-                            res.TrySetCanceled();
-                            return;
-                        }
-
-                        try
-                        {
-                            var r = await method().ConfigureAwait(false);
-                            res.SetResult(r);
-                        }
-                        catch (Exception ex)
-                        {
-                            if (ex is System.Threading.ThreadAbortException)
-                                res.TrySetCanceled();
-                            else
-                                res.TrySetException(ex);
-                        }
-                    }).ConfigureAwait(false);
-                }
-                catch (Exception ex)
-                {
-                    if (ex is System.Threading.ThreadAbortException)
-                        res.TrySetCanceled();
-                    else
-                        res.TrySetException(ex);
-                }
-            });
-
-            return res.Task;
-        }
-
         protected Task RunOnMain(Action method)
         {
             return DoRunOnMain<bool>(() =>
@@ -138,13 +79,6 @@ namespace Duplicati.Library.Main.Operation.Common
         protected virtual void Dispose(bool isDisposing)
         {
             m_workerSource.Cancel();
-
-            if (m_channel != null)
-                try { m_channel.Retire(); }
-                catch { }
-                finally { }
-
-            AutomationExtensions.RetireAllChannels(this);
         }
     }
 }

Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs

@@ -105,7 +105,14 @@ namespace Duplicati.Library.Main.Operation
             if (brokensets.Length == 0)
             {
                 m_result.BrokenFiles = new Tuple<long, DateTime, IEnumerable<Tuple<string, long>>>[0];
-                Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "No broken filesets found");
+
+                if (missing == null)
+                    Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets");
+                else if (missing.Count == 0)
+                    Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files");
+                else
+                    Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Run purge-broken-files.", missing.Count));
+
                 return;
             }
 

Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs

@@ -65,103 +65,105 @@ namespace Duplicati.Library.Main.Operation
                     else if (missing.Count == 0)
                         Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files");
                     else
-                        throw new UserInformationException(string.Format("Found no broken filesets, but {0} missing remote files", sets.Length), "NoBrokenSetsButMissingRemoteFiles");
+                        Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count));
                 }
+                else
+                { 
+                    Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.Item3));
 
-                Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.Item3));
+                    var pgoffset = 0.0f;
+                    var pgspan = 0.95f / sets.Length;
 
-                var pgoffset = 0.0f;
-                var pgspan = 0.95f / sets.Length;
+                    var filesets = db.FilesetTimes.ToList();
 
-                var filesets = db.FilesetTimes.ToList();
-
-                var compare_list = sets.Select(x => new
-                {
-                    FilesetID = x.Item2,
-                    Timestamp = x.Item1,
-                    RemoveCount = x.Item3,
-                    Version = filesets.FindIndex(y => y.Key == x.Item2),
-                    SetCount = db.GetFilesetFileCount(x.Item2, tr)
-                }).ToArray();
-
-                var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
-                var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
-
-                if (fully_emptied.Length != 0)
-                {
-                    if (fully_emptied.Length == 1)
-                        Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount);
-                    else
-                        Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())));
-
-                    m_result.DeleteResults = new DeleteResults(m_result);
-                    using (var rmdb = new Database.LocalDeleteDatabase(db))
+                    var compare_list = sets.Select(x => new
                     {
-                        var deltr = rmdb.BeginTransaction();
-                        try
+                        FilesetID = x.Item2,
+                        Timestamp = x.Item1,
+                        RemoveCount = x.Item3,
+                        Version = filesets.FindIndex(y => y.Key == x.Item2),
+                        SetCount = db.GetFilesetFileCount(x.Item2, tr)
+                    }).ToArray();
+
+                    var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
+                    var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
+
+                    if (fully_emptied.Length != 0)
+                    {
+                        if (fully_emptied.Length == 1)
+                            Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount);
+                        else
+                            Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())));
+
+                        m_result.DeleteResults = new DeleteResults(m_result);
+                        using (var rmdb = new Database.LocalDeleteDatabase(db))
                         {
-                            var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
-                            opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
-                            opts.RawOptions.Remove("time");
-                            opts.RawOptions["no-auto-compact"] = "true";
-
-                            new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults)
-                                .DoRun(rmdb, ref deltr, true, false, null);
-
-                            if (!m_options.Dryrun)
+                            var deltr = rmdb.BeginTransaction();
+                            try
                             {
-                                using (new Logging.Timer(LOGTAG, "CommitDelete", "CommitDelete"))
-                                    deltr.Commit();
+                                var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
+                                opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
+                                opts.RawOptions.Remove("time");
+                                opts.RawOptions["no-auto-compact"] = "true";
 
-                                rmdb.WriteResults();
+                                new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults)
+                                    .DoRun(rmdb, ref deltr, true, false, null);
+
+                                if (!m_options.Dryrun)
+                                {
+                                    using (new Logging.Timer(LOGTAG, "CommitDelete", "CommitDelete"))
+                                        deltr.Commit();
+
+                                    rmdb.WriteResults();
+                                }
+                                else
+                                    deltr.Rollback();
                             }
-                            else
-                                deltr.Rollback();
-                        }
-                        finally
-                        {
-                            if (deltr != null)
-                                try { deltr.Rollback(); }
-                                catch { }
+                            finally
+                            {
+                                if (deltr != null)
+                                    try { deltr.Rollback(); }
+                                    catch { }
+                            }
+
                         }
 
+                        pgoffset += (pgspan * fully_emptied.Length);
+                        m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
                     }
 
-                    pgoffset += (pgspan * fully_emptied.Length);
-                    m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
-                }
-
-                if (to_purge.Length > 0)
-                {
-                    m_result.PurgeResults = new PurgeFilesResults(m_result);
-
-                    foreach (var bs in to_purge)
+                    if (to_purge.Length > 0)
                     {
-                        Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime());
-                        var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
+                        m_result.PurgeResults = new PurgeFilesResults(m_result);
 
-                        using (var pgdb = new Database.LocalPurgeDatabase(db))
+                        foreach (var bs in to_purge)
                         {
-                            // Recompute the version number after we deleted the versions before
-                            filesets = pgdb.FilesetTimes.ToList();
-                            var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID);
-                            if (thisversion < 0)
-                                throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString()))));
+                            Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime());
+                            var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
 
-                            opts.RawOptions["version"] = thisversion.ToString();
-                            opts.RawOptions.Remove("time");
-                            opts.RawOptions["no-auto-compact"] = "true";
-
-                            new PurgeFilesHandler(m_backendurl, opts, (PurgeFilesResults)m_result.PurgeResults).Run(pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) =>
+                            using (var pgdb = new Database.LocalPurgeDatabase(db))
                             {
-                                if (filesetid != bs.FilesetID)
-                                    throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
-                                db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction);
-                            });
-                        }
+                                // Recompute the version number after we deleted the versions before
+                                filesets = pgdb.FilesetTimes.ToList();
+                                var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID);
+                                if (thisversion < 0)
+                                    throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString()))));
 
-                        pgoffset += pgspan;
-                        m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
+                                opts.RawOptions["version"] = thisversion.ToString();
+                                opts.RawOptions.Remove("time");
+                                opts.RawOptions["no-auto-compact"] = "true";
+
+                                new PurgeFilesHandler(m_backendurl, opts, (PurgeFilesResults)m_result.PurgeResults).Run(pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) =>
+                                {
+                                    if (filesetid != bs.FilesetID)
+                                        throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
+                                    db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction);
+                                });
+                            }
+
+                            pgoffset += pgspan;
+                            m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
+                        }
                     }
                 }
 
@@ -172,18 +174,6 @@ namespace Duplicati.Library.Main.Operation
 
                 m_result.OperationProgressUpdater.UpdateProgress(0.95f);
 
-                if (missing != null && missing.Count > 0)
-                {
-                    using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
-                    {
-                        foreach (var f in missing)
-                            if (m_options.Dryrun)
-                                Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size));
-                            else
-                                backend.Delete(f.Name, f.Size);
-                    }
-                }
-
                 if (!m_options.Dryrun && db.RepairInProgress)
                 {                    
                     Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files");

Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs

@@ -92,7 +92,7 @@ namespace Duplicati.Library.Main.Operation
             using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb))
             {
                 restoredb.RepairInProgress = true;
-
+                var autoDetectBlockSize = !(m_options.HasBlocksize && restoredb.GetDbOptions().ContainsKey("blocksize"));                    
                 var volumeIds = new Dictionary<string, long>();
 
                 var rawlist = backend.List();
@@ -205,7 +205,7 @@ namespace Duplicati.Library.Main.Operation
 
                                 var parsed = VolumeBase.ParseFilename(entry.Name);
 
-                                if (!hasUpdatedOptions && !updating) 
+                                if (!hasUpdatedOptions && (!updating || autoDetectBlockSize)) 
                                 {
                                     VolumeReaderBase.UpdateOptionsFromManifest(parsed.CompressionModule, tmpfile, m_options);
                                     hasUpdatedOptions = true;

Duplicati/Library/Main/Operation/RepairHandler.cs

@@ -261,6 +261,13 @@ namespace Duplicati.Library.Main.Operation
                             if (ex is System.Threading.ThreadAbortException)
                                 throw;
                         }
+
+                    if (!m_options.RebuildMissingDblockFiles)
+                    {
+                        var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks).ToArray();
+                        if (missingDblocks.Length > 0)
+                            throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles");
+                    }
                             
                     foreach(var n in tp.MissingVolumes)
                     {

Duplicati/Library/Main/Operation/TestHandler.cs

@@ -52,6 +52,7 @@ namespace Duplicati.Library.Main.Operation
                 db.SetResult(m_results);
                 Utility.UpdateOptionsFromDb(db, m_options);
                 Utility.VerifyParameters(db, m_options);
+                db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null);
                 
                 if (!m_options.NoBackendverification)
                     FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_results.BackendWriter);

Duplicati/Library/Main/Options.cs

@@ -327,6 +327,7 @@ namespace Duplicati.Library.Main
                     "log-file-log-filter",
                     "console-log-level",
                     "console-log-filter",
+                    "profile-all-database-queries"
                 };
             }
         }
@@ -464,7 +465,7 @@ namespace Duplicati.Library.Main
                     new CommandLineArgument("upload-unchanged-backups", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UploadUnchangedBackupsShort, Strings.Options.UploadUnchangedBackupsLong, "false"),
 
                     new CommandLineArgument("snapshot-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotpolicyShort, Strings.Options.SnapshotpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))),
-                    new CommandLineArgument("vss-exclude-writers", CommandLineArgument.ArgumentType.String, Strings.Options.VssexcludewritersShort, Strings.Options.VssexcludewritersLong),
+                    new CommandLineArgument("vss-exclude-writers", CommandLineArgument.ArgumentType.String, Strings.Options.VssexcludewritersShort, Strings.Options.VssexcludewritersLong, "{e8132975-6f93-4464-a53e-1050253ae220}"),
                     new CommandLineArgument("vss-use-mapping", CommandLineArgument.ArgumentType.Boolean, Strings.Options.VssusemappingShort, Strings.Options.VssusemappingLong, "false"),
                     new CommandLineArgument("usn-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.UsnpolicyShort, Strings.Options.UsnpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))),
 
@@ -477,14 +478,16 @@ namespace Duplicati.Library.Main
                     new CommandLineArgument("debug-output", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DebugoutputShort, Strings.Options.DebugoutputLong, "false"),
                     new CommandLineArgument("debug-retry-errors", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DebugretryerrorsShort, Strings.Options.DebugretryerrorsLong, "false"),
 
-                    new CommandLineArgument("log-file", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Options.LogfileShort, Strings.Options.LogfileLong),
-                    new CommandLineArgument("log-file-log-level", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration, Strings.Options.LogfileloglevelShort, Strings.Options.LogfileloglevelShort, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))),
-                    new CommandLineArgument("log-file-log-filter", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Options.LogfilelogfiltersShort, Strings.Options.LogfilelogfiltersLong(System.IO.Path.PathSeparator.ToString()), null),
+                    new CommandLineArgument("log-file", CommandLineArgument.ArgumentType.Path, Strings.Options.LogfileShort, Strings.Options.LogfileLong),
+                    new CommandLineArgument("log-file-log-level", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.LogfileloglevelShort, Strings.Options.LogfileloglevelShort, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))),
+                    new CommandLineArgument("log-file-log-filter", CommandLineArgument.ArgumentType.String, Strings.Options.LogfilelogfiltersShort, Strings.Options.LogfilelogfiltersLong(System.IO.Path.PathSeparator.ToString()), null),
 
-                    new CommandLineArgument("console-log-level", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration, Strings.Options.ConsoleloglevelShort, Strings.Options.ConsoleloglevelShort, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))),
-                    new CommandLineArgument("console-log-filter", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Options.ConsolelogfiltersShort, Strings.Options.ConsolelogfiltersLong(System.IO.Path.PathSeparator.ToString()), null),
+                    new CommandLineArgument("console-log-level", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.ConsoleloglevelShort, Strings.Options.ConsoleloglevelShort, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))),
+                    new CommandLineArgument("console-log-filter", CommandLineArgument.ArgumentType.String, Strings.Options.ConsolelogfiltersShort, Strings.Options.ConsolelogfiltersLong(System.IO.Path.PathSeparator.ToString()), null),
 
-                    new CommandLineArgument("log-level", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration, Strings.Options.LoglevelShort, Strings.Options.LoglevelLong, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType)), Strings.Options.LogLevelDeprecated("log-file-log-level", "console-log-level")),
+                    new CommandLineArgument("log-level", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.LoglevelShort, Strings.Options.LoglevelLong, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType)), Strings.Options.LogLevelDeprecated("log-file-log-level", "console-log-level")),
+
+                    new CommandLineArgument("profile-all-database-queries", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ProfilealldatabasequeriesShort, Strings.Options.ProfilealldatabasequeriesLong, "false"),
 
                     new CommandLineArgument("list-verify-uploads", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ListverifyuploadsShort, Strings.Options.ListverifyuploadsShort, "false"),
                     new CommandLineArgument("allow-sleep", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AllowsleepShort, Strings.Options.AllowsleepLong, "false"),
@@ -508,7 +511,6 @@ namespace Duplicati.Library.Main
                     new CommandLineArgument("dbpath", CommandLineArgument.ArgumentType.Path, Strings.Options.DbpathShort, Strings.Options.DbpathLong),
                     new CommandLineArgument("blocksize", CommandLineArgument.ArgumentType.Size, Strings.Options.BlocksizeShort, Strings.Options.BlocksizeLong, DEFAULT_BLOCKSIZE),
                     new CommandLineArgument("file-read-buffer-size", CommandLineArgument.ArgumentType.Size, Strings.Options.FilereadbuffersizeShort, Strings.Options.FilereadbuffersizeLong, "0kb"),
-                    new CommandLineArgument("store-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.StoremetadataShort, Strings.Options.StoremetadataLong, "true", null, null, Strings.Options.StoremetadataDeprecated),
                     new CommandLineArgument("skip-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkipmetadataShort, Strings.Options.SkipmetadataLong, "false"),
                     new CommandLineArgument("restore-permissions", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorepermissionsShort, Strings.Options.RestorepermissionsLong, "false"),
                     new CommandLineArgument("skip-restore-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkiprestoreverificationShort, Strings.Options.SkiprestoreverificationLong, "false"),
@@ -558,11 +560,14 @@ namespace Duplicati.Library.Main
                     
                     new CommandLineArgument("auto-vacuum", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AutoVacuumShort, Strings.Options.AutoVacuumLong, "false"),
                     new CommandLineArgument("disable-file-scanner", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilescannerShort, Strings.Options.DisablefilescannerLong, "false"),
+                    new CommandLineArgument("disable-filelist-consistency-checks", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilelistconsistencychecksShort, Strings.Options.DisablefilelistconsistencychecksLong, "false"),
                     new CommandLineArgument("disable-on-battery", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableOnBatteryShort, Strings.Options.DisableOnBatteryLong, "false"),
 
                     new CommandLineArgument("exclude-empty-folders", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ExcludeemptyfoldersShort, Strings.Options.ExcludeemptyfoldersLong, "false"),
                     new CommandLineArgument("ignore-filenames", CommandLineArgument.ArgumentType.Path, Strings.Options.IgnorefilenamesShort, Strings.Options.IgnorefilenamesLong),
                     new CommandLineArgument("restore-symlink-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestoresymlinkmetadataShort, Strings.Options.RestoresymlinkmetadataLong, "false"),
+                    new CommandLineArgument("rebuild-missing-dblock-files", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RebuildmissingdblockfilesShort, Strings.Options.RebuildmissingdblockfilesLong, "false"),
+
                 });
 
                 return lst;
@@ -1340,6 +1345,12 @@ namespace Duplicati.Library.Main
                 return Duplicati.Library.Logging.LogMessageType.Warning;
             }
         }
+
+        /// <summary>
+        /// A value indicating if all database queries should be logged
+        /// </summary>
+        public bool ProfileAllDatabaseQueries { get { return GetBool("profile-all-database-queries"); } }
+
         /// <summary>
         /// Gets the attribute filter used to exclude files and folders.
         /// </summary>
@@ -1457,6 +1468,11 @@ namespace Duplicati.Library.Main
             }
         }
 
+        /// <summary>
+        /// Gets a value indicating whether a blocksize has been specified
+        /// </summary>
+        public bool HasBlocksize { get { return m_options.ContainsKey("blocksize") && !string.IsNullOrEmpty(m_options["blocksize"]); } }
+
         /// <summary>
         /// Gets the size of file-blocks
         /// </summary>
@@ -1511,23 +1527,6 @@ namespace Duplicati.Library.Main
                 return (int)t;
             }
         }
-        
-        /// <summary>
-        /// Gets a flag indicating if metadata for files and folders should be ignored
-        /// </summary>
-        public bool StoreMetadata
-        {
-            get 
-            { 
-                if (m_options.ContainsKey("skip-metadata"))
-                    return !Library.Utility.Utility.ParseBoolOption(m_options, "skip-metadata");
-
-                if (m_options.ContainsKey("store-metadata"))
-                    return Library.Utility.Utility.ParseBoolOption(m_options, "store-metadata"); 
-
-                return true;
-            }
-        }
 
         /// <summary>
         /// Gets a flag indicating if metadata for files and folders should be ignored
@@ -1927,6 +1926,15 @@ namespace Duplicati.Library.Main
             get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-file-scanner"); }
         }
 
+        /// <summary>
+        /// Gets a flag indicating if the filelist consistency checks should be disabled
+        /// </summary>
+        /// <value><c>true</c> if the filelist consistency checks should be disabled; otherwise, <c>false</c>.</value>
+        public bool DisableFilelistConsistencyChecks
+        {
+            get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-filelist-consistency-checks"); }
+        }
+
         /// <summary>
         /// Gets a flag indicating whether the backup should be disabled when on battery power.
         /// </summary>
@@ -1936,6 +1944,14 @@ namespace Duplicati.Library.Main
             get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-on-battery"); }
         }
 
+        /// <summary>
+        /// Gets a value indicating if missing dblock files are attempted created
+        /// </summary>
+        public bool RebuildMissingDblockFiles
+        {
+            get { return GetBool("rebuild-missing-dblock-files"); }
+        }
+
         /// <summary>
         /// Gets the threshold for when log data should be cleaned
         /// </summary>

Duplicati/Library/Main/Strings.cs

@@ -28,6 +28,7 @@ namespace Duplicati.Library.Main.Strings
         public static string FailedForceLocaleError(string exMsg) { return LC.L(@"Failed to apply 'force-locale' setting. Please try to update .NET-Framework. Exception was: ""{0}"" ", exMsg); }
         public static string SourceVolumeNameInvalidError(string filename) { return LC.L(@"The source {0} uses an invalid volume name, aborting backup", filename); }
         public static string SourceVolumeNameNotFoundError(string filename, Guid volumeGuid) { return LC.L(@"The source {0} is on volume {1}, which could not be found, aborting backup", filename, volumeGuid); }
+        public static string NonQualifiedSizeValue(string optionname, string value) { return LC.L(@"The size ""{1}"" supplied to --{0} does not have a multiplier (b, kb, mb, etc). A multiplier is recommended to avoid unexpected changes if the program is updated.", optionname, value); }
     }
 
     internal static class Options
@@ -152,9 +153,6 @@ namespace Duplicati.Library.Main.Strings
         public static string DisablefilepathcacheShort { get { return LC.L(@"Reduce memory footprint by disabling in-memory lookups"); } }
 		public static string UseblockcacheShort { get { return LC.L(@"This option can be used to increase speed in exchange for extra memory use."); } }
 		public static string UseblockcacheLong { get { return LC.L(@"Store an in-memory block cache"); } }
-		public static string StoremetadataLong { get { return LC.L(@"Stores metadata, such as file timestamps and attributes. This increases the required storage space as well as the processing time."); } }
-        public static string StoremetadataShort { get { return LC.L(@"Enables storing file metadata"); } }
-        public static string StoremetadataDeprecated { get { return LC.L(@"This option is no longer used as metadata is now stored by default"); } }
         public static string MetadatahashlookupsizeLong { get { return LC.L(@"A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log."); } }
         public static string MetadatahashlookupsizeShort { get { return LC.L(@"Memory used by the metadata hash"); } }
         public static string NobackendverificationLong { get { return LC.L(@"If this flag is set, the local database is not compared to the remote filelist on startup. The intended usage for this option is to work correctly in cases where the filelisting is broken or unavailable."); } }
@@ -247,6 +245,8 @@ namespace Duplicati.Library.Main.Strings
         public static string AutoVacuumLong { get { return LC.L(@"Some operations that manipulate the local database leave unused entries behind. These entries are not deleted from a hard drive until a VACUUM operation is run. This operation saves disk space in the long run but needs to temporarily create a copy of all valid entries in the database. Setting this to true will allow Duplicati to perform VACUUM operations at its discretion."); } }
         public static string DisablefilescannerShort { get { return LC.L(@"Disable the read-ahead scanner"); } }
         public static string DisablefilescannerLong { get { return LC.L(@"When this flag is enabled, the scanner that computes the size of source files is disabled, and instead the reported size is read from the database. Using this flag can speed up the backup by reducing disk access, but will give a less accurate progress indicator."); } }
+        public static string DisablefilelistconsistencychecksShort { get { return LC.L(@"Disable filelist consistency checks"); } }
+        public static string DisablefilelistconsistencychecksLong { get { return LC.L(@"In backups with a large number of filesets, the verification can take up a large part of the backup time. If you disable the checks, make sure you run regular check commands to ensure that everything is working as expected."); } }
         public static string DisableOnBatteryShort { get { return LC.L("Disable the backup when on battery power"); } }
         public static string DisableOnBatteryLong { get { return LC.L("When this flag is enabled, a scheduled backup will not run if the system is detected to be running on battery power (manual or command line backups will still be run).  If the detected power source is mains (i.e., AC) or unknown, then scheduled backups will proceed as normal."); } }
 
@@ -270,6 +270,11 @@ namespace Duplicati.Library.Main.Strings
         public static string RestoresymlinkmetadataLong { get { return "If symlink metadata is applied, it will usually mean changing the symlink target, instead of the symlink itself. For this reason, metadata is not applied to symlinks, but this option can be used to override this, such that metadata is applied to symlinks as well."; } }
         public static string UnittestmodeShort { get { return "Activate unittest mode"; } }
         public static string UnittestmodeLong { get { return "When running in unittest mode, no automatic fixes are applied, which assumes that the input data is always in perfect shape. This option is not intended for use in daily backups, but required for testing purposes to reveal potential problems."; } }
+
+        public static string ProfilealldatabasequeriesShort { get { return LC.L("Activates logging of all database queries"); } }
+        public static string ProfilealldatabasequeriesLong { get { return LC.L("To improve performance of the backups, frequent database queries are not logged by default. Enable this option to log all database queries, and remember to set either --{0}={2} or --{1}={2} to report the additional log data", "console-log-level", "log-file-log-level", nameof(Logging.LogMessageType.Profiling)); } }
+        public static string RebuildmissingdblockfilesShort { get { return "Rebuild dblock files when missing"; } }
+        public static string RebuildmissingdblockfilesLong { get { return "If dblock files are missing from the destination, you can attempt to rebuild them using local source data. However, since the local data may have changed, it may not be possible to retrieve all the required data and the process may be slow. Use this option to attempt to rebuild missing dblock files."; } }
     }
 
     internal static class Common

Duplicati/Library/Main/default_compressed_extensions.txt

@@ -97,8 +97,11 @@
 .pspimage #PaintShopPro Image
 .tif #TIFF image
 .dng #Adobe Digital Negative
-.cr2 #Canon RAW Image Format
+.cr2 #Canon RAW Image Format (already compressed despite being "RAW")
 .webp #WebP Image
+.nef #Nikon RAW format
+.arw #Sony RAW format
+.heic #High Efficiency Image File Format
 
 
 # Compressed Font files

Duplicati/Library/Modules/Builtin/ResultSerialization/JsonFormatSerializer.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Duplicati.Library.Interface;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
@@ -42,7 +43,8 @@ namespace Duplicati.Library.Modules.Builtin.ResultSerialization
             {
                 return base
                     .CreateProperties(type, memberSerialization)
-                    .Where(x => !m_excludes.Contains(x.PropertyName))
+                    .Where(x => !m_excludes.Contains(x.PropertyName))
+                    .Where(x => !typeof(Task).IsAssignableFrom(x.PropertyType))
                     .ToList();
             }
         }
@@ -70,7 +72,8 @@ namespace Duplicati.Library.Modules.Builtin.ResultSerialization
                     ContractResolver = new DynamicContractResolver(
                             nameof(IBasicResults.Warnings),
                             nameof(IBasicResults.Errors),
-                            nameof(IBasicResults.Messages)
+                            nameof(IBasicResults.Messages),
+                            "TaskReader"
                     ),
                     Converters = new List<JsonConverter>()
                     {

Duplicati/Library/Modules/Builtin/RunScript.cs

@@ -24,6 +24,7 @@ using Duplicati.Library.Utility;
 using Duplicati.Library.Interface;
 using System.Linq;
 using Duplicati.Library.Modules.Builtin.ResultSerialization;
+using System.Threading.Tasks;
 
 namespace Duplicati.Library.Modules.Builtin
 {
@@ -220,7 +221,7 @@ namespace Duplicati.Library.Modules.Builtin
                 psi.EnvironmentVariables["DUPLICATI__REMOTEURL"] = remoteurl;
                 if (level != null)
                     psi.EnvironmentVariables["DUPLICATI__PARSED_RESULT"] = level.Value.ToString();
-                
+
                 if (localpath != null)
                     psi.EnvironmentVariables["DUPLICATI__LOCALPATH"] = string.Join(System.IO.Path.PathSeparator.ToString(), localpath);
 
@@ -230,9 +231,9 @@ namespace Duplicati.Library.Modules.Builtin
                 if (!string.IsNullOrEmpty(datafile))
                     psi.EnvironmentVariables["DUPLICATI__RESULTFILE"] = datafile;
 
-                using(System.Diagnostics.Process p = System.Diagnostics.Process.Start(psi))
+                using (System.Diagnostics.Process p = System.Diagnostics.Process.Start(psi))
                 {
-                    ConsoleDataHandler cs = new ConsoleDataHandler(p);
+                    var cs = new ConsoleDataHandler(p);
 
                     if (timeout <= 0)
                         p.WaitForExit();
@@ -249,14 +250,57 @@ namespace Duplicati.Library.Modules.Builtin
 
                     if (p.HasExited)
                     {
+                        cs.WaitForCompletion();
+
                         stderr = cs.StandardError;
                         stdout = cs.StandardOutput;
                         if (p.ExitCode != 0)
-                            Logging.Log.WriteWarningMessage(LOGTAG, "InvalidExitCode", null, Strings.RunScript.InvalidExitCodeError(scriptpath, p.ExitCode));
+                        {
+                            if (!requiredScript)
+                            {
+                                // We log a warning or an error depending on the exit code
+                                switch (p.ExitCode)
+                                {
+                                    case 0:
+                                    case 1:
+                                        // No messages here
+                                        break;
+
+                                    case 2:
+                                    case 3:
+                                        Logging.Log.WriteWarningMessage(LOGTAG, "InvalidExitCode", null, Strings.RunScript.ExitCodeError(scriptpath, p.ExitCode, stderr));
+                                        stderr = null;
+                                        break;
+
+                                    case 4:
+                                    case 5:
+                                    default:
+                                        Logging.Log.WriteErrorMessage(LOGTAG, "InvalidExitCode", null, Strings.RunScript.ExitCodeError(scriptpath, p.ExitCode, stderr));
+                                        stderr = null;
+                                        break;
+                                }
+
+                                // If this is the start event, we abort the backup 
+                                if (eventname == "BEFORE")
+                                {
+                                    switch (p.ExitCode)
+                                    {
+                                        case 1:
+                                            throw new OperationAbortException(OperationAbortReason.Normal, Strings.RunScript.InvalidExitCodeError(scriptpath, p.ExitCode));
+                                        case 3:
+                                            throw new OperationAbortException(OperationAbortReason.Warning, Strings.RunScript.InvalidExitCodeError(scriptpath, p.ExitCode));
+                                        case 5:
+                                            throw new OperationAbortException(OperationAbortReason.Error, Strings.RunScript.InvalidExitCodeError(scriptpath, p.ExitCode));
+                                    }
+                                }
+                            }
+                            else
+                                Logging.Log.WriteWarningMessage(LOGTAG, "InvalidExitCode", null, Strings.RunScript.InvalidExitCodeError(scriptpath, p.ExitCode));
+                        }
                     }
                     else
                     {
-                            Logging.Log.WriteWarningMessage(LOGTAG, "ScriptTimeout", null, Strings.RunScript.ScriptTimeoutError(scriptpath));
+                        Logging.Log.WriteWarningMessage(LOGTAG, "ScriptTimeout", null, Strings.RunScript.ScriptTimeoutError(scriptpath));
                     }
                 }
 
@@ -266,7 +310,7 @@ namespace Duplicati.Library.Modules.Builtin
                 //We only allow setting parameters on startup
                 if (eventname == "BEFORE" && stdout != null)
                 {
-                    foreach(string rawline in stdout.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
+                    foreach (string rawline in stdout.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
                     {
                         string line = rawline.Trim();
                         if (!line.StartsWith("--", StringComparison.Ordinal))
@@ -303,7 +347,7 @@ namespace Duplicati.Library.Modules.Builtin
                             localpath = value.Split(System.IO.Path.PathSeparator);
                         }
                         else if (
-                            string.Equals(key, "eventname", StringComparison.OrdinalIgnoreCase) || 
+                            string.Equals(key, "eventname", StringComparison.OrdinalIgnoreCase) ||
                             string.Equals(key, "operationname", StringComparison.OrdinalIgnoreCase) ||
                             string.Equals(key, "main-action", StringComparison.OrdinalIgnoreCase) ||
                             key == ""
@@ -313,10 +357,14 @@ namespace Duplicati.Library.Modules.Builtin
                         }
                         else
                             options[key] = value;
-
                     }
                 }
             }
+            catch (OperationAbortException)
+            {
+                // Do not log this, it is already logged
+                throw;
+            }
             catch (Exception ex)
             {
                 Logging.Log.WriteWarningMessage(LOGTAG, "ScriptExecuteError", ex, Strings.RunScript.ScriptExecuteError(scriptpath, ex.Message));
@@ -325,49 +373,29 @@ namespace Duplicati.Library.Modules.Builtin
             }
         }
 
+        /// <summary>
+        /// Helper class to extract output from the program
+        /// </summary>
         private class ConsoleDataHandler
         {
             public ConsoleDataHandler(System.Diagnostics.Process p)
             {
-                p.OutputDataReceived += new System.Diagnostics.DataReceivedEventHandler(HandleOutputDataReceived);
-                p.ErrorDataReceived += new System.Diagnostics.DataReceivedEventHandler(HandleErrorDataReceived);
-
-                p.BeginErrorReadLine();
-                p.BeginOutputReadLine();
+                m_task = Task.WhenAll(
+                    Task.Run(async () => StandardOutput = await p.StandardOutput.ReadToEndAsync()),
+                    Task.Run(async () => StandardError = await p.StandardError.ReadToEndAsync())
+                );
             }
 
-            private readonly StringBuilder m_standardOutput = new StringBuilder();
-            private readonly StringBuilder m_standardError = new StringBuilder();
-            private readonly object m_lock = new object();
-
-            private void HandleOutputDataReceived (object sender, System.Diagnostics.DataReceivedEventArgs e)
+            private readonly Task m_task;
+            public string StandardOutput { get; private set; }
+            public string StandardError { get; private set; }
+            public void WaitForCompletion()
             {
-                lock(m_lock)
-                    m_standardOutput.AppendLine(e.Data);
-            }
-
-            private void HandleErrorDataReceived (object sender, System.Diagnostics.DataReceivedEventArgs e)
-            {
-                lock(m_lock)
-                    m_standardError.AppendLine(e.Data);
-            }
-
-            public string StandardOutput
-            {
-                get
-                {
-                    lock(m_lock)
-                        return m_standardOutput.ToString().Trim();
-                }
-            }
-
-            public string StandardError
-            {
-                get
-                {
-                    lock(m_lock)
-                        return m_standardError.ToString().Trim();
-                }
+                // NOTE: This is ugly, but there is a race where "HasExited" is set, 
+                // but the stdout/stderr streams have not yet completed.
+                // If we wait a little here, we eventually get the data.
+                // If the streams have completed we do not wait.
+                m_task.Wait(TimeSpan.FromSeconds(5));
             }
         }
     }

Duplicati/Library/Modules/Builtin/Strings.cs

@@ -1,8 +1,8 @@
 using Duplicati.Library.Localization.Short;
 using System;
 using System.Collections;
-using System.Collections.Generic;
-
+using System.Collections.Generic;
+
 namespace Duplicati.Library.Modules.Builtin.Strings {
     internal static class ConsolePasswordInput {
         public static string ConfirmPassphrasePrompt { get { return LC.L(@"Confirm encryption passphrase"); } }
@@ -15,7 +15,7 @@ namespace Duplicati.Library.Modules.Builtin.Strings {
     internal static class CheckMonoSSL {
         public static string Description { get { return LC.L(@"When running with Mono, this module will check if any certificates are installed and suggest installing them otherwise"); } }
         public static string Displayname { get { return LC.L(@"Check for SSL certificates"); } }
-        public static string ErrorMessage { get { return LC.L(@"No certificates found, you can install some with one of these commands:{0}    cert-sync /etc/ssl/certs/ca-certificates.crt #for Debian based systems{0}    cert-sync /etc/pki/tls/certs/ca-bundle.crt #for RedHat derivatives{0}Read more: {1}", Environment.NewLine, "http://www.mono-project.com/docs/about-mono/releases/3.12.0/#cert-sync"); } }
+        public static string ErrorMessage { get { return LC.L(@"No certificates found, you can install some with one of these commands:{0}    cert-sync /etc/ssl/certs/ca-certificates.crt #for Debian based systems{0}    cert-sync /etc/pki/tls/certs/ca-bundle.crt #for RedHat derivatives{0}    curl -O https://curl.haxx.se/ca/cacert.pem; cert-sync --user cacert.pem; rm cacert.pem #for MacOS{0}Read more: {1}", Environment.NewLine, "http://www.mono-project.com/docs/about-mono/releases/3.12.0/#cert-sync"); } }
     }
     internal static class HttpOptions {
         public static string Description { get { return LC.L(@"This module exposes a number of properties that can be used to change the way http requests are issued"); } }
@@ -54,6 +54,7 @@ namespace Duplicati.Library.Modules.Builtin.Strings {
         public static string FinishoptionLong { get { return LC.L(@"Executes a script after performing an operation. The script will receive the operation results written to stdout."); } }
         public static string FinishoptionShort { get { return LC.L(@"Run a script on exit"); } }
         public static string InvalidExitCodeError(string script, int exitcode) { return LC.L(@"The script ""{0}"" returned with exit code {1}", script, exitcode); }
+        public static string ExitCodeError(string script, int exitcode, string message) { return LC.L(@"The script ""{0}"" returned with exit code {1}{2}", script, exitcode, string.IsNullOrWhiteSpace(message) ? string.Empty : string.Format(": {0}", message)); }
         public static string RequiredoptionLong { get { return LC.L(@"Executes a script before performing an operation. The operation will block until the script has completed or timed out. If the script returns a non-zero error code or times out, the operation will be aborted."); } }
         public static string RequiredoptionShort { get { return LC.L(@"Run a required script on startup"); } }
         public static string ResultFormatShort { get { return LC.L(@"Selects the output format for results"); } }

Duplicati/Library/Modules/Builtin/run-script-example.bat

@@ -16,6 +16,15 @@ REM --run-script-before = <filename>
 REM Duplicati will run the script before the backup job and waits for its 
 REM completion for 60 seconds (default timeout value). After a timeout a 
 REM warning is logged and the backup is started.
+REM The following exit codes are supported:
+REM
+REM - 0: OK, run operation
+REM - 1: OK, don't run operation
+REM - 2: Warning, run operation
+REM - 3: Warning, don't run operation
+REM - 4: Error, run operation
+REM - 5: Error don't run operation
+REM - other: Error don't run operation
 REM
 REM --run-script-before-required = <filename>
 REM Duplicati will run the script before the backup job and wait for its 
@@ -32,6 +41,10 @@ REM --run-script-after = <filename>
 REM Duplicati will run the script after the backup job and wait for its 
 REM completion for 60 seconds (default timeout value). After a timeout a 
 REM warning is logged.
+REM The same exit codes as in --run-script-before are supported, but
+REM the operation will always continue (i.e. 1 => 0, 3 => 2, 5 => 4)
+REM as it has already completed so stopping it during stop is useless.
+
 
 
 REM ###############################################################################

Duplicati/Library/Modules/Builtin/run-script-example.sh

@@ -16,6 +16,15 @@
 # Duplicati will run the script before the backup job and waits for its 
 # completion for 60 seconds (default timeout value). After a timeout a 
 # warning is logged and the backup is started.
+# The following exit codes are supported:
+#
+# - 0: OK, run operation
+# - 1: OK, don't run operation
+# - 2: Warning, run operation
+# - 3: Warning, don't run operation
+# - 4: Error, run operation
+# - 5: Error don't run operation
+# - other: Error don't run operation
 #
 # --run-script-before-required = <filename>
 # Duplicati will run the script before the backup job and wait for its 
@@ -32,6 +41,9 @@
 # Duplicati will run the script after the backup job and wait for its 
 # completion for 60 seconds (default timeout value). After a timeout a 
 # warning is logged.
+# The same exit codes as in --run-script-before are supported, but
+# the operation will always continue (i.e. 1 => 0, 3 => 2, 5 => 4)
+# as it has already completed so stopping it during stop is useless.
 
 
 ###############################################################################

Duplicati/Library/Snapshots/WindowsSnapshot.cs

@@ -75,6 +75,11 @@ namespace Duplicati.Library.Snapshots
         /// </summary>
         private static SystemIOWindows IO_WIN = new SystemIOWindows();
 
+        /// <summary>
+        /// Commonly used string element
+        /// </summary>
+        private static string SLASH = Path.DirectorySeparatorChar.ToString();
+
         /// <summary>
         /// Constructs a new backup snapshot, using all the required disks
         /// </summary>
@@ -95,10 +100,15 @@ namespace Duplicati.Library.Snapshots
                 if (vss == null)
                     throw new InvalidOperationException();
 
-                var excludedWriters = new Guid[0];
+                // Default to exclude the System State writer
+                var excludedWriters = new Guid[] { new Guid("{e8132975-6f93-4464-a53e-1050253ae220}") };
                 if (options.ContainsKey("vss-exclude-writers"))
                 {
-                    excludedWriters = options["vss-exclude-writers"].Split(';').Where(x => !string.IsNullOrWhiteSpace(x) && x.Trim().Length > 0).Select(x => new Guid(x)).ToArray();
+                    excludedWriters = options["vss-exclude-writers"]
+                        .Split(';')
+                        .Where(x => !string.IsNullOrWhiteSpace(x) && x.Trim().Length > 0)
+                        .Select(x => new Guid(x))
+                        .ToArray();
                 }
 
                 //Check if we should map any drives
@@ -406,11 +416,16 @@ namespace Duplicati.Library.Snapshots
                 throw new InvalidOperationException();
 
             var root = AlphaFS.Path.GetPathRoot(localPath);
-
             if (!m_volumeMap.TryGetValue(root, out var volumePath))
                 throw new InvalidOperationException();
 
-            return Path.Combine(volumePath, localPath.Substring(root.Length));
+            // Note: Do NOT use Path.Combine as it strips the UNC path prefix
+            var subPath = localPath.Substring(root.Length);
+            if (!volumePath.EndsWith(SLASH, StringComparison.Ordinal) && !subPath.StartsWith(SLASH, StringComparison.Ordinal))
+                volumePath += SLASH;
+
+            var mappedPath = volumePath + subPath;
+            return mappedPath;
         }
 
         /// <inheritdoc />
@@ -450,13 +465,13 @@ namespace Duplicati.Library.Snapshots
 				catch (Exception ex)
                 {
 					Logging.Log.WriteVerboseMessage(LOGTAG, "MappedDriveCleanupError", ex, "Failed during VSS mapped drive unmapping");
-                }
-
-				try
-				{
-					m_backup?.BackupComplete();
+                }
+
+				try
+				{
+					m_backup?.BackupComplete();
 				}
-				catch (Exception ex)
+				catch (Exception ex)
 				{
 					Logging.Log.WriteVerboseMessage(LOGTAG, "VSSTerminateError", ex, "Failed to signal VSS completion");
 				}
@@ -466,12 +481,12 @@ namespace Duplicati.Library.Snapshots
                     if (m_backup != null)
                     {
                         foreach (var g in m_volumes.Values)
-                        {
-							try
-							{
-								m_backup.DeleteSnapshot(g, false);
+                        {
+							try
+							{
+								m_backup.DeleteSnapshot(g, false);
 							}
-							catch (Exception ex)
+							catch (Exception ex)
 							{
 								Logging.Log.WriteVerboseMessage(LOGTAG, "VSSSnapShotDeleteError", ex, "Failed to close VSS snapshot");
 							}

Duplicati/Library/Utility/Utility.cs

@@ -653,17 +653,6 @@ namespace Duplicati.Library.Utility
             return a1 == a2;
         }
 
-        /// <summary>
-        /// Calculates the hash of a given file, and returns the results as an base64 encoded string
-        /// </summary>
-        /// <param name="path">The path to the file to calculate the hash for</param>
-        /// <returns>The base64 encoded hash</returns>
-        public static string CalculateHash(string path)
-        {
-            using (FileStream fs = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read))
-                return CalculateHash(fs);
-        }
-
         /// <summary>
         /// Calculates the hash of a given stream, and returns the results as an base64 encoded string
         /// </summary>
@@ -1062,64 +1051,15 @@ namespace Duplicati.Library.Utility
             }
         }
 
-        /// <summary>
-        /// Gets the users default UI language
-        /// </summary>
-        public static System.Globalization.CultureInfo DefaultCulture => new System.Threading.Thread(() => { }).CurrentUICulture;
-
         /// <summary>
         /// Gets a string comparer that matches the client filesystems case sensitivity
         /// </summary>
-        public static StringComparer ClientFilenameStringComparer => IsFSCaseSensitive ? StringComparer.CurrentCulture : StringComparer.CurrentCultureIgnoreCase;
+        public static StringComparer ClientFilenameStringComparer => IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase;
 
         /// <summary>
         /// Gets the string comparision that matches the client filesystems case sensitivity
         /// </summary>
-        public static StringComparison ClientFilenameStringComparison => IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase;
-
-        /// <summary>
-        /// Searches the system paths for the file specified
-        /// </summary>
-        /// <param name="filename">The file to locate</param>
-        /// <returns>The full path to the file, or null if the file was not found</returns>
-        public static string LocateFileInSystemPath(string filename)
-        {
-            try
-            {
-                if (Path.IsPathRooted(filename))
-                    return File.Exists(filename) ? filename : null;
-
-                try
-                {
-                    filename = Path.GetFileName(filename);
-                }
-                catch
-                {
-                    // ignored
-                }
-
-                string homedir = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + Path.PathSeparator.ToString();
-
-                //Look in application base folder and all system path folders
-                foreach (string s in (homedir + Environment.GetEnvironmentVariable("PATH")).Split(Path.PathSeparator))
-                    if (!string.IsNullOrEmpty(s) && s.Trim().Length > 0)
-                        try
-                        {
-                            foreach (string sx in Directory.GetFiles(ExpandEnvironmentVariables(s), filename))
-                                return sx;
-                        }
-                        catch
-                        {
-                            // ignored
-                        }
-            }
-            catch
-            {
-                // ignored
-            }
-
-            return null;
-        }
+        public static StringComparison ClientFilenameStringComparison => IsFSCaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase;
 
         /// <summary>
         /// The path to the users home directory
@@ -1141,11 +1081,6 @@ namespace Duplicati.Library.Utility
         /// </summary>
         private static readonly Regex ENVIRONMENT_VARIABLE_MATCHER_WINDOWS = new Regex(@"\%(?<name>\w+)\%");
 
-        /// <summary>
-        /// Regexp for matching environment variables on Linux ($VAR or ${VAR})
-        /// </summary>
-        private static readonly Regex ENVIRONMENT_VARIABLE_MATCHER_LINUX = new Regex(@"\$(?<name>\w+)|(\{(?<name>[^\}]+)\})");
-
         /// <summary>
         /// Expands environment variables in a RegExp safe format
         /// </summary>
@@ -1159,29 +1094,12 @@ namespace Duplicati.Library.Utility
 
             return
 
-                // TODO: Should we switch to using the native format, instead of following the Windows scheme?
-                //IsClientLinux ? ENVIRONMENT_VARIABLE_MATCHER_LINUX : ENVIRONMENT_VARIABLE_MATCHER_WINDOWS
+                // TODO: Should we switch to using the native format ($VAR or ${VAR}), instead of following the Windows scheme?
+                // IsClientLinux ? new Regex(@"\$(?<name>\w+)|(\{(?<name>[^\}]+)\})") : ENVIRONMENT_VARIABLE_MATCHER_WINDOWS
 
                 ENVIRONMENT_VARIABLE_MATCHER_WINDOWS.Replace(str, m => Regex.Escape(lookup(m.Groups["name"].Value)));
         }
 
-        /// <summary>
-        /// Checks that a hostname is valid
-        /// </summary>
-        /// <param name="hostname">The hostname to verify</param>
-        /// <returns>True if the hostname is valid, false otherwise</returns>
-        public static bool IsValidHostname(string hostname)
-        {
-            try
-            {
-                return System.Uri.CheckHostName(hostname) != UriHostNameType.Unknown;
-            }
-            catch
-            {
-                return false;
-            }
-        }
-
         /// <summary>
         /// The format string for a DateTime
         /// </summary>
@@ -1255,34 +1173,6 @@ namespace Duplicati.Library.Utility
             return uniqueItems;
         }
 
-        /// <summary>
-        /// Helper method that replaces one file with another
-        /// </summary>
-        /// <param name="target">The file to replace</param>
-        /// <param name="sourcefile">The file to replace with</param>
-        public static void ReplaceFile(string target, string sourcefile)
-        {
-            if (File.Exists(target))
-                File.Delete(target);
-
-            //Nasty workaround for the fact that a recently deleted file occasionally blocks a new write
-            long i = 5;
-            do
-            {
-                try
-                {
-                    File.Move(sourcefile, target);
-                    break;
-                }
-                catch (Exception ex)
-                {
-                    if (i == 0)
-                        throw new Exception(
-                            $"Failed to replace the file \"{target}\" volume with the \"{sourcefile}\", error: {ex.Message}");
-                    System.Threading.Thread.Sleep(250);
-                }
-            } while (i-- > 0);
-        }
         // <summary>
         // Returns the entry assembly or reasonable approximation if no entry assembly is available.
         // This is the case in NUnit tests.  The following approach does not work w/ Mono due to unimplemented members:
@@ -1328,17 +1218,6 @@ namespace Duplicati.Library.Utility
             return Base64PlainToBase64Url(Convert.ToBase64String(data));
         }
 
-        /// <summary>
-        /// Decodes a &quot;base64 for url&quot; encoded string into the raw byte array.
-        /// See https://en.wikipedia.org/wiki/Base64#URL_applications
-        /// </summary>
-        /// <param name="data">The data to decode</param>
-        /// <returns>The raw data</returns>
-        public static byte[] Base64UrlDecode(string data)
-        {
-            return Convert.FromBase64String(Base64UrlToBase64Plain(data));
-        }
-
         /// <summary>
         /// Converts a DateTime instance to a Unix timestamp
         /// </summary>
@@ -1353,16 +1232,6 @@ namespace Duplicati.Library.Utility
             return (long)Math.Floor((input - EPOCH).TotalSeconds);
         }
 
-        /// <summary>
-        /// Converts a Unix timestamp to a DateTime instance
-        /// </summary>
-        /// <returns>The DateTime instance represented by the timestamp.</returns>
-        /// <param name="input">The Unix timestamp to use.</param>
-        public static DateTime ToUnixTimestamp(long input)
-        {
-            return EPOCH.AddSeconds(input);
-        }
-
         /// <summary>
         /// Returns a value indicating if the given type should be treated as a primitive
         /// </summary>
@@ -1445,6 +1314,11 @@ namespace Duplicati.Library.Utility
                     writer.Write("{0}{1}: ", indentstring, p.Name);
                     PrintSerializeIfPrimitive(p.GetValue(item, null), writer);
                 }
+                else if (typeof(Task).IsAssignableFrom(p.PropertyType) || p.Name == "TaskReader")
+                {
+                    // Ignore Task items
+                    continue;
+                }
                 else if (typeof(System.Collections.IEnumerable).IsAssignableFrom(p.PropertyType))
                 {
                     var enumerable = (System.Collections.IEnumerable)p.GetValue(item, null);

Duplicati/Server/Database/ServerSettings.cs

@@ -53,6 +53,7 @@ namespace Duplicati.Server.Database
             public const string USAGE_REPORTER_LEVEL = "usage-reporter-level";
 			public const string HAS_ASKED_FOR_PASSWORD_PROTECTION = "has-asked-for-password-protection";
             public const string DISABLE_TRAY_ICON_LOGIN = "disable-tray-icon-login";
+            public const string SERVER_ALLOWED_HOSTNAMES = "allowed-hostnames";
 		}
 
         private readonly Dictionary<string, string> m_values;
@@ -339,10 +340,20 @@ namespace Duplicati.Server.Database
             GenerateWebserverPasswordTrayIcon();
         }
 
+        public void SetAllowedHostnames(string allowedHostnames)
+        {
+            lock (m_connection.m_lock)
+                m_values[CONST.SERVER_ALLOWED_HOSTNAMES] = allowedHostnames;
+
+            SaveSettings();
+        }
+
         public string WebserverPasswordTrayIcon => m_values[CONST.SERVER_PASSPHRASETRAYICON];
 
         public string WebserverPasswordTrayIconHash => m_values[CONST.SERVER_PASSPHRASETRAYICONHASH];
 
+        public string AllowedHostnames => m_values[CONST.SERVER_ALLOWED_HOSTNAMES];
+
         public void GenerateWebserverPasswordTrayIcon()
         {
             var password = "";

Duplicati/Server/Program.cs

@@ -313,6 +313,10 @@ namespace Duplicati.Server
 
                 Program.DataConnection.ApplicationSettings.GenerateWebserverPasswordTrayIcon();
 
+                if (commandlineOptions.ContainsKey("webservice-allowed-hostnames"))
+                    Program.DataConnection.ApplicationSettings.SetAllowedHostnames(commandlineOptions["webservice-allowed-hostnames"]);
+
+
                 ApplicationExitEvent = new System.Threading.ManualResetEvent(false);
 
                 Duplicati.Library.AutoUpdater.UpdaterManager.OnError += (Exception obj) =>
@@ -742,6 +746,7 @@ namespace Duplicati.Server
                     new Duplicati.Library.Interface.CommandLineArgument(Duplicati.Server.WebServer.Server.OPTION_SSLCERTIFICATEFILEPASSWORD, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverCertificatePasswordDescription, Strings.Program.WebserverCertificatePasswordDescription, Duplicati.Server.WebServer.Server.OPTION_SSLCERTIFICATEFILEPASSWORD),
                     new Duplicati.Library.Interface.CommandLineArgument(Duplicati.Server.WebServer.Server.OPTION_INTERFACE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverInterfaceDescription, Strings.Program.WebserverInterfaceDescription, Duplicati.Server.WebServer.Server.DEFAULT_OPTION_INTERFACE),
                     new Duplicati.Library.Interface.CommandLineArgument("webservice-password", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Password, Strings.Program.WebserverPasswordDescription, Strings.Program.WebserverPasswordDescription),
+                    new Duplicati.Library.Interface.CommandLineArgument("webservice-allowed-hostnames", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverAllowedhostnamesDescription, Strings.Program.WebserverAllowedhostnamesDescription),
                     new Duplicati.Library.Interface.CommandLineArgument("ping-pong-keepalive", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PingpongkeepaliveShort, Strings.Program.PingpongkeepaliveLong),
                     new Duplicati.Library.Interface.CommandLineArgument("log-retention", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.Program.LogretentionShort, Strings.Program.LogretentionLong, DEFAULT_LOG_RETENTION),
                     new Duplicati.Library.Interface.CommandLineArgument("server-datafolder", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ServerdatafolderShort, Strings.Program.ServerdatafolderLong(DATAFOLDER_ENV_NAME), System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), Library.AutoUpdater.AutoUpdateSettings.AppName)),

Duplicati/Server/Strings.cs

@@ -31,6 +31,7 @@ Error message: {0}", error); }
         public static string WebserverCertificatePasswordDescription { get { return LC.L(@"The password for decryption of certificate PKCS #12 file."); } }
         public static string WebserverInterfaceDescription { get { return LC.L(@"The interface the webserver listens on. The special values ""*"" and ""any"" means any interface. The special value ""loopback"" means the loopback adapter."); } }
         public static string WebserverPasswordDescription { get { return LC.L(@"The password required to access the webserver. This option is saved so you do not need to set it on each run. Setting an empty value disables the password."); } }
+        public static string WebserverAllowedhostnamesDescription { get { return LC.L(@"The hostnames that are accepted, separated with semicolons. If any of the hostnames are ""*"", all hostnames are allowed and the hostname checking is disabled."); } }
         public static string PingpongkeepaliveShort { get { return LC.L(@"Enables the ping-pong responder"); } }
         public static string PingpongkeepaliveLong { get { return LC.L(@"When running as a server, the service daemon must verify that the process is responding. If this option is enabled, the server reads stdin and writes a reply to each line read"); } }
         public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } }

Duplicati/Server/WebServer/RESTMethods/Filesystem.cs

@@ -221,8 +221,8 @@ namespace Duplicati.Server.WebServer.RESTMethods
                     var isFile = !isFolder;
                     var isHidden = (attr & FileAttributes.Hidden) != 0;
 
-                    var accesible = isFile || canAccess(s);
-                    var isLeaf = isFile || !accesible || isEmptyFolder(s);
+                    var accessible = isFile || canAccess(s);
+                    var isLeaf = isFile || !accessible || isEmptyFolder(s);
 
                     var rawid = isFolder ? Library.Utility.Utility.AppendDirSeparator(s) : s;
                     if (skipFiles && !isFolder)
@@ -237,7 +237,7 @@ namespace Duplicati.Server.WebServer.RESTMethods
                         text = systemIO.PathGetFileName(s),
                         hidden = isHidden,
                         symlink = isSymlink,
-                        iconCls = isFolder ? (accesible ? (isSymlink ? "x-tree-icon-symlink" : "x-tree-icon-parent") : "x-tree-icon-locked") : "x-tree-icon-leaf",
+                        iconCls = isFolder ? (accessible ? (isSymlink ? "x-tree-icon-symlink" : "x-tree-icon-parent") : "x-tree-icon-locked") : "x-tree-icon-leaf",
                         leaf = isLeaf
                     };
                 }

Duplicati/Server/WebServer/Server.cs

@@ -201,6 +201,8 @@ namespace Duplicati.Server.WebServer
         {
             HttpServer.HttpServer server = new HttpServer.HttpServer();
 
+            server.Add(new HostHeaderChecker());
+
             if (string.Equals(Environment.GetEnvironmentVariable("SYNO_DSM_AUTH") ?? string.Empty, "1"))
                 server.Add(new SynologyAuthenticationHandler());
 
@@ -320,5 +322,120 @@ namespace Duplicati.Server.WebServer
                 return base.Process(request, response, session);
             }
         }
+
+        /// <summary>
+        /// Module for injecting host header verification
+        /// </summary>
+        private class HostHeaderChecker : HttpModule
+        {
+            /// <summary>
+            /// The hostnames that we allow
+            /// </summary>
+            private string[] m_lastSplitNames;
+
+            /// <summary>
+            /// The string used to generate m_lastSplitNames;
+            /// </summary>
+            private string m_lastAllowed;
+
+            /// <summary>
+            /// A regex to detect potential IPv4 addresses.
+            /// Note that this also detects things that are not valid IPv4.
+            /// </summary>
+            private static readonly System.Text.RegularExpressions.Regex IPV4 = new System.Text.RegularExpressions.Regex(@"((\d){1,3}\.){3}(\d){1,3}");
+            /// <summary>
+            /// A regex to detect potential IPv6 addresses.
+            /// Note that this also detects things that are not valid IPv6.
+            /// </summary>
+            private static readonly System.Text.RegularExpressions.Regex IPV6 = new System.Text.RegularExpressions.Regex(@"(\:)?(\:?[A-Fa-f0-9]{1,4}\:?){1,8}(\:)?");
+
+            /// <summary>
+            /// The hostnames that are always allowed
+            /// </summary>
+            private static string[] DEFAULT_ALLOWED = new string[] { "localhost", "127.0.0.1", "::1", "localhost.localdomain" };
+
+            /// <summary>
+            /// Process the received request
+            /// </summary>
+            /// <returns>A flag indicating if the request is handled.</returns>
+            /// <param name="request">The received request.</param>
+            /// <param name="response">The response object.</param>
+            /// <param name="session">The session state.</param>
+            public override bool Process(HttpServer.IHttpRequest request, HttpServer.IHttpResponse response, HttpServer.Sessions.IHttpSession session)
+            {
+                string[] h = null;
+                var hstring = Program.DataConnection.ApplicationSettings.AllowedHostnames;
+
+                if (!string.IsNullOrWhiteSpace(hstring))
+                {
+                    h = m_lastSplitNames;
+                    if (hstring != m_lastAllowed)
+                    {
+                        m_lastAllowed = hstring;
+                        h = m_lastSplitNames = (hstring ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
+                    }
+
+                    if (h == null || h.Length == 0)
+                        h = null;
+                }
+
+                // For some reason, the web server strips out the host header
+                var host = request.Headers["Host"];
+                if (string.IsNullOrWhiteSpace(host))
+                    host = request.Uri.Host;
+
+                // This should not happen
+                if (string.IsNullOrWhiteSpace(host))
+                {
+                    response.Reason = "Invalid request, missing host header";
+                    response.Status = System.Net.HttpStatusCode.Forbidden;
+                    var msg = System.Text.Encoding.ASCII.GetBytes(response.Reason);
+                    response.ContentType = "text/plain";
+                    response.ContentLength = msg.Length;
+                    response.Body.Write(msg, 0, msg.Length);
+                    response.Send();
+                    return true;
+                }
+
+                // Check the hostnames we always allow
+                if (Array.IndexOf(DEFAULT_ALLOWED, host) >= 0)
+                    return false;
+
+                // Then the user specified ones
+                if (h != null && Array.IndexOf(h, host) >= 0)
+                    return false;
+
+                // Disable checks if we have an asterisk
+                if (h != null && Array.IndexOf(h, "*") >= 0)
+                    return false;
+
+                // Finally, check if we have a potential IP address
+                var v4 = IPV4.Match(host);
+                var v6 = IPV6.Match(host);
+
+                if ((v4.Success && v4.Length == host.Length) || (v6.Success && v6.Length == host.Length))
+                {
+                    try
+                    {
+                        // Verify that the hostname is indeed a valid IP address
+                        System.Net.IPAddress.Parse(host);
+                        return false;
+                    }
+                    catch
+                    { }
+                }
+
+                // Failed to find a valid header
+                response.Reason = $"The host header sent by the client is not allowed";
+                response.Status = System.Net.HttpStatusCode.Forbidden;
+                var txt = System.Text.Encoding.ASCII.GetBytes(response.Reason);
+                response.ContentType = "text/plain";
+                response.ContentLength = txt.Length;
+                response.Body.Write(txt, 0, txt.Length);
+                response.Send();
+                return true;
+
+            }
+        }
     }
 }

Duplicati/Server/webroot/login.html

@@ -22,7 +22,7 @@
             <fieldset>
 
                 <p><label for="login-password">Password</label></p>
-                <p><input type="password" id="login-password" value="password" onBlur="if(this.value=='')this.value='password'" onFocus="if(this.value=='password')this.value=''"></p> <!-- JS because of IE support; better: placeholder="password" -->
+                <p><input type="password" id="login-password" value="password" onBlur="if(this.value=='')this.value='password'" onFocus="if(this.value=='password')this.value=''" autofocus></p> <!-- JS because of IE support; better: placeholder="password" -->
 
                 <p><input type="submit" id="login-button" value="Sign In"></p>
 

Duplicati/Server/webroot/ngax/scripts/controllers/SystemSettingsController.js

@@ -60,6 +60,7 @@ backupApp.controller('SystemSettingsController', function($rootScope, $scope, $l
         $scope.originalUpdateChannel = data.data['update-channel'];
         $scope.usageReporterLevel = data.data['usage-reporter-level'];
         $scope.disableTrayIconLogin =  AppUtils.parseBoolString(data.data['disable-tray-icon-login']);
+        $scope.remoteHostnames = data.data['allowed-hostnames'];
         $scope.advancedOptions = AppUtils.serializeAdvancedOptionsToArray(data.data);
         $scope.servermodulesettings = {};
 
@@ -75,7 +76,7 @@ backupApp.controller('SystemSettingsController', function($rootScope, $scope, $l
 
         var patchdata = {
             'server-passphrase': $scope.requireRemotePassword ? $scope.remotePassword : '',
-
+            'allowed-hostnames': $scope.remoteHostnames,
             'server-listen-interface': $scope.allowRemoteAccess ? 'any' : 'loopback',
             'startup-delay': $scope.startupDelayDurationValue + '' + $scope.startupDelayDurationMultiplier,
             'update-channel': $scope.updateChannel,

Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js

@@ -78,7 +78,7 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
 
     function reloadTexts() {
         apputils.fileSizeMultipliers = [
-            {name: gettextCatalog.getString('byte'), value: ''},
+            {name: gettextCatalog.getString('byte'), value: 'b'},
             {name: gettextCatalog.getString('KByte'), value: 'KB'},
             {name: gettextCatalog.getString('MByte'), value: 'MB'},
             {name: gettextCatalog.getString('GByte'), value: 'GB'},
@@ -111,7 +111,7 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
         ];
 
         apputils.speedMultipliers = [
-            {name: gettextCatalog.getString('byte/s'), value: ''},
+            {name: gettextCatalog.getString('byte/s'), value: 'b'},
             {name: gettextCatalog.getString('KByte/s'), value: 'KB'},
             {name: gettextCatalog.getString('MByte/s'), value: 'MB'},
             {name: gettextCatalog.getString('GByte/s'), value: 'GB'},

Duplicati/Server/webroot/ngax/scripts/services/ServerStatus.js

@@ -36,7 +36,7 @@ backupApp.service('ServerStatus', function($rootScope, $timeout, AppService, App
             'Backup_PreviousBackupFinalize': gettextCatalog.getString('Completing previous backup ...'),
             'Backup_ProcessingFiles': null,
             'Backup_Finalize': gettextCatalog.getString('Completing backup ...'),
-            'Backup_WaitForUpload': gettextCatalog.getString('Waiting for upload ...'),
+            'Backup_WaitForUpload': gettextCatalog.getString('Waiting for upload to finish ...'),
             'Backup_Delete': gettextCatalog.getString('Deleting unwanted files ...'),
             'Backup_Compact': gettextCatalog.getString('Compacting remote data ...'),
             'Backup_VerificationUpload': gettextCatalog.getString('Uploading verification file ...'),

Duplicati/Server/webroot/ngax/templates/backends/b2.html

@@ -9,7 +9,7 @@
 </div>
 
 <div class="input text">
-    <label for="b2_username" translate>B2 Account ID</label>
+    <label for="b2_username" translate>B2 Account ID or Application ID</label>
     <input type="text" name="b2_username" id="b2_username" ng-model="$parent.Username" placeholder="{{'B2 Cloud Storage Account ID' | translate}}" />
 </div>
 <div class="input password">

Duplicati/Server/webroot/ngax/templates/pause.html

@@ -26,6 +26,21 @@
     			<label for="onehour" class="pause-60" translate translate-params-number="1">{{number}} Hour</label>
     		</li>
 
+        <li class="input" ng-click="selection.time = '4h'">
+	    		<input type="radio" name="fourhour" id="fourhour" ng-model="selection.time" value="4h">
+    			<label for="fourhour" class="pause-240" translate translate-params-number="4">{{number}} Hours</label>
+    		</li>
+
+        <li class="input" ng-click="selection.time = '8h'">
+	    		<input type="radio" name="eighthour" id="eighthour" ng-model="selection.time" value="8h">
+    			<label for="eighthour" class="pause-480" translate translate-params-number="8">{{number}} Hours</label>
+    		</li>
+
+        <li class="input" ng-click="selection.time = '24h'">
+	    		<input type="radio" name="twentyfourhour" id="twentyfourhour" ng-model="selection.time" value="24h">
+    			<label for="twentyfourhour" class="pause-1440" translate translate-params-number="24">{{number}} Hours</label>
+    		</li>
+
     		<li class="input" ng-click="selection.time = 'infinite'">
 	    		<input type="radio" name="infinite" id="infinite" ng-model="selection.time" value="infinite">
     			<label for="infinite" class="pause-x" translate>Until resumed</label>
@@ -33,4 +48,4 @@
     	</ul>
 	</form>
 
-</div>
+</div>

Duplicati/Server/webroot/ngax/templates/settings.html

@@ -12,7 +12,12 @@
             <input type="checkbox" class="checkbox" id="remote" name="remote" ng-model="allowRemoteAccess" />
             <label for="remote" translate>Allow remote access (requires restart)</label>     
             <div class="sublabel" translate>By allowing remote access, the server listens to requests from any machine on your network. If you enable this option, make sure you are always using the computer on a secure firewall protected network.</div>
-        </div>            
+        </div>
+        <div class="input mixed multiple" ng-show="allowRemoteAccess">
+            <label for="hostnames" translate>Hostnames</label>
+            <input type="text" id="hostnames" name="hostnames" ng-model="remoteHostnames" />
+            <div class="sublabel" translate>To prevent various DNS based attacks, Duplicati limits the allowed hostnames to the ones listed here. Direct IP access and localhost is always allowed. Multiple hostnames can be supplied with a semicolon separator. If any of the allowed hostnames is an asterisk (*), all hostnames are allowed and this feature is disabled. If the field is empty, only IP address and localhost access is allowed.</div>
+        </div>
 
         <div class="input checkbox">
             <input type="checkbox" class="checkbox" id="trayiconlogin" name="trayiconlogin" ng-model="disableTrayIconLogin" />

Duplicati/UnitTest/BorderTests.cs

@@ -169,6 +169,18 @@ namespace Duplicati.UnitTest
                 opts["check-filetime-only"] = "true";
             });
         }
+
+        [Test]
+        [Category("Border")]
+        public void RunFullScan()
+        {
+            PrepareSourceData();
+            RunCommands(1024 * 10, modifyOptions: opts =>
+            {
+                opts["disable-filetime-check"] = "true";
+            });
+        }
+
         public static Dictionary<string, int> WriteTestFilesToFolder(string targetfolder, int blocksize, int basedatasize = 0)
         {
             if (basedatasize <= 0)
@@ -227,13 +239,27 @@ namespace Duplicati.UnitTest
                 //Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
             }
 
+            // Do a "touch" on files to trigger a re-scan, which should do nothing
+            //foreach (var k in filenames)
+                //if (File.Exists(Path.Combine(DATAFOLDER, "a" + k.Key)))
+                    //File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a" + k.Key), DateTime.Now.AddSeconds(5));
+
             var data = new byte[filenames.Select(x => x.Value).Max()];
             new Random().NextBytes(data);
             foreach(var k in filenames)
                 File.WriteAllBytes(Path.Combine(DATAFOLDER, "b" + k.Key), data.Take(k.Value).ToArray());
 
-            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
-                c.Backup(new string[] { DATAFOLDER });
+            using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
+            {
+                var r = c.Backup(new string[] { DATAFOLDER });
+                if (!Library.Utility.Utility.ParseBoolOption(testopts, "disable-filetime-check"))
+                {
+                    if (r.OpenedFiles != filenames.Count)
+                        throw new Exception($"Opened {r.OpenedFiles}, but should open {filenames.Count}");
+                    if (r.ExaminedFiles != filenames.Count * 2)
+                        throw new Exception($"Examined {r.ExaminedFiles}, but should examine open {filenames.Count * 2}");
+                }
+            }
 
             var rn = new Random();
             foreach(var k in filenames)

Duplicati/UnitTest/Duplicati.UnitTest.csproj

@@ -52,6 +52,7 @@
     <Compile Include="ResultFormatSerializerProviderTest.cs" />
     <Compile Include="UriUtilityTests.cs" />
     <Compile Include="WebApiTests.cs" />
+    <Compile Include="RunScriptTests.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <ItemGroup>

Duplicati/UnitTest/RunScriptTests.cs

@@ -0,0 +1,200 @@
+//  Copyright (C) 2018, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Duplicati.Library.Interface;
+using NUnit.Framework;
+
+namespace Duplicati.UnitTest
+{
+    public class RunScriptTests : BasicSetupHelper
+    {
+        public override void PrepareSourceData()
+        {
+            base.PrepareSourceData();
+
+            Directory.CreateDirectory(DATAFOLDER);
+            Directory.CreateDirectory(TARGETFOLDER);
+        }
+
+        [Test]
+        [Category("Border")]
+        public void RunScriptBefore()
+        {
+            PrepareSourceData();
+
+            var blocksize = 10 * 1024;
+            var options = TestOptions;
+            options["blocksize"] = blocksize.ToString() + "b";
+            options["run-script-timeout"] = "5s";
+
+            // We need a small delay as we run very small backups back-to-back
+            var PAUSE_TIME = TimeSpan.FromSeconds(3);
+
+            BorderTests.WriteTestFilesToFolder(DATAFOLDER, blocksize, 0);
+
+            using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, options, null))
+            {
+                var res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Success)
+                    throw new Exception("Unexpected result from base backup");
+                
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(0);
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Success)
+                    throw new Exception("Unexpected result from backup with return code 0");
+                if (res.ExaminedFiles <= 0)
+                    throw new Exception("Backup did not examine any files for code 0?");
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(1);
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Success)
+                    throw new Exception("Unexpected result from backup with return code 1");
+                if (res.ExaminedFiles > 0)
+                    throw new Exception("Backup did examine files for code 1?");
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(2);
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Warning)
+                    throw new Exception("Unexpected result from backup with return code 2");
+                if (res.ExaminedFiles <= 0)
+                    throw new Exception("Backup did not examine any files for code 2?");
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(3);
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Warning)
+                    throw new Exception("Unexpected result from backup with return code 3");
+                if (res.ExaminedFiles > 0)
+                    throw new Exception("Backup did examine files for code 3?");
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(4);
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Error)
+                    throw new Exception("Unexpected result from backup with return code 4");
+                if (res.ExaminedFiles <= 0)
+                    throw new Exception("Backup did not examine any files for code 4?");
+                
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(5);
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Error)
+                    throw new Exception("Unexpected result from backup with return code 5");
+                if (res.ExaminedFiles > 0)
+                    throw new Exception("Backup did examine files for code 5?");
+
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(2, "TEST WARNING MESSAGE");
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Warning)
+                    throw new Exception("Unexpected result from backup with return code 2");
+                if (res.ExaminedFiles <= 0)
+                    throw new Exception("Backup did examine files for code 2?");
+                if (!res.Warnings.Any(x => x.IndexOf("TEST WARNING MESSAGE", StringComparison.Ordinal) >= 0))
+                    throw new Exception("Found no warning message in output for code 2");
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(3, "TEST WARNING MESSAGE");
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Warning)
+                    throw new Exception("Unexpected result from backup with return code 3");
+                if (res.ExaminedFiles > 0)
+                    throw new Exception("Backup did examine files for code 3?");
+                if (!res.Warnings.Any(x => x.IndexOf("TEST WARNING MESSAGE", StringComparison.Ordinal) >= 0))
+                    throw new Exception("Found no warning message in output for code 3");
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(4, "TEST ERROR MESSAGE");
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Error)
+                    throw new Exception("Unexpected result from backup with return code 4");
+                if (res.ExaminedFiles <= 0)
+                    throw new Exception("Backup did examine files for code 4?");
+                if (!res.Errors.Any(x => x.IndexOf("TEST ERROR MESSAGE", StringComparison.Ordinal) >= 0))
+                    throw new Exception("Found no error message in output for code 4");
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(5, "TEST ERROR MESSAGE");
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Error)
+                    throw new Exception("Unexpected result from backup with return code 5");
+                if (res.ExaminedFiles > 0)
+                    throw new Exception("Backup did examine files for code 5?");
+                if (!res.Errors.Any(x => x.IndexOf("TEST ERROR MESSAGE", StringComparison.Ordinal) >= 0))
+                    throw new Exception("Found no error message in output for code 5");
+
+                System.Threading.Thread.Sleep(PAUSE_TIME);
+                options["run-script-before"] = CreateScript(0, sleeptime: 10);
+                res = c.Backup(new string[] { DATAFOLDER });
+                if (res.ParsedResult != ParsedResultType.Warning)
+                    throw new Exception("Unexpected result from backup with timeout script");
+                if (res.ExaminedFiles <= 0)
+                    throw new Exception("Backup did not examine any files after timeout?");
+            }
+        }
+
+
+        private string CreateScript(int exitcode, string stderr = null, string stdout = null, int sleeptime = 0)
+        {
+            var id = Guid.NewGuid().ToString("N").Substring(0, 6);
+            if (Library.Utility.Utility.IsClientWindows)
+            {
+                var commands = new List<string>();
+                if (!string.IsNullOrWhiteSpace(stdout))
+                    commands.Add($@"echo {stdout}");
+                if (!string.IsNullOrWhiteSpace(stderr))
+                    commands.Add($@"echo {stderr} 1>&2");
+                if (sleeptime > 0)
+                    commands.Add($@"sleep {sleeptime}");
+
+                commands.Add($"exit {exitcode}");
+
+                var filename = Path.GetFullPath(Path.Combine(DATAFOLDER, $"run-script-{id}.bat"));
+                File.WriteAllLines(filename, commands);
+
+                return filename;
+            }
+            else
+            {
+                var commands = new List<string>();
+                commands.Add("#!/bin/sh");
+
+                if (!string.IsNullOrWhiteSpace(stdout))
+                    commands.Add($@"echo {stdout}");
+                if (!string.IsNullOrWhiteSpace(stderr))
+                    commands.Add($@"(>&2 echo {stderr})");
+                if (sleeptime > 0)
+                    commands.Add($@"sleep {sleeptime}");
+
+                commands.Add($"exit {exitcode}");
+                var filename = Path.GetFullPath(Path.Combine(DATAFOLDER, $"run-script-{id}.sh"));
+                File.WriteAllLines(filename, commands);
+
+                System.Diagnostics.Process.Start("chmod", $@"+x ""{filename}""").WaitForExit();
+
+                return filename;
+            }
+        }
+    }
+}

Duplicati/UnitTest/UtilityTests.cs

@@ -24,6 +24,46 @@ namespace Duplicati.UnitTest
 {
     public class UtilityTests
     {
+        [Test]
+        [Category("Utility")]
+        [TestCase("da-DK")]
+        [TestCase("en-US")]
+        [TestCase("hu-HU")]
+        [TestCase("tr-TR")]
+        public static void FilenameStringComparison(string cultureName)
+        {
+            Action<string, string> checkStringComparison = (x, y) => Assert.IsFalse(String.Equals(x, y, Utility.ClientFilenameStringComparison));
+            Action<string, string> checkStringComparer = (x, y) => Assert.IsFalse(new HashSet<string>(new[] { x }).Contains(y, Utility.ClientFilenameStringComparer));
+
+            System.Globalization.CultureInfo originalCulture = System.Globalization.CultureInfo.CurrentCulture;
+            try
+            {
+                System.Threading.Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo(cultureName, false);
+
+                // These are equivalent with respect to hu-HU, but different with respect to en-US.
+                string ddzs = "ddzs";
+                string dzsdzs = "dzsdzs";
+                checkStringComparison(ddzs, dzsdzs);
+                checkStringComparer(ddzs, dzsdzs);
+
+                // Many cultures treat the following as equivalent.
+                string eAcuteOneCharacter = System.Text.Encoding.GetEncoding("iso-8859-1").GetString(new byte[] { 233 }); // 'é' as one character (ALT+0233).
+                string eAcuteTwoCharacters = "\u0065\u0301"; // 'e', combined with an acute accent (U+301).
+                checkStringComparison(eAcuteOneCharacter, eAcuteTwoCharacters);
+                checkStringComparer(eAcuteOneCharacter, eAcuteTwoCharacters);
+
+                // These are equivalent with respect to en-US, but different with respect to da-DK.
+                string aDiaeresisOneCharacter = "\u00C4"; // 'A' with a diaeresis.
+                string aDiaeresisTwoCharacters = "\u0041\u0308"; // 'A', combined with a diaeresis.
+                checkStringComparison(aDiaeresisOneCharacter, aDiaeresisTwoCharacters);
+                checkStringComparer(aDiaeresisOneCharacter, aDiaeresisTwoCharacters);
+            }
+            finally
+            {
+                System.Threading.Thread.CurrentThread.CurrentCulture = originalCulture;
+            }
+        }
+
         [Test]
         [Category("Utility")]
         public static void ForceStreamRead()
@@ -82,7 +122,7 @@ namespace Duplicati.UnitTest
 
             // Test with custom comparer.
             IEqualityComparer<string> comparer = StringComparer.OrdinalIgnoreCase;
-            uniqueItems = new string[] {"a", "b", "c"};
+            uniqueItems = new string[] { "a", "b", "c" };
             duplicateItems = new string[] { "a", "c" };
 
             actualDuplicateItems = null;

Duplicati/UnitTest/WebApiTests.cs

@@ -27,9 +27,7 @@ namespace Duplicati.UnitTest
         public static void GoogleCloudPutUrl()
         {
             string bucketId = "my_bucket";
-
-            var putUrl = "https://www.googleapis.com/upload/storage/v1/b/" +
-                bucketId + "?uploadType=resumable";
+            string putUrl = $"https://www.googleapis.com/upload/storage/v1/b/{bucketId}/o?uploadType=resumable";
             Assert.AreEqual(putUrl, GoogleCloudStorage.PutUrl(bucketId));
         }
 

Installer/OSX/make-dmg.sh

@@ -38,7 +38,8 @@ if [ ! -f "$1" ]; then
     exit
 fi
 
-VERSION_NUMBER=$(echo "$1" | awk -F- '{print $2}' | awk -F_ '{print $1}')
+ZIPNAME=$(basename "$1")
+VERSION_NUMBER=$(echo "$ZIPNAME" | awk -F- '{print $2}' | awk -F_ '{print $1}')
 
 VERSION_NAME="Duplicati"
 if [ -e "${OUTPUT_DMG}" ]; then
@@ -63,11 +64,9 @@ mkdir "Duplicati.app/Contents/Resources"
 # Extract the zip into the Resouces folder
 unzip -q "$1" -d "Duplicati.app/Contents/Resources"
 
-# Install the Info.plist and icon
-SHORT_VERSION_NUMBER=$(echo ${VERSION_NUMBER} | awk -F. '{printf $1; printf "."; printf $2; printf "."; print $3}')
+# Install the Info.plist and icon, patch the plist file as well
 PLIST=$(cat "Info.plist")
-PLIST=${PLIST/!LONG_VERSION!/${VERSION_NUMBER}}
-PLIST=${PLIST/!SHORT_VERSION!/${SHORT_VERSION_NUMBER}}
+PLIST=${PLIST//!LONG_VERSION!/${VERSION_NUMBER}}
 echo ${PLIST} > "Duplicati.app/Contents/Info.plist"
 cp "Duplicati.icns" "Duplicati.app/Contents/Resources"
 

Installer/Windows/UpdateVersion/Properties/AssemblyInfo.cs

@@ -1,4 +1,4 @@
-using System.Reflection;
+using System.Reflection;
 using System.Runtime.CompilerServices;
 
 // Information about this assembly is defined by the following attributes. 
@@ -17,7 +17,7 @@ using System.Runtime.CompilerServices;
 // The form "{Major}.{Minor}.*" will automatically update the build and revision,
 // and "{Major}.{Minor}.{Build}.*" will update just the revision.
 
-[assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("2.0.0.7")]
 
 // The following attributes are used to specify the signing key for the assembly, 
 // if desired. See the Mono documentation for more information about signing.