Merge remote-tracking branch master into feature/net5-avalonia

# Conflicts:
#	Duplicati CommandLine Only.sln
#	Duplicati.sln
#	Duplicati/CommandLine/BackendTester/Duplicati.CommandLine.BackendTester.csproj
#	Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj
#	Duplicati/CommandLine/Duplicati.CommandLine.csproj
#	Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj
#	Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj
#	Duplicati/GUI/Duplicati.GUI.TrayIcon/app.config
#	Duplicati/Library/AutoUpdater/UpdaterManager.cs
#	Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj
#	Duplicati/Library/Main/Duplicati.Library.Main.csproj
#	Duplicati/Library/Utility/Duplicati.Library.Utility.csproj
#	Duplicati/Library/Utility/Utility.cs
#	Duplicati/Server/Duplicati.Server.csproj
#	Duplicati/Server/app.config
#	Duplicati/Service/app.config
#	Duplicati/UnitTest/BackendToolTests.cs
#	Duplicati/UnitTest/Duplicati.UnitTest.csproj
#	guiTests/guiTest.py

Duplicati.sln

@@ -141,6 +141,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.CommandLine", "Ex
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "UnixSupport", "Duplicati\UnixSupport\UnixSupport.csproj", "{2B605B81-353F-43CD-BF3A-29B3926A204F}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Backend.Idrivee2", "Duplicati\Library\Backend\Idrivee2\Duplicati.Library.Backend.Idrivee2.csproj", "{6B594D23-B629-465C-B799-70EE9E56C218}"
+	ProjectSection(ProjectDependencies) = postProject
+		{C03F6DFD-805A-4BE0-9338-64870ADDB4A2} = {C03F6DFD-805A-4BE0-9338-64870ADDB4A2}
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -403,6 +408,10 @@ Global
 		{2B605B81-353F-43CD-BF3A-29B3926A204F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2B605B81-353F-43CD-BF3A-29B3926A204F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2B605B81-353F-43CD-BF3A-29B3926A204F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6B594D23-B629-465C-B799-70EE9E56C218}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6B594D23-B629-465C-B799-70EE9E56C218}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6B594D23-B629-465C-B799-70EE9E56C218}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6B594D23-B629-465C-B799-70EE9E56C218}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Duplicati/CommandLine/BackendTester/Duplicati.CommandLine.BackendTester.csproj

@@ -13,6 +13,8 @@
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Dropbox\Duplicati.Library.Backend.Dropbox.csproj">
     </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\Idrivee2\Duplicati.Library.Backend.Idrivee2.csproj">
+    </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Jottacloud\Duplicati.Library.Backend.Jottacloud.csproj">
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Rclone\Duplicati.Library.Backend.Rclone.csproj">
@@ -22,10 +24,8 @@
     <ProjectReference Include="..\..\Library\Backend\Sia\Duplicati.Library.Backend.Sia.csproj">
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
-
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
-
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\TencentCOS\Duplicati.Library.Backend.TencentCOS.csproj">
     </ProjectReference>

Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj

@@ -16,6 +16,8 @@
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\FTP\Duplicati.Library.Backend.FTP.csproj">
     </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\Idrivee2\Duplicati.Library.Backend.Idrivee2.csproj">
+    </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Jottacloud\Duplicati.Library.Backend.Jottacloud.csproj">
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Rclone\Duplicati.Library.Backend.Rclone.csproj">
@@ -31,10 +33,8 @@
     <ProjectReference Include="..\..\Library\Backend\TahoeLAFS\Duplicati.Library.Backend.TahoeLAFS.csproj">
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
-
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
-
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\TencentCOS\Duplicati.Library.Backend.TencentCOS.csproj">
     </ProjectReference>

Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj

@@ -23,6 +23,8 @@
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\HubiC\Duplicati.Library.Backend.HubiC.csproj">
     </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\Idrivee2\Duplicati.Library.Backend.Idrivee2.csproj">
+    </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Jottacloud\Duplicati.Library.Backend.Jottacloud.csproj">
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\OneDrive\Duplicati.Library.Backend.OneDrive.csproj">

Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj

@@ -71,6 +71,8 @@
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\FTP\Duplicati.Library.Backend.FTP.csproj">
     </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\Idrivee2\Duplicati.Library.Backend.Idrivee2.csproj">
+    </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Jottacloud\Duplicati.Library.Backend.Jottacloud.csproj">
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\OAuthHelper\Duplicati.Library.OAuthHelper.csproj">
@@ -86,12 +88,10 @@
     <ProjectReference Include="..\..\Library\Backend\TahoeLAFS\Duplicati.Library.Backend.TahoeLAFS.csproj">
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
-
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\TencentCOS\Duplicati.Library.Backend.TencentCOS.csproj">
-
     </ProjectReference>
     <ProjectReference Include="..\..\Library\Backend\WEBDAV\Duplicati.Library.Backend.WEBDAV.csproj">
     </ProjectReference>
@@ -160,12 +160,4 @@
     <ProjectReference Include="..\..\Tools\Duplicati.Tools.csproj">
     </ProjectReference>
   </ItemGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
-  </ItemGroup>
-
 </Project>

Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs

@@ -223,7 +223,16 @@ namespace Duplicati.GUI.TrayIcon
                 {
                     try
                     {
-                        System.Net.ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
+                        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
+
+                        try
+                        {
+                            //try TLS 1.3 (type not available on .NET < 4.8)
+                            ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | (SecurityProtocolType)12288;
+                        }
+                        catch (NotSupportedException)
+                        {
+                        }
 
                         using (Connection = new HttpServerConnection(serverURL, password, saltedpassword, databaseConnection != null ? PasswordSource.Database : PasswordSource.HostedServer, disableTrayIconLogin, options))
                         {

Duplicati/Library/AutoUpdater/UpdaterManager.cs

@@ -58,7 +58,7 @@ namespace Duplicati.Library.AutoUpdater
             : Environment.ExpandEnvironmentVariables(System.Environment.GetEnvironmentVariable(string.Format(BASEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)));
 
         //FIXME: Auto Update doesn't work with the .net5 version
-        private static readonly bool DISABLE_UPDATE_DOMAIN = true;//Utility.Utility.ParseBool(Environment.GetEnvironmentVariable(string.Format(SKIPUPDATE_ENVNAME_TEMPLATE, APPNAME)), false);
+        private static readonly bool DISABLE_UPDATE_DOMAIN = true;//Debugger.IsAttached || Utility.Utility.ParseBool(Environment.GetEnvironmentVariable(string.Format(SKIPUPDATE_ENVNAME_TEMPLATE, APPNAME)), false);
 
         public static bool RequiresRespawn { get; set; }
 

Duplicati/Library/Backend/Backblaze/Strings.cs

@@ -14,9 +14,9 @@ namespace Duplicati.Library.Backend.Strings {
         public static string NoB2UserIDError { get { return LC.L(@"No ""B2 Cloud Storage Account ID"" given"); } }
         public static string Description { get { return LC.L(@"This backend can read and write data to the Backblaze B2 Cloud Storage. Allowed formats are: ""b2://bucketname/prefix"""); } }
         public static string B2createbuckettypeDescriptionLong { get { return LC.L(@"By default, a private bucket is created. Use this option to set the bucket type. Refer to the B2 documentation for allowed types "); } }
-        public static string B2createbuckettypeDescriptionShort { get { return LC.L(@"The bucket type used when creating a bucket"); } }
-		public static string B2pagesizeDescriptionLong { get { return LC.L(@"Use this option to set the page size for listing contents of B2 buckets. A lower number means less data, but can increase the number of Class C transaction on B2. Suggested values are between 100 and 1000"); } }
-		public static string B2pagesizeDescriptionShort { get { return LC.L(@"The size of file-listing pages"); } }
+        public static string B2createbuckettypeDescriptionShort { get { return LC.L(@"The bucket type used when creating a bucket"); } }
+		public static string B2pagesizeDescriptionLong { get { return LC.L(@"Use this option to set the page size for listing contents of B2 buckets. A lower number means less data, but can increase the number of Class C transaction on B2. Suggested values are between 100 and 1000"); } }
+		public static string B2pagesizeDescriptionShort { get { return LC.L(@"The size of file-listing pages"); } }
         public static string B2downloadurlDescriptionLong { get { return LC.L(@"Change this if you want to use your custom domain to download files, and uploading will not be affected. The default download url depends on your account and looks like ""https://f00X.backblazeb2.com"""); } }
         public static string B2downloadurlDescriptionShort { get { return LC.L(@"The base URL to use for downloading files"); } }
         public static string InvalidPageSizeError(string argname, string value) { return LC.L(@"The setting ""{0}"" is invalid for ""{1}"", it must be an integer larger than zero", value, argname); }

Duplicati/Library/Backend/Dropbox/DropboxHelper.cs

@@ -18,7 +18,9 @@ namespace Duplicati.Library.Backend
             : base(accessToken, "dropbox")
         {
             base.AutoAuthHeader = true;
-            base.AccessTokenOnly = true;
+            // Pre 2022 tokens are direct Dropbox tokens (no ':')
+            // Post 2022-02-21 tokens are regular authid tokens (with a ':')
+            base.AccessTokenOnly = !accessToken.Contains(":");
         }
 
         public ListFolderResult ListFiles(string path)

Duplicati/Library/Backend/Dropbox/WebApi.cs

@@ -41,7 +41,7 @@ namespace Duplicati.Library.Backend.WebApi
             return Uri.UriBuilder(Url.API, Path.DeleteFolder);
         }
 
-        public static string UploadSessionStartUrl()
+        public static string UploadSessionStartUrl()
         {
             return Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionStart);
         }

Duplicati/Library/Backend/File/FileBackend.cs

@@ -22,6 +22,7 @@ using Duplicati.Library.Common.IO;
 using Duplicati.Library.Interface;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
@@ -36,6 +37,7 @@ namespace Duplicati.Library.Backend
         private const string OPTION_ALTERNATE_PATHS = "alternate-target-paths";
         private const string OPTION_MOVE_FILE = "use-move-for-put";
         private const string OPTION_FORCE_REAUTH = "force-smb-authentication";
+        private const string OPTION_DISABLE_LENGTH_VERIFICATION = "disable-length-verification";
 
         private readonly string m_path;
         private string m_username;
@@ -43,6 +45,7 @@ namespace Duplicati.Library.Backend
         private readonly bool m_moveFile;
         private bool m_hasAutenticated;
         private readonly bool m_forceReauth;
+        private readonly bool m_verifyDestinationLength;
 
         private readonly byte[] m_copybuffer = new byte[Utility.Utility.DEFAULT_BUFFER_SIZE];
 
@@ -56,7 +59,7 @@ namespace Duplicati.Library.Backend
         {
             var uri = new Utility.Uri(url);
             m_path = uri.HostAndPath;
-            
+
             if (options.ContainsKey("auth-username"))
                 m_username = options["auth-username"];
             if (options.ContainsKey("auth-password"))
@@ -65,7 +68,7 @@ namespace Duplicati.Library.Backend
                 m_username = uri.Username;
             if (!string.IsNullOrEmpty(uri.Password))
                 m_password = uri.Password;
-            
+
             if (!System.IO.Path.IsPathRooted(m_path))
                 m_path = systemIO.PathGetFullPath(m_path);
 
@@ -81,7 +84,7 @@ namespace Duplicati.Library.Backend
                 if (!Platform.IsClientPosix)
                 {
                     System.IO.DriveInfo[] drives = System.IO.DriveInfo.GetDrives();
-                    
+
                     for (int i = 0; i < paths.Count; i++)
                     {
                         if (paths[i].StartsWith("*:", StringComparison.Ordinal))
@@ -126,6 +129,7 @@ namespace Duplicati.Library.Backend
 
             m_moveFile = Utility.Utility.ParseBoolOption(options, OPTION_MOVE_FILE);
             m_forceReauth = Utility.Utility.ParseBoolOption(options, OPTION_FORCE_REAUTH);
+            m_verifyDestinationLength = Utility.Utility.ParseBoolOption(options, OPTION_DISABLE_LENGTH_VERIFICATION);
             m_hasAutenticated = false;
         }
 
@@ -195,10 +199,14 @@ namespace Duplicati.Library.Backend
             }
         }
 #else
-        public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
+        public async Task PutAsync(string targetFilename, Stream sourceStream, CancellationToken cancelToken)
         {
-            using (System.IO.FileStream writestream = systemIO.FileCreate(GetRemoteName(remotename)))
-                await Utility.Utility.CopyStreamAsync(stream, writestream, true, cancelToken, m_copybuffer);
+            string targetFilePath = GetRemoteName(targetFilename);
+            long copiedBytes = 0;
+            using (var targetStream = systemIO.FileCreate(targetFilePath))
+                copiedBytes = await Utility.Utility.CopyStreamAsync(sourceStream, targetStream, true, cancelToken, m_copybuffer);
+
+            VerifyMatchingSize(targetFilePath, sourceStream, copiedBytes);
         }
 #endif
 
@@ -209,18 +217,27 @@ namespace Duplicati.Library.Backend
                 Utility.Utility.CopyStream(readstream, stream, true, m_copybuffer);
         }
 
-        public Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
+        public Task PutAsync(string targetFilename, string sourceFilePath, CancellationToken cancelToken)
         {
-            string path = GetRemoteName(remotename);
+            string targetFilePath = GetRemoteName(targetFilename);
             if (m_moveFile)
             {
-                if (systemIO.FileExists(path))
-                    systemIO.FileDelete(path);
-                
-                systemIO.FileMove(filename, path);
+                if (systemIO.FileExists(targetFilePath))
+                    systemIO.FileDelete(targetFilePath);
+
+                var sourceFileInfo = new FileInfo(sourceFilePath);
+                var sourceFileLength = sourceFileInfo.Exists ? (long?)sourceFileInfo.Length : null;
+
+                systemIO.FileMove(sourceFilePath, targetFilePath);
+                if (m_verifyDestinationLength)
+                    VerifyMatchingSize(targetFilePath, null, sourceFileLength);
             }
             else
-                systemIO.FileCopy(filename, path, true);
+            {
+                systemIO.FileCopy(sourceFilePath, targetFilePath, true);
+                if (m_verifyDestinationLength)
+                    VerifyMatchingSize(targetFilePath, sourceFilePath);
+            }
 
             return Task.FromResult(true);
         }
@@ -246,6 +263,8 @@ namespace Duplicati.Library.Backend
                     new CommandLineArgument(OPTION_ALTERNATE_PATHS, CommandLineArgument.ArgumentType.Path, Strings.FileBackend.AlternateTargetPathsShort, Strings.FileBackend.AlternateTargetPathsLong(OPTION_DESTINATION_MARKER, System.IO.Path.PathSeparator)),
                     new CommandLineArgument(OPTION_MOVE_FILE, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.UseMoveForPutShort, Strings.FileBackend.UseMoveForPutLong),
                     new CommandLineArgument(OPTION_FORCE_REAUTH, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.ForceReauthShort, Strings.FileBackend.ForceReauthLong),
+                    new CommandLineArgument(OPTION_DISABLE_LENGTH_VERIFICATION, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.DisableLengthVerificationShort, Strings.FileBackend.DisableLengthVerificationShort),
+
                 });
 
             }
@@ -386,5 +405,51 @@ namespace Duplicati.Library.Backend
                 out ulong lpTotalNumberOfBytes,
                 out ulong lpTotalNumberOfFreeBytes);
         }
+
+        private static void VerifyMatchingSize(string targetFilePath, string sourceFilePath)
+        {
+            try
+            {
+                var targetFileInfo = new FileInfo(targetFilePath);
+                if (!targetFileInfo.Exists)
+                    throw new FileMissingException($"Target file does not exist. Target: {targetFilePath}");
+
+                var sourceFileInfo = new FileInfo(sourceFilePath);
+                if (!sourceFileInfo.Exists)
+                    throw new FileMissingException($"Source file does not exist. Source: {sourceFilePath}");
+
+                if (targetFileInfo.Length != sourceFileInfo.Length)
+                    throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from source file size ({sourceFileInfo.Length:n0}). Target: {targetFilePath}");
+            }
+            catch
+            {
+                try { System.IO.File.Delete(targetFilePath); } catch { }
+                throw;
+            }
+        }
+
+        private static void VerifyMatchingSize(string targetFilePath, Stream sourceStream, long? expectedLength)
+        {
+            try
+            {
+                var targetFileInfo = new FileInfo(targetFilePath);
+                if (!targetFileInfo.Exists)
+                    throw new FileMissingException($"Target file does not exist. Target: {targetFilePath}");
+
+                bool isStreamPostion = false;
+                long? sourceStreamLength = sourceStream == null ? null : Utility.Utility.GetStreamLength(sourceStream, out isStreamPostion);
+
+                if (sourceStreamLength.HasValue && targetFileInfo.Length != sourceStreamLength.Value)
+                    throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position)" : "")}. Target: {targetFilePath}");
+
+                if (expectedLength.HasValue && targetFileInfo.Length != expectedLength.Value)
+                    throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the expected length ({expectedLength.Value:n0}). Target: {targetFilePath}");
+            }
+            catch
+            {
+                try { System.IO.File.Delete(targetFilePath); } catch { }
+                throw;
+            }
+        }
     }
 }

Duplicati/Library/Backend/File/Strings.cs

@@ -17,5 +17,7 @@ namespace Duplicati.Library.Backend.Strings {
         public static string UseMoveForPutShort { get { return LC.L(@"Move the file instead of copying it"); } }
         public static string ForceReauthShort { get { return LC.L(@"Force authentication against remote share"); } }
         public static string ForceReauthLong { get { return LC.L(@"If this option is set, any existing authentication against the remote share is dropped before attempting to authenticate"); } }
+        public static string DisableLengthVerificationShort { get { return LC.L(@"Disable length verification"); } }
+        public static string DisableLengthVerificationLong { get { return LC.L(@"As an extra precaution the uploaded file length will be checked against the local source length."); } }
     }
 }

Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>    
+    <Copyright>LGPL, Copyright © Duplicati Team 2021</Copyright>
+    <Description>iDrivee2 backend for Duplicati</Description>
+  </PropertyGroup>
+  
+  <ItemGroup>
+    <ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
+    <ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
+    <ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
+    <ProjectReference Include="..\..\Modules\Builtin\Duplicati.Library.Modules.Builtin.csproj" />
+    <ProjectReference Include="..\S3\Duplicati.Library.Backend.S3.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+</Project>

Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs

@@ -0,0 +1,249 @@
+#region Disclaimer / License
+// Copyright (C) 2015, The Duplicati Team
+// http://www.duplicati.com, info@duplicati.com
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+//
+#endregion
+using Duplicati.Library.Common.IO;
+using Duplicati.Library.Interface;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Backend
+{
+    public class Idrivee2Backend : IBackend, IStreamingBackend
+    {
+        private static readonly string LOGTAG = Logging.Log.LogTagFromType<Idrivee2Backend>();
+
+        static Idrivee2Backend()
+        {
+            
+        }
+
+        private readonly string m_prefix;
+        private readonly string m_bucket;
+
+        private IS3Client m_s3Client;
+
+        public Idrivee2Backend()
+        {
+        }
+
+        public Idrivee2Backend(string url, Dictionary<string, string> options)
+        {
+            var uri = new Utility.Uri(url);
+            m_bucket = uri.Host;
+            m_prefix = uri.Path;
+            m_prefix = m_prefix.Trim();
+            if (m_prefix.Length != 0)
+            {
+                m_prefix = Util.AppendDirSeparator(m_prefix, "/");
+            }
+            string accessKeyId = null;
+            string accessKeySecret = null;
+
+            if (options.ContainsKey("auth-username"))
+                accessKeyId = options["auth-username"];
+            if (options.ContainsKey("auth-password"))
+                accessKeySecret = options["auth-password"];
+
+            if (options.ContainsKey("access_key_id"))
+                accessKeyId = options["access_key_id"];
+            if (options.ContainsKey("secret_access_key"))
+                accessKeySecret = options["secret_access_key"];
+
+            if (string.IsNullOrEmpty(accessKeyId))
+                throw new UserInformationException(Strings.Idrivee2Backend.NoKeyIdError, "Idrivee2NoKeyId");
+            if (string.IsNullOrEmpty(accessKeySecret))
+                throw new UserInformationException(Strings.Idrivee2Backend.NoKeySecretError, "Idrivee2NoKeySecret");
+            string host= GetRegionEndpoint("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId);
+
+
+            m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, options);
+
+        }
+
+        public string GetRegionEndpoint(string url)
+        {
+            try
+            {
+                System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
+                req.Method = System.Net.WebRequestMethods.Http.Get;
+                
+                Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
+               
+                using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
+                {
+                    int code = (int)resp.StatusCode;
+                    if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
+                        throw new Exception("Failed to fetch region endpoint");
+                    using (var s = areq.GetResponseStream())
+                    {
+                        using (var reader = new StreamReader(s))
+                        {
+                            string endpoint = reader.ReadToEnd();
+                            return endpoint;
+                        }
+                    }
+                }
+            }
+            catch (System.Net.WebException wex)
+            {
+                //Convert to better exception
+                throw new Exception("Failed to fetch region endpoint");
+            }
+        }
+
+        #region IBackend Members
+
+        public string DisplayName
+        {
+            get { return Strings.Idrivee2Backend.DisplayName; }
+        }
+
+        public string ProtocolKey => "e2";
+
+        public bool SupportsStreaming => true;
+
+
+        public IEnumerable<IFileEntry> List()
+        {
+            foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix))
+            {
+                ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length);
+                if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal))
+                    ((FileEntry)file).Name = file.Name.Substring(1);
+
+                yield return file;
+            }
+        }
+
+        public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken)
+        {
+            using (FileStream fs = File.Open(localname, FileMode.Open, FileAccess.Read, FileShare.Read))
+                await PutAsync(remotename, fs, cancelToken);
+        }
+
+        public async Task PutAsync(string remotename, Stream input, CancellationToken cancelToken)
+        {
+            await Connection.AddFileStreamAsync(m_bucket, GetFullKey(remotename), input, cancelToken);
+        }
+
+        public void Get(string remotename, string localname)
+        {
+            using (var fs = File.Open(localname, FileMode.Create, FileAccess.Write, FileShare.None))
+                Get(remotename, fs);
+        }
+
+        public void Get(string remotename, Stream output)
+        {
+            Connection.GetFileStream(m_bucket, GetFullKey(remotename), output);
+        }
+
+        public void Delete(string remotename)
+        {
+            Connection.DeleteObject(m_bucket, GetFullKey(remotename));
+        }
+
+        public IList<ICommandLineArgument> SupportedCommands
+        {
+            get
+            {
+                
+                var defaults = new Amazon.S3.AmazonS3Config();
+
+                var exts =
+                    typeof(Amazon.S3.AmazonS3Config).GetProperties().Where(x => x.CanRead && x.CanWrite && (x.PropertyType == typeof(string) || x.PropertyType == typeof(bool) || x.PropertyType == typeof(int) || x.PropertyType == typeof(long) || x.PropertyType.IsEnum))
+                        .Select(x => (ICommandLineArgument)new CommandLineArgument(
+                            "s3-ext-" + x.Name.ToLowerInvariant(),
+                            x.PropertyType == typeof(bool) ? CommandLineArgument.ArgumentType.Boolean : x.PropertyType.IsEnum ? CommandLineArgument.ArgumentType.Enumeration : CommandLineArgument.ArgumentType.String,
+                            x.Name,
+                            string.Format("Extended option {0}", x.Name),
+                            string.Format("{0}", x.GetValue(defaults)),
+                            null,
+                            x.PropertyType.IsEnum ? Enum.GetNames(x.PropertyType) : null));
+
+
+                var normal = new ICommandLineArgument[] {
+                  
+                    new CommandLineArgument("access_key_secret", CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, new[]{"auth-password"}, null),
+                    new CommandLineArgument("access_key_id", CommandLineArgument.ArgumentType.String, Strings.Idrivee2Backend.KeyIDDescriptionShort, Strings.Idrivee2Backend.KeyIDDescriptionLong,null, new[]{"auth-username"}, null)
+                 
+                };
+
+                return normal.Union(exts).ToList();
+
+            }
+        }
+
+        public string Description
+        {
+            get
+            {
+                return Strings.Idrivee2Backend.Description;
+            }
+        }
+
+        public void Test()
+        {
+            this.TestList();
+        }
+
+        public void CreateFolder()
+        {
+            //S3 does not complain if the bucket already exists
+            Connection.AddBucket(m_bucket);
+        }
+
+        #endregion
+
+        #region IRenameEnabledBackend Members
+
+        public void Rename(string source, string target)
+        {
+            Connection.RenameFile(m_bucket, GetFullKey(source), GetFullKey(target));
+        }
+
+        #endregion
+
+        #region IDisposable Members
+
+        public void Dispose()
+        {
+            m_s3Client?.Dispose();
+            m_s3Client = null;
+        }
+
+        #endregion
+
+        private IS3Client Connection => m_s3Client;
+
+        public string[] DNSName
+        {
+            get { return new[] { m_s3Client.GetDnsHost() }; }
+        }
+
+        private string GetFullKey(string name)
+        {
+            //AWS SDK encodes the filenames correctly
+            return m_prefix + name;
+        }
+    }
+}

Duplicati/Library/Backend/Idrivee2/Strings.cs

@@ -0,0 +1,17 @@
+using Duplicati.Library.Localization.Short;
+namespace Duplicati.Library.Backend.Strings {
+    internal static class Idrivee2Backend {
+        public static string KeySecretDescriptionLong { get { return LC.L(@"The ""Access Key Secret "" can be obtained after logging into your IDrive e2 account, this can also be supplied through the ""auth-password"" property."); } }
+        public static string KeySecretDescriptionShort { get { return LC.L(@"The ""Access Key Secret"""); } }
+        public static string KeyIDDescriptionLong { get { return LC.L(@"The ""Access Key ID"" can be obtained after logging into your IDrive e2 account., this can also be supplied through the ""auth-username"" property."); } }
+        public static string KeyIDDescriptionShort { get { return LC.L(@"The ""Access Key ID"""); } }
+
+        public static string BucketNameOrPathDescriptionLong { get { return LC.L(@"The ""Bucket Name or Complete Path"" is name of target bucket or complete of a folder inside the bucket."); } }
+        public static string BucketNameOrPathDescriptionShort { get { return LC.L(@"The ""Bucket Name or Complete Path"""); } }
+
+        public static string DisplayName { get { return LC.L(@"IDrive e2"); } }
+        public static string NoKeySecretError { get { return LC.L(@"No Access key secret given"); } }
+        public static string NoKeyIdError { get { return LC.L(@"No Access key Id given"); } }
+        public static string Description { get; set; }
+    }
+}

Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj

@@ -18,6 +18,6 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
   </ItemGroup>
-
-</Project>
+</Project>

Duplicati/Library/Backend/Jottacloud/Jottacloud.cs

@@ -32,9 +32,9 @@ namespace Duplicati.Library.Backend
     // This class is instantiated dynamically in the BackendLoader.
     public class Jottacloud : IBackend, IStreamingBackend
     {
+        private const string AUTHID_OPTION = "authid";
         private const string JFS_ROOT = "https://jfs.jottacloud.com/jfs";
         private const string JFS_ROOT_UPLOAD = "https://up.jottacloud.com/jfs"; // Separate host for uploading files
-        private const string API_VERSION = "2.4"; // Hard coded per 09. March 2017.
         private const string JFS_BUILTIN_DEVICE = "Jotta"; // The built-in device used for the built-in Sync and Archive mount points.
         private static readonly string JFS_DEFAULT_BUILTIN_MOUNT_POINT = "Archive"; // When using the built-in device we pick this mount point as our default.
         private static readonly string JFS_DEFAULT_CUSTOM_MOUNT_POINT = "Duplicati"; // When custom device is specified then we pick this mount point as our default.
@@ -52,7 +52,6 @@ namespace Duplicati.Library.Backend
         private readonly string m_url_device;
         private readonly string m_url;
         private readonly string m_url_upload;
-        private readonly System.Net.NetworkCredential m_userInfo;
         private readonly byte[] m_copybuffer = new byte[Duplicati.Library.Utility.Utility.DEFAULT_BUFFER_SIZE];
 
         private static readonly string JFS_DEFAULT_CHUNKSIZE = "5mb";
@@ -60,6 +59,8 @@ namespace Duplicati.Library.Backend
         private readonly int m_threads;
         private readonly long m_chunksize;
 
+        private readonly JottacloudAuthHelper m_oauth;
+
         /// <summary>
         /// The default maximum number of concurrent connections allowed by a ServicePoint object is 2.
         /// It should be increased to allow multiple download threads.
@@ -133,40 +134,21 @@ namespace Duplicati.Library.Backend
                     m_mountPoint = JFS_DEFAULT_CUSTOM_MOUNT_POINT; // Set a suitable default mount point for custom (backup) devices.
             }
 
+            string authid = null;
+            if (options.ContainsKey(AUTHID_OPTION))
+                authid = options[AUTHID_OPTION];
+            m_oauth = new JottacloudAuthHelper(authid);
+
             // Build URL
             var u = new Utility.Uri(url);
             m_path = u.HostAndPath; // Host and path of "jottacloud://folder/subfolder" is "folder/subfolder", so the actual folder path within the mount point.
             if (string.IsNullOrEmpty(m_path)) // Require a folder. Actually it is possible to store files directly on the root level of the mount point, but that does not seem to be a good option.
                 throw new UserInformationException(Strings.Jottacloud.NoPathError, "JottaNoPath");
             m_path = Util.AppendDirSeparator(m_path, "/");
-            if (!string.IsNullOrEmpty(u.Username))
-            {
-                m_userInfo = new System.Net.NetworkCredential();
-                m_userInfo.UserName = u.Username;
-                if (!string.IsNullOrEmpty(u.Password))
-                    m_userInfo.Password = u.Password;
-                else if (options.ContainsKey("auth-password"))
-                    m_userInfo.Password = options["auth-password"];
-            }
-            else
-            {
-                if (options.ContainsKey("auth-username"))
-                {
-                    m_userInfo = new System.Net.NetworkCredential();
-                    m_userInfo.UserName = options["auth-username"];
-                    if (options.ContainsKey("auth-password"))
-                        m_userInfo.Password = options["auth-password"];
-                }
-            }
-            if (m_userInfo == null || string.IsNullOrEmpty(m_userInfo.UserName))
-                throw new UserInformationException(Strings.Jottacloud.NoUsernameError, "JottaNoUsername");
-            if (m_userInfo == null || string.IsNullOrEmpty(m_userInfo.Password))
-                throw new UserInformationException(Strings.Jottacloud.NoPasswordError, "JottaNoPassword");
-            if (m_userInfo != null) // Bugfix, see http://connect.microsoft.com/VisualStudio/feedback/details/695227/networkcredential-default-constructor-leaves-domain-null-leading-to-null-object-reference-exceptions-in-framework-code
-                m_userInfo.Domain = "";
-            m_url_device = JFS_ROOT + "/" + m_userInfo.UserName + "/" + m_device;
+
+            m_url_device = JFS_ROOT + "/" + m_oauth.Username + "/" + m_device;
             m_url        = m_url_device + "/" + m_mountPoint + "/" + m_path;
-            m_url_upload = JFS_ROOT_UPLOAD + "/" + m_userInfo.UserName + "/" + m_device + "/" + m_mountPoint + "/" + m_path; // Different hostname, else identical to m_url.
+            m_url_upload = JFS_ROOT_UPLOAD + "/" + m_oauth.Username + "/" + m_device + "/" + m_mountPoint + "/" + m_path; // Different hostname, else identical to m_url.
 
             m_threads = int.Parse(options.ContainsKey(JFS_THREADS) ? options[JFS_THREADS] : JFS_DEFAULT_THREADS);
 
@@ -187,7 +169,7 @@ namespace Duplicati.Library.Backend
             m_chunksize = chunksize;
         }
 
-        #region IBackend Members
+#region IBackend Members
 
         public string DisplayName
         {
@@ -330,8 +312,7 @@ namespace Duplicati.Library.Backend
             get 
             {
                 return new List<ICommandLineArgument>(new ICommandLineArgument[] {
-                    new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.Jottacloud.DescriptionAuthPasswordShort, Strings.Jottacloud.DescriptionAuthPasswordLong),
-                    new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionAuthUsernameShort, Strings.Jottacloud.DescriptionAuthUsernameLong),
+                    new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Jottacloud.AuthidShort, Strings.Jottacloud.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("jottacloud"))),
                     new CommandLineArgument(JFS_DEVICE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionDeviceShort, Strings.Jottacloud.DescriptionDeviceLong(JFS_MOUNT_POINT_OPTION)),
                     new CommandLineArgument(JFS_MOUNT_POINT_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionMountPointShort, Strings.Jottacloud.DescriptionMountPointLong(JFS_DEVICE_OPTION)),
                     new CommandLineArgument(JFS_THREADS, CommandLineArgument.ArgumentType.Integer, Strings.Jottacloud.ThreadsShort, Strings.Jottacloud.ThreadsLong, JFS_DEFAULT_THREADS),
@@ -369,26 +350,19 @@ namespace Duplicati.Library.Backend
             }
         }
 
-        #endregion
+#endregion
 
-        #region IDisposable Members
+#region IDisposable Members
 
         public void Dispose()
         {
         }
 
-        #endregion
+#endregion
 
         private System.Net.HttpWebRequest CreateRequest(string method, string url, string queryparams)
         {
-            System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(url + (string.IsNullOrEmpty(queryparams) || queryparams.Trim().Length == 0 ? "" : "?" + queryparams));
-            req.Method = method;
-            req.Credentials = m_userInfo;
-            req.PreAuthenticate = true; // We need this under Mono for some reason, and it appears some servers require this as well
-            req.KeepAlive = false;
-            req.UserAgent = "Duplicati Jottacloud Client v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
-            req.Headers.Add("x-jftp-version", API_VERSION);
-            return req;
+            return m_oauth.CreateRequest(url + (string.IsNullOrEmpty(queryparams) || queryparams.Trim().Length == 0 ? "" : "?" + queryparams), method);
         }
 
         private System.Net.HttpWebRequest CreateRequest(string method, string remotename, string queryparams, bool upload)

Duplicati/Library/Backend/Jottacloud/JottacloudAuthHelper.cs

@@ -0,0 +1,71 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using Duplicati.Library.Interface;
+using Newtonsoft.Json;
+
+namespace Duplicati.Library.Backend
+{
+    public class JottacloudAuthHelper : OAuthHelper
+    {
+        private const string USERINFO_URL = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/userinfo";
+        private string m_username;
+
+        public JottacloudAuthHelper(string accessToken)
+            : base(accessToken, "jottacloud")
+        {
+            base.AutoAuthHeader = true;
+            base.AutoV2 = false; // Jottacloud is not v2 compatible because it generates a new refresh token with every access token refresh and invalidates the old.
+
+            var userinfo = GetJSONData<UserInfo>(USERINFO_URL);
+            if (userinfo == null || string.IsNullOrEmpty(userinfo.Username))
+                throw new UserInformationException(Strings.Jottacloud.NoUsernameError, "JottaNoUsername");
+            m_username = userinfo.Username;
+        }
+
+        public string Username
+        {
+            get
+            {
+                return m_username;
+            }
+        }
+
+        private class UserInfo
+        {
+            [JsonProperty("sub")]
+            public string Subject { get; set; }
+            [JsonProperty("email_verified")]
+            public bool EmailVerified { get; set; }
+            [JsonProperty("name")]
+            public string Name { get; set; }
+            [JsonProperty("realm")]
+            public string Realm { get; set; }
+            [JsonProperty("preferred_username")]
+            public string PreferredUsername { get; set; } // The numeric internal username, same as Username
+            [JsonProperty("given_name")]
+            public string GivenName { get; set; }
+            [JsonProperty("family_name")]
+            public string FamilyName { get; set; }
+            [JsonProperty("email")]
+            public string Email { get; set; }
+            [JsonProperty("username")]
+            public string Username { get; set; } // The numeric internal username, same as PreferredUsername
+        }
+    }
+
+}
+

Duplicati/Library/Backend/Jottacloud/Strings.cs

@@ -4,15 +4,12 @@ namespace Duplicati.Library.Backend.Strings {
     {
         public static string DisplayName { get { return LC.L(@"Jottacloud"); } }
         public static string Description { get { return LC.L(@"This backend can read and write data to Jottacloud using it's REST protocol. Allowed format is ""jottacloud://folder/subfolder""."); } }
-        public static string NoUsernameError { get { return LC.L(@"No username given"); } }
-        public static string NoPasswordError { get { return LC.L(@"No password given"); } }
+        public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
+        public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
+        public static string NoUsernameError { get { return LC.L(@"No username found"); } }
         public static string NoPathError { get { return LC.L(@"No path given, cannot upload files to the root folder"); } }
         public static string IllegalMountPoint { get { return LC.L(@"Illegal mount point given."); } }
         public static string FileUploadError { get { return LC.L(@"Failed to upload file"); } }
-        public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supplies the username used to connect to the server"); } }
-        public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
-        public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supplies the password used to connect to the server"); } }
-        public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
         public static string DescriptionDeviceShort { get { return LC.L(@"Supplies the backup device to use"); } }
         public static string DescriptionDeviceLong(string mountPointOption) { return LC.L(@"The backup device to use. Will be created if not already exists. You can manage your devices from the backup panel in the Jottacloud web interface. When you specify a custom device you should also specify the mount point to use on this device with the ""{0}"" option.", mountPointOption); }
         public static string DescriptionMountPointShort { get { return LC.L(@"Supplies the mount point to use on the server"); } }

Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs

@@ -83,6 +83,16 @@ namespace Duplicati.Library
         /// Set to true if the provider does not use refresh tokens, but only access tokens
         /// </summary>
         public bool AccessTokenOnly { get; set; }
+        /// <summary>
+        /// If true (the default), when a v1 authid is being used it will be swapped
+        /// with a v2 authid, when the OAuth service returns one (which it dypically
+        /// does after a provider token refresh has been performed). Some providers
+        /// are not compatible with v2 authid, tyically because they generate a new
+        /// refresh token with every access token refresh and invalidates the old.
+        /// If the oauth service still returns a v2 authid for such a provider,
+        /// set this property to false to make Duplicati ignore it.
+        /// </summary>
+        public bool AutoV2 { get; set; } = true;
 
         public OAuthHelper(string authid, string servicename, string useragent = null)
             : base(useragent)
@@ -134,7 +144,7 @@ namespace Duplicati.Library
                             var res = GetTokenResponse<OAuth_Service_Response>();
 
                             m_tokenExpires = DateTime.UtcNow.AddSeconds(res.expires - 30);
-                            if (!string.IsNullOrWhiteSpace(res.v2_authid))
+                            if (AutoV2 && !string.IsNullOrWhiteSpace(res.v2_authid))
                                 m_authid = res.v2_authid;
                             return m_token = res.access_token;
                         }

Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs

@@ -68,6 +68,10 @@ namespace Duplicati.Library.Backend.OpenStack
             new KeyValuePair<string, string>("OVH Cloud Storage", "https://auth.cloud.ovh.net/v3"),
             new KeyValuePair<string, string>("Selectel Cloud Storage", "https://auth.selcdn.ru"),
             new KeyValuePair<string, string>("Memset Cloud Storage", "https://auth.storage.memset.com"),
+            new KeyValuePair<string, string>("Infomaniak Swiss Backup cluster 1", "https://swiss-backup.infomaniak.com/identity/v3"),
+            new KeyValuePair<string, string>("Infomaniak Swiss Backup cluster 2", "https://swiss-backup02.infomaniak.com/identity/v3"),
+            new KeyValuePair<string, string>("Infomaniak Swiss Backup cluster 3", "https://swiss-backup03.infomaniak.com/identity/v3"),
+            new KeyValuePair<string, string>("Infomaniak Public Cloud 1", "https://api.pub1.infomaniak.cloud/identity/v3"),
         };
 
         public static readonly KeyValuePair<string, string>[] OPENSTACK_VERSIONS = {

Duplicati/Library/Backend/S3/S3Backend.cs

@@ -56,6 +56,10 @@ namespace Duplicati.Library.Backend
             { "Wasabi Hot Storage", "s3.wasabisys.com" },
             { "Wasabi Hot Storage (US West)", "s3.us-west-1.wasabisys.com" },
             { "Wasabi Hot Storage (EU Central)", "s3.eu-central-1.wasabisys.com" },
+            { "Infomaniak Swiss Backup cluster 1", "s3.swiss-backup.infomaniak.com" },
+            { "Infomaniak Swiss Backup cluster 2", "s3.swiss-backup02.infomaniak.com" },
+            { "Infomaniak Swiss Backup cluster 3", "s3.swiss-backup03.infomaniak.com" },
+            { "Infomaniak Public Cloud 1", "s3.pub1.infomaniak.cloud" },
         };
 
         //Updated list: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region

Duplicati/Library/Common/IO/SystemIOWindows.cs

@@ -508,7 +508,19 @@ namespace Duplicati.Library.Common.IO
         public IFileEntry FileEntry(string path)
         {
             var fileInfo = new FileInfo(AddExtendedDevicePathPrefix(path));
-            return new FileEntry(fileInfo.Name, fileInfo.Length, fileInfo.LastAccessTime, fileInfo.LastWriteTime);
+            var lastAccess = new DateTime();
+            try
+            {
+                // Internally this will convert the FILETIME value from Windows API to a
+                // DateTime. If the value represents a date after 12/31/9999 it will throw
+                // ArgumentOutOfRangeException, because this is not supported by DateTime.
+                // Some file systems seem to set strange access timestamps on files, which
+                // may lead to this exception being thrown. Since the last accessed
+                // timestamp is not important such exeptions are just silently ignored.
+                lastAccess = fileInfo.LastAccessTime;
+            }
+            catch { }
+            return new FileEntry(fileInfo.Name, fileInfo.Length, lastAccess, fileInfo.LastWriteTime);
         }
 
         public Dictionary<string, string> GetMetadata(string path, bool isSymlink, bool followSymlink)

Duplicati/Library/Main/AsyncDownloader.cs

@@ -13,6 +13,8 @@ namespace Duplicati.Library.Main
 
     internal class AsyncDownloader : IEnumerable<IAsyncDownloadedFile>
     {
+        private static readonly string LOGTAG = Logging.Log.LogTagFromType<AsyncDownloader>();
+
         private class AsyncDownloaderEnumerator : IEnumerator<IAsyncDownloadedFile>
         {
             private class AsyncDownloadedFile : IAsyncDownloadedFile
@@ -110,6 +112,7 @@ namespace Duplicati.Library.Main
                 }
                 catch (Exception ex)
                 {
+                    Logging.Log.WriteErrorMessage(LOGTAG, "FailedToRetrieveFile", ex, "Failed to retrieve file {0}", m_volumes[m_index].Name);
                     exception = ex;
                 }
                 

Duplicati/Library/Main/Database/Database schema/12. Performance Indexes.sql

@@ -0,0 +1,12 @@
+CREATE INDEX "nnc_Metadataset" ON Metadataset ("ID","BlocksetID");
+CREATE INDEX "nn_FilesetentryFile" on FilesetEntry ("FilesetID","FileID");
+
+-- Line 602 & 603 LocalBackupDatabase.cs 
+-- CREATE INDEX "tmpName1" ON "{0}" ("Path"),tmpName1
+-- CREATE INDEX "tmpName2" ON "{0}" ("Path"),tmpName2
+
+CREATE INDEX "nn_FileLookup_BlockMeta" ON FileLookup ("BlocksetID", "MetadataID");
+
+CREATE INDEX "nnc_BlocksetEntry" ON "BlocksetEntry" ("Index", "BlocksetID", "BlockID");
+
+UPDATE "Version" SET "Version" = 12;

Duplicati/Library/Main/Database/Database schema/Schema.sql

@@ -81,6 +81,8 @@ CREATE TABLE "FilesetEntry" (
 
 /* Improved reverse lookup for joining Fileset and File table */
 CREATE INDEX "FilesetentryFileIdIndex" on "FilesetEntry" ("FileID");
+CREATE INDEX "nn_FilesetentryFile" on FilesetEntry ("FilesetID","FileID");
+
 
 
 /*
@@ -109,6 +111,9 @@ CREATE TABLE "FileLookup" (
 
 /* Fast path based lookup, single properties are auto-indexed */
 CREATE UNIQUE INDEX "FileLookupPath" ON "FileLookup" ("PrefixID", "Path", "BlocksetID", "MetadataID");
+CREATE INDEX "nn_FileLookup_BlockMeta" ON FileLookup ("BlocksetID", "MetadataID");
+
+
 
 /*
 The File view contains an ID
@@ -167,7 +172,7 @@ CREATE TABLE "BlocksetEntry" (
 
 /* As this table is a cross table we need fast lookup */
 CREATE INDEX "BlocksetEntry_IndexIdsBackwards" ON "BlocksetEntry" ("BlockID");
-
+CREATE INDEX "nnc_BlocksetEntry" ON "BlocksetEntry" ("Index", "BlocksetID", "BlockID");
 
 /*
 The individual block hashes,
@@ -222,6 +227,8 @@ CREATE TABLE "Metadataset" (
 );
 
 CREATE INDEX "MetadatasetBlocksetID" ON "Metadataset" ("BlocksetID");
+CREATE INDEX "nnc_Metadataset" ON Metadataset ("ID","BlocksetID");
+
 
 /*
 Operations performed on the backend,
@@ -280,4 +287,4 @@ CREATE TABLE "ChangeJournalData" (
     "ConfigHash" TEXT NOT NULL  
 );
 
-INSERT INTO "Version" ("Version") VALUES (11);
+INSERT INTO "Version" ("Version") VALUES (12);

Duplicati/Library/Main/Database/LocalBackupDatabase.cs

@@ -601,6 +601,9 @@ namespace Duplicati.Library.Main.Database
 
                     cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName1), lastFilesetId);
                     cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName2), m_filesetId);
+                    cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""nn_tmpName1"" ON ""{0}"" (""Path"")",tmpName1));
+                    cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""nn_tmpName2"" ON ""{0}"" (""Path"")",tmpName2));
+
 
                     results.AddedFiles = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{0}"")", tmpName1), 0, m_filesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
                     results.DeletedFiles = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""{0}"" WHERE ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", tmpName1), 0, m_filesetId);
@@ -691,9 +694,9 @@ namespace Duplicati.Library.Main.Database
         public void AppendFilesFromPreviousSetWithPredicate(System.Data.IDbTransaction transaction,
             Func<string, long, bool> exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp)
         {
-            if (exclusionPredicate == null)
-            {
-                AppendFilesFromPreviousSet(transaction, null, fileSetId, prevFileSetId, timestamp);
+            if (exclusionPredicate == null)
+            {
+                AppendFilesFromPreviousSet(transaction, null, fileSetId, prevFileSetId, timestamp);
                 return;
             }
 
@@ -704,7 +707,7 @@ namespace Duplicati.Library.Main.Database
                 long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId;
 
                 // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup
-                var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
+                var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                 cmd.Transaction = tr.Parent;
                 cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?))", lastFilesetId, fileSetId);
 
@@ -736,10 +739,10 @@ namespace Duplicati.Library.Main.Database
                 cmd.ExecuteNonQuery($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") 
                                       SELECT ?, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}""", fileSetId);
 
-                tr.Commit();
+                tr.Commit();
             }
-        }
-
+        }
+
         /// <summary>
         /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
         /// </summary>
@@ -782,23 +785,23 @@ namespace Duplicati.Library.Main.Database
                     );
                 else
                     return default(RemoteVolumeEntry);
-        }
-
-        public IEnumerable<string> GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction transaction = null)
-        {
-            var incompleteFilesetIDs = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).Select(x => x.Key).ToArray();
-
-            if (!incompleteFilesetIDs.Any())
-                return Enumerable.Empty<string>();
-
-            if (latestOnly)
-                incompleteFilesetIDs = new long[] { incompleteFilesetIDs.Last() };
-
-            var volumeNames = new List<string>();
-            foreach (var filesetID in incompleteFilesetIDs)
-                volumeNames.Add(GetRemoteVolumeFromFilesetID(filesetID).Name);
-
-            return volumeNames;
+        }
+
+        public IEnumerable<string> GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction transaction = null)
+        {
+            var incompleteFilesetIDs = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).Select(x => x.Key).ToArray();
+
+            if (!incompleteFilesetIDs.Any())
+                return Enumerable.Empty<string>();
+
+            if (latestOnly)
+                incompleteFilesetIDs = new long[] { incompleteFilesetIDs.Last() };
+
+            var volumeNames = new List<string>();
+            foreach (var filesetID in incompleteFilesetIDs)
+                volumeNames.Add(GetRemoteVolumeFromFilesetID(filesetID).Name);
+
+            return volumeNames;
         }
 
         public IEnumerable<string> GetMissingIndexFiles(System.Data.IDbTransaction transaction)

Duplicati/Library/Main/DatabaseLocator.cs

@@ -73,9 +73,6 @@ namespace Duplicati.Library.Main
 					folder = newlocation;
 			}
 
-            if (!System.IO.Directory.Exists(folder))
-                System.IO.Directory.CreateDirectory(folder);
-                
             var file = System.IO.Path.Combine(folder, "dbconfig.json");
             List<BackendEntry> configs;
             if (!System.IO.File.Exists(file))
@@ -179,7 +176,10 @@ namespace Duplicati.Library.Main
                     Databasepath = newpath, 
                     ParameterFile = null
                 });
-                
+
+                if (!System.IO.Directory.Exists(folder))
+                    System.IO.Directory.CreateDirectory(folder);
+
                 var settings = new Newtonsoft.Json.JsonSerializerSettings();
                 settings.Formatting = Newtonsoft.Json.Formatting.Indented;
                 System.IO.File.WriteAllText(file, Newtonsoft.Json.JsonConvert.SerializeObject(configs, settings), System.Text.Encoding.UTF8);

Duplicati/Library/Main/Duplicati.Library.Main.csproj

@@ -46,6 +46,7 @@
     <EmbeddedResource Include="Database\Database schema\9. Refactor Paths.sql" />
     <EmbeddedResource Include="Database\Database schema\10. Add IsFullBackup to Fileset table.sql" />
     <EmbeddedResource Include="Database\Database schema\11. Add Block indices.sql" />
+    <EmbeddedResource Include="Database\Database schema\12. Performance Indexes.sql" />
   </ItemGroup>
   
   <ItemGroup>
@@ -61,4 +62,4 @@
     </PackageReference>
   </ItemGroup>
 
-</Project>
+</Project>

Duplicati/Library/Main/Options.cs

@@ -1646,7 +1646,7 @@ namespace Duplicati.Library.Main
         /// </summary>
         public bool PatchWithLocalBlocks
         {
-            get { return m_options.ContainsKey("patch-with-local-blocks"); }
+            get { return Library.Utility.Utility.ParseBoolOption(m_options, "patch-with-local-blocks"); }
         }
 
         /// <summary>

Duplicati/Library/Main/default_compressed_extensions.txt

@@ -171,8 +171,12 @@
 # Other Compressed or High-Entropy Files
 .apk #Android Application Package
 .eftx #Microsoft Office Theme Effects
+.mrimg #Macrium Reflect disk image format
+.rvz #Dolphin compression storage format
 .sdg #OpenOffice.org Configuration File	
 .thmx #Microsoft Office Theme
+.tib #Acronis True Image disk image format
+.tibx #Acronis True Image disk image format
 .vsix #Microsoft Visual Studio Extension
 .vsv #Microsoft Virtual Machine Saved State	
 .wmz #Windows Media Player Skin Package	

Duplicati/Library/Snapshots/HyperVUtility.cs

@@ -302,7 +302,16 @@ namespace Duplicati.Library.Snapshots
                                        select ((string[])systemBaseObj["Connection"])[0]).ToList();
 
                         foreach (var vhd in tempvhd)
-                            result.Add(vhd);
+                        {
+                            if (File.Exists(vhd))
+                            {
+                                result.Add(vhd);
+                            }
+                            else
+                            {
+                                Logging.Log.WriteWarningMessage(LOGTAG, "HyperVInvalidVhd", null, "Invalid VHD file detected, file does not exist: {0}", vhd);
+                            }
+                        }
                     }
             }
 
@@ -319,11 +328,16 @@ namespace Duplicati.Library.Snapshots
                         if (outParams != null)
                         {
                             var doc = new System.Xml.XmlDocument();
-                            doc.LoadXml((string)outParams[_wmiv2Namespace ? "SettingData" : "Info"]);
-                            var node = doc.SelectSingleNode("//PROPERTY[@NAME = 'ParentPath']/VALUE/child::text()");
+                            var propertyValue = (string)outParams[_wmiv2Namespace ? "SettingData" : "Info"];
 
-                            if (node != null && File.Exists(node.Value))
-                                ParentPaths.Add(node.Value);
+                            if (propertyValue != null)
+                            {
+                                doc.LoadXml(propertyValue);
+                                var node = doc.SelectSingleNode("//PROPERTY[@NAME = 'ParentPath']/VALUE/child::text()");
+
+                                if (node != null && File.Exists(node.Value))
+                                    ParentPaths.Add(node.Value);
+                            }
                         }
                     }
                 }

Duplicati/Library/Utility/Duplicati.Library.Utility.csproj

@@ -24,4 +24,4 @@
     </PackageReference>
   </ItemGroup>
 
-</Project>
+</Project>

Duplicati/Library/Utility/KeyGenerator.cs

@@ -1,92 +0,0 @@
-#region Disclaimer / License
-// Copyright (C) 2015, The Duplicati Team
-// http://www.duplicati.com, info@duplicati.com
-// 
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-// 
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-// Lesser General Public License for more details.
-// 
-// You should have received a copy of the GNU Lesser General Public
-// License along with this library; if not, write to the Free Software
-// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-// 
-#endregion
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace Duplicati.Library.Utility
-{
-    public static class KeyGenerator
-    {
-        public static readonly char[] ALPHA_LOWER = new char[] { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' };
-        public static readonly char[] ALPHA_UPPER = new char[] { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z' };
-        public static readonly char[] DIGITS = new char[] { '0', '2', '3', '4', '5', '6', '7', '8', '9' };
-        public static readonly char[] PUNCTUATION = new char[] { '.', ',', ':', ';', '!', '?' };
-        public static readonly char[] MISC = new char[] { '*', '@', '#', '$', '%', '&', '/', '(', ')', '=', '+', '>', '<', '-', '_' };
-
-        public static readonly char[] HEX_CHARS = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
-
-        private static char[] _allowed = null;
-        private static Random _rnd = null;
-
-        public static char[] AllowedChars
-        {
-            get
-            {
-                if (_allowed == null)
-                {
-                    List<char> l = new List<char>();
-                    l.AddRange(ALPHA_LOWER);
-                    l.AddRange(ALPHA_UPPER);
-                    l.AddRange(DIGITS);
-                    l.AddRange(PUNCTUATION);
-                    l.AddRange(MISC);
-                    _allowed = l.ToArray();
-                }
-
-                return _allowed;
-            }
-        }
-
-        public static Random Rnd
-        {
-            get
-            {
-                if (_rnd == null)
-                    _rnd = new Random();
-                return _rnd;
-            }
-        }
-
-        public static string GenerateKey(int minChars, int maxChars)
-        {
-            char[] tmp = new char[Rnd.Next(minChars, maxChars + 1)];
-            for (int i = 0; i < tmp.Length; i++)
-                tmp[i] = AllowedChars[Rnd.Next(0, AllowedChars.Length)];
-
-            return new string(tmp);
-        }
-
-        public static string GenerateSignKey()
-        {
-            byte[] tmp = new byte[4];
-            Rnd.NextBytes(tmp);
-
-            char[] res = new char[8];
-            for (int i = 0; i < tmp.Length; i++)
-            {
-                res[i * 2] = HEX_CHARS[tmp[i] & 0xF];
-                res[i * 2 + 1] = HEX_CHARS[(tmp[i] >> 4) & 0xF];
-            }
-
-            return new string(res);
-        }
-    }
-}

Duplicati/Library/Utility/Utility.cs

@@ -135,10 +135,49 @@ namespace Duplicati.Library.Utility
                 await target.WriteAsync(buf, 0, read, cancelToken).ConfigureAwait(false);
                 total += read;
             }
-            
+
             return total;
         }
 
+        /// <summary>
+        /// Get the length of a stream.
+        /// Attempt to use the stream's Position property if allowPositionFallback is <c>true</c> (only valid if stream is at the end).
+        /// </summary>
+        /// <param name="stream">Stream to get the length of.</param>
+        /// <param name="allowPositionFallback">Attempt to use the Position property if <c>true</c> and the Length property is not available (only valid if stream is at the end).</param>
+        /// <returns>Returns the stream's length, if available, or null if not supported by the stream.</returns>
+        public static long? GetStreamLength(Stream stream, bool allowPositionFallback = true)
+        {
+            return GetStreamLength(stream, out bool _, allowPositionFallback);
+        }
+
+        /// <summary>
+        /// Get the length of a stream.
+        /// Attempt to use the stream's Position property if allowPositionFallback is <c>true</c> (only valid if stream is at the end).
+        /// </summary>
+        /// <param name="stream">Stream to get the length of.</param>
+        /// <param name="isStreamPosition">Indicates if the Position value was used instead of Length.</param>
+        /// <param name="allowPositionFallback">Attempt to use the Position property if <c>true</c> and the Length property is not available (only valid if stream is at the end).</param>
+        /// <returns>Returns the stream's length, if available, or null if not supported by the stream.</returns>
+        public static long? GetStreamLength(Stream stream, out bool isStreamPosition, bool allowPositionFallback = true)
+        {
+            isStreamPosition = false;
+            long? streamLength = null;
+            try { streamLength = stream.Length; } catch { }
+            if (!streamLength.HasValue && allowPositionFallback)
+            {
+                try
+                {
+                    // Hack: This is a fall-back method to detect the source stream size, assuming the current position is the end of the stream.
+                    streamLength = stream.Position;
+                    isStreamPosition = true;
+                }
+                catch { } // 
+            }
+
+            return streamLength;
+        }
+
         /// <summary>
         /// These are characters that must be escaped when using a globbing expression
         /// </summary>

Duplicati/Server/Duplicati.Server.csproj

@@ -35,6 +35,8 @@
     </ProjectReference>
     <ProjectReference Include="..\Library\Backend\FTP\Duplicati.Library.Backend.FTP.csproj">
     </ProjectReference>
+    <ProjectReference Include="..\Library\Backend\Idrivee2\Duplicati.Library.Backend.Idrivee2.csproj">
+    </ProjectReference>
     <ProjectReference Include="..\Library\Backend\Jottacloud\Duplicati.Library.Backend.Jottacloud.csproj">
     </ProjectReference>
     <ProjectReference Include="..\Library\Backend\OAuthHelper\Duplicati.Library.OAuthHelper.csproj">
@@ -50,7 +52,6 @@
     <ProjectReference Include="..\Library\Backend\SSHv2\Duplicati.Library.Backend.SSHv2.csproj">
     </ProjectReference>
     <ProjectReference Include="..\Library\Backend\TahoeLAFS\Duplicati.Library.Backend.TahoeLAFS.csproj">
-
     </ProjectReference>
     <ProjectReference Include="..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
     </ProjectReference>

Duplicati/Server/WebServer/RESTMethods/ServerSettings.cs

@@ -75,10 +75,10 @@ namespace Duplicati.Server.WebServer.RESTMethods
                 var serversettings = data.Where(x => !string.IsNullOrWhiteSpace(x.Key)).ToDictionary(x => x.Key, x => x.Key.StartsWith("--", StringComparison.Ordinal) ? null : x.Value);
                 var globalsettings = data.Where(x => !string.IsNullOrWhiteSpace(x.Key) && x.Key.StartsWith("--", StringComparison.Ordinal));
 
-                serversettings.Remove("server-ssl-certificate");
-				serversettings.Remove("ServerSSLCertificate");
+                serversettings.Remove("server-ssl-certificate");
+				serversettings.Remove("ServerSSLCertificate");
 
-                if (serversettings.Any())
+                if (serversettings.Any())
 				    Program.DataConnection.ApplicationSettings.UpdateSettings(serversettings, false);
 
                 if (globalsettings.Any())

Duplicati/Server/WebServer/Server.cs

@@ -158,11 +158,23 @@ namespace Duplicati.Server.WebServer
                     // so we create a new server for each attempt
                 
                     var server = CreateServer(options);
-                    
+
                     if (!certValid)
                         server.Start(listenInterface, p);
                     else
-                        server.Start(listenInterface, p, cert, System.Security.Authentication.SslProtocols.Tls11 | System.Security.Authentication.SslProtocols.Tls12, null, false);
+                    {
+                        var secProtocols = System.Security.Authentication.SslProtocols.Tls12;
+
+                        try
+                        { 
+                            //try TLS 1.3 (type not available on .NET < 4.8)
+                            secProtocols = System.Security.Authentication.SslProtocols.Tls12 | (System.Security.Authentication.SslProtocols)12288;
+                        }
+                        catch (NotSupportedException)
+                        {
+                        }
+                        server.Start(listenInterface, p, cert, secProtocols, null, false);
+                    }
 
                     m_server = server;
                     m_server.ServerName = string.Format("{0} v{1}", Library.AutoUpdater.AutoUpdateSettings.AppName, System.Reflection.Assembly.GetExecutingAssembly().GetName().Version);

Duplicati/Server/webroot/ngax/index.html

@@ -283,9 +283,9 @@
     <notification-area></notification-area>
 
     <div ng-controller="DialogController">
-        <div ng-show="state.CurrentItem != null" class="blocker" ng-click=""></div>
+        <div ng-show="state.CurrentItem != null && state.CurrentItem.title != null" class="blocker" ng-click=""></div>
 
-        <div ng-show="state.CurrentItem != null" class="modal-dialog ng-cloak">
+        <div ng-show="state.CurrentItem != null && state.CurrentItem.title != null" class="modal-dialog ng-cloak">
             <div class="info">
                 <div class="title">{{state.CurrentItem.title}}</div>
                 <div class="content" ng-show="state.CurrentItem.message || state.CurrentItem.enableTextarea">

Duplicati/Server/webroot/ngax/scripts/controllers/StateController.js

@@ -29,7 +29,7 @@ backupApp.controller('StateController', function($scope, $timeout, ServerStatus,
             text = ServerStatus.progress_state_text[$scope.state.lastPgEvent.Phase || ''] || $scope.state.lastPgEvent.Phase;
 
 
-            if ($scope.state.lastPgEvent.Phase == 'Backup_ProcessingFiles') {
+            if ($scope.state.lastPgEvent.Phase == 'Backup_ProcessingFiles' || $scope.state.lastPgEvent.Phase == 'Restore_DownloadingRemoteFiles') {
                 if ($scope.state.lastPgEvent.StillCounting) {
                     text = gettextCatalog.getString('Counting ({{files}} files found, {{size}})', { files: $scope.state.lastPgEvent.TotalFileCount, size: AppUtils.formatSizeString($scope.state.lastPgEvent.TotalFileSize) });
                     pg = 0;
@@ -47,8 +47,10 @@ backupApp.controller('StateController', function($scope, $timeout, ServerStatus,
                     // If we have a speed append it
                     var speed_txt = ($scope.state.lastPgEvent.BackendSpeed < 0) ? "" : " at "+AppUtils.formatSizeString($scope.state.lastPgEvent.BackendSpeed)+"/s";
                     
+                    var restoring_text = $scope.state.lastPgEvent.Phase == 'Restore_DownloadingRemoteFiles' ? 'Restoring: ' : '';
+
                     // Finally construct the whole text
-                    text = gettextCatalog.getString('{{files}} files ({{size}}) to go {{speed_txt}}', { files: filesleft, size: AppUtils.formatSizeString(sizeleft), speed_txt: speed_txt});
+                    text = gettextCatalog.getString(restoring_text + '{{files}} files ({{size}}) to go {{speed_txt}}', { files: filesleft, size: AppUtils.formatSizeString(sizeleft), speed_txt: speed_txt});
                 }
             }
             else if ($scope.state.lastPgEvent.Phase == 'Backup_Finalize' || $scope.state.lastPgEvent.Phase == 'Backup_WaitForUpload')

Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js

@@ -25,14 +25,16 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
     EditUriBackendConfig.templates['gcs']         = 'templates/backends/gcs.html';
     EditUriBackendConfig.templates['b2']          = 'templates/backends/b2.html';
     EditUriBackendConfig.templates['mega']        = 'templates/backends/mega.html';
-    EditUriBackendConfig.templates['jottacloud']  = 'templates/backends/jottacloud.html';
+    EditUriBackendConfig.templates['jottacloud']  = 'templates/backends/oauth.html';
+    EditUriBackendConfig.templates['idrive']      = 'templates/backends/idrive.html';
     EditUriBackendConfig.templates['box']         = 'templates/backends/oauth.html';
-    EditUriBackendConfig.templates['dropbox'] = 'templates/backends/oauth.html';
-    EditUriBackendConfig.templates['sia']       = 'templates/backends/sia.html';
-    EditUriBackendConfig.templates['storj']  = 'templates/backends/storj.html';
+    EditUriBackendConfig.templates['dropbox']     = 'templates/backends/oauth.html';
+    EditUriBackendConfig.templates['sia']         = 'templates/backends/sia.html';
+    EditUriBackendConfig.templates['storj']       = 'templates/backends/storj.html';
     EditUriBackendConfig.templates['tardigrade']  = 'templates/backends/tardigrade.html';
 	EditUriBackendConfig.templates['rclone']       = 'templates/backends/rclone.html';
 	EditUriBackendConfig.templates['cos']       = 'templates/backends/cos.html';
+    EditUriBackendConfig.templates['e2'] = 'templates/backends/e2.html';
 
     EditUriBackendConfig.testers['s3'] = function(scope, callback) {
 
@@ -267,6 +269,7 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
     EditUriBackendConfig.loaders['onedrivev2']  = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.loaders['sharepoint']  = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.loaders['msgroup']     = function() { return this['oauth-base'].apply(this, arguments); };
+    EditUriBackendConfig.loaders['jottacloud']  = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.loaders['box']         = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.loaders['dropbox']     = function() { return this['oauth-base'].apply(this, arguments); };
 
@@ -421,6 +424,7 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
     EditUriBackendConfig.parsers['onedrive']    = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.parsers['onedrivev2']  = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.parsers['sharepoint']  = function() { return this['oauth-base'].apply(this, arguments); };
+    EditUriBackendConfig.parsers['jottacloud']  = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.parsers['box']         = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.parsers['dropbox']     = function() { return this['oauth-base'].apply(this, arguments); };
 
@@ -478,11 +482,18 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
             delete options[nukeopts[x]];
     };
 
-    EditUriBackendConfig.parsers['mega'] = function (scope, module, server, port, path, options) {
-        EditUriBackendConfig.mergeServerAndPath(scope);
+    EditUriBackendConfig.parsers['e2'] = function (scope, module, server, port, path, options) {
+        if (options['--access-key-id'])
+            scope.Username = options['--access-key-id'];
+        if (options['--access-key-secret'])
+            scope.Password = options['--access-key-secret'];
+
+        var nukeopts = ['--access-key-id', '--access-key-secret'];
+        for (var x in nukeopts)
+            delete options[nukeopts[x]];
     };
 
-    EditUriBackendConfig.parsers['jottacloud'] = function (scope, module, server, port, path, options) {
+    EditUriBackendConfig.parsers['mega'] = function (scope, module, server, port, path, options) {
         EditUriBackendConfig.mergeServerAndPath(scope);
     };
 
@@ -643,6 +654,7 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
     EditUriBackendConfig.builders['onedrivev2']  = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.builders['sharepoint']  = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.builders['msgroup']     = function() { return this['oauth-base'].apply(this, arguments); };
+    EditUriBackendConfig.builders['jottacloud']  = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.builders['box']         = function() { return this['oauth-base'].apply(this, arguments); };
     EditUriBackendConfig.builders['dropbox']     = function() { return this['oauth-base'].apply(this, arguments); };
 
@@ -759,6 +771,24 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
         return url;
     };
 
+    EditUriBackendConfig.builders['e2'] = function (scope) {
+        var opts = {};
+
+        EditUriBackendConfig.merge_in_advanced_options(scope, opts);
+
+        // Slightly better error message
+        scope.Folder = scope.Server;
+
+        var url = AppUtils.format('{0}://{1}/{2}{3}',
+            scope.Backend.Key,
+            scope.Server || '',
+            scope.Path || '',
+            AppUtils.encodeDictAsUrl(opts)
+        );
+
+        return url;
+    };
+
     EditUriBackendConfig.builders['mega'] = function (scope) {
         var opts = {};
 
@@ -865,23 +895,6 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
 
     }
 
-    EditUriBackendConfig.builders['jottacloud'] = function (scope) {
-        var opts = {};
-
-        EditUriBackendConfig.merge_in_advanced_options(scope, opts);
-
-        // Slightly better error message
-        scope.Folder = scope.Path;
-
-        var url = AppUtils.format('{0}://{1}{2}',
-            scope.Backend.Key,
-            scope.Path,
-            AppUtils.encodeDictAsUrl(opts)
-        );
-
-        return url;
-    };
-
 
     EditUriBackendConfig.builders['cos'] = function (scope) {
         var opts = {
@@ -948,6 +961,7 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
 
     EditUriBackendConfig.validaters['googledrive'] = EditUriBackendConfig.validaters['authid-base'];
     EditUriBackendConfig.validaters['gcs']         = EditUriBackendConfig.validaters['authid-base'];
+    EditUriBackendConfig.validaters['jottacloud']  = EditUriBackendConfig.validaters['authid-base'];
     EditUriBackendConfig.validaters['box']         = EditUriBackendConfig.validaters['authid-base'];
     EditUriBackendConfig.validaters['dropbox']     = EditUriBackendConfig.validaters['authid-base'];
     EditUriBackendConfig.validaters['onedrive']    = EditUriBackendConfig.validaters['authid-base'];
@@ -1157,16 +1171,6 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
             continuation();
     };
 
-    EditUriBackendConfig.validaters['jottacloud'] = function (scope, continuation) {
-        scope.Path = scope.Path || '';
-        var res =
-            EditUriBackendConfig.require_field(scope, 'Username', gettextCatalog.getString('Username')) &&
-            EditUriBackendConfig.require_field(scope, 'Password', gettextCatalog.getString('Password'));
-
-        if (res)
-            continuation();
-    };
-
     EditUriBackendConfig.validaters['sia'] = function (scope, continuation) {
         var res =
             EditUriBackendConfig.require_field(scope, 'Server', gettextCatalog.getString('Server'));
@@ -1272,4 +1276,48 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo,
 		if (res)
             continuation();
     };
+
+    EditUriBackendConfig.validaters['e2'] = function (scope, continuation) {
+        var res =
+                EditUriBackendConfig.require_field(scope, 'Username', gettextCatalog.getString('Idrivee2 Access Key Id')) &&
+            EditUriBackendConfig.require_field(scope, 'Password', gettextCatalog.getString('Idrivee2 Access Key Secret')) &&
+            EditUriBackendConfig.require_field(scope, 'Server', gettextCatalog.getString('Bucket Name'));
+
+        if (res) {
+            var re = new RegExp('[^A-Za-z0-9-]');
+            var bucketname = scope['Server'] || '';
+            var ix = bucketname.search(/[^A-Za-z0-9-]/g);
+
+            if (ix >= 0) {
+                EditUriBackendConfig.show_error_dialog(gettextCatalog.getString('The \'{{fieldname}}\' field contains an invalid character: {{character}} (value: {{value}}, index: {{pos}})', {
+                    value: bucketname[ix].charCodeAt(),
+                    pos: ix,
+                    character: bucketname[ix],
+                    fieldname: gettextCatalog.getString('Bucket Name')
+                }));
+                res = false;
+            }
+        }
+
+        if (res) {
+            var pathname = scope['Path'] || '';
+            for (var i = pathname.length - 1; i >= 0; i--) {
+                var char = pathname.charCodeAt(i);
+
+                if (char == '\\'.charCodeAt(0) || char == 127 || char < 32) {
+                    EditUriBackendConfig.show_error_dialog(gettextCatalog.getString('The \'{{fieldname}}\' field contains an invalid character: {{character}} (value: {{value}}, index: {{pos}})', {
+                        value: char,
+                        pos: i,
+                        character: pathname[i],
+                        fieldname: gettextCatalog.getString('Path')
+                    }));
+                    res = false;
+                    break;
+                }
+            }
+        }
+
+        if (res)
+            continuation();
+    };
 });

Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js

@@ -53,6 +53,7 @@ backupApp.service('SystemInfo', function($rootScope, $timeout, $cookies, AppServ
             },
             local: {'file': null},
             prop: {
+                'e2':null,
                 's3': null,
                 'azure': null,
                 'googledrive': null,
@@ -66,6 +67,7 @@ backupApp.service('SystemInfo', function($rootScope, $timeout, $cookies, AppServ
                 'hubic': null,
                 'b2': null,
                 'mega': null,
+                'idrive': null,
                 'box': null,
                 'od4b': null,
                 'mssp': null,

Duplicati/Server/webroot/ngax/templates/backends/e2.html

@@ -0,0 +1,18 @@
+<div class="input text">
+    <label for="e2_access_key" translate>Access Key Id</label>
+    <input type="text" name="e2_access_key" id="e2_access_key" ng-model="$parent.Username" placeholder="{{'Idrivee2 Access Id' | translate}}" />
+</div>
+<div class="input password">
+    <label for="e2_access_secret" translate>Access Key Secret</label>
+    <input autocomplete="new-password" type="password" name="e2_access_secret" id="e2_access_secret" ng-model="$parent.Password" placeholder="{{'Idrivee2 Access Secret' | translate}}" />
+</div>
+
+<div class="input text">
+    <label for="e2_bucket" translate>Bucket name</label>
+    <input type="text" id="b2_bucket" ng-model="$parent.Server" placeholder="{{'Bucket name' | translate}}" />
+</div>
+
+<div class="input text">
+    <label for="e2_path" translate>Folder path</label>
+    <input type="text" name="e2_path" id="e2_path" ng-model="$parent.Path" placeholder="{{'Path or subfolder in the bucket' | translate}}" />
+</div>

Duplicati/Server/webroot/ngax/templates/backends/idrive.html

@@ -0,0 +1,13 @@
+<div class="input text">
+    <label for="idrive_path" translate>IDrive Sync directory path</label>
+    <input type="text" name="idrive_path" id="idrive_path" ng-model="$parent.Path" placeholder="{{'Enter directory path' | translate}}" />
+</div>
+
+<div class="input text">
+    <label for="idrive_username" translate>Username</label>
+    <input type="text" name="idrive_username" id="idrive_username" ng-model="$parent.Username" placeholder="{{'Username' | translate}}" />
+</div>
+<div class="input password">
+    <label for="idrive_password" translate>Password</label>
+    <input autocomplete="new-password" type="password" name="idrive_password" id="idrive_password" ng-model="$parent.Password" placeholder="{{'Password' | translate}}" />
+</div>

Duplicati/Server/webroot/ngax/templates/backends/jottacloud.html

@@ -1,13 +1,12 @@
 <div class="input text">
     <label for="jottacloud_path" translate>Folder path</label>
-    <input type="text" name="jottacloud_path" id="jottacloud_path" ng-model="$parent.Path" placeholder="{{'Enter folder path name' | translate}}" />
+    <input type="text" name="jottacloud_path" id="jottacloud_path" ng-model="$parent.Path" placeholder="{{'Enter the destination path' | translate}}" />
 </div>
 
 <div class="input text">
-    <label for="jottacloud_username" translate>Username</label>
-    <input type="text" name="jottacloud_username" id="jottacloud_username" ng-model="$parent.Username" placeholder="{{'Username' | translate}}" />
-</div>
-<div class="input password">
-    <label for="jottacloud_password" translate>Password</label>
-    <input autocomplete="new-password" type="password" name="jottacloud_password" id="jottacloud_password" ng-model="$parent.Password" placeholder="{{'Password' | translate}}" />
+    <label for="jottacloud_authid">
+        <a ng-show="oauth_in_progress" href="{{oauth_start_link}}" target="_blank" translate>AuthID</a> 
+        <a ng-hide="oauth_in_progress" href ng-click="oauth_start_token_creation()" translate>AuthID</a>
+    </label>
+    <input type="text" name="jottacloud_authid" id="jottacloud_authid" ng-model="$parent.AuthID"  placeholder="{{'Click the AuthID link to create an AuthID' | translate}}" />
 </div>

Duplicati/Service/Runner.cs

@@ -79,13 +79,13 @@ namespace Duplicati.Service
 
                         m_reportMessage(string.Format("Starting process {0} with cmd args {1}", exec, cmdargs), false);
 
-                        var pr = new System.Diagnostics.ProcessStartInfo(exec, cmdargs)
-                        {
-                            UseShellExecute = false,
-                            RedirectStandardInput = true,
+                        var pr = new System.Diagnostics.ProcessStartInfo(exec, cmdargs)
+                        {
+                            UseShellExecute = false,
+                            RedirectStandardInput = true,
                             RedirectStandardOutput = true,
-                            RedirectStandardError = false,
-                            WorkingDirectory = path
+                            RedirectStandardError = false,
+                            WorkingDirectory = path
                         };
 
                         if (!m_terminate)

Duplicati/UnitTest/BackendToolTests.cs

@@ -31,6 +31,10 @@ namespace Duplicati.UnitTest
             using (Controller c = new Controller(backendURL, options, null))
             {
                 var backupResults = c.Backup(new[] {DATAFOLDER});
+                foreach (var backupResultsWarning in backupResults.Warnings)
+                {
+                    TestContext.WriteLine("Backend result warning:" + backupResultsWarning);
+                }
                 Assert.AreEqual(0, backupResults.Errors.Count(), $"Errors: {String.Join(", ", backupResults.Errors)}");
                 Assert.AreEqual(0, backupResults.Warnings.Count(), $"Warnings: {String.Join(", ", backupResults.Warnings)}");
             }
@@ -71,4 +75,4 @@ namespace Duplicati.UnitTest
             }
         }
     }
-}
+}

Duplicati/UnitTest/Duplicati.UnitTest.csproj

@@ -29,6 +29,8 @@
     </ProjectReference>
     <ProjectReference Include="..\Library\Backend\FTP\Duplicati.Library.Backend.FTP.csproj">
     </ProjectReference>
+    <ProjectReference Include="..\Library\Backend\Idrivee2\Duplicati.Library.Backend.Idrivee2.csproj">
+    </ProjectReference>
     <ProjectReference Include="..\Library\Backend\Jottacloud\Duplicati.Library.Backend.Jottacloud.csproj">
     </ProjectReference>
     <ProjectReference Include="..\Library\Backend\OAuthHelper\Duplicati.Library.OAuthHelper.csproj">

Installer/Docker/README.md

@@ -1,7 +1,7 @@
 # [Duplicati](https://www.duplicati.com)
 Duplicati is a free, open source, backup client that securely stores encrypted, incremental, compressed backups on cloud storage services and remote file servers. It works with:
 
-*Amazon S3, OneDrive, Google Drive, Rackspace Cloud Files, HubiC, Backblaze (B2), Swift / OpenStack, WebDAV, SSH (SFTP), FTP, and more!*
+*Amazon S3, IDrive e2, OneDrive, Google Drive, Rackspace Cloud Files, HubiC, Backblaze (B2), Swift / OpenStack, WebDAV, SSH (SFTP), FTP, and more!*
 
 Duplicati is licensed under LGPL and available for Windows, OSX and Linux (.NET 4.7.1+ or Mono 4.8.0+ required).
 

README.md

@@ -21,7 +21,7 @@ Removed Gitter
 
 Duplicati is a free, open source, backup client that securely stores encrypted, incremental, compressed backups on cloud storage services and remote file servers. It works with:
 
-   *Amazon S3, [Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage"), Box, Dropbox, FTP, Google Cloud and Drive, HubiC, MEGA, Microsoft Azure and OneDrive, Rackspace Cloud Files, OpenStack Storage (Swift), Sia, Storj DCS, SSH (SFTP), WebDAV, Tencent Cloud Object Storage (COS), [and more!](https://duplicati.readthedocs.io/en/latest/01-introduction/#supported-backends)*
+   *Amazon S3, [IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2"), [Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage"), Box, Dropbox, FTP, Google Cloud and Drive, HubiC, MEGA, Microsoft Azure and OneDrive, Rackspace Cloud Files, OpenStack Storage (Swift), Sia, Storj DCS, SSH (SFTP), WebDAV, Tencent Cloud Object Storage (COS), [and more!](https://duplicati.readthedocs.io/en/latest/01-introduction/#supported-backends)*
 
 Duplicati is licensed under LGPL and available for Windows, OSX and Linux (.NET 4.7.1+ or Mono 5.10.0+ required). 
 

SECURITY.md

@@ -0,0 +1,3 @@
+Any security issues can be reported to the main author at kenneth@duplicati.com.
+
+If the issue is sensitive, the [PGP signing key](https://pgp.mit.edu/pks/lookup?op=get&search=0xC20E90473DAC703D) can be used for encryption.

Tools/Verification/DuplicatiVerify.py

@@ -46,11 +46,13 @@ def verifyHashes(filename, quiet = True, buffer_size=8192):
         filename = file["Name"]
         hash = file["Hash"]
         size = file["Size"]
+        state = file["State"]
 
         fullpath = os.path.join(folder, filename)
         if not os.path.exists(fullpath):
-            print("File missing:", fullpath)
-            errorCount += 1
+            if state != 5:
+                print("File missing:", fullpath)
+                errorCount += 1
         else:
             checked += 1
             if not quiet:
@@ -82,22 +84,142 @@ if __name__ == "__main__":
     parser.add_argument("path", type=str, nargs='?',
                     help="""path to the verification file or folder containing the verification file.\
                         Defaulf is curent path""")
+    parser.add_argument("--stats_only", action='store_true', help="display statistics from json only, no verification")
     args = parser.parse_args()
 
-    if args.path is None:
-        args.path = os.getcwd()
+    if not args.stats_only: # if stats_only arg present skip file verification
+        if args.path is None:
+            args.path = os.getcwd()
 
-    if not os.path.exists(args.path):
-        print("No such file or directory: ", args.path)
-    else:
-        if os.path.isfile(args.path):
-            verifyHashes(args.path, args.quiet, args.buffer_size * 1024)
+        if not os.path.exists(args.path):
+            print("No such file or directory: ", args.path)
         else:
-            files = 0
-            for f in os.listdir(args.path):
-                if (f.endswith("-verification.json")):
-                    print("Verifying file:", f)
-                    files += 1
-                    verifyHashes(os.path.join(args.path, f), args.quiet, args.buffer_size *1024)
-            if files == 0:
-                print("No verification files in folder:", args.path)
+            if os.path.isfile(args.path):
+                verifyHashes(args.path, args.quiet, args.buffer_size * 1024)
+            else:
+                files = 0
+                for f in os.listdir(args.path):
+                    if (f.endswith("-verification.json")):
+                        print("Verifying file:", f)
+                        files += 1
+                        verifyHashes(os.path.join(args.path, f), args.quiet, args.buffer_size *1024)
+                if files == 0:
+                    print("No verification files in folder:", args.path)
+    else:
+        print("Verify skipped, displaying statistics only ")
+
+
+""" Calculate statistics """   
+class Statistics:
+
+    def get_json_for_statistics(): #copied from main
+        files = 0
+        for f in os.listdir(args.path):
+            if (f.endswith("-verification.json")):
+                print("Statistics for file", f)
+                files += 1
+                statistics_source = Statistics.load_json_for_statistics(os.path.join(args.path, f), args.quiet, args.buffer_size *1024)
+                return statistics_source
+        if files == 0:
+            print("No verification files in folder:", args.path)
+
+    def load_json_for_statistics(filename, quiet = True, buffer_size=8192): #copied from verifyHashes
+        if (not os.path.exists(filename)):
+            print("Specified file does not exist:", filename)
+            return -1
+        folder = os.path.dirname(filename)
+        with codecs.open(filename, "r", "utf-8-sig") as f:
+            doc = json.load(f)
+            statistics_source = doc
+        return statistics_source
+
+    """ Statistics legend """ 
+    RemoteVolumeState = [
+        # Indicates that the remote volume is being created
+        "Temporary",
+        # Indicates that the remote volume is being uploaded
+        "Uploading",
+        # Indicates that the remote volume has been uploaded
+        "Uploaded",
+        # Indicates that the remote volume has been uploaded, and seen by a list operation
+        "Verified",
+        # Indicattes that the remote volume should be deleted
+        "Deleting",
+        # Indicates that the remote volume was successfully deleted from the remote location
+        "Deleted"]
+    RemoteVolumeType = [
+        # Contains data blocks
+        "Blocks",
+        # Contains file lists
+        "Files",
+        # Contains redundant lookup information
+        "Index"]
+
+    """ Utility convert bytes to readable units for statistics """
+    def convert_bytes_to_readable(size):
+        if size is None:
+            size = 0
+            return size
+        if size < 0: 
+            size = 0 #because for "RemoteVolumeState 5" (Deleted) is size in json "-1"
+            return size
+        for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:
+            if size < 1024.0:
+                return "%3.1f %s" % (size, x)
+            size /= 1024.0  
+        return size
+
+
+    def compute_statistics():
+        index_main_loop = 0
+        index_sub_loop = 0
+        list_of_all_types = []
+        list_of_all_states = []
+        dict_with_sizes_of_all_items= {}
+        total_size = 0
+
+        #main statistics loop 
+        statistics_source = Statistics.get_json_for_statistics() # call load data from json
+        for file in statistics_source:
+            #loop for geting size for each RemoteVolumeType to dict
+            for each_type in Statistics.RemoteVolumeType: #for each json file for each type ID
+                current_type = (statistics_source[index_main_loop]["Type"])
+                if current_type == index_sub_loop:
+                    dict_with_sizes_of_all_items[each_type] = dict_with_sizes_of_all_items.get(each_type,0) + (statistics_source[index_main_loop]["Size"])
+                index_sub_loop += 1
+                if index_sub_loop >2:
+                    index_sub_loop = 0
+            #loop for geting size for each RemoteVolumeState to dict
+            for each_state in Statistics.RemoteVolumeState:
+                current_type = (statistics_source[index_main_loop]["State"])
+                if current_type == index_sub_loop:
+                    dict_with_sizes_of_all_items[each_state] = dict_with_sizes_of_all_items.get(each_state,0) + (statistics_source[index_main_loop]["Size"])
+                index_sub_loop += 1
+                if index_sub_loop >5:
+                    index_sub_loop = 0
+            #store Type and State to list
+            list_of_all_types.append(statistics_source[index_main_loop]["Type"])
+            list_of_all_states.append(statistics_source[index_main_loop]["State"])
+            index_main_loop += 1
+
+        print("\n")
+        index_main_loop = 0
+        #Print number of files and size from dict
+        for state in Statistics.RemoteVolumeState:
+            print(state,"count",list_of_all_states.count(Statistics.RemoteVolumeState.index(state)),"Size ",Statistics.convert_bytes_to_readable(dict_with_sizes_of_all_items.get(state)))
+            index_main_loop += 1
+        print("\n")
+
+        #Print number of files and size from dict
+        for type in Statistics.RemoteVolumeType:
+            print(type,"count",list_of_all_types.count(Statistics.RemoteVolumeType.index(type)),"size:",Statistics.convert_bytes_to_readable(dict_with_sizes_of_all_items.get(type)))
+
+        #Print total size
+        for key in Statistics.RemoteVolumeType:
+            total_size += dict_with_sizes_of_all_items.get(key)
+        print("\n""Total size:",Statistics.convert_bytes_to_readable(total_size) )
+
+#to run statistics
+Statistics.compute_statistics()
+
+

Updates/build_version.txt

@@ -1 +1 @@
-100
+104

changelog.txt

@@ -1,3 +1,64 @@
+2022-06-15 - 2.0.6.104_canary_2022-06-15
+==========
+* Updated compressed files exclusion list, thanks @martengooz
+* Updated list of Infomaniak servers, thanks @linitio
+* Fixed irregular parsing of `--patch-with-local-blocks`, thanks @warwickmm
+* Added `IDrivee2` backend, thanks @MuhammadAbrar1984
+* Fixed `--use-move-for-put` which broken in previous build, thanks @kenkendk
+
+2022-06-12 - 2.0.6.103_canary_2022-06-12
+==========
+* Avoid creating empty data folders, thanks @albertony
+* Disable updates for debug builds, thanks @AxeOfMen
+* Fixed issue with empty popups, thanks @AxeOfMen
+* Improved Python verification script, thanks @mr-flibble
+* Fixed issue where invalid timestamps would prevent files from being backed up, thanks @albertony
+* Improved restore progress information, thanks @flawiddsouza
+* Python verification script now has statistics, thanks @mr-flibble
+* Added Swiss Backup to list of OpenStack and S3 providers, thanks @PAPAMICA
+* Fixed errors on updated Hyper-V versions, thanks @awerghcpc
+* Added check to detect partially written files on network drives, thanks @dkrahmer
+* Added IDrive as a backup destination, thanks @dkrahmer
+* Removed insecure TLS versions, thanks @mnaiman
+* Changed Jottacloud backend to use OAuth, thanks @albertony
+
+2022-04-06 - 2.0.6.102_canary_2022-04-06
+==========
+**NOTE**: This is a rebuild of 2.0.6.101, now signed.
+
+- Fixed duplicity documentation Url, thanks @beppo-dd
+- Added option to set Tencent storage class, thanks @moesoha
+- Added Poli Systems as S3 provider, thanks @IIPoliII
+- Improved automated unittests, thanks @samuel-w
+- Improved contrast ratios in themes, thanks @samuel-w
+- Improved handling of relative paths and extended device paths, thanks @dferreyra
+- UI performance improvements, thanks @ltGuillaume
+- Added exponential backup for retries, thanks @vmsh0
+- Improved filter-text-box and made log-view resizable, thanks @idotj
+- Improved error messages, thanks @warwickmm
+- Added support for old and new Dropbox AuthIDs, thanks @kenkendk
+- Improved database query performance, thanks @jedthe3rd
+
+2022-03-13 - 2.0.6.101_canary_2022-03-13
+==========
+**NOTE**: This is a preliminary build that is not signed with AuthentiCode.
+
+It is primarily intended for users that need access to Dropbox which stopped working after an update.
+A signed version will be published asap.
+
+- Fixed duplicity documentation Url, thanks @beppo-dd
+- Added option to set Tencent storage class, thanks @moesoha
+- Added Poli Systems as S3 provider, thanks @IIPoliII
+- Improved automated unittests, thanks @samuel-w
+- Improved contrast ratios in themes, thanks @samuel-w
+- Improved handling of relative paths and extended device paths, thanks @dferreyra
+- UI performance improvements, thanks @ltGuillaume
+- Added exponential backup for retries, thanks @vmsh0
+- Improved filter-text-box and made log-view resizable, thanks @idotj
+- Improved error messages, thanks @warwickmm
+- Added support for old and new Dropbox AuthIDs, thanks @kenkendk
+- Improved database query performance, thanks @jedthe3rd
+
 2021-08-11 - 2.0.6.100_canary_2021-08-11
 ==========
 * Switched to improved CI model, thanks @warwickmm

guiTests/guiTest.py

@@ -120,14 +120,14 @@ wait_for_load(10, By.ID, "save").click()
 
 # Run the backup job and wait for finish
 wait_for_load(10, By.LINK_TEXT, BACKUP_NAME).click()
-[n for n in driver.find_elements_by_xpath("//dl[@class='taskmenu']/dd/p/span[contains(text(),'Run now')]") if n.is_displayed()][0].click()
+[n for n in driver.find_elements("xpath", "//dl[@class='taskmenu']/dd/p/span[contains(text(),'Run now')]") if n.is_displayed()][0].click()
 wait_for_text(60, "//div[@class='task ng-scope']/dl[2]/dd[1]", "(took ")
 
 # Restore
-if len([n for n in driver.find_elements_by_xpath("//span[contains(text(),'Restore files \u2026')]") if n.is_displayed()]) == 0:
+if len([n for n in driver.find_elements("xpath", u"//span[contains(text(),'Restore files \u2026')]") if n.is_displayed()]) == 0:
     wait_for_load(10, By.LINK_TEXT, BACKUP_NAME).click()
 
-[n for n in driver.find_elements_by_xpath("//span[contains(text(),'Restore files \u2026')]") if n.is_displayed()][0].click()
+[n for n in driver.find_elements("xpath", u"//span[contains(text(),'Restore files \u2026')]") if n.is_displayed()][0].click()
 wait_for_load(10, By.XPATH, "//span[contains(text(),'" + SOURCE_FOLDER + "')]")  # wait for filelist
 time.sleep(1) # Delay so page has time to load
 wait_for_load(10, By.XPATH, "//restore-file-picker/ul/li/div/a[2]").click()  # select root folder checkbox