Merge branch 'feature/use-cng-hashalgorithms'

Duplicati/CommandLine/RecoveryTool/Recompress.cs

@@ -204,7 +204,7 @@ namespace Duplicati.CommandLine.RecoveryTool
                                             textJSON = sourceStreamReader.ReadToEnd();
                                             JToken token = JObject.Parse(textJSON);
                                             var fileInfoBlocks = new FileInfo(Path.Combine(targetfolder, cmfileNew.Replace("vol/", "")));
-                                            var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_Options.FileHashAlgorithm);
+                                            var filehasher = HashAlgorithmHelper.Create(m_Options.FileHashAlgorithm);
                                             
                                             using (var fileStream = fileInfoBlocks.Open(FileMode.Open))
                                             {

Duplicati/CommandLine/RecoveryTool/Restore.cs

@@ -90,8 +90,8 @@ namespace Duplicati.CommandLine.RecoveryTool
                 return 100;
             }
 
-            var blockhasher = string.IsNullOrWhiteSpace(blockhash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(blockhash_str);
-            var filehasher = string.IsNullOrWhiteSpace(filehash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(filehash_str);
+            var blockhasher = string.IsNullOrWhiteSpace(blockhash_str) ? null : Library.Utility.HashAlgorithmHelper.Create(blockhash_str);
+            var filehasher = string.IsNullOrWhiteSpace(filehash_str) ? null : Library.Utility.HashAlgorithmHelper.Create(filehash_str);
 
             if (blockhasher == null)
                 throw new Duplicati.Library.Interface.UserInformationException(string.Format("Block hash algorithm not valid: {0}", blockhash_str));

Duplicati/Library/Backend/Backblaze/B2.cs

@@ -177,7 +177,7 @@ namespace Duplicati.Library.Backend.Backblaze
                 var p = measure.Position;
 
                 // Compute the hash
-                using(var hashalg = System.Security.Cryptography.HashAlgorithm.Create("sha1"))
+                using(var hashalg = Duplicati.Library.Utility.HashAlgorithmHelper.Create("sha1"))
                     sha1 = Library.Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure));
 
                 // Reset the stream position

Duplicati/Library/Main/BackendManager.cs

@@ -451,14 +451,14 @@ namespace Duplicati.Library.Main
         public static string CalculateFileHash(string filename)
         {
             using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
-            using (var hasher = System.Security.Cryptography.HashAlgorithm.Create(VOLUME_HASH))
+            using (var hasher = HashAlgorithmHelper.Create(VOLUME_HASH))
                 return Convert.ToBase64String(hasher.ComputeHash(fs));
         }
 
         /// <summary> Calculate file hash directly on stream object (for piping) </summary>
         public static string CalculateFileHash(System.IO.Stream stream)
         {
-            using (var hasher = System.Security.Cryptography.HashAlgorithm.Create(VOLUME_HASH))
+            using (var hasher = HashAlgorithmHelper.Create(VOLUME_HASH))
                 return Convert.ToBase64String(hasher.ComputeHash(stream));
         }
 
@@ -469,7 +469,7 @@ namespace Duplicati.Library.Main
         public static System.Security.Cryptography.CryptoStream GetFileHasherStream
             (System.IO.Stream stream, System.Security.Cryptography.CryptoStreamMode mode, out Func<string> getHash)
         {
-            var hasher = System.Security.Cryptography.HashAlgorithm.Create(VOLUME_HASH);
+            var hasher = HashAlgorithmHelper.Create(VOLUME_HASH);
             System.Security.Cryptography.CryptoStream retHasherStream =
                 new System.Security.Cryptography.CryptoStream(stream, hasher, mode);
             getHash = () =>
@@ -678,7 +678,7 @@ namespace Duplicati.Library.Main
                 IndexVolumeWriter wr = null;
                 try
                 {
-                    var hashsize = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm).HashSize / 8;
+                    var hashsize = HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm).HashSize / 8;
                     wr = new IndexVolumeWriter(m_options);
                     using(var rd = new IndexVolumeReader(p.CompressionModule, item.Indexfile.Item2.LocalFilename, m_options, hashsize))
                         wr.CopyFrom(rd, x => x == oldname ? newname : x);

Duplicati/Library/Main/Database/LocalRepairDatabase.cs

@@ -334,7 +334,7 @@ namespace Duplicati.Library.Main.Database
 
         public void FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize)
         {
-            var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(blockhashalgorithm);
+            var blockhasher = Library.Utility.HashAlgorithmHelper.Create(blockhashalgorithm);
             var hashsize = blockhasher.HashSize / 8;
             var blocklistbuffer = new byte[blocksize];
             int blocklistoffset = 0;

Duplicati/Library/Main/Operation/BackupHandler.cs

@@ -378,7 +378,7 @@ namespace Duplicati.Library.Main.Operation
         {
             if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
             {
-                var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
+                var blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
                 var hashsize = blockhasher.HashSize / 8;
 
                 foreach (var blockfile in m_database.GetMissingIndexFiles())
@@ -696,8 +696,8 @@ namespace Duplicati.Library.Main.Operation
                 m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
                 m_blocklistbuffer = new byte[m_options.Blocksize];
 
-                m_blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
-                m_filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
+                m_blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
+                m_filehasher = Library.Utility.HashAlgorithmHelper.Create(m_options.FileHashAlgorithm);
 
                 if (m_blockhasher == null)
                     throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));

Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs

@@ -299,7 +299,7 @@ namespace Duplicati.Library.Main.Operation
             
                 if (!m_options.RepairOnlyPaths)
                 {
-                    var hashalg = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
+                    var hashalg = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
                     if (hashalg == null)
                         throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
                     var hashsize = hashalg.HashSize / 8;

Duplicati/Library/Main/Operation/RepairHandler.cs

@@ -113,7 +113,7 @@ namespace Duplicati.Library.Main.Operation
 
                 var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter, null);
                 var buffer = new byte[m_options.Blocksize];
-                var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
+                var blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
                 var hashsize = blockhasher.HashSize / 8;
 
                 if (blockhasher == null)
@@ -300,7 +300,7 @@ namespace Duplicati.Library.Main.Operation
                                 newEntry = w;
                                 w.SetRemoteFilename(n.Name);
 
-                                var h = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
+                                var h = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
                                 
                                 foreach(var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
                                 {                               

Duplicati/Library/Main/Operation/RestoreHandler.cs

@@ -124,8 +124,8 @@ namespace Duplicati.Library.Main.Operation
                                     //Figure out what files are to be patched, and what blocks are needed
                                     PrepareBlockAndFileList(database, m_options, filter, m_result);
 
-                                    blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
-                                    filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
+                                    blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
+                                    filehasher = Library.Utility.HashAlgorithmHelper.Create(m_options.FileHashAlgorithm);
                                     if (blockhasher == null)
                                         throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
                                     if (!blockhasher.CanReuseTransform)
@@ -194,7 +194,7 @@ namespace Duplicati.Library.Main.Operation
             var blocksize = options.Blocksize;
             var updateCounter = 0L;
             var fullblockverification = options.FullBlockVerification;
-            var blockhasher = fullblockverification ? System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm) : null;
+            var blockhasher = fullblockverification ? Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm) : null;
 
             using (var blockmarker = database.CreateBlockMarker())
             using(var volumekeeper = database.GetMissingBlockData(blocks, options.Blocksize))
@@ -345,8 +345,8 @@ namespace Duplicati.Library.Main.Operation
                 Utility.VerifyParameters(database, m_options);
                 m_blockbuffer = new byte[m_options.Blocksize];
                 
-                var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
-                var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
+                var blockhasher = Library.Utility.HashAlgorithmHelper.Create(m_options.BlockHashAlgorithm);
+                var filehasher = Library.Utility.HashAlgorithmHelper.Create(m_options.FileHashAlgorithm);
                 if (blockhasher == null)
                     throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
                 if (!blockhasher.CanReuseTransform)

Duplicati/Library/Main/Operation/TestHandler.cs

@@ -198,7 +198,7 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="sample_percent">A value between 0 and 1 that indicates how many blocks are tested in a dblock file</param>
         public static KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, ILogWriter log, double sample_percent)
         {
-            var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm);
+            var blockhasher = Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm);
  
             if (blockhasher == null)
                 throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm));

Duplicati/Library/Main/Options.cs

@@ -1372,7 +1372,7 @@ namespace Duplicati.Library.Main
         {
             get
             {
-                return System.Security.Cryptography.HashAlgorithm.Create(BlockHashAlgorithm).HashSize / 8;
+                return Duplicati.Library.Utility.HashAlgorithmHelper.Create(BlockHashAlgorithm).HashSize / 8;
             }
         }
 

Duplicati/Library/Main/Utility.cs

@@ -50,7 +50,7 @@ namespace Duplicati.Library.Main
                     
                 using (var ms = new System.IO.MemoryStream())
                 using (var w = new StreamWriter(ms, Encoding.UTF8))
-                using(var filehasher = System.Security.Cryptography.HashAlgorithm.Create(options.FileHashAlgorithm))
+                using(var filehasher = Library.Utility.HashAlgorithmHelper.Create(options.FileHashAlgorithm))
                 {
                     if (filehasher == null)
                         throw new Duplicati.Library.Interface.UserInformationException(Strings.Common.InvalidHashAlgorithm(options.FileHashAlgorithm));

Duplicati/Library/Utility/Duplicati.Library.Utility.csproj

@@ -72,6 +72,7 @@
     <Compile Include="DirectStreamLink.cs" />
     <Compile Include="HostKeyException.cs" />
     <Compile Include="CallContextSettings.cs" />
+    <Compile Include="HashAlgorithmHelper.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="app.config" />

Duplicati/Library/Utility/HashAlgorithmHelper.cs

@@ -0,0 +1,64 @@
+//  Copyright (C) 2017, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+using System.Security.Cryptography;
+
+namespace Duplicati.Library.Utility
+{
+	/// <summary>
+	/// This class helps picking the fastest hash algorithm implementation,
+	/// which is what <seealso cref="System.Security.Cryptography.HashAlgorithm.Create"/> should do, but does not.
+	/// </summary>
+	public static class HashAlgorithmHelper
+    {
+        /// <summary>
+        /// Cache of known types
+        /// </summary>
+        private static readonly Dictionary<string, Type> _knownTypes = new Dictionary<string, Type>(StringComparer.OrdinalIgnoreCase);
+
+        /// <summary>
+        /// Static initializer to pre-populate the type table
+        /// </summary>
+        static HashAlgorithmHelper()
+        {
+            _knownTypes["MD5"] = typeof(MD5Cng);
+			_knownTypes["SHA1"] = typeof(SHA1Cng);
+			_knownTypes["SHA256"] = typeof(SHA256Cng);
+			_knownTypes["SHA384"] = typeof(SHA384Cng);
+			_knownTypes["SHA512"] = typeof(SHA512Cng);
+		}
+
+        /// <summary>
+        /// Create the hash algorithm with the specified name.
+        /// </summary>
+        /// <returns>The hash algorithm.</returns>
+        /// <param name="name">The name of the algorithm to create.</param>
+        public static HashAlgorithm Create(string name)
+        {
+            Type known;
+            if (!_knownTypes.TryGetValue(name, out known))
+                known = _knownTypes[name] = Type.GetType("System.Security.Cryptography." + name.ToUpperInvariant() + "Cng", false, true);
+
+            if (known == null || known.GetConstructor(Type.EmptyTypes) == null)
+                return HashAlgorithm.Create(name);
+            else
+			    return (HashAlgorithm)Activator.CreateInstance(known);
+
+        }
+    }
+}

Duplicati/Library/Utility/HashCalculatingStream.cs

@@ -49,7 +49,7 @@ namespace Duplicati.Library.Utility
         private int m_hashbufferLength = 0;
 
         public HashCalculatingStream(System.IO.Stream basestream, string hashalgorithm)
-            : this(basestream, System.Security.Cryptography.HashAlgorithm.Create(hashalgorithm))
+            : this(basestream, HashAlgorithmHelper.Create(hashalgorithm))
         {
         }
 

Duplicati/Library/Utility/Utility.cs

@@ -551,7 +551,7 @@ namespace Duplicati.Library.Utility
         /// <returns>The base64 encoded hash</returns>
         public static string CalculateHash(System.IO.Stream stream)
         {
-            return Convert.ToBase64String(System.Security.Cryptography.HashAlgorithm.Create(HashAlgorithm).ComputeHash(stream));
+            return Convert.ToBase64String(HashAlgorithmHelper.Create(HashAlgorithm).ComputeHash(stream));
         }
 
         /// <summary>