Merge pull request #2022 from duplicati/feature/fix_for_different_blockhash_and_filehash_plus_multiblocks_metadata

Support different block-hash and file-hash algorithms
2016-10-18 11:02:27 +02:00
parent adf7c6fbf1 60c972cf0b
commit c8e8ebc44a
13 changed files with 506 additions and 180 deletions
@@ -458,26 +458,26 @@ namespace Duplicati.Library.Main.Database
        /// <param name="hash">The metadata hash</param>
        /// <param name="metadataid">The id of the metadata set</param>
        /// <returns>True if the set was added to the database, false otherwise</returns>
-        public bool AddMetadataset(string hash, long size, out long metadataid, System.Data.IDbTransaction transaction = null)
+        public bool AddMetadataset(string filehash, long size, int blocksize, IEnumerable<string> blockhashes, IEnumerable<string> blocklisthashes, out long metadataid, System.Data.IDbTransaction transaction = null)
        {
            if (size > 0)
            {
                if (m_metadataLookup != null)
                {
-                    if(m_metadataLookup.TryGet(hash, size, out metadataid))
+                    if(m_metadataLookup.TryGet(filehash, size, out metadataid))
                        return false;
                }
                else
                {
                    m_findmetadatasetCommand.Transaction = transaction;
-                    metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, hash, size);
+                    metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size);
                    if (metadataid != -1)
                        return false;
                }
            

                long blocksetid;
-                AddBlockset(hash, size, (int)size, new string[] { hash }, null, out blocksetid, transaction);
+                AddBlockset(filehash, size, blocksize, blockhashes, blocklisthashes, out blocksetid, transaction);

                using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
                {
@@ -486,7 +486,7 @@ namespace Duplicati.Library.Main.Database
                    metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64();
                    tr.Commit();
                    if (m_metadataLookup != null)
-                        m_metadataLookup.Add(hash, size, metadataid);
+                        m_metadataLookup.Add(filehash, size, metadataid);
                }

                return true;
@@ -723,7 +723,7 @@ namespace Duplicati.Library.Main.Database
                return cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > ?", -1, fhblocksize);
        }

-        public void VerifyConsistency(System.Data.IDbTransaction transaction, long blocksize, long hashsize)
+        public void VerifyConsistency(System.Data.IDbTransaction transaction, long blocksize, long hashsize, bool verifyfilelists)
        {
            using (var cmd = m_connection.CreateCommand())
            {
@@ -768,6 +768,20 @@ namespace Duplicati.Library.Main.Database
                if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0)
                    throw new Exception("Detected file entries with not associated blocks");

+
+                if (verifyfilelists)
+                {
+                    using(var cmd2 = m_connection.CreateCommand(transaction))
+                    foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1)))
+                    {
+                        var expandedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid);
+                        //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
+                        var storedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?"), 0, filesetid);
+
+                        if (expandedlist != storedlist)
+                            throw new Exception(string.Format("Unexpected difference in fileset {0}, found {1} entries, but expected {2}", filesetid, expandedlist, storedlist));
+                    }
+                }
            }
        }

@@ -885,31 +899,168 @@ namespace Duplicati.Library.Main.Database
            }
        }

+        public const string LIST_FILESETS = @"
+SELECT
+    ""L"".""Path"", 
+    ""L"".""Lastmodified"", 
+    ""L"".""Filelength"", 
+    ""L"".""Filehash"", 
+    ""L"".""Metahash"", 
+    ""L"".""Metalength"",
+    ""L"".""BlocklistHash"", 
+    ""L"".""FirstBlockHash"",
+    ""L"".""FirstBlockSize"",
+    ""L"".""FirstMetaBlockHash"",
+    ""L"".""FirstMetaBlockSize"",
+    ""M"".""Hash"" AS ""MetaBlocklistHash""
+FROM
+    (
+    SELECT 
+        ""J"".""Path"", 
+        ""J"".""Lastmodified"", 
+        ""J"".""Filelength"", 
+        ""J"".""Filehash"", 
+        ""J"".""Metahash"", 
+        ""J"".""Metalength"",
+        ""K"".""Hash"" AS ""BlocklistHash"", 
+        ""J"".""FirstBlockHash"",
+        ""J"".""FirstBlockSize"",
+        ""J"".""FirstMetaBlockHash"",
+        ""J"".""FirstMetaBlockSize"",
+        ""J"".""MetablocksetID""
+    FROM 
+        (
+        SELECT 
+            ""A"".""Path"" AS ""Path"", 
+            ""D"".""Lastmodified"" AS ""Lastmodified"", 
+            ""B"".""Length"" AS ""Filelength"", 
+            ""B"".""FullHash"" AS ""Filehash"", 
+            ""E"".""FullHash"" AS ""Metahash"", 
+            ""E"".""Length"" AS ""Metalength"", 
+            ""A"".""BlocksetID"" AS ""BlocksetID"",
+            ""F"".""Hash"" AS ""FirstBlockHash"",
+            ""F"".""Size"" AS ""FirstBlockSize"",
+            ""H"".""Hash"" AS ""FirstMetaBlockHash"",
+            ""H"".""Size"" AS ""FirstMetaBlockSize"",
+            ""C"".""BlocksetID"" AS ""MetablocksetID""
+        FROM 
+            ""File"" A, 
+            ""Blockset"" B, 
+            ""Metadataset"" C, 
+            ""FilesetEntry"" D, 
+            ""Blockset"" E, 
+            ""Block"" F,
+            ""BlocksetEntry"" G,
+            ""Block"" H,
+            ""BlocksetEntry"" I
+        WHERE 
+            ""A"".""ID"" = ""D"".""FileID"" 
+            AND ""D"".""FilesetID"" = ? 
+            AND ""A"".""BlocksetID"" = ""B"".""ID"" 
+            AND ""A"".""MetadataID"" = ""C"".""ID"" 
+            AND ""E"".""ID"" = ""C"".""BlocksetID""
+            AND ""B"".""ID"" = ""G"".""BlocksetID""
+            AND ""G"".""BlockID"" = ""F"".""ID""
+            AND ""G"".""Index"" = 0
+            AND ""I"".""BlocksetID"" = ""E"".""ID""
+            AND ""I"".""BlockID"" = ""H"".""ID""
+            AND ""I"".""Index"" = 0
+        ) J
+    LEFT OUTER JOIN 
+        ""BlocklistHash"" K 
+    ON 
+        ""K"".""BlocksetID"" = ""J"".""BlocksetID"" 
+    ORDER BY ""J"".""Path"", ""K"".""Index""
+    ) L
+
+LEFT OUTER JOIN
+    ""BlocklistHash"" M
+ON
+    ""M"".""BlocksetID"" = ""L"".""MetablocksetID""
+";
+
+        public const string LIST_FOLDERS_AND_SYMLINKS = @"
+SELECT
+    ""G"".""BlocksetID"",
+    ""G"".""ID"",
+    ""G"".""Path"",
+    ""G"".""Length"",
+    ""G"".""FullHash"",
+    ""G"".""Lastmodified"",
+    ""G"".""FirstMetaBlockHash"",
+    ""H"".""Hash"" AS ""MetablocklistHash""
+FROM
+    (
+    SELECT
+        ""B"".""BlocksetID"",
+        ""B"".""ID"",
+        ""B"".""Path"",
+        ""D"".""Length"",
+        ""D"".""FullHash"",
+        ""A"".""Lastmodified"",
+        ""F"".""Hash"" AS ""FirstMetaBlockHash"",
+        ""C"".""BlocksetID"" AS ""MetaBlocksetID""
+    FROM
+        ""FilesetEntry"" A, 
+        ""File"" B, 
+        ""Metadataset"" C, 
+        ""Blockset"" D,
+        ""BlocksetEntry"" E,
+        ""Block"" F
+    WHERE 
+        ""A"".""FileID"" = ""B"".""ID"" 
+        AND ""B"".""MetadataID"" = ""C"".""ID"" 
+        AND ""C"".""BlocksetID"" = ""D"".""ID"" 
+        AND ""E"".""BlocksetID"" = ""C"".""BlocksetID""
+        AND ""E"".""BlockID"" = ""F"".""ID""
+        AND ""E"".""Index"" = 0
+        AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) 
+        AND ""A"".""FilesetID"" = ?
+    ) G
+LEFT OUTER JOIN
+   ""BlocklistHash"" H
+ON
+   ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID""
+ORDER BY
+   ""G"".""Path"", ""H"".""Index""
+
+";
+
        public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, System.Data.IDbTransaction transaction, long filesetId)
        {
            using (var cmd = m_connection.CreateCommand())
            {
-                cmd.Transaction = transaction;
-                cmd.CommandText = @"SELECT ""B"".""BlocksetID"", ""B"".""ID"", ""B"".""Path"", ""D"".""Length"", ""D"".""FullHash"", ""A"".""Lastmodified"" FROM ""FilesetEntry"" A, ""File"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) AND ""A"".""FilesetID"" = ? ";
+            	cmd.Transaction = transaction;
+                cmd.CommandText = LIST_FOLDERS_AND_SYMLINKS;
                cmd.AddParameter(FOLDER_BLOCKSET_ID);
                cmd.AddParameter(SYMLINK_BLOCKSET_ID);
                cmd.AddParameter(filesetId);

+                string lastpath = null;
                using (var rd = cmd.ExecuteReader())
                while(rd.Read())
                {
-                        var blocksetID = rd.GetInt64(0);
-                        var path = rd.GetValue(2).ToString();
-                        var metalength = rd.GetInt64(3);
-                        var metahash = rd.GetValue(4).ToString();
+                    var blocksetID = rd.ConvertValueToInt64(0, -1);
+                    var path = rd.GetValue(2).ToString();
+                    var metalength = rd.ConvertValueToInt64(3, -1);
+                    var metahash = rd.GetValue(4).ToString();
+                    var metablockhash = rd.GetValue(6).ToString();
+                    var metablocklisthash = rd.GetValue(7).ToString();
+
+                    if (path == lastpath)
+                        m_result.AddWarning(string.Format("Duplicate path detected: {0}!", path), null);
+
+                    lastpath = path;

                    if (blocksetID == FOLDER_BLOCKSET_ID)
-                        filesetvolume.AddDirectory(path, metahash, metalength);
+                        filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
                    else if (blocksetID == SYMLINK_BLOCKSET_ID)
-                        filesetvolume.AddSymlink(path, metahash, metalength);
+                        filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
                }
+                        
+                // TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x

-                cmd.CommandText = @"SELECT ""F"".""Path"", ""F"".""Lastmodified"", ""F"".""Filelength"", ""F"".""Filehash"", ""F"".""Metahash"", ""F"".""Metalength"", ""G"".""Hash"" FROM (SELECT ""A"".""Path"" AS ""Path"", ""D"".""Lastmodified"" AS ""Lastmodified"", ""B"".""Length"" AS ""Filelength"", ""B"".""FullHash"" AS ""Filehash"", ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"" FROM ""File"" A, ""Blockset"" B, ""Metadataset"" C, ""FilesetEntry"" D, ""Blockset"" E WHERE ""A"".""ID"" = ""D"".""FileID"" AND ""D"".""FilesetID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"") F LEFT OUTER JOIN ""BlocklistHash"" G ON ""G"".""BlocksetID"" = ""F"".""BlocksetID"" ORDER BY ""F"".""Path"", ""G"".""Index"" ";
+                cmd.CommandText = LIST_FILESETS;
                cmd.Parameters.Clear();
                cmd.AddParameter(filesetId);

@@ -927,8 +1078,19 @@ namespace Duplicati.Library.Main.Database
                        var metasize = rd.ConvertValueToInt64(5, -1);
                        var p = rd.GetValue(6);
                        var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd);
+                        var blockhash = rd.GetValue(7).ToString();
+                        var blocksize = rd.ConvertValueToInt64(8, -1);
+                        var metablockhash = rd.GetValue(9).ToString();
+                        //var metablocksize = rd.ConvertValueToInt64(10, -1);
+                        var metablocklisthash = rd.GetValue(11).ToString();

-                        filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, blrd);
+                        if (blockhash == filehash)
+                            blockhash = null;
+
+                        if (metablockhash == metahash)
+                            metablockhash = null;
+
+                        filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
                        if (blrd == null)
                            more = rd.Read();
                        else
@@ -51,6 +51,7 @@ namespace Duplicati.Library.Main.Database
        private System.Data.IDbCommand m_insertBlocklistHashCommand;
        private System.Data.IDbCommand m_updateBlockVolumeCommand;
        private System.Data.IDbCommand m_insertBlockset;
+        private System.Data.IDbCommand m_insertSmallBlockset;
        private System.Data.IDbCommand m_findBlocksetCommand;
        private System.Data.IDbCommand m_findMetadatasetCommand;
        private System.Data.IDbCommand m_findFilesetCommand;
@@ -66,6 +67,7 @@ namespace Duplicati.Library.Main.Database
        private PathLookupHelper<PathEntryKeeper> m_filesetLookup;
        
        private string m_tempblocklist;
+        private string m_tempsmalllist;
        
        /// <summary>
        /// A lookup table that prevents multiple downloads of the same volume
@@ -122,11 +124,16 @@ namespace Duplicati.Library.Main.Database
            : base(parentdb)
        {
            m_tempblocklist = "TempBlocklist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
+            m_tempsmalllist = "TempSmalllist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                        
            using(var cmd = m_connection.CreateCommand())
            {
                cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist));
                cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist));
+
+                cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)", m_tempsmalllist));
+                cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_File_{0}"" ON ""{0}"" (""FileHash"", ""BlockSize"");", m_tempsmalllist));
+                cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_Block_{0}"" ON ""{0}"" (""BlockHash"", ""BlockSize"");", m_tempsmalllist));
            }

            m_insertFileCommand = m_connection.CreateCommand();
@@ -136,6 +143,7 @@ namespace Duplicati.Library.Main.Database
            m_insertBlocklistHashCommand = m_connection.CreateCommand();
            m_updateBlockVolumeCommand = m_connection.CreateCommand();
            m_insertBlockset = m_connection.CreateCommand();
+            m_insertSmallBlockset = m_connection.CreateCommand();
            m_findBlocksetCommand = m_connection.CreateCommand();
            m_findMetadatasetCommand = m_connection.CreateCommand();
            m_findFilesetCommand = m_connection.CreateCommand();
@@ -164,7 +172,10 @@ namespace Duplicati.Library.Main.Database

            m_insertBlockset.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist);
            m_insertBlockset.AddParameters(3);
-            
+
+            m_insertSmallBlockset.CommandText = string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) ", m_tempsmalllist);
+            m_insertSmallBlockset.AddParameters(3);
+
            m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? ";
            m_findBlocksetCommand.AddParameters(2);
            
@@ -206,7 +217,8 @@ namespace Duplicati.Library.Main.Database
                cmd.Transaction = transaction;
                
                //Update all small blocklists and matching blocks
-                var selectSmallBlocks = string.Format(@"SELECT ""Blockset"".""Fullhash"", ""Blockset"".""Length"" FROM ""Blockset"" WHERE ""Blockset"".""Length"" <= {0}", blocksize);
+
+                var selectSmallBlocks = string.Format(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist);
            
                var selectBlockHashes = string.Format(
                    @"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " +
@@ -257,9 +269,12 @@ namespace Duplicati.Library.Main.Database
                    blocksize / hashsize
                    );
                    
+
+
                var selectBlocksetEntries = string.Format(
-                    @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" <= {0} ",
-                    blocksize
+                    @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{1}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {0} ",
+                    blocksize,
+                    m_tempsmalllist
                    );
                    
                var selectAllBlocksetEntries =
@@ -279,25 +294,24 @@ namespace Duplicati.Library.Main.Database
            }
        }
        
-        public void AddDirectoryEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+        public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
        {
-            AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metahash, metahashsize, transaction);
+            AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction);
        }

-        public void AddSymlinkEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+        public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
        {
-            AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metahash, metahashsize, transaction);
+            AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
        }
        
-        public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+        public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
        {
-            AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metahash, metahashsize, transaction);
+            AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metadataid, transaction);
        }
        
-        private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+        private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
        {
            var fileid = -1L;
-            var metadataid = AddMetadataset(metahash, metahashsize, transaction);
                        
            if (m_filesetLookup != null)
            {
@@ -342,7 +356,7 @@ namespace Duplicati.Library.Main.Database
            m_insertFilesetEntryCommand.ExecuteNonQuery();
        }
        
-        private long AddMetadataset(string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+        public long AddMetadataset(string metahash, long metahashsize, IEnumerable<string> metablocklisthashes, long expectedmetablocklisthashes, System.Data.IDbTransaction transaction)
        {
            var metadataid = -1L;
            if (metahash == null)
@@ -364,8 +378,8 @@ namespace Duplicati.Library.Main.Database
                if (metadataid != -1)
                    return metadataid;
            }
-            
-            var blocksetid = AddBlockset(metahash, metahashsize, null, 0, transaction);
+
+            var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction);
            
            m_insertMetadatasetCommand.Transaction = transaction;
            m_insertMetadatasetCommand.SetParameterValue(0, blocksetid);
@@ -405,13 +419,13 @@ namespace Duplicati.Library.Main.Database
            if (m_fileHashLookup != null)
                m_fileHashLookup.Add(fullhash, size, blocksetid);
        
+            long c = 0;
            if (blocklisthashes != null)
            {
                var index = 0L;
                m_insertBlocklistHashCommand.Transaction = transaction;
                m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid);

-                long c = 0;
                foreach(var hash in blocklisthashes)
                {
                    if (!string.IsNullOrEmpty(hash))
@@ -425,12 +439,11 @@ namespace Duplicati.Library.Main.Database
                        }
                    }
                }
-
-                if (c != expectedblocklisthashes) // or is there a legacy with single element blocklist hashes? --> && !(c == 1 && hash == computeBlockHash(fullhash))
-                    m_result.AddWarning(string.Format("Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid), null);
-
            }
                            
+            if (c != expectedblocklisthashes)
+                m_result.AddWarning(string.Format("Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid), null);
+            
            return blocksetid;
        }

@@ -494,6 +507,15 @@ namespace Duplicati.Library.Main.Database
                return false;
            }            
        }
+
+        public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, System.Data.IDbTransaction transaction)
+        {
+            m_insertSmallBlockset.Transaction = transaction;
+            m_insertSmallBlockset.SetParameterValue(0, filehash);
+            m_insertSmallBlockset.SetParameterValue(1, blockhash);
+            m_insertSmallBlockset.SetParameterValue(2, blocksize);
+            m_insertSmallBlockset.ExecuteNonQuery();
+        }
        
        public bool UpdateBlockset(string hash, IEnumerable<string> blocklisthashes, System.Data.IDbTransaction transaction)
        {
@@ -643,8 +665,17 @@ namespace Duplicati.Library.Main.Database
                    catch { }
                    finally { m_tempblocklist = null; }
                    
+                if (m_tempsmalllist != null)
+                    try
+                {
+                    cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempsmalllist);
+                    cmd.ExecuteNonQuery();
+                }
+                catch { }
+                finally { m_tempsmalllist = null; }
+
            }
-                    
+
            base.Dispose();
        }
    }
@@ -478,7 +478,7 @@ namespace Duplicati.Library.Main.Database

                    try
                    {
-                        VerifyConsistency(tr, blocksize, hashsize);
+                        VerifyConsistency(tr, blocksize, hashsize, true);
                    }
                    catch(Exception ex)
                    {
@@ -28,11 +28,7 @@ namespace Duplicati.Library.Main
        /// <summary>
        /// The base64 encoded hash of the metadata
        /// </summary>
-        string Hash { get; }
-        /// <summary>
-        /// The size of the metadata in bytes
-        /// </summary>
-        long Size { get; }
+        string FileHash { get; }
        /// <summary>
        /// The UTF-8 encoded json element with the metadata
        /// </summary>
@@ -41,6 +41,7 @@ namespace Duplicati.Library.Main.Operation
        private readonly FileAttributes m_attributeFilter;
        private readonly Options.SymlinkStrategy m_symlinkPolicy;
        private int m_blocksize;
+        private long m_maxmetadatasize;

        public BackupHandler(string backendurl, Options options, BackupResults results)
        {
@@ -695,6 +696,7 @@ namespace Duplicati.Library.Main.Operation
                    throw new Exception("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.");

                m_blocksize = m_options.Blocksize;
+                m_maxmetadatasize = (m_blocksize / m_options.BlockhashSize) * m_blocksize;

                m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
                m_blocklistbuffer = new byte[m_options.Blocksize];
@@ -712,7 +714,7 @@ namespace Duplicati.Library.Main.Operation
                if (!m_filehasher.CanReuseTransform)
                    throw new Exception(Strings.Common.InvalidCryptoSystem(m_options.FileHashAlgorithm));

-                m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
+                m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, false);
                // If there is no filter, we set an empty filter to simplify the code
                // If there is a filter, we make sure that the sources are included
                m_filter = filter ?? new Library.Utility.FilterExpression();
@@ -743,7 +745,6 @@ namespace Duplicati.Library.Main.Operation
                            PreBackupVerify(backend);

                            // Verify before uploading a synthetic list
-                            m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
                            UploadSyntheticFilelist(backend);

                            m_database.BuildLookupTable(m_options);
@@ -772,7 +773,7 @@ namespace Duplicati.Library.Main.Operation
                        using(new Logging.Timer("UpdateChangeStatistics"))
                            m_database.UpdateChangeStatistics(m_result);
                        using(new Logging.Timer("VerifyConsistency"))
-                            m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize);
+                            m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize, false);
    
                        UploadRealFileList(backend, filesetvolume);
                                        
@@ -987,7 +988,7 @@ namespace Duplicati.Library.Main.Operation
                var timestampChanged = lastwrite != oldModified || lastwrite.Ticks == 0 || oldModified.Ticks == 0;
                var filesizeChanged = filestatsize < 0 || lastFileSize < 0 || filestatsize != lastFileSize;
                var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan;
-                var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Size != oldMetasize || metahashandsize.Hash != oldMetahash);
+                var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Blob.Length != oldMetasize || metahashandsize.FileHash != oldMetahash);

                if ((oldId < 0 || m_options.DisableFiletimeCheck || timestampChanged || filesizeChanged || metadatachanged) && !tooLargeFile)
                {
@@ -1003,67 +1004,16 @@ namespace Duplicati.Library.Main.Operation
                    using (var blocklisthashes = new Library.Utility.FileBackedStringList())
                    using (var hashcollector = new Library.Utility.FileBackedStringList())
                    {
-                        using (var fs = new Blockprocessor(snapshot.OpenRead(path), m_blockbuffer))
+                        using (var fs = snapshot.OpenRead(path))
                        {
                            try { m_result.OperationProgressUpdater.StartFile(path, fs.Length); }
                            catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read file length for file {0}", path), ex); }
                            
-                            int blocklistoffset = 0;
-
-                            m_filehasher.Initialize();
-
-                            
-                            var offset = 0;
-                            var remaining = fs.Readblock();
-                            
-                            do
-                            {
-                                var size = Math.Min(m_blocksize, remaining);
-
-                                m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset);
-                                var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size);
-                                if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length)
-                                {
-                                    var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
-                                    blocklisthashes.Add(blkey);
-                                    AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
-                                    blocklistoffset = 0;
-                                }
-
-                                Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length);
-                                blocklistoffset += blockkey.Length;
-
-                                var key = Convert.ToBase64String(blockkey);
-                                AddBlockToOutput(backend, key, m_blockbuffer, offset, size, hint, false);
-                                hashcollector.Add(key);
-                                filesize += size;
-
-                                m_result.OperationProgressUpdater.UpdateFileProgress(filesize);
-                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
-                                    return false;
-                                
-                                remaining -= size;
-                                offset += size;
-                                
-                                if (remaining == 0)
-                                {
-                                    offset = 0;
-                                    remaining = fs.Readblock();
-                                }
-
-                            } while (remaining > 0);
-
-                            //If all fits in a single block, don't bother with blocklists
-                            if (hashcollector.Count > 1)
-                            {
-                                var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
-                                blocklisthashes.Add(blkeyfinal);
-                                AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
-                            }
+                            if ((filesize = ProcessStream(fs, hint, backend, blocklisthashes, hashcollector, false)) < 0)
+                                return false;
                        }

                        m_result.SizeOfOpenedFiles += filesize;
-                        m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0);

                        var filekey = Convert.ToBase64String(m_filehasher.Hash);
                        if (oldHash != filekey)
@@ -1131,6 +1081,76 @@ namespace Duplicati.Library.Main.Operation
            return true;
        }

+        private long ProcessStream(System.IO.Stream stream, Library.Interface.CompressionHint hint, BackendManager backend, Library.Utility.FileBackedStringList blocklisthashes, Library.Utility.FileBackedStringList hashcollector, bool skipfilehash)
+        {
+            int blocklistoffset = 0;
+            long filesize = 0;
+
+            using(var fs = new Blockprocessor(stream, m_blockbuffer))
+            {
+                m_filehasher.Initialize();
+
+                var offset = 0;
+                var remaining = fs.Readblock();
+
+                do
+                {
+                    var size = Math.Min(m_blocksize, remaining);
+
+                    if (!skipfilehash)
+                        m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset);
+                    
+                    var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size);
+                    if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length)
+                    {
+                        var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
+                        blocklisthashes.Add(blkey);
+                        AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
+                        blocklistoffset = 0;
+                    }
+
+                    Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length);
+                    blocklistoffset += blockkey.Length;
+
+                    var key = Convert.ToBase64String(blockkey);
+                    AddBlockToOutput(backend, key, m_blockbuffer, offset, size, hint, false);
+                    hashcollector.Add(key);
+                    filesize += size;
+
+                    if (!skipfilehash)
+                    {
+                        m_result.OperationProgressUpdater.UpdateFileProgress(filesize);                    
+                        if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+                            return -1;
+                    }
+
+                    remaining -= size;
+                    offset += size;
+
+                    if (remaining == 0)
+                    {
+                        offset = 0;
+                        remaining = fs.Readblock();
+                    }
+
+                } while (remaining > 0);
+
+                //If all fits in a single block, don't bother with blocklists
+                if (hashcollector.Count > 1)
+                {
+                    var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
+                    blocklisthashes.Add(blkeyfinal);
+                    AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
+                }
+            }
+
+            if (!skipfilehash)
+                m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0);
+
+            return filesize;
+
+        }
+
        /// <summary>
        /// Adds the found file data to the output unless the block already exists
        /// </summary>
@@ -1223,29 +1243,32 @@ namespace Duplicati.Library.Main.Operation
            m_database.AddUnmodifiedFile(oldId, lastModified, m_transaction);
        }

-
-        /// <summary>
-        /// Adds a file to the output, 
-        /// </summary>
-        /// <param name="filename">The name of the file to record</param>
-        /// <param name="lastModified">The value of the lastModified timestamp</param>
-        /// <param name="hashlist">The list of hashes that make up the file</param>
-        /// <param name="size">The size of the file</param>
-        /// <param name="fragmentoffset">The offset into a fragment block where the last few bytes are stored</param>
-        /// <param name="metadata">A lookup table with various metadata values describing the file</param>
-        private bool AddFolderToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash meta)
+        private long AddMetadataToOutput(BackendManager backend, IMetahash meta)
        {
            long metadataid;
-            bool r = false;

-            if (meta.Size > m_blocksize)
-                throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize));
+            if (meta.Blob.Length > m_maxmetadatasize)
+            {
+                //TODO: To fix this, the "WriteFileset" method in BackupHandler needs to
+                // be updated such that it can select sets even when there are multiple
+                // blocklist hashes for the metadata.
+                // This could be done such that an extra query is made if the metadata
+                // spans multiple blocklist hashes, as it is not expected to be common

-            r |= AddBlockToOutput(backend, meta.Hash, meta.Blob, 0, (int)meta.Size, CompressionHint.Default, false);
-            r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction);
+                m_result.AddWarning(string.Format("Metadata size is {0}, but the largest accepted size is {1}, recording empty metadata", meta.Blob.Length, m_maxmetadatasize), null);
+                meta = EMPTY_METADATA;
+            }

-            m_database.AddDirectoryEntry(filename, metadataid, lastModified, m_transaction);
-            return r;
+            using(var blocklisthashes = new Library.Utility.FileBackedStringList())
+            using(var hashcollector = new Library.Utility.FileBackedStringList())
+            {
+                using(var ms = new MemoryStream(meta.Blob))
+                    ProcessStream(ms, CompressionHint.Compressible, backend, blocklisthashes, hashcollector, true);
+
+                m_database.AddMetadataset(meta.FileHash, meta.Blob.Length, m_blocksize, hashcollector, blocklisthashes, out metadataid, m_transaction);
+            }  
+
+            return metadataid;
        }

        /// <summary>
@@ -1257,19 +1280,25 @@ namespace Duplicati.Library.Main.Operation
        /// <param name="size">The size of the file</param>
        /// <param name="fragmentoffset">The offset into a fragment block where the last few bytes are stored</param>
        /// <param name="metadata">A lookup table with various metadata values describing the file</param>
-        private bool AddSymlinkToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash meta)
+        private void AddFolderToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash metadata)
        {
-            long metadataid;
-            bool r = false;
-
-            if (meta.Size > m_blocksize)
-                throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize));
-
-            r |= AddBlockToOutput(backend, meta.Hash, meta.Blob, 0, (int)meta.Size, CompressionHint.Default, false);
-            r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction);
+            var metadataid = AddMetadataToOutput(backend, metadata);
+            m_database.AddDirectoryEntry(filename, metadataid, lastModified, m_transaction);
+        }

+        /// <summary>
+        /// Adds a file to the output, 
+        /// </summary>
+        /// <param name="filename">The name of the file to record</param>
+        /// <param name="lastModified">The value of the lastModified timestamp</param>
+        /// <param name="hashlist">The list of hashes that make up the file</param>
+        /// <param name="size">The size of the file</param>
+        /// <param name="fragmentoffset">The offset into a fragment block where the last few bytes are stored</param>
+        /// <param name="metadata">A lookup table with various metadata values describing the file</param>
+        private void AddSymlinkToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash metadata)
+        {
+            var metadataid = AddMetadataToOutput(backend, metadata);
            m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction);
-            return r;
        }

        /// <summary>
@@ -1283,17 +1312,10 @@ namespace Duplicati.Library.Main.Operation
        /// <param name="metadata">A lookup table with various metadata values describing the file</param>
        private void AddFileToOutput(BackendManager backend, string filename, long size, DateTime lastmodified, IMetahash metadata, IEnumerable<string> hashlist, string filehash, IEnumerable<string> blocklisthashes)
        {
-            long metadataid;
            long blocksetid;
-            
-            if (metadata.Size > m_blocksize)
-                throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize));
-
-            AddBlockToOutput(backend, metadata.Hash, metadata.Blob, 0, (int)metadata.Size, CompressionHint.Default, false);
-            m_database.AddMetadataset(metadata.Hash, metadata.Size, out metadataid, m_transaction);

+            var metadataid = AddMetadataToOutput(backend, metadata);
            m_database.AddBlockset(filehash, size, m_blocksize, hashlist, blocklisthashes, out blocksetid, m_transaction);
-
            m_database.AddFile(filename, lastmodified, blocksetid, metadataid, m_transaction);
        }

@@ -215,9 +215,14 @@ namespace Duplicati.Library.Main.Operation
                                    {
                                        try
                                        {
+                                            var expectedmetablocks = (fe.Metasize + blocksize - 1)  / blocksize;
+                                            var expectedmetablocklisthashes = (expectedmetablocks + hashes_pr_block - 1) / hashes_pr_block;
+                                            if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0;
+
                                            if (fe.Type == FilelistEntryType.Folder)
                                            {
-                                                restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
+                                                var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
+                                                restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
                                            }
                                            else if (fe.Type == FilelistEntryType.File)
                                            {
@@ -226,16 +231,38 @@ namespace Duplicati.Library.Main.Operation
                                                if (expectedblocks <= 1) expectedblocklisthashes = 0;

                                                var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr);
-                                                restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
+                                                var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
+                                                restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, metadataid, tr);
+                                                
+                                                if (fe.Size <= blocksize)
+                                                {
+                                                    if (!string.IsNullOrWhiteSpace(fe.Blockhash))
+                                                        restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, tr);
+                                                    else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
+                                                        restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, tr);
+                                                    else
+                                                        m_result.AddWarning(string.Format("No block hash found for file: {0}", fe.Path), null);
+                                                }
                                            }
                                            else if (fe.Type == FilelistEntryType.Symlink)
                                            {
-                                                restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
+                                                var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
+                                                restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
                                            }
                                            else
                                            {
                                                m_result.AddWarning(string.Format("Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path), null);
                                            }
+
+                                            if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink))
+                                            {
+                                                if (!string.IsNullOrWhiteSpace(fe.Metablockhash))
+                                                    restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, tr);
+                                                else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
+                                                    restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, tr);
+                                                else
+                                                    m_result.AddWarning(string.Format("No block hash found for file metadata: {0}", fe.Path), null);                                                
+                                            }
                                        }
                                        catch (Exception ex)
                                        {
@@ -435,7 +462,7 @@ namespace Duplicati.Library.Main.Operation
                    //All done, we must verify that we have all blocklist fully intact
                    // if this fails, the db will not be deleted, so it can be used,
                    // except to continue a backup
-                    restoredb.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
+                    restoredb.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, true);

                    m_result.AddMessage("Recreate completed, and consistency checks completed, marking database as complete");

@@ -34,7 +34,7 @@ namespace Duplicati.Library.Main
            /// <summary>
            /// The base64 encoded hash
            /// </summary>
-            private readonly string m_hash;
+            private readonly string m_filehash;
            /// <summary>
            /// The UTF-8 encoded json element with the metadata
            /// </summary>
@@ -47,30 +47,29 @@ namespace Duplicati.Library.Main
            public Metahash(Dictionary<string, string> values, Options options)
            {
                m_values = values;
-                var hasher = System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm);
-                if (hasher == null)
-                    throw new Exception(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm));
-                if (!hasher.CanReuseTransform)
-                    throw new Exception(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm));
                    
                using (var ms = new System.IO.MemoryStream())
                using (var w = new StreamWriter(ms, Encoding.UTF8))
+                using(var filehasher = System.Security.Cryptography.HashAlgorithm.Create(options.FileHashAlgorithm))
                {
+                    if (filehasher == null)
+                        throw new Exception(Strings.Common.InvalidHashAlgorithm(options.FileHashAlgorithm));
+                    
                    w.Write(JsonConvert.SerializeObject(values));
                    w.Flush();
    
                    m_blob = ms.ToArray();
    
                    ms.Position = 0;
-                    m_hash = Convert.ToBase64String(hasher.ComputeHash(ms));
+                    m_filehash = Convert.ToBase64String(filehasher.ComputeHash(ms));
                }
            }
    
-            public string Hash
+            public string FileHash
            {
-                get { return m_hash; }
+                get { return m_filehash; }
            }
-    
+                
            public long Size
            {
                get { return m_blob.Length; }
@@ -97,8 +97,12 @@ namespace Duplicati.Library.Main.Volumes
                    public long Size { get; private set; }
                    public DateTime Time { get; private set; }
                    public string Metahash { get; private set; }
+                    public string Metablockhash { get; private set; }
                    public long Metasize { get; private set; }
+                    public string Blockhash { get; private set; }
+                    public long Blocksize { get; private set; }
                    public IEnumerable<string> BlocklistHashes { get; private set; }
+                    public IEnumerable<string> MetaBlocklistHashes { get; private set; }
                    private JsonReader m_reader;

                    public FileEntry(JsonReader reader)
@@ -150,16 +154,60 @@ namespace Duplicati.Library.Main.Volumes

                                if (!m_reader.Read())
                                    throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+
+                                if (m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "metablocklists")
+                                {
+                                    var metadatablocklisthashes = new List<string>();
+                                    SkipJsonToken(m_reader, JsonToken.StartArray);
+
+                                    if (!m_reader.Read())
+                                        throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+                                    
+                                    while(m_reader.TokenType == JsonToken.String)
+                                    {
+                                        metadatablocklisthashes.Add(m_reader.Value.ToString());
+                                        if (!m_reader.Read())
+                                            throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+                                    }
+                                    
+                                    if (m_reader.TokenType != JsonToken.EndArray)
+                                        throw new InvalidDataException(string.Format("Invalid JSON, unexpected token {1} found while reading entry {0}", this.Path, m_reader.TokenType));
+                                    if (!m_reader.Read())
+                                        throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+
+                                    this.MetaBlocklistHashes = metadatablocklisthashes;
+                                    this.Metablockhash = null;
+                                }
+                                else if (m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "metablockhash")
+                                {
+                                    if (!m_reader.Read())
+                                        throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+                                    this.Metablockhash = m_reader.Value.ToString();
+
+                                    if (!m_reader.Read())
+                                        throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+
+                                    this.MetaBlocklistHashes = null;
+                                }
                            }

                            if ((this.Type == FilelistEntryType.File || this.Type == FilelistEntryType.AlternateStream) && m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "blocklists")
                            {
                                SkipJsonToken(m_reader, JsonToken.StartArray);
-                                BlocklistHashes = new BlocklistHashEnumerable(m_reader);
+                                this.BlocklistHashes = new BlocklistHashEnumerable(m_reader);
+                            }
+                            else if ((this.Type == FilelistEntryType.File || this.Type == FilelistEntryType.AlternateStream) && m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "blockhash")
+                            {
+                                if (!m_reader.Read())
+                                    throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+                                
+                                this.Blockhash = m_reader.Value.ToString();
+                                this.Blocksize = ReadJsonInt64Property(m_reader, "blocksize");
+                                this.BlocklistHashes = null;
                            }
                            else
                            {
-                                BlocklistHashes = null;
+                                this.BlocklistHashes = null;
                                if (m_reader.TokenType != JsonToken.EndObject)
                                    throw new InvalidDataException(string.Format("Invalid JSON, expected EndObject, but found {0}, \"{1}\" while reading entry {2}", m_reader.TokenType, m_reader.Value, this.Path));
                            }
@@ -25,17 +25,45 @@ namespace Duplicati.Library.Main.Volumes
            m_writer.WriteStartArray();
        }

-        public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable<string> blocklisthashes)
+        private void WriteMetaProperties(string metahash, long metasize, string metablockhash, IEnumerable<string> metablocklisthashes)
        {
-            AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, blocklisthashes);
+            m_writer.WritePropertyName("metahash");
+            m_writer.WriteValue(metahash);
+            m_writer.WritePropertyName("metasize");
+            m_writer.WriteValue(metasize);
+
+            if (metablocklisthashes != null)
+            {
+                //Slightly akward, but we avoid writing if there are no entries 
+                var en = metablocklisthashes.GetEnumerator();
+                if (en.MoveNext() && !string.IsNullOrEmpty(en.Current))
+                {
+                    m_writer.WritePropertyName("metablocklists");
+                    m_writer.WriteStartArray();
+                    m_writer.WriteValue(en.Current);
+                    while (en.MoveNext())
+                        m_writer.WriteValue(en.Current);
+                    m_writer.WriteEndArray();
+                }
+            }
+            else if (!string.IsNullOrWhiteSpace(metablockhash))
+            {
+                m_writer.WritePropertyName("metablockhash");
+                m_writer.WriteValue(metablockhash);
+            }
        }

-        public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable<string> blocklisthashes)
+        public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
        {
-            AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, blocklisthashes);
+            AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes);
        }

-        private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable<string> blocklisthashes)
+        public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
+        {
+            AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes);
+        }
+
+        private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
        {
            m_filecount++;
            m_writer.WriteStartObject();
@@ -50,12 +78,7 @@ namespace Duplicati.Library.Main.Volumes
            m_writer.WritePropertyName("time");
            m_writer.WriteValue(Library.Utility.Utility.SerializeDateTime(lastmodified));
            if (metahash != null)
-            {
-                m_writer.WritePropertyName("metahash");
-                m_writer.WriteValue(metahash);
-                m_writer.WritePropertyName("metasize");
-                m_writer.WriteValue(metasize);
-            }
+                WriteMetaProperties(metahash, metasize, metablockhash, metablocklisthashes);

            if (blocklisthashes != null)
            {
@@ -66,21 +89,28 @@ namespace Duplicati.Library.Main.Volumes
                    m_writer.WritePropertyName("blocklists");
                    m_writer.WriteStartArray();
                    m_writer.WriteValue(en.Current);
-                    while(en.MoveNext())
+                    while (en.MoveNext())
                        m_writer.WriteValue(en.Current);
                    m_writer.WriteEndArray();
                }
            }
+            else if (!string.IsNullOrWhiteSpace(blockhash))
+            {
+                m_writer.WritePropertyName("blockhash");
+                m_writer.WriteValue(blockhash);
+                m_writer.WritePropertyName("blocksize");
+                m_writer.WriteValue(blocksize);
+            }

            m_writer.WriteEndObject();
        }

-        public void AddDirectory(string name, string metahash, long metasize)
+        public void AddDirectory(string name, string metahash, long metasize, string metablockhash, IEnumerable<string> metablocklisthashes)
        {
-            AddMetaEntry(FilelistEntryType.Folder, name, metahash, metasize);
+            AddMetaEntry(FilelistEntryType.Folder, name, metahash, metasize, metablockhash, metablocklisthashes);
        }

-        public void AddMetaEntry(FilelistEntryType type, string name, string metahash, long metasize)
+        public void AddMetaEntry(FilelistEntryType type, string name, string metahash, long metasize, string metablockhash, IEnumerable<string> metablocklisthashes)
        {
            m_foldercount++;
            m_writer.WriteStartObject();
@@ -89,12 +119,8 @@ namespace Duplicati.Library.Main.Volumes
            m_writer.WritePropertyName("path");
            m_writer.WriteValue(name);
            if (metahash != null)
-            {
-                m_writer.WritePropertyName("metahash");
-                m_writer.WriteValue(metahash);
-                m_writer.WritePropertyName("metasize");
-                m_writer.WriteValue(metasize);
-            }
+                WriteMetaProperties(metahash, metasize, metablockhash, metablocklisthashes);
+            
            m_writer.WriteEndObject();
        }

@@ -127,9 +153,9 @@ namespace Duplicati.Library.Main.Volumes
        public long FileCount { get { return m_filecount; } }
        public long FolderCount { get { return m_foldercount; } }

-        public void AddSymlink(string name, string metahash, long metasize)
+        public void AddSymlink(string name, string metahash, long metasize, string metablockhash, IEnumerable<string> metablocklisthashes)
        {
-            AddMetaEntry(FilelistEntryType.Symlink, name, metahash, metasize);
+            AddMetaEntry(FilelistEntryType.Symlink, name, metahash, metasize, metablockhash, metablocklisthashes);
        }
    }
 }
@@ -14,7 +14,11 @@ namespace Duplicati.Library.Main.Volumes
        long Size { get; }
        DateTime Time { get; }
        string Metahash { get; }
+        string Metablockhash { get; }
        long Metasize { get; }
+        string Blockhash { get; }
+        long Blocksize { get; }
        IEnumerable<string> BlocklistHashes { get; }
+        IEnumerable<string> MetaBlocklistHashes { get; }
    }
 }
@@ -11,7 +11,7 @@ namespace Duplicati.Library.Main.Volumes
        protected class ManifestData
        {
            public const string ENCODING = "utf8";
-            public const long VERSION = 1;
+            public const long VERSION = 2;

            public long Version;
            public string Created;
@@ -65,6 +65,17 @@ namespace Duplicati.UnitTest
            RunCommands(12345, 1024 * 1024 * 10);
        }

+        [Test]
+        [Category("Border")]
+        public void RunNoMetadata()
+        {
+            PrepareSourceData();
+            RunCommands(1024 * 10, modifyOptions: opts => {
+                opts["skip-metadata"] = "true";
+            });
+        }
+
+
        [Test]
        [Category("Border")]
        public void RunMD5()
@@ -87,7 +98,7 @@ namespace Duplicati.UnitTest
            });
        }

-        //[Test]
+        [Test]
        [Category("Border")]
        public void RunMixedBlockFile_1()
        {
@@ -98,7 +109,7 @@ namespace Duplicati.UnitTest
            });
        }

-        //[Test]
+        [Test]
        [Category("Border")]
        public void RunMixedBlockFile_2()
        {
@@ -261,7 +272,7 @@ namespace Duplicati.UnitTest
                Assert.AreEqual(filenames.Count * 3, r.FilesRestored);
            }

-            TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, true);
+            TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"));

            using(var tf = new Library.Utility.TempFolder())
            {