From e40bd7f02a92dc159eaf06f8833dc473b494db41 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 3 Apr 2016 23:10:47 +0200 Subject: [PATCH 1/9] Proposed version, not complete with handling large metadata chunks --- .../Main/Database/LocalBackupDatabase.cs | 10 +- .../Library/Main/Database/LocalDatabase.cs | 65 +++++- .../Main/Database/LocalRecreateDatabase.cs | 56 +++++- Duplicati/Library/Main/IMetahash.cs | 6 +- .../Library/Main/Operation/BackupHandler.cs | 190 +++++++++--------- .../Main/Operation/RecreateDatabaseHandler.cs | 21 ++ Duplicati/Library/Main/Utility.cs | 19 +- .../Main/Volumes/FilesetVolumeReader.cs | 26 ++- .../Main/Volumes/FilesetVolumeWriter.cs | 24 ++- Duplicati/Library/Main/Volumes/IFileEntry.cs | 3 + Duplicati/Library/Main/Volumes/VolumeBase.cs | 2 +- 11 files changed, 293 insertions(+), 129 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 62c0f3d35..6321754d3 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -441,26 +441,26 @@ namespace Duplicati.Library.Main.Database /// The metadata hash /// The id of the metadata set /// True if the set was added to the database, false otherwise - public bool AddMetadataset(string hash, long size, out long metadataid, System.Data.IDbTransaction transaction = null) + public bool AddMetadataset(string filehash, long size, IEnumerable blockhashes, IEnumerable blocklisthashes, out long metadataid, System.Data.IDbTransaction transaction = null) { if (size > 0) { if (m_metadataLookup != null) { - if(m_metadataLookup.TryGet(hash, size, out metadataid)) + if(m_metadataLookup.TryGet(filehash, size, out metadataid)) return false; } else { m_findmetadatasetCommand.Transaction = transaction; - metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, hash, size); + metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size); if (metadataid != -1) return false; } long blocksetid; - AddBlockset(hash, size, (int)size, new string[] { hash }, null, out blocksetid, transaction); + AddBlockset(filehash, size, (int)size, blockhashes, blocklisthashes, out blocksetid, transaction); using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { @@ -469,7 +469,7 @@ namespace Duplicati.Library.Main.Database metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64(); tr.Commit(); if (m_metadataLookup != null) - m_metadataLookup.Add(hash, size, metadataid); + m_metadataLookup.Add(filehash, size, metadataid); } return true; diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 2b467c995..54148a72f 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -887,7 +887,64 @@ namespace Duplicati.Library.Main.Database filesetvolume.AddSymlink(path, metahash, metalength); } - cmd.CommandText = @"SELECT ""F"".""Path"", ""F"".""Lastmodified"", ""F"".""Filelength"", ""F"".""Filehash"", ""F"".""Metahash"", ""F"".""Metalength"", ""G"".""Hash"" FROM (SELECT ""A"".""Path"" AS ""Path"", ""D"".""Lastmodified"" AS ""Lastmodified"", ""B"".""Length"" AS ""Filelength"", ""B"".""FullHash"" AS ""Filehash"", ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"" FROM ""File"" A, ""Blockset"" B, ""Metadataset"" C, ""FilesetEntry"" D, ""Blockset"" E WHERE ""A"".""ID"" = ""D"".""FileID"" AND ""D"".""FilesetID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"") F LEFT OUTER JOIN ""BlocklistHash"" G ON ""G"".""BlocksetID"" = ""F"".""BlocksetID"" ORDER BY ""F"".""Path"", ""G"".""Index"" "; + var query = @" +SELECT + ""J"".""Path"", + ""J"".""Lastmodified"", + ""J"".""Filelength"", + ""J"".""Filehash"", + ""J"".""Metahash"", + ""J"".""Metalength"", + ""K"".""Hash"" AS ""BlocklistHash"", + ""J"".""FirstBlockHash"", + ""J"".""FirstBlockSize"", + ""J"".""FirstMetaBlockHash"", + ""J"".""FirstMetaBlockSize"" +FROM + ( + SELECT + ""A"".""Path"" AS ""Path"", + ""D"".""Lastmodified"" AS ""Lastmodified"", + ""B"".""Length"" AS ""Filelength"", + ""B"".""FullHash"" AS ""Filehash"", + ""E"".""FullHash"" AS ""Metahash"", + ""E"".""Length"" AS ""Metalength"", + ""A"".""BlocksetID"" AS ""BlocksetID"", + ""F"".""Hash"" AS ""FirstBlockHash"", + ""F"".""Size"" AS ""FirstBlockSize"", + ""H"".""Hash"" AS ""FirstMetaBlockHash"", + ""H"".""Size"" AS ""FirstMetaBlockSize"" + FROM + ""File"" A, + ""Blockset"" B, + ""Metadataset"" C, + ""FilesetEntry"" D, + ""Blockset"" E, + ""Block"" F, + ""BlocksetEntry"" G, + ""Block"" H, + ""BlocksetEntry"" I + WHERE + ""A"".""ID"" = ""D"".""FileID"" + AND ""D"".""FilesetID"" = ? + AND ""A"".""BlocksetID"" = ""B"".""ID"" + AND ""A"".""MetadataID"" = ""C"".""ID"" + AND ""E"".""ID"" = ""C"".""BlocksetID"" + AND ""B"".""ID"" = ""G"".""BlocksetID"" + AND ""G"".""BlockID"" = ""F"".""ID"" + AND ""G"".""Index"" = 0 + AND ""I"".""BlocksetID"" = ""E"".""ID"" + AND ""I"".""BlockID"" = ""H"".""ID"" + AND ""I"".""Index"" = 0 + ) J +LEFT OUTER JOIN + ""BlocklistHash"" K +ON + ""K"".""BlocksetID"" = ""J"".""BlocksetID"" +ORDER BY ""J"".""Path"", ""K"".""Index"" +"; + + cmd.CommandText = query; cmd.Parameters.Clear(); cmd.AddParameter(filesetId); @@ -905,8 +962,12 @@ namespace Duplicati.Library.Main.Database var metasize = rd.ConvertValueToInt64(5, -1); var p = rd.GetValue(6); var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd); + var blockhash = rd.GetValue(7).ToString(); + var blocksize = rd.ConvertValueToInt64(8, -1); + var metablockhash = rd.GetValue(9).ToString(); + //var metablocksize = rd.ConvertValueToInt64(10, -1); - filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, blrd); + filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd); if (blrd == null) more = rd.Read(); else diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index fb876b226..ad3941ddf 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -51,6 +51,7 @@ namespace Duplicati.Library.Main.Database private System.Data.IDbCommand m_insertBlocklistHashCommand; private System.Data.IDbCommand m_updateBlockVolumeCommand; private System.Data.IDbCommand m_insertBlockset; + private System.Data.IDbCommand m_insertSmallBlockset; private System.Data.IDbCommand m_findBlocksetCommand; private System.Data.IDbCommand m_findMetadatasetCommand; private System.Data.IDbCommand m_findFilesetCommand; @@ -66,6 +67,7 @@ namespace Duplicati.Library.Main.Database private PathLookupHelper m_filesetLookup; private string m_tempblocklist; + private string m_tempsmalllist; /// /// A lookup table that prevents multiple downloads of the same volume @@ -122,11 +124,16 @@ namespace Duplicati.Library.Main.Database : base(parentdb) { m_tempblocklist = "TempBlocklist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + m_tempsmalllist = "TempSmalllist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using(var cmd = m_connection.CreateCommand()) { cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist)); cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist)); + + cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)", m_tempsmalllist)); + cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_File_{0}"" ON ""{0}"" (""FileHash"", ""BlockSize"");", m_tempsmalllist)); + cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_Block_{0}"" ON ""{0}"" (""BlockHash"", ""BlockSize"");", m_tempsmalllist)); } m_insertFileCommand = m_connection.CreateCommand(); @@ -136,6 +143,7 @@ namespace Duplicati.Library.Main.Database m_insertBlocklistHashCommand = m_connection.CreateCommand(); m_updateBlockVolumeCommand = m_connection.CreateCommand(); m_insertBlockset = m_connection.CreateCommand(); + m_insertSmallBlockset = m_connection.CreateCommand(); m_findBlocksetCommand = m_connection.CreateCommand(); m_findMetadatasetCommand = m_connection.CreateCommand(); m_findFilesetCommand = m_connection.CreateCommand(); @@ -164,7 +172,10 @@ namespace Duplicati.Library.Main.Database m_insertBlockset.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist); m_insertBlockset.AddParameters(3); - + + m_insertSmallBlockset.CommandText = string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) ", m_tempsmalllist); + m_insertSmallBlockset.AddParameters(3); + m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? "; m_findBlocksetCommand.AddParameters(2); @@ -206,7 +217,8 @@ namespace Duplicati.Library.Main.Database cmd.Transaction = transaction; //Update all small blocklists and matching blocks - var selectSmallBlocks = string.Format(@"SELECT ""Blockset"".""Fullhash"", ""Blockset"".""Length"" FROM ""Blockset"" WHERE ""Blockset"".""Length"" <= {0}", blocksize); + + var selectSmallBlocks = string.Format(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist); var selectBlockHashes = string.Format( @"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " + @@ -257,9 +269,12 @@ namespace Duplicati.Library.Main.Database blocksize / hashsize ); + + var selectBlocksetEntries = string.Format( - @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" <= {0} ", - blocksize + @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{1}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {0} ", + blocksize, + m_tempsmalllist ); var selectAllBlocksetEntries = @@ -364,7 +379,9 @@ namespace Duplicati.Library.Main.Database if (metadataid != -1) return metadataid; } - + + // Strange ..... metahash is hashed with block hash algorithm, but should be filehash? + var blocksetid = AddBlockset(metahash, metahashsize, null, 0, transaction); m_insertMetadatasetCommand.Transaction = transaction; @@ -426,8 +443,12 @@ namespace Duplicati.Library.Main.Database } } - if (c != expectedblocklisthashes) // or is there a legacy with single element blocklist hashes? --> && !(c == 1 && hash == computeBlockHash(fullhash)) - m_result.AddWarning(string.Format("Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid), null); + if (c != expectedblocklisthashes) + { + // Older versions of the filelists do not have a blocklisthash for files smaller than blocksize + if (c != 0 || expectedblocklisthashes != 1) + m_result.AddWarning(string.Format("Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid), null); + } } @@ -494,6 +515,15 @@ namespace Duplicati.Library.Main.Database return false; } } + + public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, System.Data.IDbTransaction transaction) + { + m_insertSmallBlockset.Transaction = transaction; + m_insertSmallBlockset.SetParameterValue(0, filehash); + m_insertSmallBlockset.SetParameterValue(1, blockhash); + m_insertSmallBlockset.SetParameterValue(2, blocksize); + m_insertSmallBlockset.ExecuteNonQuery(); + } public bool UpdateBlockset(string hash, IEnumerable blocklisthashes, System.Data.IDbTransaction transaction) { @@ -640,6 +670,15 @@ namespace Duplicati.Library.Main.Database catch { } finally { m_tempblocklist = null; } + if (m_tempsmalllist != null) + try + { + cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempsmalllist); + cmd.ExecuteNonQuery(); + } + catch { } + finally { m_tempsmalllist = null; } + } foreach(var cmd in new IDisposable [] { @@ -656,7 +695,8 @@ namespace Duplicati.Library.Main.Database m_findblocklisthashCommand, m_findHashBlockCommand, m_insertBlockCommand, - m_insertDuplicateBlockCommand + m_insertDuplicateBlockCommand, + m_insertSmallBlockset }) try { diff --git a/Duplicati/Library/Main/IMetahash.cs b/Duplicati/Library/Main/IMetahash.cs index e9a830900..2fdfd7880 100644 --- a/Duplicati/Library/Main/IMetahash.cs +++ b/Duplicati/Library/Main/IMetahash.cs @@ -28,11 +28,7 @@ namespace Duplicati.Library.Main /// /// The base64 encoded hash of the metadata /// - string Hash { get; } - /// - /// The size of the metadata in bytes - /// - long Size { get; } + string FileHash { get; } /// /// The UTF-8 encoded json element with the metadata /// diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index e0f975691..5aa94a036 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -930,7 +930,7 @@ namespace Duplicati.Library.Main.Operation var timestampChanged = lastwrite != oldModified || lastwrite.Ticks == 0 || oldModified.Ticks == 0; var filesizeChanged = filestatsize < 0 || lastFileSize < 0 || filestatsize != lastFileSize; var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan; - var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Size != oldMetasize || metahashandsize.Hash != oldMetahash); + var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Blob.Length != oldMetasize || metahashandsize.FileHash != oldMetahash); if ((oldId < 0 || m_options.DisableFiletimeCheck || timestampChanged || filesizeChanged || metadatachanged) && !tooLargeFile) { @@ -946,67 +946,16 @@ namespace Duplicati.Library.Main.Operation using (var blocklisthashes = new Library.Utility.FileBackedStringList()) using (var hashcollector = new Library.Utility.FileBackedStringList()) { - using (var fs = new Blockprocessor(snapshot.OpenRead(path), m_blockbuffer)) + using (var fs = snapshot.OpenRead(path)) { try { m_result.OperationProgressUpdater.StartFile(path, fs.Length); } catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read file length for file {0}", path), ex); } - int blocklistoffset = 0; - - m_filehasher.Initialize(); - - - var offset = 0; - var remaining = fs.Readblock(); - - do - { - var size = Math.Min(m_blocksize, remaining); - - m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset); - var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size); - if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length) - { - var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); - blocklisthashes.Add(blkey); - AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); - blocklistoffset = 0; - } - - Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length); - blocklistoffset += blockkey.Length; - - var key = Convert.ToBase64String(blockkey); - AddBlockToOutput(backend, key, m_blockbuffer, offset, size, hint, false); - hashcollector.Add(key); - filesize += size; - - m_result.OperationProgressUpdater.UpdateFileProgress(filesize); - if (m_result.TaskControlRendevouz() == TaskControlState.Stop) - return false; - - remaining -= size; - offset += size; - - if (remaining == 0) - { - offset = 0; - remaining = fs.Readblock(); - } - - } while (remaining > 0); - - //If all fits in a single block, don't bother with blocklists - if (hashcollector.Count > 1) - { - var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); - blocklisthashes.Add(blkeyfinal); - AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); - } + if ((filesize = ProcessStream(fs, hint, backend, blocklisthashes, hashcollector, false)) < 0) + return false; } m_result.SizeOfOpenedFiles += filesize; - m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0); var filekey = Convert.ToBase64String(m_filehasher.Hash); if (oldHash != filekey) @@ -1074,6 +1023,76 @@ namespace Duplicati.Library.Main.Operation return true; } + private long ProcessStream(System.IO.Stream stream, Library.Interface.CompressionHint hint, BackendManager backend, Library.Utility.FileBackedStringList blocklisthashes, Library.Utility.FileBackedStringList hashcollector, bool skipfilehash) + { + int blocklistoffset = 0; + long filesize = 0; + + using(var fs = new Blockprocessor(stream, m_blockbuffer)) + { + m_filehasher.Initialize(); + + var offset = 0; + var remaining = fs.Readblock(); + + do + { + var size = Math.Min(m_blocksize, remaining); + + if (!skipfilehash) + m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset); + + var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size); + if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length) + { + var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); + blocklisthashes.Add(blkey); + AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); + blocklistoffset = 0; + } + + Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length); + blocklistoffset += blockkey.Length; + + var key = Convert.ToBase64String(blockkey); + AddBlockToOutput(backend, key, m_blockbuffer, offset, size, hint, false); + hashcollector.Add(key); + filesize += size; + + if (!skipfilehash) + { + m_result.OperationProgressUpdater.UpdateFileProgress(filesize); + if (m_result.TaskControlRendevouz() == TaskControlState.Stop) + return -1; + } + + remaining -= size; + offset += size; + + if (remaining == 0) + { + offset = 0; + remaining = fs.Readblock(); + } + + } while (remaining > 0); + + //If all fits in a single block, don't bother with blocklists + if (hashcollector.Count > 1) + { + var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); + blocklisthashes.Add(blkeyfinal); + AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); + } + } + + if (!skipfilehash) + m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0); + + return filesize; + + } + /// /// Adds the found file data to the output unless the block already exists /// @@ -1167,28 +1186,20 @@ namespace Duplicati.Library.Main.Operation } - /// - /// Adds a file to the output, - /// - /// The name of the file to record - /// The value of the lastModified timestamp - /// The list of hashes that make up the file - /// The size of the file - /// The offset into a fragment block where the last few bytes are stored - /// A lookup table with various metadata values describing the file - private bool AddFolderToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash meta) + private long AddMetadataToOutput(BackendManager backend, IMetahash meta) { long metadataid; - bool r = false; - if (meta.Size > m_blocksize) - throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize)); + using(var blocklisthashes = new Library.Utility.FileBackedStringList()) + using(var hashcollector = new Library.Utility.FileBackedStringList()) + { + using(var ms = new MemoryStream(meta.Blob)) + ProcessStream(ms, CompressionHint.Compressible, backend, blocklisthashes, hashcollector, true); - r |= AddBlockToOutput(backend, meta.Hash, meta.Blob, 0, (int)meta.Size, CompressionHint.Default, false); - r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction); + m_database.AddMetadataset(meta.FileHash, meta.Blob.Length, hashcollector, blocklisthashes, out metadataid, m_transaction); + } - m_database.AddDirectoryEntry(filename, metadataid, lastModified, m_transaction); - return r; + return metadataid; } /// @@ -1200,19 +1211,25 @@ namespace Duplicati.Library.Main.Operation /// The size of the file /// The offset into a fragment block where the last few bytes are stored /// A lookup table with various metadata values describing the file - private bool AddSymlinkToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash meta) + private void AddFolderToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash metadata) { - long metadataid; - bool r = false; - - if (meta.Size > m_blocksize) - throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize)); - - r |= AddBlockToOutput(backend, meta.Hash, meta.Blob, 0, (int)meta.Size, CompressionHint.Default, false); - r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction); + var metadataid = AddMetadataToOutput(backend, metadata); + m_database.AddDirectoryEntry(filename, metadataid, lastModified, m_transaction); + } + /// + /// Adds a file to the output, + /// + /// The name of the file to record + /// The value of the lastModified timestamp + /// The list of hashes that make up the file + /// The size of the file + /// The offset into a fragment block where the last few bytes are stored + /// A lookup table with various metadata values describing the file + private void AddSymlinkToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash metadata) + { + var metadataid = AddMetadataToOutput(backend, metadata); m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction); - return r; } /// @@ -1226,17 +1243,10 @@ namespace Duplicati.Library.Main.Operation /// A lookup table with various metadata values describing the file private void AddFileToOutput(BackendManager backend, string filename, long size, DateTime lastmodified, IMetahash metadata, IEnumerable hashlist, string filehash, IEnumerable blocklisthashes) { - long metadataid; long blocksetid; - - if (metadata.Size > m_blocksize) - throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize)); - - AddBlockToOutput(backend, metadata.Hash, metadata.Blob, 0, (int)metadata.Size, CompressionHint.Default, false); - m_database.AddMetadataset(metadata.Hash, metadata.Size, out metadataid, m_transaction); + var metadataid = AddMetadataToOutput(backend, metadata); m_database.AddBlockset(filehash, size, m_blocksize, hashlist, blocklisthashes, out blocksetid, m_transaction); - m_database.AddFile(filename, lastmodified, blocksetid, metadataid, m_transaction); } diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 8bf18503e..412bf07c8 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -225,6 +225,27 @@ namespace Duplicati.Library.Main.Operation var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr); restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr); + + if (fe.Size <= blocksize) + { + if (!string.IsNullOrWhiteSpace(fe.Blockhash)) + restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, tr); + else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm) + restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, tr); + else + m_result.AddWarning(string.Format("No block hash found for file: {0}", fe.Path), null); + } + + if (fe.Metasize <= blocksize) + { + if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) + restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, tr); + else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm) + restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, tr); + else + m_result.AddWarning(string.Format("No block hash found for file metadata: {0}", fe.Path), null); + } + } else if (fe.Type == FilelistEntryType.Symlink) { diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index 663f31196..9f78a41d5 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -34,7 +34,7 @@ namespace Duplicati.Library.Main /// /// The base64 encoded hash /// - private readonly string m_hash; + private readonly string m_filehash; /// /// The UTF-8 encoded json element with the metadata /// @@ -47,30 +47,29 @@ namespace Duplicati.Library.Main public Metahash(Dictionary values, Options options) { m_values = values; - var hasher = System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm); - if (hasher == null) - throw new Exception(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm)); - if (!hasher.CanReuseTransform) - throw new Exception(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm)); using (var ms = new System.IO.MemoryStream()) using (var w = new StreamWriter(ms, Encoding.UTF8)) + using(var filehasher = System.Security.Cryptography.HashAlgorithm.Create(options.FileHashAlgorithm)) { + if (filehasher == null) + throw new Exception(Strings.Common.InvalidHashAlgorithm(options.FileHashAlgorithm)); + w.Write(JsonConvert.SerializeObject(values)); w.Flush(); m_blob = ms.ToArray(); ms.Position = 0; - m_hash = Convert.ToBase64String(hasher.ComputeHash(ms)); + m_filehash = Convert.ToBase64String(filehasher.ComputeHash(ms)); } } - public string Hash + public string FileHash { - get { return m_hash; } + get { return m_filehash; } } - + public long Size { get { return m_blob.Length; } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs index 742fa653c..23436f29b 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs @@ -97,7 +97,10 @@ namespace Duplicati.Library.Main.Volumes public long Size { get; private set; } public DateTime Time { get; private set; } public string Metahash { get; private set; } + public string Metablockhash { get; private set; } public long Metasize { get; private set; } + public string Blockhash { get; private set; } + public long Blocksize { get; private set; } public IEnumerable BlocklistHashes { get; private set; } private JsonReader m_reader; @@ -150,16 +153,35 @@ namespace Duplicati.Library.Main.Volumes if (!m_reader.Read()) throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); + + if (m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "metablockhash") + { + if (!m_reader.Read()) + throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); + this.Metablockhash = m_reader.Value.ToString(); + + if (!m_reader.Read()) + throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); + } } if ((this.Type == FilelistEntryType.File || this.Type == FilelistEntryType.AlternateStream) && m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "blocklists") { SkipJsonToken(m_reader, JsonToken.StartArray); - BlocklistHashes = new BlocklistHashEnumerable(m_reader); + this.BlocklistHashes = new BlocklistHashEnumerable(m_reader); + } + else if ((this.Type == FilelistEntryType.File || this.Type == FilelistEntryType.AlternateStream) && m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "blockhash") + { + if (!m_reader.Read()) + throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); + + this.Blockhash = m_reader.Value.ToString(); + this.Blocksize = ReadJsonInt64Property(m_reader, "blocksize"); + this.BlocklistHashes = null; } else { - BlocklistHashes = null; + this.BlocklistHashes = null; if (m_reader.TokenType != JsonToken.EndObject) throw new InvalidDataException(string.Format("Invalid JSON, expected EndObject, but found {0}, \"{1}\" while reading entry {2}", m_reader.TokenType, m_reader.Value, this.Path)); } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 1a86d2f09..62f26ea9f 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -25,17 +25,17 @@ namespace Duplicati.Library.Main.Volumes m_writer.WriteStartArray(); } - public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable blocklisthashes) + public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes) { - AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, blocklisthashes); + AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes); } - public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable blocklisthashes) + public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, IEnumerable blocklisthashes) { - AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, blocklisthashes); + AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, null, -1, blocklisthashes); } - private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable blocklisthashes) + private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes) { m_filecount++; m_writer.WriteStartObject(); @@ -55,6 +55,11 @@ namespace Duplicati.Library.Main.Volumes m_writer.WriteValue(metahash); m_writer.WritePropertyName("metasize"); m_writer.WriteValue(metasize); + if (!string.IsNullOrWhiteSpace(metablockhash)) + { + m_writer.WritePropertyName("metablockhash"); + m_writer.WriteValue(metablockhash); + } } if (blocklisthashes != null) @@ -66,11 +71,18 @@ namespace Duplicati.Library.Main.Volumes m_writer.WritePropertyName("blocklists"); m_writer.WriteStartArray(); m_writer.WriteValue(en.Current); - while(en.MoveNext()) + while (en.MoveNext()) m_writer.WriteValue(en.Current); m_writer.WriteEndArray(); } } + else if (blockhash != null) + { + m_writer.WritePropertyName("blockhash"); + m_writer.WriteValue(blockhash); + m_writer.WritePropertyName("blocksize"); + m_writer.WriteValue(blocksize); + } m_writer.WriteEndObject(); } diff --git a/Duplicati/Library/Main/Volumes/IFileEntry.cs b/Duplicati/Library/Main/Volumes/IFileEntry.cs index 836f99d44..97b9d6572 100644 --- a/Duplicati/Library/Main/Volumes/IFileEntry.cs +++ b/Duplicati/Library/Main/Volumes/IFileEntry.cs @@ -14,7 +14,10 @@ namespace Duplicati.Library.Main.Volumes long Size { get; } DateTime Time { get; } string Metahash { get; } + string Metablockhash { get; } long Metasize { get; } + string Blockhash { get; } + long Blocksize { get; } IEnumerable BlocklistHashes { get; } } } diff --git a/Duplicati/Library/Main/Volumes/VolumeBase.cs b/Duplicati/Library/Main/Volumes/VolumeBase.cs index 16cce8aa1..33b774cc3 100644 --- a/Duplicati/Library/Main/Volumes/VolumeBase.cs +++ b/Duplicati/Library/Main/Volumes/VolumeBase.cs @@ -11,7 +11,7 @@ namespace Duplicati.Library.Main.Volumes protected class ManifestData { public const string ENCODING = "utf8"; - public const long VERSION = 1; + public const long VERSION = 2; public long Version; public string Created; From 72b33bcf935e60108f495965c13dfb218e589d7f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 3 Apr 2016 23:54:52 +0200 Subject: [PATCH 2/9] Extra fixes to support larger metadata (up to appx 300 mb with default blocksize) --- Duplicati/Library/Main/Database/LocalBackupDatabase.cs | 4 ++-- Duplicati/Library/Main/Operation/BackupHandler.cs | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 6321754d3..c1fa66d0a 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -441,7 +441,7 @@ namespace Duplicati.Library.Main.Database /// The metadata hash /// The id of the metadata set /// True if the set was added to the database, false otherwise - public bool AddMetadataset(string filehash, long size, IEnumerable blockhashes, IEnumerable blocklisthashes, out long metadataid, System.Data.IDbTransaction transaction = null) + public bool AddMetadataset(string filehash, long size, int blocksize, IEnumerable blockhashes, IEnumerable blocklisthashes, out long metadataid, System.Data.IDbTransaction transaction = null) { if (size > 0) { @@ -460,7 +460,7 @@ namespace Duplicati.Library.Main.Database long blocksetid; - AddBlockset(filehash, size, (int)size, blockhashes, blocklisthashes, out blocksetid, transaction); + AddBlockset(filehash, size, blocksize, blockhashes, blocklisthashes, out blocksetid, transaction); using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 5aa94a036..c5dc099d0 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -41,6 +41,7 @@ namespace Duplicati.Library.Main.Operation private readonly FileAttributes m_attributeFilter; private readonly Options.SymlinkStrategy m_symlinkPolicy; private int m_blocksize; + private long m_maxmetadatasize; public BackupHandler(string backendurl, Options options, BackupResults results) { @@ -647,6 +648,7 @@ namespace Duplicati.Library.Main.Operation throw new Exception("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again."); m_blocksize = m_options.Blocksize; + m_maxmetadatasize = (m_blocksize / m_options.BlockhashSize) * m_blocksize; m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)]; m_blocklistbuffer = new byte[m_options.Blocksize]; @@ -1190,13 +1192,19 @@ namespace Duplicati.Library.Main.Operation { long metadataid; + if (meta.Blob.Length > m_maxmetadatasize) + { + m_result.AddWarning(string.Format("Metadata size is {0}, but the largest accepted size is {1}, recording empty metadata", meta.Blob.Length, m_maxmetadatasize), null); + meta = EMPTY_METADATA; + } + using(var blocklisthashes = new Library.Utility.FileBackedStringList()) using(var hashcollector = new Library.Utility.FileBackedStringList()) { using(var ms = new MemoryStream(meta.Blob)) ProcessStream(ms, CompressionHint.Compressible, backend, blocklisthashes, hashcollector, true); - m_database.AddMetadataset(meta.FileHash, meta.Blob.Length, hashcollector, blocklisthashes, out metadataid, m_transaction); + m_database.AddMetadataset(meta.FileHash, meta.Blob.Length, m_blocksize, hashcollector, blocklisthashes, out metadataid, m_transaction); } return metadataid; From d101f91c949bc93c10935a6c366979948da74156 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 4 Apr 2016 18:11:48 +0200 Subject: [PATCH 3/9] Fixed support for metadata larger than blocksize. Now supports one full blocklist of metadata, appx 300mb with default settings. Added correct handling of metadata hashes, such that a metadata blockset is now handled the same was as a regular file blockset. The query that creates the filesets does not support more than a single blocklist of metadata, hence the limitiation. --- .../Library/Main/Database/LocalDatabase.cs | 235 +++++++++++++----- .../Main/Database/LocalRecreateDatabase.cs | 34 +-- .../Main/Database/LocalRepairDatabase.cs | 2 +- .../Library/Main/Operation/BackupHandler.cs | 5 +- .../Main/Operation/RecreateDatabaseHandler.cs | 15 +- .../Main/Volumes/FilesetVolumeReader.cs | 28 ++- .../Main/Volumes/FilesetVolumeWriter.cs | 70 +++--- Duplicati/Library/Main/Volumes/IFileEntry.cs | 1 + 8 files changed, 265 insertions(+), 125 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 54148a72f..5af357b93 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -701,7 +701,7 @@ namespace Duplicati.Library.Main.Database return cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > ?", -1, fhblocksize); } - public void VerifyConsistency(System.Data.IDbTransaction transaction, long blocksize, long hashsize) + public void VerifyConsistency(System.Data.IDbTransaction transaction, long blocksize, long hashsize, bool verifyfilelists) { using (var cmd = m_connection.CreateCommand()) { @@ -746,6 +746,20 @@ namespace Duplicati.Library.Main.Database if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0) throw new Exception("Detected file entries with not associated blocks"); + + if (verifyfilelists) + { + using(var cmd2 = m_connection.CreateCommand(transaction)) + foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) + { + var expandedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, 1); + //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + var storedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?"), 0, filesetid); + + if (expandedlist != storedlist) + throw new Exception(string.Format("Unexpected difference in fileset {0}, found {1} entries, but expected {2}", filesetid, expandedlist, storedlist)); + } + } } } @@ -863,88 +877,168 @@ namespace Duplicati.Library.Main.Database } } + public const string LIST_FILESETS = @" +SELECT + ""L"".""Path"", + ""L"".""Lastmodified"", + ""L"".""Filelength"", + ""L"".""Filehash"", + ""L"".""Metahash"", + ""L"".""Metalength"", + ""L"".""BlocklistHash"", + ""L"".""FirstBlockHash"", + ""L"".""FirstBlockSize"", + ""L"".""FirstMetaBlockHash"", + ""L"".""FirstMetaBlockSize"", + ""M"".""Hash"" AS ""MetaBlocklistHash"" +FROM + ( + SELECT + ""J"".""Path"", + ""J"".""Lastmodified"", + ""J"".""Filelength"", + ""J"".""Filehash"", + ""J"".""Metahash"", + ""J"".""Metalength"", + ""K"".""Hash"" AS ""BlocklistHash"", + ""J"".""FirstBlockHash"", + ""J"".""FirstBlockSize"", + ""J"".""FirstMetaBlockHash"", + ""J"".""FirstMetaBlockSize"", + ""J"".""MetablocksetID"" + FROM + ( + SELECT + ""A"".""Path"" AS ""Path"", + ""D"".""Lastmodified"" AS ""Lastmodified"", + ""B"".""Length"" AS ""Filelength"", + ""B"".""FullHash"" AS ""Filehash"", + ""E"".""FullHash"" AS ""Metahash"", + ""E"".""Length"" AS ""Metalength"", + ""A"".""BlocksetID"" AS ""BlocksetID"", + ""F"".""Hash"" AS ""FirstBlockHash"", + ""F"".""Size"" AS ""FirstBlockSize"", + ""H"".""Hash"" AS ""FirstMetaBlockHash"", + ""H"".""Size"" AS ""FirstMetaBlockSize"", + ""C"".""BlocksetID"" AS ""MetablocksetID"" + FROM + ""File"" A, + ""Blockset"" B, + ""Metadataset"" C, + ""FilesetEntry"" D, + ""Blockset"" E, + ""Block"" F, + ""BlocksetEntry"" G, + ""Block"" H, + ""BlocksetEntry"" I + WHERE + ""A"".""ID"" = ""D"".""FileID"" + AND ""D"".""FilesetID"" = ? + AND ""A"".""BlocksetID"" = ""B"".""ID"" + AND ""A"".""MetadataID"" = ""C"".""ID"" + AND ""E"".""ID"" = ""C"".""BlocksetID"" + AND ""B"".""ID"" = ""G"".""BlocksetID"" + AND ""G"".""BlockID"" = ""F"".""ID"" + AND ""G"".""Index"" = 0 + AND ""I"".""BlocksetID"" = ""E"".""ID"" + AND ""I"".""BlockID"" = ""H"".""ID"" + AND ""I"".""Index"" = 0 + ) J + LEFT OUTER JOIN + ""BlocklistHash"" K + ON + ""K"".""BlocksetID"" = ""J"".""BlocksetID"" + ORDER BY ""J"".""Path"", ""K"".""Index"" + ) L + +LEFT OUTER JOIN + ""BlocklistHash"" M +ON + ""M"".""BlocksetID"" = ""L"".""MetablocksetID"" +"; + + public const string LIST_FOLDERS_AND_SYMLINKS = @" +SELECT + ""G"".""BlocksetID"", + ""G"".""ID"", + ""G"".""Path"", + ""G"".""Length"", + ""G"".""FullHash"", + ""G"".""Lastmodified"", + ""G"".""FirstMetaBlockHash"", + ""H"".""Hash"" AS ""MetablocklistHash"" +FROM + ( + SELECT + ""B"".""BlocksetID"", + ""B"".""ID"", + ""B"".""Path"", + ""D"".""Length"", + ""D"".""FullHash"", + ""A"".""Lastmodified"", + ""F"".""Hash"" AS ""FirstMetaBlockHash"", + ""C"".""BlocksetID"" AS ""MetaBlocksetID"" + FROM + ""FilesetEntry"" A, + ""File"" B, + ""Metadataset"" C, + ""Blockset"" D, + ""BlocksetEntry"" E, + ""Block"" F + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""MetadataID"" = ""C"".""ID"" + AND ""C"".""BlocksetID"" = ""D"".""ID"" + AND ""E"".""BlocksetID"" = ""C"".""BlocksetID"" + AND ""E"".""BlockID"" = ""F"".""ID"" + AND ""E"".""Index"" = 0 + AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) + AND ""A"".""FilesetID"" = ? + ) G +LEFT OUTER JOIN + ""BlocklistHash"" H +ON + ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID"" +ORDER BY + ""G"".""Path"", ""H"".""Index"" + +"; + public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, System.Data.IDbTransaction transaction, long filesetId) { using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = transaction; - cmd.CommandText = @"SELECT ""B"".""BlocksetID"", ""B"".""ID"", ""B"".""Path"", ""D"".""Length"", ""D"".""FullHash"", ""A"".""Lastmodified"" FROM ""FilesetEntry"" A, ""File"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) AND ""A"".""FilesetID"" = ? "; + cmd.CommandText = LIST_FOLDERS_AND_SYMLINKS; cmd.AddParameter(FOLDER_BLOCKSET_ID); cmd.AddParameter(SYMLINK_BLOCKSET_ID); cmd.AddParameter(filesetId); + string lastpath = null; using (var rd = cmd.ExecuteReader()) while(rd.Read()) { - var blocksetID = rd.GetInt64(0); - var path = rd.GetValue(2).ToString(); - var metalength = rd.GetInt64(3); - var metahash = rd.GetValue(4).ToString(); + var blocksetID = rd.ConvertValueToInt64(0, -1); + var path = rd.GetValue(2).ToString(); + var metalength = rd.ConvertValueToInt64(3, -1); + var metahash = rd.GetValue(4).ToString(); + var metablockhash = rd.GetValue(6).ToString(); + var metablocklisthash = rd.GetValue(7).ToString(); + + if (path == lastpath) + m_result.AddWarning(string.Format("Duplicate path detected: {0}!", path), null); + + lastpath = path; if (blocksetID == FOLDER_BLOCKSET_ID) - filesetvolume.AddDirectory(path, metahash, metalength); + filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); else if (blocksetID == SYMLINK_BLOCKSET_ID) - filesetvolume.AddSymlink(path, metahash, metalength); + filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); } + + // TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x - var query = @" -SELECT - ""J"".""Path"", - ""J"".""Lastmodified"", - ""J"".""Filelength"", - ""J"".""Filehash"", - ""J"".""Metahash"", - ""J"".""Metalength"", - ""K"".""Hash"" AS ""BlocklistHash"", - ""J"".""FirstBlockHash"", - ""J"".""FirstBlockSize"", - ""J"".""FirstMetaBlockHash"", - ""J"".""FirstMetaBlockSize"" -FROM - ( - SELECT - ""A"".""Path"" AS ""Path"", - ""D"".""Lastmodified"" AS ""Lastmodified"", - ""B"".""Length"" AS ""Filelength"", - ""B"".""FullHash"" AS ""Filehash"", - ""E"".""FullHash"" AS ""Metahash"", - ""E"".""Length"" AS ""Metalength"", - ""A"".""BlocksetID"" AS ""BlocksetID"", - ""F"".""Hash"" AS ""FirstBlockHash"", - ""F"".""Size"" AS ""FirstBlockSize"", - ""H"".""Hash"" AS ""FirstMetaBlockHash"", - ""H"".""Size"" AS ""FirstMetaBlockSize"" - FROM - ""File"" A, - ""Blockset"" B, - ""Metadataset"" C, - ""FilesetEntry"" D, - ""Blockset"" E, - ""Block"" F, - ""BlocksetEntry"" G, - ""Block"" H, - ""BlocksetEntry"" I - WHERE - ""A"".""ID"" = ""D"".""FileID"" - AND ""D"".""FilesetID"" = ? - AND ""A"".""BlocksetID"" = ""B"".""ID"" - AND ""A"".""MetadataID"" = ""C"".""ID"" - AND ""E"".""ID"" = ""C"".""BlocksetID"" - AND ""B"".""ID"" = ""G"".""BlocksetID"" - AND ""G"".""BlockID"" = ""F"".""ID"" - AND ""G"".""Index"" = 0 - AND ""I"".""BlocksetID"" = ""E"".""ID"" - AND ""I"".""BlockID"" = ""H"".""ID"" - AND ""I"".""Index"" = 0 - ) J -LEFT OUTER JOIN - ""BlocklistHash"" K -ON - ""K"".""BlocksetID"" = ""J"".""BlocksetID"" -ORDER BY ""J"".""Path"", ""K"".""Index"" -"; - - cmd.CommandText = query; + cmd.CommandText = LIST_FILESETS; cmd.Parameters.Clear(); cmd.AddParameter(filesetId); @@ -966,8 +1060,15 @@ ORDER BY ""J"".""Path"", ""K"".""Index"" var blocksize = rd.ConvertValueToInt64(8, -1); var metablockhash = rd.GetValue(9).ToString(); //var metablocksize = rd.ConvertValueToInt64(10, -1); + var metablocklisthash = rd.GetValue(11).ToString(); - filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd); + if (blockhash == filehash) + blockhash = null; + + if (metablockhash == metahash) + metablockhash = null; + + filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); if (blrd == null) more = rd.Read(); else diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index ad3941ddf..42e3d5690 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -294,25 +294,24 @@ namespace Duplicati.Library.Main.Database } } - public void AddDirectoryEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction) + public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction) { - AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metahash, metahashsize, transaction); + AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction); } - public void AddSymlinkEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction) + public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction) { - AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metahash, metahashsize, transaction); + AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); } - public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction) + public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) { - AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metahash, metahashsize, transaction); + AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metadataid, transaction); } - private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction) + private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) { var fileid = -1L; - var metadataid = AddMetadataset(metahash, metahashsize, transaction); if (m_filesetLookup != null) { @@ -357,7 +356,7 @@ namespace Duplicati.Library.Main.Database m_insertFilesetEntryCommand.ExecuteNonQuery(); } - private long AddMetadataset(string metahash, long metahashsize, System.Data.IDbTransaction transaction) + public long AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, System.Data.IDbTransaction transaction) { var metadataid = -1L; if (metahash == null) @@ -380,9 +379,7 @@ namespace Duplicati.Library.Main.Database return metadataid; } - // Strange ..... metahash is hashed with block hash algorithm, but should be filehash? - - var blocksetid = AddBlockset(metahash, metahashsize, null, 0, transaction); + var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction); m_insertMetadatasetCommand.Transaction = transaction; m_insertMetadatasetCommand.SetParameterValue(0, blocksetid); @@ -422,13 +419,13 @@ namespace Duplicati.Library.Main.Database if (m_fileHashLookup != null) m_fileHashLookup.Add(fullhash, size, blocksetid); + long c = 0; if (blocklisthashes != null) { var index = 0L; m_insertBlocklistHashCommand.Transaction = transaction; m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid); - long c = 0; foreach(var hash in blocklisthashes) { if (!string.IsNullOrEmpty(hash)) @@ -442,16 +439,11 @@ namespace Duplicati.Library.Main.Database } } } - - if (c != expectedblocklisthashes) - { - // Older versions of the filelists do not have a blocklisthash for files smaller than blocksize - if (c != 0 || expectedblocklisthashes != 1) - m_result.AddWarning(string.Format("Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid), null); - } - } + if (c != expectedblocklisthashes) + m_result.AddWarning(string.Format("Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid), null); + return blocksetid; } diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 32f3ab1cc..54f893019 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -478,7 +478,7 @@ namespace Duplicati.Library.Main.Database try { - VerifyConsistency(tr, blocksize, hashsize); + VerifyConsistency(tr, blocksize, hashsize, true); } catch(Exception ex) { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index c5dc099d0..6fb2e1592 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -666,7 +666,7 @@ namespace Duplicati.Library.Main.Operation if (!m_filehasher.CanReuseTransform) throw new Exception(Strings.Common.InvalidCryptoSystem(m_options.FileHashAlgorithm)); - m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize); + m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, false); // If there is no filter, we set an empty filter to simplify the code // If there is a filter, we make sure that the sources are included m_filter = filter ?? new Library.Utility.FilterExpression(); @@ -697,7 +697,6 @@ namespace Duplicati.Library.Main.Operation PreBackupVerify(backend); // Verify before uploading a synthetic list - m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize); UploadSyntheticFilelist(backend); m_database.BuildLookupTable(m_options); @@ -726,7 +725,7 @@ namespace Duplicati.Library.Main.Operation using(new Logging.Timer("UpdateChangeStatistics")) m_database.UpdateChangeStatistics(m_result); using(new Logging.Timer("VerifyConsistency")) - m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize); + m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize, false); UploadRealFileList(backend, filesetvolume); diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 412bf07c8..a6741a2b4 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -213,9 +213,14 @@ namespace Duplicati.Library.Main.Operation { try { + var expectedmetablocks = (fe.Metasize + blocksize - 1) / blocksize; + var expectedmetablocklisthashes = (expectedmetablocks + hashes_pr_block - 1) / hashes_pr_block; + if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0; + if (fe.Type == FilelistEntryType.Folder) { - restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr); + var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); + restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, metadataid, tr); } else if (fe.Type == FilelistEntryType.File) { @@ -224,7 +229,8 @@ namespace Duplicati.Library.Main.Operation if (expectedblocks <= 1) expectedblocklisthashes = 0; var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr); - restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr); + var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); + restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, metadataid, tr); if (fe.Size <= blocksize) { @@ -249,7 +255,8 @@ namespace Duplicati.Library.Main.Operation } else if (fe.Type == FilelistEntryType.Symlink) { - restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr); + var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); + restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, metadataid, tr); } else { @@ -448,7 +455,7 @@ namespace Duplicati.Library.Main.Operation //All done, we must verify that we have all blocklist fully intact // if this fails, the db will not be deleted, so it can be used, // except to continue a backup - restoredb.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize); + restoredb.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, true); m_result.AddMessage("Recreate completed, and consistency checks completed, marking database as complete"); diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs index 23436f29b..483d35981 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs @@ -102,6 +102,7 @@ namespace Duplicati.Library.Main.Volumes public string Blockhash { get; private set; } public long Blocksize { get; private set; } public IEnumerable BlocklistHashes { get; private set; } + public IEnumerable MetaBlocklistHashes { get; private set; } private JsonReader m_reader; public FileEntry(JsonReader reader) @@ -154,7 +155,30 @@ namespace Duplicati.Library.Main.Volumes if (!m_reader.Read()) throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); - if (m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "metablockhash") + if (m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "metablocklists") + { + var metadatablocklisthashes = new List(); + SkipJsonToken(m_reader, JsonToken.StartArray); + + if (!m_reader.Read()) + throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); + + while(m_reader.TokenType == JsonToken.String) + { + metadatablocklisthashes.Add(m_reader.Value.ToString()); + if (!m_reader.Read()) + throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); + } + + if (m_reader.TokenType != JsonToken.EndArray) + throw new InvalidDataException(string.Format("Invalid JSON, unexpected token {1} found while reading entry {0}", this.Path, m_reader.TokenType)); + if (!m_reader.Read()) + throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); + + this.MetaBlocklistHashes = metadatablocklisthashes; + this.Metablockhash = null; + } + else if (m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "metablockhash") { if (!m_reader.Read()) throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); @@ -162,6 +186,8 @@ namespace Duplicati.Library.Main.Volumes if (!m_reader.Read()) throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path)); + + this.MetaBlocklistHashes = null; } } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 62f26ea9f..83b2eddb0 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -25,17 +25,45 @@ namespace Duplicati.Library.Main.Volumes m_writer.WriteStartArray(); } - public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes) + private void WriteMetaProperties(string metahash, long metasize, string metablockhash, IEnumerable metablocklisthashes) { - AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes); + m_writer.WritePropertyName("metahash"); + m_writer.WriteValue(metahash); + m_writer.WritePropertyName("metasize"); + m_writer.WriteValue(metasize); + + if (metablocklisthashes != null) + { + //Slightly akward, but we avoid writing if there are no entries + var en = metablocklisthashes.GetEnumerator(); + if (en.MoveNext() && !string.IsNullOrEmpty(en.Current)) + { + m_writer.WritePropertyName("metablocklists"); + m_writer.WriteStartArray(); + m_writer.WriteValue(en.Current); + while (en.MoveNext()) + m_writer.WriteValue(en.Current); + m_writer.WriteEndArray(); + } + } + else if (!string.IsNullOrWhiteSpace(metablockhash)) + { + m_writer.WritePropertyName("metablockhash"); + m_writer.WriteValue(metablockhash); + } } - public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, IEnumerable blocklisthashes) + public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) { - AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, null, -1, blocklisthashes); + AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); } - private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes) + public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) + { + AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); + } + + private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) { m_filecount++; m_writer.WriteStartObject(); @@ -50,17 +78,7 @@ namespace Duplicati.Library.Main.Volumes m_writer.WritePropertyName("time"); m_writer.WriteValue(Library.Utility.Utility.SerializeDateTime(lastmodified)); if (metahash != null) - { - m_writer.WritePropertyName("metahash"); - m_writer.WriteValue(metahash); - m_writer.WritePropertyName("metasize"); - m_writer.WriteValue(metasize); - if (!string.IsNullOrWhiteSpace(metablockhash)) - { - m_writer.WritePropertyName("metablockhash"); - m_writer.WriteValue(metablockhash); - } - } + WriteMetaProperties(metahash, metasize, metablockhash, metablocklisthashes); if (blocklisthashes != null) { @@ -76,7 +94,7 @@ namespace Duplicati.Library.Main.Volumes m_writer.WriteEndArray(); } } - else if (blockhash != null) + else if (!string.IsNullOrWhiteSpace(blockhash)) { m_writer.WritePropertyName("blockhash"); m_writer.WriteValue(blockhash); @@ -87,12 +105,12 @@ namespace Duplicati.Library.Main.Volumes m_writer.WriteEndObject(); } - public void AddDirectory(string name, string metahash, long metasize) + public void AddDirectory(string name, string metahash, long metasize, string metablockhash, IEnumerable metablocklisthashes) { - AddMetaEntry(FilelistEntryType.Folder, name, metahash, metasize); + AddMetaEntry(FilelistEntryType.Folder, name, metahash, metasize, metablockhash, metablocklisthashes); } - public void AddMetaEntry(FilelistEntryType type, string name, string metahash, long metasize) + public void AddMetaEntry(FilelistEntryType type, string name, string metahash, long metasize, string metablockhash, IEnumerable metablocklisthashes) { m_foldercount++; m_writer.WriteStartObject(); @@ -101,12 +119,8 @@ namespace Duplicati.Library.Main.Volumes m_writer.WritePropertyName("path"); m_writer.WriteValue(name); if (metahash != null) - { - m_writer.WritePropertyName("metahash"); - m_writer.WriteValue(metahash); - m_writer.WritePropertyName("metasize"); - m_writer.WriteValue(metasize); - } + WriteMetaProperties(metahash, metasize, metablockhash, metablocklisthashes); + m_writer.WriteEndObject(); } @@ -139,9 +153,9 @@ namespace Duplicati.Library.Main.Volumes public long FileCount { get { return m_filecount; } } public long FolderCount { get { return m_foldercount; } } - public void AddSymlink(string name, string metahash, long metasize) + public void AddSymlink(string name, string metahash, long metasize, string metablockhash, IEnumerable metablocklisthashes) { - AddMetaEntry(FilelistEntryType.Symlink, name, metahash, metasize); + AddMetaEntry(FilelistEntryType.Symlink, name, metahash, metasize, metablockhash, metablocklisthashes); } } } diff --git a/Duplicati/Library/Main/Volumes/IFileEntry.cs b/Duplicati/Library/Main/Volumes/IFileEntry.cs index 97b9d6572..8be20f37b 100644 --- a/Duplicati/Library/Main/Volumes/IFileEntry.cs +++ b/Duplicati/Library/Main/Volumes/IFileEntry.cs @@ -19,5 +19,6 @@ namespace Duplicati.Library.Main.Volumes string Blockhash { get; } long Blocksize { get; } IEnumerable BlocklistHashes { get; } + IEnumerable MetaBlocklistHashes { get; } } } From f0631432794ad2c0ea1ce32084c72b610b71ba4d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 4 Apr 2016 18:19:35 +0200 Subject: [PATCH 4/9] Re-enable mixed block/filehash tests as it now works --- Duplicati/UnitTest/BorderTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs index 3f27f0663..7b9822d55 100644 --- a/Duplicati/UnitTest/BorderTests.cs +++ b/Duplicati/UnitTest/BorderTests.cs @@ -81,7 +81,7 @@ namespace Duplicati.UnitTest }); } - //[Test] + [Test] public void RunMixedBlockFile_1() { PrepareSourceData(); @@ -91,7 +91,7 @@ namespace Duplicati.UnitTest }); } - //[Test] + [Test] public void RunMixedBlockFile_2() { PrepareSourceData(); From d137efa5e38accda0a2141516a3dfa6308835911 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 5 Apr 2016 00:20:16 +0200 Subject: [PATCH 5/9] Fix for recreate database with small chunks as metadata on folders and symlinks --- .../Main/Operation/RecreateDatabaseHandler.cs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index a6741a2b4..4ece85100 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -241,17 +241,6 @@ namespace Duplicati.Library.Main.Operation else m_result.AddWarning(string.Format("No block hash found for file: {0}", fe.Path), null); } - - if (fe.Metasize <= blocksize) - { - if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) - restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, tr); - else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm) - restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, tr); - else - m_result.AddWarning(string.Format("No block hash found for file metadata: {0}", fe.Path), null); - } - } else if (fe.Type == FilelistEntryType.Symlink) { @@ -262,6 +251,16 @@ namespace Duplicati.Library.Main.Operation { m_result.AddWarning(string.Format("Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path), null); } + + if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink)) + { + if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) + restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, tr); + else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm) + restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, tr); + else + m_result.AddWarning(string.Format("No block hash found for file metadata: {0}", fe.Path), null); + } } catch (Exception ex) { From d5e419497829f1ab6e1f260e83c0182f0ae8ff6d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 5 Apr 2016 00:27:14 +0200 Subject: [PATCH 6/9] Added extra test for testing with metadata disabled --- Duplicati/UnitTest/BorderTests.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs index 7b9822d55..4b84dd5c9 100644 --- a/Duplicati/UnitTest/BorderTests.cs +++ b/Duplicati/UnitTest/BorderTests.cs @@ -61,6 +61,16 @@ namespace Duplicati.UnitTest RunCommands(12345, 1024 * 1024 * 10); } + [Test] + public void RunNoMetadata() + { + PrepareSourceData(); + RunCommands(1024 * 10, modifyOptions: opts => { + opts["skip-metadata"] = "true"; + }); + } + + [Test] public void RunMD5() { @@ -230,7 +240,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(filenames.Count * 3, r.FilesRestored); } - TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, true); + TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata")); using(var tf = new Library.Utility.TempFolder()) { From 32d8c18257cc142ab23090bd2f718b8f40abdbdf Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 5 Apr 2016 14:00:12 +0200 Subject: [PATCH 7/9] Fixed repair consistency check --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 5af357b93..2d1ec6c23 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -752,7 +752,7 @@ namespace Duplicati.Library.Main.Database using(var cmd2 = m_connection.CreateCommand(transaction)) foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) { - var expandedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, 1); + var expandedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid); //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); var storedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?"), 0, filesetid); From 3a034c65fffbe3acbcc9ebb99ae72da47defec60 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Oct 2016 15:32:30 +0200 Subject: [PATCH 8/9] Removed some extra dispose code --- .../Main/Database/LocalRecreateDatabase.cs | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index aefe3b8a6..0353902c5 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -675,33 +675,7 @@ namespace Duplicati.Library.Main.Database finally { m_tempsmalllist = null; } } - - foreach(var cmd in new IDisposable [] { - m_insertFileCommand, - m_insertFilesetEntryCommand, - m_insertMetadatasetCommand, - m_insertBlocksetCommand, - m_insertBlocklistHashCommand, - m_updateBlockVolumeCommand, - m_insertBlockset, - m_findBlocksetCommand, - m_findMetadatasetCommand, - m_findFilesetCommand, - m_findblocklisthashCommand, - m_findHashBlockCommand, - m_insertBlockCommand, - m_insertDuplicateBlockCommand, - m_insertSmallBlockset - }) - try - { - if (cmd != null) - cmd.Dispose(); - } - catch - { - } - + base.Dispose(); } } From 60c972cf0b763c54bf35171e0536dc6a3a5f890f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Oct 2016 15:43:41 +0200 Subject: [PATCH 9/9] Added a comment that explains how to get more than 300mb metadata support, should we need it --- Duplicati/Library/Main/Operation/BackupHandler.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 6d208110c..0fa191725 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -1243,13 +1243,18 @@ namespace Duplicati.Library.Main.Operation m_database.AddUnmodifiedFile(oldId, lastModified, m_transaction); } - private long AddMetadataToOutput(BackendManager backend, IMetahash meta) { long metadataid; if (meta.Blob.Length > m_maxmetadatasize) { + //TODO: To fix this, the "WriteFileset" method in BackupHandler needs to + // be updated such that it can select sets even when there are multiple + // blocklist hashes for the metadata. + // This could be done such that an extra query is made if the metadata + // spans multiple blocklist hashes, as it is not expected to be common + m_result.AddWarning(string.Format("Metadata size is {0}, but the largest accepted size is {1}, recording empty metadata", meta.Blob.Length, m_maxmetadatasize), null); meta = EMPTY_METADATA; }