diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
index 2de6fe011..d68a87c9e 100644
--- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
@@ -458,26 +458,26 @@ namespace Duplicati.Library.Main.Database
/// The metadata hash
/// The id of the metadata set
/// True if the set was added to the database, false otherwise
- public bool AddMetadataset(string hash, long size, out long metadataid, System.Data.IDbTransaction transaction = null)
+ public bool AddMetadataset(string filehash, long size, int blocksize, IEnumerable blockhashes, IEnumerable blocklisthashes, out long metadataid, System.Data.IDbTransaction transaction = null)
{
if (size > 0)
{
if (m_metadataLookup != null)
{
- if(m_metadataLookup.TryGet(hash, size, out metadataid))
+ if(m_metadataLookup.TryGet(filehash, size, out metadataid))
return false;
}
else
{
m_findmetadatasetCommand.Transaction = transaction;
- metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, hash, size);
+ metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size);
if (metadataid != -1)
return false;
}
long blocksetid;
- AddBlockset(hash, size, (int)size, new string[] { hash }, null, out blocksetid, transaction);
+ AddBlockset(filehash, size, blocksize, blockhashes, blocklisthashes, out blocksetid, transaction);
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
@@ -486,7 +486,7 @@ namespace Duplicati.Library.Main.Database
metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64();
tr.Commit();
if (m_metadataLookup != null)
- m_metadataLookup.Add(hash, size, metadataid);
+ m_metadataLookup.Add(filehash, size, metadataid);
}
return true;
diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs
index 589ee7ae7..5c580a2fa 100644
--- a/Duplicati/Library/Main/Database/LocalDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalDatabase.cs
@@ -723,7 +723,7 @@ namespace Duplicati.Library.Main.Database
return cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > ?", -1, fhblocksize);
}
- public void VerifyConsistency(System.Data.IDbTransaction transaction, long blocksize, long hashsize)
+ public void VerifyConsistency(System.Data.IDbTransaction transaction, long blocksize, long hashsize, bool verifyfilelists)
{
using (var cmd = m_connection.CreateCommand())
{
@@ -768,6 +768,20 @@ namespace Duplicati.Library.Main.Database
if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0)
throw new Exception("Detected file entries with not associated blocks");
+
+ if (verifyfilelists)
+ {
+ using(var cmd2 = m_connection.CreateCommand(transaction))
+ foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1)))
+ {
+ var expandedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid);
+ //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
+ var storedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?"), 0, filesetid);
+
+ if (expandedlist != storedlist)
+ throw new Exception(string.Format("Unexpected difference in fileset {0}, found {1} entries, but expected {2}", filesetid, expandedlist, storedlist));
+ }
+ }
}
}
@@ -885,31 +899,168 @@ namespace Duplicati.Library.Main.Database
}
}
+ public const string LIST_FILESETS = @"
+SELECT
+ ""L"".""Path"",
+ ""L"".""Lastmodified"",
+ ""L"".""Filelength"",
+ ""L"".""Filehash"",
+ ""L"".""Metahash"",
+ ""L"".""Metalength"",
+ ""L"".""BlocklistHash"",
+ ""L"".""FirstBlockHash"",
+ ""L"".""FirstBlockSize"",
+ ""L"".""FirstMetaBlockHash"",
+ ""L"".""FirstMetaBlockSize"",
+ ""M"".""Hash"" AS ""MetaBlocklistHash""
+FROM
+ (
+ SELECT
+ ""J"".""Path"",
+ ""J"".""Lastmodified"",
+ ""J"".""Filelength"",
+ ""J"".""Filehash"",
+ ""J"".""Metahash"",
+ ""J"".""Metalength"",
+ ""K"".""Hash"" AS ""BlocklistHash"",
+ ""J"".""FirstBlockHash"",
+ ""J"".""FirstBlockSize"",
+ ""J"".""FirstMetaBlockHash"",
+ ""J"".""FirstMetaBlockSize"",
+ ""J"".""MetablocksetID""
+ FROM
+ (
+ SELECT
+ ""A"".""Path"" AS ""Path"",
+ ""D"".""Lastmodified"" AS ""Lastmodified"",
+ ""B"".""Length"" AS ""Filelength"",
+ ""B"".""FullHash"" AS ""Filehash"",
+ ""E"".""FullHash"" AS ""Metahash"",
+ ""E"".""Length"" AS ""Metalength"",
+ ""A"".""BlocksetID"" AS ""BlocksetID"",
+ ""F"".""Hash"" AS ""FirstBlockHash"",
+ ""F"".""Size"" AS ""FirstBlockSize"",
+ ""H"".""Hash"" AS ""FirstMetaBlockHash"",
+ ""H"".""Size"" AS ""FirstMetaBlockSize"",
+ ""C"".""BlocksetID"" AS ""MetablocksetID""
+ FROM
+ ""File"" A,
+ ""Blockset"" B,
+ ""Metadataset"" C,
+ ""FilesetEntry"" D,
+ ""Blockset"" E,
+ ""Block"" F,
+ ""BlocksetEntry"" G,
+ ""Block"" H,
+ ""BlocksetEntry"" I
+ WHERE
+ ""A"".""ID"" = ""D"".""FileID""
+ AND ""D"".""FilesetID"" = ?
+ AND ""A"".""BlocksetID"" = ""B"".""ID""
+ AND ""A"".""MetadataID"" = ""C"".""ID""
+ AND ""E"".""ID"" = ""C"".""BlocksetID""
+ AND ""B"".""ID"" = ""G"".""BlocksetID""
+ AND ""G"".""BlockID"" = ""F"".""ID""
+ AND ""G"".""Index"" = 0
+ AND ""I"".""BlocksetID"" = ""E"".""ID""
+ AND ""I"".""BlockID"" = ""H"".""ID""
+ AND ""I"".""Index"" = 0
+ ) J
+ LEFT OUTER JOIN
+ ""BlocklistHash"" K
+ ON
+ ""K"".""BlocksetID"" = ""J"".""BlocksetID""
+ ORDER BY ""J"".""Path"", ""K"".""Index""
+ ) L
+
+LEFT OUTER JOIN
+ ""BlocklistHash"" M
+ON
+ ""M"".""BlocksetID"" = ""L"".""MetablocksetID""
+";
+
+ public const string LIST_FOLDERS_AND_SYMLINKS = @"
+SELECT
+ ""G"".""BlocksetID"",
+ ""G"".""ID"",
+ ""G"".""Path"",
+ ""G"".""Length"",
+ ""G"".""FullHash"",
+ ""G"".""Lastmodified"",
+ ""G"".""FirstMetaBlockHash"",
+ ""H"".""Hash"" AS ""MetablocklistHash""
+FROM
+ (
+ SELECT
+ ""B"".""BlocksetID"",
+ ""B"".""ID"",
+ ""B"".""Path"",
+ ""D"".""Length"",
+ ""D"".""FullHash"",
+ ""A"".""Lastmodified"",
+ ""F"".""Hash"" AS ""FirstMetaBlockHash"",
+ ""C"".""BlocksetID"" AS ""MetaBlocksetID""
+ FROM
+ ""FilesetEntry"" A,
+ ""File"" B,
+ ""Metadataset"" C,
+ ""Blockset"" D,
+ ""BlocksetEntry"" E,
+ ""Block"" F
+ WHERE
+ ""A"".""FileID"" = ""B"".""ID""
+ AND ""B"".""MetadataID"" = ""C"".""ID""
+ AND ""C"".""BlocksetID"" = ""D"".""ID""
+ AND ""E"".""BlocksetID"" = ""C"".""BlocksetID""
+ AND ""E"".""BlockID"" = ""F"".""ID""
+ AND ""E"".""Index"" = 0
+ AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?)
+ AND ""A"".""FilesetID"" = ?
+ ) G
+LEFT OUTER JOIN
+ ""BlocklistHash"" H
+ON
+ ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID""
+ORDER BY
+ ""G"".""Path"", ""H"".""Index""
+
+";
+
public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, System.Data.IDbTransaction transaction, long filesetId)
{
using (var cmd = m_connection.CreateCommand())
{
- cmd.Transaction = transaction;
- cmd.CommandText = @"SELECT ""B"".""BlocksetID"", ""B"".""ID"", ""B"".""Path"", ""D"".""Length"", ""D"".""FullHash"", ""A"".""Lastmodified"" FROM ""FilesetEntry"" A, ""File"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) AND ""A"".""FilesetID"" = ? ";
+ cmd.Transaction = transaction;
+ cmd.CommandText = LIST_FOLDERS_AND_SYMLINKS;
cmd.AddParameter(FOLDER_BLOCKSET_ID);
cmd.AddParameter(SYMLINK_BLOCKSET_ID);
cmd.AddParameter(filesetId);
+ string lastpath = null;
using (var rd = cmd.ExecuteReader())
while(rd.Read())
{
- var blocksetID = rd.GetInt64(0);
- var path = rd.GetValue(2).ToString();
- var metalength = rd.GetInt64(3);
- var metahash = rd.GetValue(4).ToString();
+ var blocksetID = rd.ConvertValueToInt64(0, -1);
+ var path = rd.GetValue(2).ToString();
+ var metalength = rd.ConvertValueToInt64(3, -1);
+ var metahash = rd.GetValue(4).ToString();
+ var metablockhash = rd.GetValue(6).ToString();
+ var metablocklisthash = rd.GetValue(7).ToString();
+
+ if (path == lastpath)
+ m_result.AddWarning(string.Format("Duplicate path detected: {0}!", path), null);
+
+ lastpath = path;
if (blocksetID == FOLDER_BLOCKSET_ID)
- filesetvolume.AddDirectory(path, metahash, metalength);
+ filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
else if (blocksetID == SYMLINK_BLOCKSET_ID)
- filesetvolume.AddSymlink(path, metahash, metalength);
+ filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
}
+
+ // TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x
- cmd.CommandText = @"SELECT ""F"".""Path"", ""F"".""Lastmodified"", ""F"".""Filelength"", ""F"".""Filehash"", ""F"".""Metahash"", ""F"".""Metalength"", ""G"".""Hash"" FROM (SELECT ""A"".""Path"" AS ""Path"", ""D"".""Lastmodified"" AS ""Lastmodified"", ""B"".""Length"" AS ""Filelength"", ""B"".""FullHash"" AS ""Filehash"", ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"" FROM ""File"" A, ""Blockset"" B, ""Metadataset"" C, ""FilesetEntry"" D, ""Blockset"" E WHERE ""A"".""ID"" = ""D"".""FileID"" AND ""D"".""FilesetID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"") F LEFT OUTER JOIN ""BlocklistHash"" G ON ""G"".""BlocksetID"" = ""F"".""BlocksetID"" ORDER BY ""F"".""Path"", ""G"".""Index"" ";
+ cmd.CommandText = LIST_FILESETS;
cmd.Parameters.Clear();
cmd.AddParameter(filesetId);
@@ -927,8 +1078,19 @@ namespace Duplicati.Library.Main.Database
var metasize = rd.ConvertValueToInt64(5, -1);
var p = rd.GetValue(6);
var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd);
+ var blockhash = rd.GetValue(7).ToString();
+ var blocksize = rd.ConvertValueToInt64(8, -1);
+ var metablockhash = rd.GetValue(9).ToString();
+ //var metablocksize = rd.ConvertValueToInt64(10, -1);
+ var metablocklisthash = rd.GetValue(11).ToString();
- filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, blrd);
+ if (blockhash == filehash)
+ blockhash = null;
+
+ if (metablockhash == metahash)
+ metablockhash = null;
+
+ filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
if (blrd == null)
more = rd.Read();
else
diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs
index 1d1ce6ab8..0353902c5 100644
--- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs
@@ -51,6 +51,7 @@ namespace Duplicati.Library.Main.Database
private System.Data.IDbCommand m_insertBlocklistHashCommand;
private System.Data.IDbCommand m_updateBlockVolumeCommand;
private System.Data.IDbCommand m_insertBlockset;
+ private System.Data.IDbCommand m_insertSmallBlockset;
private System.Data.IDbCommand m_findBlocksetCommand;
private System.Data.IDbCommand m_findMetadatasetCommand;
private System.Data.IDbCommand m_findFilesetCommand;
@@ -66,6 +67,7 @@ namespace Duplicati.Library.Main.Database
private PathLookupHelper m_filesetLookup;
private string m_tempblocklist;
+ private string m_tempsmalllist;
///
/// A lookup table that prevents multiple downloads of the same volume
@@ -122,11 +124,16 @@ namespace Duplicati.Library.Main.Database
: base(parentdb)
{
m_tempblocklist = "TempBlocklist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
+ m_tempsmalllist = "TempSmalllist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using(var cmd = m_connection.CreateCommand())
{
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist));
cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist));
+
+ cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)", m_tempsmalllist));
+ cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_File_{0}"" ON ""{0}"" (""FileHash"", ""BlockSize"");", m_tempsmalllist));
+ cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_Block_{0}"" ON ""{0}"" (""BlockHash"", ""BlockSize"");", m_tempsmalllist));
}
m_insertFileCommand = m_connection.CreateCommand();
@@ -136,6 +143,7 @@ namespace Duplicati.Library.Main.Database
m_insertBlocklistHashCommand = m_connection.CreateCommand();
m_updateBlockVolumeCommand = m_connection.CreateCommand();
m_insertBlockset = m_connection.CreateCommand();
+ m_insertSmallBlockset = m_connection.CreateCommand();
m_findBlocksetCommand = m_connection.CreateCommand();
m_findMetadatasetCommand = m_connection.CreateCommand();
m_findFilesetCommand = m_connection.CreateCommand();
@@ -164,7 +172,10 @@ namespace Duplicati.Library.Main.Database
m_insertBlockset.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist);
m_insertBlockset.AddParameters(3);
-
+
+ m_insertSmallBlockset.CommandText = string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) ", m_tempsmalllist);
+ m_insertSmallBlockset.AddParameters(3);
+
m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? ";
m_findBlocksetCommand.AddParameters(2);
@@ -206,7 +217,8 @@ namespace Duplicati.Library.Main.Database
cmd.Transaction = transaction;
//Update all small blocklists and matching blocks
- var selectSmallBlocks = string.Format(@"SELECT ""Blockset"".""Fullhash"", ""Blockset"".""Length"" FROM ""Blockset"" WHERE ""Blockset"".""Length"" <= {0}", blocksize);
+
+ var selectSmallBlocks = string.Format(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist);
var selectBlockHashes = string.Format(
@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " +
@@ -257,9 +269,12 @@ namespace Duplicati.Library.Main.Database
blocksize / hashsize
);
+
+
var selectBlocksetEntries = string.Format(
- @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" <= {0} ",
- blocksize
+ @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{1}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {0} ",
+ blocksize,
+ m_tempsmalllist
);
var selectAllBlocksetEntries =
@@ -279,25 +294,24 @@ namespace Duplicati.Library.Main.Database
}
}
- public void AddDirectoryEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+ public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
{
- AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metahash, metahashsize, transaction);
+ AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction);
}
- public void AddSymlinkEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+ public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
{
- AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metahash, metahashsize, transaction);
+ AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
}
- public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+ public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
{
- AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metahash, metahashsize, transaction);
+ AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metadataid, transaction);
}
- private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+ private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
{
var fileid = -1L;
- var metadataid = AddMetadataset(metahash, metahashsize, transaction);
if (m_filesetLookup != null)
{
@@ -342,7 +356,7 @@ namespace Duplicati.Library.Main.Database
m_insertFilesetEntryCommand.ExecuteNonQuery();
}
- private long AddMetadataset(string metahash, long metahashsize, System.Data.IDbTransaction transaction)
+ public long AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, System.Data.IDbTransaction transaction)
{
var metadataid = -1L;
if (metahash == null)
@@ -364,8 +378,8 @@ namespace Duplicati.Library.Main.Database
if (metadataid != -1)
return metadataid;
}
-
- var blocksetid = AddBlockset(metahash, metahashsize, null, 0, transaction);
+
+ var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction);
m_insertMetadatasetCommand.Transaction = transaction;
m_insertMetadatasetCommand.SetParameterValue(0, blocksetid);
@@ -405,13 +419,13 @@ namespace Duplicati.Library.Main.Database
if (m_fileHashLookup != null)
m_fileHashLookup.Add(fullhash, size, blocksetid);
+ long c = 0;
if (blocklisthashes != null)
{
var index = 0L;
m_insertBlocklistHashCommand.Transaction = transaction;
m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid);
- long c = 0;
foreach(var hash in blocklisthashes)
{
if (!string.IsNullOrEmpty(hash))
@@ -425,12 +439,11 @@ namespace Duplicati.Library.Main.Database
}
}
}
-
- if (c != expectedblocklisthashes) // or is there a legacy with single element blocklist hashes? --> && !(c == 1 && hash == computeBlockHash(fullhash))
- m_result.AddWarning(string.Format("Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid), null);
-
}
+ if (c != expectedblocklisthashes)
+ m_result.AddWarning(string.Format("Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid), null);
+
return blocksetid;
}
@@ -494,6 +507,15 @@ namespace Duplicati.Library.Main.Database
return false;
}
}
+
+ public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, System.Data.IDbTransaction transaction)
+ {
+ m_insertSmallBlockset.Transaction = transaction;
+ m_insertSmallBlockset.SetParameterValue(0, filehash);
+ m_insertSmallBlockset.SetParameterValue(1, blockhash);
+ m_insertSmallBlockset.SetParameterValue(2, blocksize);
+ m_insertSmallBlockset.ExecuteNonQuery();
+ }
public bool UpdateBlockset(string hash, IEnumerable blocklisthashes, System.Data.IDbTransaction transaction)
{
@@ -643,8 +665,17 @@ namespace Duplicati.Library.Main.Database
catch { }
finally { m_tempblocklist = null; }
+ if (m_tempsmalllist != null)
+ try
+ {
+ cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempsmalllist);
+ cmd.ExecuteNonQuery();
+ }
+ catch { }
+ finally { m_tempsmalllist = null; }
+
}
-
+
base.Dispose();
}
}
diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs
index b10679f61..4ca17a860 100644
--- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs
@@ -478,7 +478,7 @@ namespace Duplicati.Library.Main.Database
try
{
- VerifyConsistency(tr, blocksize, hashsize);
+ VerifyConsistency(tr, blocksize, hashsize, true);
}
catch(Exception ex)
{
diff --git a/Duplicati/Library/Main/IMetahash.cs b/Duplicati/Library/Main/IMetahash.cs
index e9a830900..2fdfd7880 100644
--- a/Duplicati/Library/Main/IMetahash.cs
+++ b/Duplicati/Library/Main/IMetahash.cs
@@ -28,11 +28,7 @@ namespace Duplicati.Library.Main
///
/// The base64 encoded hash of the metadata
///
- string Hash { get; }
- ///
- /// The size of the metadata in bytes
- ///
- long Size { get; }
+ string FileHash { get; }
///
/// The UTF-8 encoded json element with the metadata
///
diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs
index 3e2f33101..0fa191725 100644
--- a/Duplicati/Library/Main/Operation/BackupHandler.cs
+++ b/Duplicati/Library/Main/Operation/BackupHandler.cs
@@ -41,6 +41,7 @@ namespace Duplicati.Library.Main.Operation
private readonly FileAttributes m_attributeFilter;
private readonly Options.SymlinkStrategy m_symlinkPolicy;
private int m_blocksize;
+ private long m_maxmetadatasize;
public BackupHandler(string backendurl, Options options, BackupResults results)
{
@@ -695,6 +696,7 @@ namespace Duplicati.Library.Main.Operation
throw new Exception("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.");
m_blocksize = m_options.Blocksize;
+ m_maxmetadatasize = (m_blocksize / m_options.BlockhashSize) * m_blocksize;
m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
m_blocklistbuffer = new byte[m_options.Blocksize];
@@ -712,7 +714,7 @@ namespace Duplicati.Library.Main.Operation
if (!m_filehasher.CanReuseTransform)
throw new Exception(Strings.Common.InvalidCryptoSystem(m_options.FileHashAlgorithm));
- m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
+ m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, false);
// If there is no filter, we set an empty filter to simplify the code
// If there is a filter, we make sure that the sources are included
m_filter = filter ?? new Library.Utility.FilterExpression();
@@ -743,7 +745,6 @@ namespace Duplicati.Library.Main.Operation
PreBackupVerify(backend);
// Verify before uploading a synthetic list
- m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
UploadSyntheticFilelist(backend);
m_database.BuildLookupTable(m_options);
@@ -772,7 +773,7 @@ namespace Duplicati.Library.Main.Operation
using(new Logging.Timer("UpdateChangeStatistics"))
m_database.UpdateChangeStatistics(m_result);
using(new Logging.Timer("VerifyConsistency"))
- m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize);
+ m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize, false);
UploadRealFileList(backend, filesetvolume);
@@ -987,7 +988,7 @@ namespace Duplicati.Library.Main.Operation
var timestampChanged = lastwrite != oldModified || lastwrite.Ticks == 0 || oldModified.Ticks == 0;
var filesizeChanged = filestatsize < 0 || lastFileSize < 0 || filestatsize != lastFileSize;
var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan;
- var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Size != oldMetasize || metahashandsize.Hash != oldMetahash);
+ var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Blob.Length != oldMetasize || metahashandsize.FileHash != oldMetahash);
if ((oldId < 0 || m_options.DisableFiletimeCheck || timestampChanged || filesizeChanged || metadatachanged) && !tooLargeFile)
{
@@ -1003,67 +1004,16 @@ namespace Duplicati.Library.Main.Operation
using (var blocklisthashes = new Library.Utility.FileBackedStringList())
using (var hashcollector = new Library.Utility.FileBackedStringList())
{
- using (var fs = new Blockprocessor(snapshot.OpenRead(path), m_blockbuffer))
+ using (var fs = snapshot.OpenRead(path))
{
try { m_result.OperationProgressUpdater.StartFile(path, fs.Length); }
catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read file length for file {0}", path), ex); }
- int blocklistoffset = 0;
-
- m_filehasher.Initialize();
-
-
- var offset = 0;
- var remaining = fs.Readblock();
-
- do
- {
- var size = Math.Min(m_blocksize, remaining);
-
- m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset);
- var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size);
- if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length)
- {
- var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
- blocklisthashes.Add(blkey);
- AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
- blocklistoffset = 0;
- }
-
- Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length);
- blocklistoffset += blockkey.Length;
-
- var key = Convert.ToBase64String(blockkey);
- AddBlockToOutput(backend, key, m_blockbuffer, offset, size, hint, false);
- hashcollector.Add(key);
- filesize += size;
-
- m_result.OperationProgressUpdater.UpdateFileProgress(filesize);
- if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
- return false;
-
- remaining -= size;
- offset += size;
-
- if (remaining == 0)
- {
- offset = 0;
- remaining = fs.Readblock();
- }
-
- } while (remaining > 0);
-
- //If all fits in a single block, don't bother with blocklists
- if (hashcollector.Count > 1)
- {
- var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
- blocklisthashes.Add(blkeyfinal);
- AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
- }
+ if ((filesize = ProcessStream(fs, hint, backend, blocklisthashes, hashcollector, false)) < 0)
+ return false;
}
m_result.SizeOfOpenedFiles += filesize;
- m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0);
var filekey = Convert.ToBase64String(m_filehasher.Hash);
if (oldHash != filekey)
@@ -1131,6 +1081,76 @@ namespace Duplicati.Library.Main.Operation
return true;
}
+ private long ProcessStream(System.IO.Stream stream, Library.Interface.CompressionHint hint, BackendManager backend, Library.Utility.FileBackedStringList blocklisthashes, Library.Utility.FileBackedStringList hashcollector, bool skipfilehash)
+ {
+ int blocklistoffset = 0;
+ long filesize = 0;
+
+ using(var fs = new Blockprocessor(stream, m_blockbuffer))
+ {
+ m_filehasher.Initialize();
+
+ var offset = 0;
+ var remaining = fs.Readblock();
+
+ do
+ {
+ var size = Math.Min(m_blocksize, remaining);
+
+ if (!skipfilehash)
+ m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset);
+
+ var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size);
+ if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length)
+ {
+ var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
+ blocklisthashes.Add(blkey);
+ AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
+ blocklistoffset = 0;
+ }
+
+ Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length);
+ blocklistoffset += blockkey.Length;
+
+ var key = Convert.ToBase64String(blockkey);
+ AddBlockToOutput(backend, key, m_blockbuffer, offset, size, hint, false);
+ hashcollector.Add(key);
+ filesize += size;
+
+ if (!skipfilehash)
+ {
+ m_result.OperationProgressUpdater.UpdateFileProgress(filesize);
+ if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
+ return -1;
+ }
+
+ remaining -= size;
+ offset += size;
+
+ if (remaining == 0)
+ {
+ offset = 0;
+ remaining = fs.Readblock();
+ }
+
+ } while (remaining > 0);
+
+ //If all fits in a single block, don't bother with blocklists
+ if (hashcollector.Count > 1)
+ {
+ var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
+ blocklisthashes.Add(blkeyfinal);
+ AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
+ }
+ }
+
+ if (!skipfilehash)
+ m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0);
+
+ return filesize;
+
+ }
+
///
/// Adds the found file data to the output unless the block already exists
///
@@ -1223,29 +1243,32 @@ namespace Duplicati.Library.Main.Operation
m_database.AddUnmodifiedFile(oldId, lastModified, m_transaction);
}
-
- ///
- /// Adds a file to the output,
- ///
- /// The name of the file to record
- /// The value of the lastModified timestamp
- /// The list of hashes that make up the file
- /// The size of the file
- /// The offset into a fragment block where the last few bytes are stored
- /// A lookup table with various metadata values describing the file
- private bool AddFolderToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash meta)
+ private long AddMetadataToOutput(BackendManager backend, IMetahash meta)
{
long metadataid;
- bool r = false;
- if (meta.Size > m_blocksize)
- throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize));
+ if (meta.Blob.Length > m_maxmetadatasize)
+ {
+ //TODO: To fix this, the "WriteFileset" method in BackupHandler needs to
+ // be updated such that it can select sets even when there are multiple
+ // blocklist hashes for the metadata.
+ // This could be done such that an extra query is made if the metadata
+ // spans multiple blocklist hashes, as it is not expected to be common
- r |= AddBlockToOutput(backend, meta.Hash, meta.Blob, 0, (int)meta.Size, CompressionHint.Default, false);
- r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction);
+ m_result.AddWarning(string.Format("Metadata size is {0}, but the largest accepted size is {1}, recording empty metadata", meta.Blob.Length, m_maxmetadatasize), null);
+ meta = EMPTY_METADATA;
+ }
- m_database.AddDirectoryEntry(filename, metadataid, lastModified, m_transaction);
- return r;
+ using(var blocklisthashes = new Library.Utility.FileBackedStringList())
+ using(var hashcollector = new Library.Utility.FileBackedStringList())
+ {
+ using(var ms = new MemoryStream(meta.Blob))
+ ProcessStream(ms, CompressionHint.Compressible, backend, blocklisthashes, hashcollector, true);
+
+ m_database.AddMetadataset(meta.FileHash, meta.Blob.Length, m_blocksize, hashcollector, blocklisthashes, out metadataid, m_transaction);
+ }
+
+ return metadataid;
}
///
@@ -1257,19 +1280,25 @@ namespace Duplicati.Library.Main.Operation
/// The size of the file
/// The offset into a fragment block where the last few bytes are stored
/// A lookup table with various metadata values describing the file
- private bool AddSymlinkToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash meta)
+ private void AddFolderToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash metadata)
{
- long metadataid;
- bool r = false;
-
- if (meta.Size > m_blocksize)
- throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize));
-
- r |= AddBlockToOutput(backend, meta.Hash, meta.Blob, 0, (int)meta.Size, CompressionHint.Default, false);
- r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction);
+ var metadataid = AddMetadataToOutput(backend, metadata);
+ m_database.AddDirectoryEntry(filename, metadataid, lastModified, m_transaction);
+ }
+ ///
+ /// Adds a file to the output,
+ ///
+ /// The name of the file to record
+ /// The value of the lastModified timestamp
+ /// The list of hashes that make up the file
+ /// The size of the file
+ /// The offset into a fragment block where the last few bytes are stored
+ /// A lookup table with various metadata values describing the file
+ private void AddSymlinkToOutput(BackendManager backend, string filename, DateTime lastModified, IMetahash metadata)
+ {
+ var metadataid = AddMetadataToOutput(backend, metadata);
m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction);
- return r;
}
///
@@ -1283,17 +1312,10 @@ namespace Duplicati.Library.Main.Operation
/// A lookup table with various metadata values describing the file
private void AddFileToOutput(BackendManager backend, string filename, long size, DateTime lastmodified, IMetahash metadata, IEnumerable hashlist, string filehash, IEnumerable blocklisthashes)
{
- long metadataid;
long blocksetid;
-
- if (metadata.Size > m_blocksize)
- throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize));
-
- AddBlockToOutput(backend, metadata.Hash, metadata.Blob, 0, (int)metadata.Size, CompressionHint.Default, false);
- m_database.AddMetadataset(metadata.Hash, metadata.Size, out metadataid, m_transaction);
+ var metadataid = AddMetadataToOutput(backend, metadata);
m_database.AddBlockset(filehash, size, m_blocksize, hashlist, blocklisthashes, out blocksetid, m_transaction);
-
m_database.AddFile(filename, lastmodified, blocksetid, metadataid, m_transaction);
}
diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs
index e95429545..92140597a 100644
--- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs
+++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs
@@ -215,9 +215,14 @@ namespace Duplicati.Library.Main.Operation
{
try
{
+ var expectedmetablocks = (fe.Metasize + blocksize - 1) / blocksize;
+ var expectedmetablocklisthashes = (expectedmetablocks + hashes_pr_block - 1) / hashes_pr_block;
+ if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0;
+
if (fe.Type == FilelistEntryType.Folder)
{
- restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
+ var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
+ restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
}
else if (fe.Type == FilelistEntryType.File)
{
@@ -226,16 +231,38 @@ namespace Duplicati.Library.Main.Operation
if (expectedblocks <= 1) expectedblocklisthashes = 0;
var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr);
- restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
+ var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
+ restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, metadataid, tr);
+
+ if (fe.Size <= blocksize)
+ {
+ if (!string.IsNullOrWhiteSpace(fe.Blockhash))
+ restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, tr);
+ else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
+ restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, tr);
+ else
+ m_result.AddWarning(string.Format("No block hash found for file: {0}", fe.Path), null);
+ }
}
else if (fe.Type == FilelistEntryType.Symlink)
{
- restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
+ var metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
+ restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
}
else
{
m_result.AddWarning(string.Format("Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path), null);
}
+
+ if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink))
+ {
+ if (!string.IsNullOrWhiteSpace(fe.Metablockhash))
+ restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, tr);
+ else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
+ restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, tr);
+ else
+ m_result.AddWarning(string.Format("No block hash found for file metadata: {0}", fe.Path), null);
+ }
}
catch (Exception ex)
{
@@ -435,7 +462,7 @@ namespace Duplicati.Library.Main.Operation
//All done, we must verify that we have all blocklist fully intact
// if this fails, the db will not be deleted, so it can be used,
// except to continue a backup
- restoredb.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
+ restoredb.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, true);
m_result.AddMessage("Recreate completed, and consistency checks completed, marking database as complete");
diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs
index 663f31196..9f78a41d5 100644
--- a/Duplicati/Library/Main/Utility.cs
+++ b/Duplicati/Library/Main/Utility.cs
@@ -34,7 +34,7 @@ namespace Duplicati.Library.Main
///
/// The base64 encoded hash
///
- private readonly string m_hash;
+ private readonly string m_filehash;
///
/// The UTF-8 encoded json element with the metadata
///
@@ -47,30 +47,29 @@ namespace Duplicati.Library.Main
public Metahash(Dictionary values, Options options)
{
m_values = values;
- var hasher = System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm);
- if (hasher == null)
- throw new Exception(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm));
- if (!hasher.CanReuseTransform)
- throw new Exception(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm));
using (var ms = new System.IO.MemoryStream())
using (var w = new StreamWriter(ms, Encoding.UTF8))
+ using(var filehasher = System.Security.Cryptography.HashAlgorithm.Create(options.FileHashAlgorithm))
{
+ if (filehasher == null)
+ throw new Exception(Strings.Common.InvalidHashAlgorithm(options.FileHashAlgorithm));
+
w.Write(JsonConvert.SerializeObject(values));
w.Flush();
m_blob = ms.ToArray();
ms.Position = 0;
- m_hash = Convert.ToBase64String(hasher.ComputeHash(ms));
+ m_filehash = Convert.ToBase64String(filehasher.ComputeHash(ms));
}
}
- public string Hash
+ public string FileHash
{
- get { return m_hash; }
+ get { return m_filehash; }
}
-
+
public long Size
{
get { return m_blob.Length; }
diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs
index 742fa653c..483d35981 100644
--- a/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs
+++ b/Duplicati/Library/Main/Volumes/FilesetVolumeReader.cs
@@ -97,8 +97,12 @@ namespace Duplicati.Library.Main.Volumes
public long Size { get; private set; }
public DateTime Time { get; private set; }
public string Metahash { get; private set; }
+ public string Metablockhash { get; private set; }
public long Metasize { get; private set; }
+ public string Blockhash { get; private set; }
+ public long Blocksize { get; private set; }
public IEnumerable BlocklistHashes { get; private set; }
+ public IEnumerable MetaBlocklistHashes { get; private set; }
private JsonReader m_reader;
public FileEntry(JsonReader reader)
@@ -150,16 +154,60 @@ namespace Duplicati.Library.Main.Volumes
if (!m_reader.Read())
throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+
+ if (m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "metablocklists")
+ {
+ var metadatablocklisthashes = new List();
+ SkipJsonToken(m_reader, JsonToken.StartArray);
+
+ if (!m_reader.Read())
+ throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+
+ while(m_reader.TokenType == JsonToken.String)
+ {
+ metadatablocklisthashes.Add(m_reader.Value.ToString());
+ if (!m_reader.Read())
+ throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+ }
+
+ if (m_reader.TokenType != JsonToken.EndArray)
+ throw new InvalidDataException(string.Format("Invalid JSON, unexpected token {1} found while reading entry {0}", this.Path, m_reader.TokenType));
+ if (!m_reader.Read())
+ throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+
+ this.MetaBlocklistHashes = metadatablocklisthashes;
+ this.Metablockhash = null;
+ }
+ else if (m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "metablockhash")
+ {
+ if (!m_reader.Read())
+ throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+ this.Metablockhash = m_reader.Value.ToString();
+
+ if (!m_reader.Read())
+ throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+
+ this.MetaBlocklistHashes = null;
+ }
}
if ((this.Type == FilelistEntryType.File || this.Type == FilelistEntryType.AlternateStream) && m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "blocklists")
{
SkipJsonToken(m_reader, JsonToken.StartArray);
- BlocklistHashes = new BlocklistHashEnumerable(m_reader);
+ this.BlocklistHashes = new BlocklistHashEnumerable(m_reader);
+ }
+ else if ((this.Type == FilelistEntryType.File || this.Type == FilelistEntryType.AlternateStream) && m_reader.TokenType == JsonToken.PropertyName && m_reader.Value != null && m_reader.Value.ToString() == "blockhash")
+ {
+ if (!m_reader.Read())
+ throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading entry {0}", this.Path));
+
+ this.Blockhash = m_reader.Value.ToString();
+ this.Blocksize = ReadJsonInt64Property(m_reader, "blocksize");
+ this.BlocklistHashes = null;
}
else
{
- BlocklistHashes = null;
+ this.BlocklistHashes = null;
if (m_reader.TokenType != JsonToken.EndObject)
throw new InvalidDataException(string.Format("Invalid JSON, expected EndObject, but found {0}, \"{1}\" while reading entry {2}", m_reader.TokenType, m_reader.Value, this.Path));
}
diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs
index 1a86d2f09..83b2eddb0 100644
--- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs
+++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs
@@ -25,17 +25,45 @@ namespace Duplicati.Library.Main.Volumes
m_writer.WriteStartArray();
}
- public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable blocklisthashes)
+ private void WriteMetaProperties(string metahash, long metasize, string metablockhash, IEnumerable metablocklisthashes)
{
- AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, blocklisthashes);
+ m_writer.WritePropertyName("metahash");
+ m_writer.WriteValue(metahash);
+ m_writer.WritePropertyName("metasize");
+ m_writer.WriteValue(metasize);
+
+ if (metablocklisthashes != null)
+ {
+ //Slightly akward, but we avoid writing if there are no entries
+ var en = metablocklisthashes.GetEnumerator();
+ if (en.MoveNext() && !string.IsNullOrEmpty(en.Current))
+ {
+ m_writer.WritePropertyName("metablocklists");
+ m_writer.WriteStartArray();
+ m_writer.WriteValue(en.Current);
+ while (en.MoveNext())
+ m_writer.WriteValue(en.Current);
+ m_writer.WriteEndArray();
+ }
+ }
+ else if (!string.IsNullOrWhiteSpace(metablockhash))
+ {
+ m_writer.WritePropertyName("metablockhash");
+ m_writer.WriteValue(metablockhash);
+ }
}
- public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable blocklisthashes)
+ public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes)
{
- AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, blocklisthashes);
+ AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes);
}
- private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, IEnumerable blocklisthashes)
+ public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes)
+ {
+ AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes);
+ }
+
+ private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes)
{
m_filecount++;
m_writer.WriteStartObject();
@@ -50,12 +78,7 @@ namespace Duplicati.Library.Main.Volumes
m_writer.WritePropertyName("time");
m_writer.WriteValue(Library.Utility.Utility.SerializeDateTime(lastmodified));
if (metahash != null)
- {
- m_writer.WritePropertyName("metahash");
- m_writer.WriteValue(metahash);
- m_writer.WritePropertyName("metasize");
- m_writer.WriteValue(metasize);
- }
+ WriteMetaProperties(metahash, metasize, metablockhash, metablocklisthashes);
if (blocklisthashes != null)
{
@@ -66,21 +89,28 @@ namespace Duplicati.Library.Main.Volumes
m_writer.WritePropertyName("blocklists");
m_writer.WriteStartArray();
m_writer.WriteValue(en.Current);
- while(en.MoveNext())
+ while (en.MoveNext())
m_writer.WriteValue(en.Current);
m_writer.WriteEndArray();
}
}
+ else if (!string.IsNullOrWhiteSpace(blockhash))
+ {
+ m_writer.WritePropertyName("blockhash");
+ m_writer.WriteValue(blockhash);
+ m_writer.WritePropertyName("blocksize");
+ m_writer.WriteValue(blocksize);
+ }
m_writer.WriteEndObject();
}
- public void AddDirectory(string name, string metahash, long metasize)
+ public void AddDirectory(string name, string metahash, long metasize, string metablockhash, IEnumerable metablocklisthashes)
{
- AddMetaEntry(FilelistEntryType.Folder, name, metahash, metasize);
+ AddMetaEntry(FilelistEntryType.Folder, name, metahash, metasize, metablockhash, metablocklisthashes);
}
- public void AddMetaEntry(FilelistEntryType type, string name, string metahash, long metasize)
+ public void AddMetaEntry(FilelistEntryType type, string name, string metahash, long metasize, string metablockhash, IEnumerable metablocklisthashes)
{
m_foldercount++;
m_writer.WriteStartObject();
@@ -89,12 +119,8 @@ namespace Duplicati.Library.Main.Volumes
m_writer.WritePropertyName("path");
m_writer.WriteValue(name);
if (metahash != null)
- {
- m_writer.WritePropertyName("metahash");
- m_writer.WriteValue(metahash);
- m_writer.WritePropertyName("metasize");
- m_writer.WriteValue(metasize);
- }
+ WriteMetaProperties(metahash, metasize, metablockhash, metablocklisthashes);
+
m_writer.WriteEndObject();
}
@@ -127,9 +153,9 @@ namespace Duplicati.Library.Main.Volumes
public long FileCount { get { return m_filecount; } }
public long FolderCount { get { return m_foldercount; } }
- public void AddSymlink(string name, string metahash, long metasize)
+ public void AddSymlink(string name, string metahash, long metasize, string metablockhash, IEnumerable metablocklisthashes)
{
- AddMetaEntry(FilelistEntryType.Symlink, name, metahash, metasize);
+ AddMetaEntry(FilelistEntryType.Symlink, name, metahash, metasize, metablockhash, metablocklisthashes);
}
}
}
diff --git a/Duplicati/Library/Main/Volumes/IFileEntry.cs b/Duplicati/Library/Main/Volumes/IFileEntry.cs
index 836f99d44..8be20f37b 100644
--- a/Duplicati/Library/Main/Volumes/IFileEntry.cs
+++ b/Duplicati/Library/Main/Volumes/IFileEntry.cs
@@ -14,7 +14,11 @@ namespace Duplicati.Library.Main.Volumes
long Size { get; }
DateTime Time { get; }
string Metahash { get; }
+ string Metablockhash { get; }
long Metasize { get; }
+ string Blockhash { get; }
+ long Blocksize { get; }
IEnumerable BlocklistHashes { get; }
+ IEnumerable MetaBlocklistHashes { get; }
}
}
diff --git a/Duplicati/Library/Main/Volumes/VolumeBase.cs b/Duplicati/Library/Main/Volumes/VolumeBase.cs
index c96e03e93..a4091dc80 100644
--- a/Duplicati/Library/Main/Volumes/VolumeBase.cs
+++ b/Duplicati/Library/Main/Volumes/VolumeBase.cs
@@ -11,7 +11,7 @@ namespace Duplicati.Library.Main.Volumes
protected class ManifestData
{
public const string ENCODING = "utf8";
- public const long VERSION = 1;
+ public const long VERSION = 2;
public long Version;
public string Created;
diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs
index ac7ed0380..c7cb561fe 100644
--- a/Duplicati/UnitTest/BorderTests.cs
+++ b/Duplicati/UnitTest/BorderTests.cs
@@ -65,6 +65,17 @@ namespace Duplicati.UnitTest
RunCommands(12345, 1024 * 1024 * 10);
}
+ [Test]
+ [Category("Border")]
+ public void RunNoMetadata()
+ {
+ PrepareSourceData();
+ RunCommands(1024 * 10, modifyOptions: opts => {
+ opts["skip-metadata"] = "true";
+ });
+ }
+
+
[Test]
[Category("Border")]
public void RunMD5()
@@ -87,7 +98,7 @@ namespace Duplicati.UnitTest
});
}
- //[Test]
+ [Test]
[Category("Border")]
public void RunMixedBlockFile_1()
{
@@ -98,7 +109,7 @@ namespace Duplicati.UnitTest
});
}
- //[Test]
+ [Test]
[Category("Border")]
public void RunMixedBlockFile_2()
{
@@ -261,7 +272,7 @@ namespace Duplicati.UnitTest
Assert.AreEqual(filenames.Count * 3, r.FilesRestored);
}
- TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, true);
+ TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"));
using(var tf = new Library.Utility.TempFolder())
{