haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added faster local-block restore that only restores blocks from the original source files, if restoring to somewhere else.

Browse Source
This commit is contained in:
Kenneth Skovhede
2013-04-28 12:19:21 +02:00
parent 0470e6200a
commit d91980dda1
2 changed files with 150 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs
+75
View File
@@ -404,5 +404,80 @@ namespace Duplicati.Library.Main.ForestHash.Database
{
return new FolderListEnumerable(m_connection, m_tempfiletable);
}
public interface IFastSource
{
string TargetPath { get; }
long TargetFileID { get; }
string SourcePath { get; }
IEnumerable<IBlockEntry> Blocks { get; }
}
public interface IBlockEntry
{
long Offset { get; }
long Size { get; }
long Index { get; }
string Hash { get; }
}
private class FastSource : IFastSource
{
private class BlockEntry : IBlockEntry
{
private System.Data.IDataReader m_rd;
private long m_blocksize;
public BlockEntry(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; }
public long Offset { get { return Convert.ToInt64(m_rd.GetValue(3)) * m_blocksize; } }
public long Index { get { return Convert.ToInt64(m_rd.GetValue(3)); } }
public long Size { get { return Convert.ToInt64(m_rd.GetValue(5)); } }
public string Hash { get { return m_rd.GetValue(4).ToString(); } }
}
private System.Data.IDataReader m_rd;
private long m_blocksize;
public FastSource(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; }
public bool MoreData { get; private set; }
public string TargetPath { get { return m_rd.GetValue(0).ToString(); } }
public long TargetFileID { get { return Convert.ToInt64(m_rd.GetValue(2)); } }
public string SourcePath { get { return m_rd.GetValue(1).ToString(); } }
public IEnumerable<IBlockEntry> Blocks
{
get
{
var tid = this.TargetFileID;
do
{
yield return new BlockEntry(m_rd, m_blocksize);
} while((MoreData = m_rd.Read()) && tid == this.TargetFileID);
}
}
}
public IEnumerable<IFastSource> GetFilesAndSourceBlocksFast()
{
var whereclause = string.Format(@" ""{0}"".""ID"" = ""{1}"".""FileID"" AND ""{1}"".""Restored"" = 0 AND ""{0}"".""TargetPath"" != ""{0}"".""Path"" ", m_tempfiletable, m_tempblocktable);
var sourepaths = string.Format(@"SELECT DISTINCT ""{0}"".""Path"" FROM ""{0}"", ""{1}"" WHERE " + whereclause, m_tempfiletable, m_tempblocktable);
var latestBlocksetIds = @"SELECT ""File"".""Path"", ""File"".""BlocksetID"", MAX(""FilesetEntry"".""Scantime"") FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""File"".""Path"" IN (" + sourepaths + @") GROUP BY ""File"".""Path"" ";
var sources = string.Format(@"SELECT DISTINCT ""{0}"".""TargetPath"", ""{0}"".""Path"", ""{0}"".""ID"", ""{1}"".""Index"", ""{1}"".""Hash"", ""{1}"".""Size"" FROM ""{0}"", ""{1}"", ""File"", (" + latestBlocksetIds + @") S, ""Block"", ""BlocksetEntry"" WHERE ""BlocksetEntry"".""BlocksetID"" = ""S"".""BlocksetID"" AND ""BlocksetEntry"".""BlocksetID"" = ""File"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""{1}"".""Index"" = ""BlocksetEntry"".""Index"" AND ""{1}"".""Hash"" = ""Block"".""Hash"" AND ""{1}"".""Size"" = ""Block"".""Size"" AND ""S"".""Path"" = ""{0}"".""Path"" AND " + whereclause + @" ORDER BY ""{0}"".""ID"", ""{1}"".""Index"" ", m_tempfiletable, m_tempblocktable);
using(var cmd = m_connection.CreateCommand())
using(var rd = cmd.ExecuteReader(sources))
{
if (rd.Read())
{
var more = false;
do
{
var n = new FastSource(rd, m_blocksize);
yield return n;
more = n.MoreData;
} while (more);
}
}
}
}
}
Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs
+75
View File
@@ -223,6 +223,12 @@ namespace Duplicati.Library.Main.ForestHash.Operation
//If we are patching an existing target folder, do not touch stuff that is already updated
ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, m_stat);
#if DEBUG
if (!m_options.NoLocalBlocks)
#endif
//Look for existing blocks in the original source files only
ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, m_stat);
// If other local files already have the blocks we want, we use them instead of downloading
if (m_options.FhPatchWithLocalBlocks)
ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_stat);
@@ -269,6 +275,75 @@ namespace Duplicati.Library.Main.ForestHash.Operation
//TODO: Implement writing metadata
}
private static void ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, FhOptions options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, CommunicationStatistics stat)
{
// Fill BLOCKS with data from known local source files
using (var blockmarker = database.CreateBlockMarker())
{
foreach(var entry in database.GetFilesAndSourceBlocksFast())
{
var targetpath = entry.TargetPath;
var targetfileid = entry.TargetFileID;
var sourcepath = entry.SourcePath;
var patched = false;
try
{
using(var targetstream = System.IO.File.Open(targetpath, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.Read))
{
try
{
using(var sourcestream = System.IO.File.Open(sourcepath, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite))
{
foreach(var block in entry.Blocks)
{
if (sourcestream.Length > block.Offset)
{
sourcestream.Position = block.Offset;
var size = sourcestream.Read(blockbuffer, 0, blockbuffer.Length);
if (size == block.Size)
{
var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size));
if (key == block.Hash)
{
if (options.FhDryrun)
patched = true;
else
{
targetstream.Position = block.Offset;
targetstream.Write(blockbuffer, 0, size);
}
blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size);
}
}
}
}
}
}
catch (Exception ex)
{
stat.LogWarning(string.Format("Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message), ex);
database.LogMessage("Warning", string.Format("Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message), ex, null);
}
}
}
catch (Exception ex)
{
stat.LogWarning(string.Format("Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message), ex);
database.LogMessage("Warning", string.Format("Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message), ex, null);
}
if (patched && options.FhDryrun)
stat.LogMessage("[Dryrun] Would patch file with local data: {0}", targetpath);
}
blockmarker.Commit();
}
}
private static void ScanForExistingSourceBlocks(LocalRestoreDatabase database, FhOptions options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, CommunicationStatistics stat)
{
// Fill BLOCKS with data from known local source files