diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs index 5feca92b7..98082b05b 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs @@ -404,5 +404,80 @@ namespace Duplicati.Library.Main.ForestHash.Database { return new FolderListEnumerable(m_connection, m_tempfiletable); } + + public interface IFastSource + { + string TargetPath { get; } + long TargetFileID { get; } + string SourcePath { get; } + IEnumerable Blocks { get; } + } + + public interface IBlockEntry + { + long Offset { get; } + long Size { get; } + long Index { get; } + string Hash { get; } + } + + private class FastSource : IFastSource + { + private class BlockEntry : IBlockEntry + { + private System.Data.IDataReader m_rd; + private long m_blocksize; + public BlockEntry(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } + public long Offset { get { return Convert.ToInt64(m_rd.GetValue(3)) * m_blocksize; } } + public long Index { get { return Convert.ToInt64(m_rd.GetValue(3)); } } + public long Size { get { return Convert.ToInt64(m_rd.GetValue(5)); } } + public string Hash { get { return m_rd.GetValue(4).ToString(); } } + } + + private System.Data.IDataReader m_rd; + private long m_blocksize; + public FastSource(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } + public bool MoreData { get; private set; } + public string TargetPath { get { return m_rd.GetValue(0).ToString(); } } + public long TargetFileID { get { return Convert.ToInt64(m_rd.GetValue(2)); } } + public string SourcePath { get { return m_rd.GetValue(1).ToString(); } } + + public IEnumerable Blocks + { + get + { + var tid = this.TargetFileID; + + do + { + yield return new BlockEntry(m_rd, m_blocksize); + } while((MoreData = m_rd.Read()) && tid == this.TargetFileID); + + } + } + } + + public IEnumerable GetFilesAndSourceBlocksFast() + { + var whereclause = string.Format(@" ""{0}"".""ID"" = ""{1}"".""FileID"" AND ""{1}"".""Restored"" = 0 AND ""{0}"".""TargetPath"" != ""{0}"".""Path"" ", m_tempfiletable, m_tempblocktable); + var sourepaths = string.Format(@"SELECT DISTINCT ""{0}"".""Path"" FROM ""{0}"", ""{1}"" WHERE " + whereclause, m_tempfiletable, m_tempblocktable); + var latestBlocksetIds = @"SELECT ""File"".""Path"", ""File"".""BlocksetID"", MAX(""FilesetEntry"".""Scantime"") FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""File"".""Path"" IN (" + sourepaths + @") GROUP BY ""File"".""Path"" "; + var sources = string.Format(@"SELECT DISTINCT ""{0}"".""TargetPath"", ""{0}"".""Path"", ""{0}"".""ID"", ""{1}"".""Index"", ""{1}"".""Hash"", ""{1}"".""Size"" FROM ""{0}"", ""{1}"", ""File"", (" + latestBlocksetIds + @") S, ""Block"", ""BlocksetEntry"" WHERE ""BlocksetEntry"".""BlocksetID"" = ""S"".""BlocksetID"" AND ""BlocksetEntry"".""BlocksetID"" = ""File"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""{1}"".""Index"" = ""BlocksetEntry"".""Index"" AND ""{1}"".""Hash"" = ""Block"".""Hash"" AND ""{1}"".""Size"" = ""Block"".""Size"" AND ""S"".""Path"" = ""{0}"".""Path"" AND " + whereclause + @" ORDER BY ""{0}"".""ID"", ""{1}"".""Index"" ", m_tempfiletable, m_tempblocktable); + using(var cmd = m_connection.CreateCommand()) + using(var rd = cmd.ExecuteReader(sources)) + { + if (rd.Read()) + { + var more = false; + do + { + var n = new FastSource(rd, m_blocksize); + yield return n; + more = n.MoreData; + } while (more); + } + } + } + } } diff --git a/Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs index e9ad37cf2..f0f9aa5fc 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs @@ -223,6 +223,12 @@ namespace Duplicati.Library.Main.ForestHash.Operation //If we are patching an existing target folder, do not touch stuff that is already updated ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, m_stat); +#if DEBUG + if (!m_options.NoLocalBlocks) +#endif + //Look for existing blocks in the original source files only + ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, m_stat); + // If other local files already have the blocks we want, we use them instead of downloading if (m_options.FhPatchWithLocalBlocks) ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_stat); @@ -269,6 +275,75 @@ namespace Duplicati.Library.Main.ForestHash.Operation //TODO: Implement writing metadata } + private static void ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, FhOptions options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, CommunicationStatistics stat) + { + // Fill BLOCKS with data from known local source files + using (var blockmarker = database.CreateBlockMarker()) + { + foreach(var entry in database.GetFilesAndSourceBlocksFast()) + { + var targetpath = entry.TargetPath; + var targetfileid = entry.TargetFileID; + var sourcepath = entry.SourcePath; + var patched = false; + + try + { + using(var targetstream = System.IO.File.Open(targetpath, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.Read)) + { + try + { + using(var sourcestream = System.IO.File.Open(sourcepath, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite)) + { + foreach(var block in entry.Blocks) + { + if (sourcestream.Length > block.Offset) + { + sourcestream.Position = block.Offset; + + var size = sourcestream.Read(blockbuffer, 0, blockbuffer.Length); + if (size == block.Size) + { + var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); + if (key == block.Hash) + { + if (options.FhDryrun) + patched = true; + else + { + targetstream.Position = block.Offset; + targetstream.Write(blockbuffer, 0, size); + } + + blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size); + } + } + } + } + } + } + catch (Exception ex) + { + stat.LogWarning(string.Format("Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message), ex); + database.LogMessage("Warning", string.Format("Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message), ex, null); + } + } + } + catch (Exception ex) + { + stat.LogWarning(string.Format("Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message), ex); + database.LogMessage("Warning", string.Format("Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message), ex, null); + } + + if (patched && options.FhDryrun) + stat.LogMessage("[Dryrun] Would patch file with local data: {0}", targetpath); + } + + blockmarker.Commit(); + } + } + + private static void ScanForExistingSourceBlocks(LocalRestoreDatabase database, FhOptions options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, CommunicationStatistics stat) { // Fill BLOCKS with data from known local source files