From 23d42e24e45eefdad3cb698102c8deb3f5356bc5 Mon Sep 17 00:00:00 2001 From: "kenneth.skovhede@gmail.com" Date: Wed, 12 Jan 2011 19:05:33 +0000 Subject: [PATCH] Added code to pre-size the memory streams to avoid repeated reallocation and memory copy. git-svn-id: https://duplicati.googlecode.com/svn/trunk@671 59da171f-624f-0410-aa54-27559c288bec --- Duplicati/Library/Main/RSyncDir.cs | 18 ++++++++-- .../Library/SharpRSync/Adler32Checksum.cs | 4 --- .../Library/SharpRSync/ChecksumFileWriter.cs | 34 ++++++++++++++++++- .../SharpRSync/ChecksumGeneratingStream.cs | 10 ++++++ 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/RSyncDir.cs b/Duplicati/Library/Main/RSyncDir.cs index 89de62361..f5f05b844 100644 --- a/Duplicati/Library/Main/RSyncDir.cs +++ b/Duplicati/Library/Main/RSyncDir.cs @@ -39,6 +39,12 @@ namespace Duplicati.Library.Main.RSync /// private static readonly TimeSpan PROGRESS_TIMESPAN = TimeSpan.FromSeconds(1); + /// + /// The margin that a file is allowed to grow during signature generation, + /// without incurring a performance penalty + /// + private const double FILESIZE_GROW_MARGIN_MULTIPLIER = 1.01; + /// /// The possible filetypes in an archive /// @@ -1354,6 +1360,7 @@ namespace Duplicati.Library.Main.RSync fs.Position = 0; using (SharpRSync.ChecksumGeneratingStream ts = new SharpRSync.ChecksumGeneratingStream(newSig, fs)) { + newSig.Capacity = ts.BytesGeneratedForSignature((int)(fs.Length * FILESIZE_GROW_MARGIN_MULTIPLIER)); fs = new Utility.TempFileStream(); Utility.Utility.CopyStream(ts, fs, false); } @@ -1376,7 +1383,9 @@ namespace Duplicati.Library.Main.RSync //Set up for a new round signature = new System.IO.MemoryStream(); fs.Position = 0; + long filelen = fs.Length; fs = new SharpRSync.ChecksumGeneratingStream(signature, fs); + ((MemoryStream)signature).Capacity = ((SharpRSync.ChecksumGeneratingStream)fs).BytesGeneratedForSignature(filelen); } } @@ -1430,10 +1439,15 @@ namespace Duplicati.Library.Main.RSync { string relpath = GetRelativeName(s); - System.IO.MemoryStream ms = new MemoryStream(); m_examinedfilesize += fs.Length; m_examinedfiles++; - SharpRSync.Interface.GenerateSignature(fs, ms); + + System.IO.MemoryStream ms = new MemoryStream(); + SharpRSync.ChecksumFileWriter ws = new Duplicati.Library.SharpRSync.ChecksumFileWriter(ms); + + //Expand the memorystream to contain all bytes, which avoids re-allocation + ms.Capacity = ws.BytesGeneratedForSignature((int)(fs.Length * FILESIZE_GROW_MARGIN_MULTIPLIER)); + ws.AddStream(fs); ms.Position = 0; if (!m_oldSignatures.ContainsKey(relpath)) diff --git a/Duplicati/Library/SharpRSync/Adler32Checksum.cs b/Duplicati/Library/SharpRSync/Adler32Checksum.cs index e10857c71..1e38dbdc6 100644 --- a/Duplicati/Library/SharpRSync/Adler32Checksum.cs +++ b/Duplicati/Library/SharpRSync/Adler32Checksum.cs @@ -29,10 +29,6 @@ namespace Duplicati.Library.SharpRSync /// public class Adler32Checksum { - /// - /// The number of bytes to include in each checksum - /// - public const int DEFAULT_BLOCK_SIZE = 2048; /// /// The charated offset used in the checksum /// diff --git a/Duplicati/Library/SharpRSync/ChecksumFileWriter.cs b/Duplicati/Library/SharpRSync/ChecksumFileWriter.cs index 2c04351a0..9fe6a08f5 100644 --- a/Duplicati/Library/SharpRSync/ChecksumFileWriter.cs +++ b/Duplicati/Library/SharpRSync/ChecksumFileWriter.cs @@ -9,6 +9,23 @@ namespace Duplicati.Library.SharpRSync /// public class ChecksumFileWriter { + /// + /// The number of bytes to include in each checksum + /// + public const int DEFAULT_BLOCK_SIZE = 2048; + /// + /// The default number of bytes to use from the strong hash + /// + public const int DEFAULT_STRONG_LEN = 8; + /// + /// The default number of bytes generated per input block + /// + public const int DEFAULT_BYTES_PER_BLOCK = DEFAULT_STRONG_LEN + 4; + /// + /// The number of bytes used for the rdiff header + /// + public static readonly int HEADER_SIZE = RDiffBinary.SIGNATURE_MAGIC.Length + 4 + 4; + /// /// The length of a datablock /// @@ -31,7 +48,7 @@ namespace Duplicati.Library.SharpRSync /// /// The stream into which the checksum data is written public ChecksumFileWriter(System.IO.Stream outputstream) - : this(outputstream, Adler32Checksum.DEFAULT_BLOCK_SIZE, 8) + : this(outputstream, DEFAULT_BLOCK_SIZE, DEFAULT_STRONG_LEN) { } @@ -112,5 +129,20 @@ namespace Duplicati.Library.SharpRSync /// Gets the number of bytes in a signature file /// public int StrongLength { get { return m_stronglen; } } + /// + /// Gets the number of bytes generated per input block + /// + public int BytesPrBlock { get { return m_stronglen + 4; } } + + /// + /// Returns the number of bytes generated when processing the specified amount of bytes + /// + /// The size of the file to process + /// The expected size of the signature file + public int BytesGeneratedForSignature(long filesize) + { + return (int)(SharpRSync.ChecksumFileWriter.HEADER_SIZE + + (((filesize + m_blocklen - 1) / m_blocklen) * (m_stronglen + 4))); + } } } diff --git a/Duplicati/Library/SharpRSync/ChecksumGeneratingStream.cs b/Duplicati/Library/SharpRSync/ChecksumGeneratingStream.cs index 73b727a11..a7ce51394 100644 --- a/Duplicati/Library/SharpRSync/ChecksumGeneratingStream.cs +++ b/Duplicati/Library/SharpRSync/ChecksumGeneratingStream.cs @@ -138,5 +138,15 @@ namespace Duplicati.Library.SharpRSync base.Dispose(disposing); } + + /// + /// Returns the number of bytes generated when processing the specified amount of bytes + /// + /// The size of the file to process + /// The expected size of the signature file + public int BytesGeneratedForSignature(long filesize) + { + return m_outstream.BytesGeneratedForSignature(filesize); + } } }