Added code to pre-size the memory streams to avoid repeated reallocation and memory copy.

git-svn-id: https://duplicati.googlecode.com/svn/trunk@671 59da171f-624f-0410-aa54-27559c288bec

Duplicati/Library/Main/RSyncDir.cs

@@ -39,6 +39,12 @@ namespace Duplicati.Library.Main.RSync
         /// </summary>
         private static readonly TimeSpan PROGRESS_TIMESPAN = TimeSpan.FromSeconds(1);
 
+        /// <summary>
+        /// The margin that a file is allowed to grow during signature generation, 
+        /// without incurring a performance penalty
+        /// </summary>
+        private const double FILESIZE_GROW_MARGIN_MULTIPLIER = 1.01;
+
         /// <summary>
         /// The possible filetypes in an archive
         /// </summary>
@@ -1354,6 +1360,7 @@ namespace Duplicati.Library.Main.RSync
                                                 fs.Position = 0;
                                                 using (SharpRSync.ChecksumGeneratingStream ts = new SharpRSync.ChecksumGeneratingStream(newSig, fs))
                                                 {
+                                                    newSig.Capacity = ts.BytesGeneratedForSignature((int)(fs.Length * FILESIZE_GROW_MARGIN_MULTIPLIER));
                                                     fs = new Utility.TempFileStream();
                                                     Utility.Utility.CopyStream(ts, fs, false);
                                                 }
@@ -1376,7 +1383,9 @@ namespace Duplicati.Library.Main.RSync
                                             //Set up for a new round
                                             signature = new System.IO.MemoryStream();
                                             fs.Position = 0;
+                                            long filelen = fs.Length;
                                             fs = new SharpRSync.ChecksumGeneratingStream(signature, fs);
+                                            ((MemoryStream)signature).Capacity = ((SharpRSync.ChecksumGeneratingStream)fs).BytesGeneratedForSignature(filelen);
                                         }
                                     }
 
@@ -1430,10 +1439,15 @@ namespace Duplicati.Library.Main.RSync
         {
             string relpath = GetRelativeName(s);
 
-            System.IO.MemoryStream ms = new MemoryStream();
             m_examinedfilesize += fs.Length;
             m_examinedfiles++;
-            SharpRSync.Interface.GenerateSignature(fs, ms);
+
+            System.IO.MemoryStream ms = new MemoryStream();
+            SharpRSync.ChecksumFileWriter ws = new Duplicati.Library.SharpRSync.ChecksumFileWriter(ms);
+            
+            //Expand the memorystream to contain all bytes, which avoids re-allocation
+            ms.Capacity = ws.BytesGeneratedForSignature((int)(fs.Length * FILESIZE_GROW_MARGIN_MULTIPLIER));
+            ws.AddStream(fs);
             ms.Position = 0;
 
             if (!m_oldSignatures.ContainsKey(relpath))

Duplicati/Library/SharpRSync/Adler32Checksum.cs

@@ -29,10 +29,6 @@ namespace Duplicati.Library.SharpRSync
     /// </summary>
     public class Adler32Checksum
     {
-        /// <summary>
-        /// The number of bytes to include in each checksum
-        /// </summary>
-        public const int DEFAULT_BLOCK_SIZE = 2048;
         /// <summary>
         /// The charated offset used in the checksum
         /// </summary>

Duplicati/Library/SharpRSync/ChecksumFileWriter.cs

@@ -9,6 +9,23 @@ namespace Duplicati.Library.SharpRSync
     /// </summary>
     public class ChecksumFileWriter
     {
+        /// <summary>
+        /// The number of bytes to include in each checksum
+        /// </summary>
+        public const int DEFAULT_BLOCK_SIZE = 2048;
+        /// <summary>
+        /// The default number of bytes to use from the strong hash
+        /// </summary>
+        public const int DEFAULT_STRONG_LEN = 8;
+        /// <summary>
+        /// The default number of bytes generated per input block
+        /// </summary>
+        public const int DEFAULT_BYTES_PER_BLOCK = DEFAULT_STRONG_LEN + 4;
+        /// <summary>
+        /// The number of bytes used for the rdiff header
+        /// </summary>
+        public static readonly int HEADER_SIZE = RDiffBinary.SIGNATURE_MAGIC.Length + 4 + 4;
+
         /// <summary>
         /// The length of a datablock
         /// </summary>
@@ -31,7 +48,7 @@ namespace Duplicati.Library.SharpRSync
         /// </summary>
         /// <param name="outputstream">The stream into which the checksum data is written</param>
         public ChecksumFileWriter(System.IO.Stream outputstream)
-            : this(outputstream, Adler32Checksum.DEFAULT_BLOCK_SIZE, 8)
+            : this(outputstream, DEFAULT_BLOCK_SIZE, DEFAULT_STRONG_LEN)
         {
         }
 
@@ -112,5 +129,20 @@ namespace Duplicati.Library.SharpRSync
         /// Gets the number of bytes in a signature file
         /// </summary>
         public int StrongLength { get { return m_stronglen; } }
+        /// <summary>
+        /// Gets the number of bytes generated per input block
+        /// </summary>
+        public int BytesPrBlock { get { return m_stronglen + 4; } }
+
+        /// <summary>
+        /// Returns the number of bytes generated when processing the specified amount of bytes
+        /// </summary>
+        /// <param name="filesize">The size of the file to process</param>
+        /// <returns>The expected size of the signature file</returns>
+        public int BytesGeneratedForSignature(long filesize)
+        {
+            return (int)(SharpRSync.ChecksumFileWriter.HEADER_SIZE +
+                (((filesize + m_blocklen - 1) / m_blocklen) * (m_stronglen + 4)));
+        }
     }
 }

Duplicati/Library/SharpRSync/ChecksumGeneratingStream.cs

@@ -138,5 +138,15 @@ namespace Duplicati.Library.SharpRSync
 
             base.Dispose(disposing);
         }
+
+        /// <summary>
+        /// Returns the number of bytes generated when processing the specified amount of bytes
+        /// </summary>
+        /// <param name="filesize">The size of the file to process</param>
+        /// <returns>The expected size of the signature file</returns>
+        public int BytesGeneratedForSignature(long filesize)
+        {
+            return m_outstream.BytesGeneratedForSignature(filesize);
+        }
     }
 }