Status: Fixed After a lot of work, I have now re-written the entire SharpRSync library. My tests indicate that generating a diff file for an 8gb file took 10 minutes before, and now takes 8 minutes. A delta for the same (unchanged) 8GB file took 58 minutes, and now takes 7 minutes. The implementation seems to run faster under 64 bit, but I'm unsure if it is related to the actual instructions issued, or the address simulation/conversion done by the OS. I have also greatly optimized the amount of memory required by the program. On windows, a delta for an 8GB file can be now be created with a total of 125 mb required memory. The base app takes 65mb, so the used memory is around 60mb, and the raw diff file is approximately 44mb. I compared it a bit to the rdiff program on linux. For a 150mb file, rdiff takes about 1 second to generate a diff, and the same for a delta. SharpRSync runs the same in about 7 seconds, although some of that is the mono startup and library loading. For a 8gb file rdiff can generate a diff in about 500 seconds, where SharpRSync takes about 590 seconds. When generating a delta, rdiff uses 530 seconds, and SharpRSync uses 770 seconds. The last measurement is inaccurate as the memory requirement under mono is greater, and the test machine had low memory, so there was a lot of swapping going on. An interesting side effect is that the rdiff generated delta file is about 380kb, where the SharpRSync delta is 15 bytes. This is because rdiff does not do the sequence matching optimization mentioned on the librsync page (last paragraph): http://rsync.samba.org/tech_report/node4.html Further optimization should probably focus on getting the MD4 algorithm to run faster. git-svn-id: https://duplicati.googlecode.com/svn/trunk@480 59da171f-624f-0410-aa54-27559c288bec
123 lines
4.8 KiB
C#
123 lines
4.8 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Text;
|
|
|
|
namespace Duplicati.Library.SharpRSync
|
|
{
|
|
/// <summary>
|
|
/// This class writes signature files in the same format as RDiff
|
|
/// </summary>
|
|
public class ChecksumFileWriter
|
|
{
|
|
/// <summary>
|
|
/// The length of a datablock
|
|
/// </summary>
|
|
private int m_blocklen;
|
|
/// <summary>
|
|
/// The number of bytes used for storing the strong signature
|
|
/// </summary>
|
|
private int m_stronglen;
|
|
/// <summary>
|
|
/// The re-used MD4 algorithm
|
|
/// </summary>
|
|
private System.Security.Cryptography.HashAlgorithm m_hashAlgorithm;
|
|
/// <summary>
|
|
/// The stream into which the signature data is written
|
|
/// </summary>
|
|
private System.IO.Stream m_outstream;
|
|
|
|
/// <summary>
|
|
/// Constructs a new CheckSum file with default sizes
|
|
/// </summary>
|
|
/// <param name="outputstream">The stream into which the checksum data is written</param>
|
|
public ChecksumFileWriter(System.IO.Stream outputstream)
|
|
: this(outputstream, Adler32Checksum.DEFAULT_BLOCK_SIZE, 8)
|
|
{
|
|
}
|
|
|
|
/// <summary>
|
|
/// Constructs a new CheckSum file
|
|
/// <param name="blocklength">The length of a single block</param>
|
|
/// <param name="stronglength">The number of bytes in the MD4 checksum</param>
|
|
/// <param name="outputstream">The stream into which the checksum data is written</param>
|
|
/// </summary>
|
|
public ChecksumFileWriter(System.IO.Stream outputstream, int blocklength, int stronglength)
|
|
{
|
|
if (outputstream == null)
|
|
throw new ArgumentNullException("outputstream");
|
|
|
|
m_blocklen = blocklength;
|
|
m_stronglen = stronglength;
|
|
m_outstream = outputstream;
|
|
m_hashAlgorithm = System.Security.Cryptography.HashAlgorithm.Create("MD4");
|
|
if (m_hashAlgorithm == null)
|
|
m_hashAlgorithm = Mono.Security.Cryptography.MD4.Create("MD4");
|
|
|
|
m_hashAlgorithm.Initialize();
|
|
|
|
m_outstream.Write(RDiffBinary.SIGNATURE_MAGIC, 0, RDiffBinary.SIGNATURE_MAGIC.Length);
|
|
m_outstream.Write(RDiffBinary.FixEndian(BitConverter.GetBytes(m_blocklen)), 0, 4);
|
|
m_outstream.Write(RDiffBinary.FixEndian(BitConverter.GetBytes(m_stronglen)), 0, 4);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds a chunck of data to checksum list
|
|
/// </summary>
|
|
/// <param name="buffer">The data to add a checksum entry for</param>
|
|
public void AddChunk(byte[] buffer)
|
|
{
|
|
AddChunk(buffer, 0, buffer.Length);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds a chunck of data to checksum list
|
|
/// </summary>
|
|
/// <param name="buffer">The data to add a checksum entry for</param>
|
|
/// <param name="index">The index in the buffer to start reading from</param>
|
|
/// <param name="count">The number of bytes to extract from the array</param>
|
|
public void AddChunk(byte[] buffer, int index, int count)
|
|
{
|
|
if (!m_hashAlgorithm.CanReuseTransform)
|
|
{
|
|
m_hashAlgorithm = System.Security.Cryptography.HashAlgorithm.Create("MD4");
|
|
if (m_hashAlgorithm == null)
|
|
m_hashAlgorithm = Mono.Security.Cryptography.MD4.Create("MD4");
|
|
}
|
|
|
|
m_outstream.Write(RDiffBinary.FixEndian(BitConverter.GetBytes(Adler32Checksum.Calculate(buffer, index, count))), 0, 4);
|
|
m_outstream.Write(m_hashAlgorithm.ComputeHash(buffer, index, count), 0, m_stronglen);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds all checksums in an entire stream
|
|
/// </summary>
|
|
/// <param name="stream">The stream to read checksums from</param>
|
|
public void AddStream(System.IO.Stream stream)
|
|
{
|
|
byte[] buffer = new byte[m_blocklen];
|
|
int a;
|
|
|
|
//These streams do not fragment data
|
|
if (stream is System.IO.FileStream || stream is System.IO.MemoryStream)
|
|
{
|
|
while ((a = stream.Read(buffer, 0, buffer.Length)) != 0)
|
|
AddChunk(buffer, 0, a);
|
|
}
|
|
else
|
|
{
|
|
while ((a = Utility.ForceStreamRead(stream, buffer, buffer.Length)) != 0)
|
|
AddChunk(buffer, 0, a);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the number of bytes in each hashed block
|
|
/// </summary>
|
|
public int BlockLength { get { return m_blocklen; } }
|
|
/// <summary>
|
|
/// Gets the number of bytes in a signature file
|
|
/// </summary>
|
|
public int StrongLength { get { return m_stronglen; } }
|
|
}
|
|
}
|