kenneth.skovhede@gmail.com
a02cdfc6cd
Status: Fixed After a lot of work, I have now re-written the entire SharpRSync library. My tests indicate that generating a diff file for an 8gb file took 10 minutes before, and now takes 8 minutes. A delta for the same (unchanged) 8GB file took 58 minutes, and now takes 7 minutes. The implementation seems to run faster under 64 bit, but I'm unsure if it is related to the actual instructions issued, or the address simulation/conversion done by the OS. I have also greatly optimized the amount of memory required by the program. On windows, a delta for an 8GB file can be now be created with a total of 125 mb required memory. The base app takes 65mb, so the used memory is around 60mb, and the raw diff file is approximately 44mb. I compared it a bit to the rdiff program on linux. For a 150mb file, rdiff takes about 1 second to generate a diff, and the same for a delta. SharpRSync runs the same in about 7 seconds, although some of that is the mono startup and library loading. For a 8gb file rdiff can generate a diff in about 500 seconds, where SharpRSync takes about 590 seconds. When generating a delta, rdiff uses 530 seconds, and SharpRSync uses 770 seconds. The last measurement is inaccurate as the memory requirement under mono is greater, and the test machine had low memory, so there was a lot of swapping going on. An interesting side effect is that the rdiff generated delta file is about 380kb, where the SharpRSync delta is 15 bytes. This is because rdiff does not do the sequence matching optimization mentioned on the librsync page (last paragraph): http://rsync.samba.org/tech_report/node4.html Further optimization should probably focus on getting the MD4 algorithm to run faster. git-svn-id: https://duplicati.googlecode.com/svn/trunk@480 59da171f-624f-0410-aa54-27559c288bec
360 lines
16 KiB
C#
360 lines
16 KiB
C#
#region Disclaimer / License
|
|
// Copyright (C) 2010, Kenneth Skovhede
|
|
// http://www.hexad.dk, opensource@hexad.dk
|
|
//
|
|
// This library is free software; you can redistribute it and/or
|
|
// modify it under the terms of the GNU Lesser General Public
|
|
// License as published by the Free Software Foundation; either
|
|
// version 2.1 of the License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
// Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this library; if not, write to the Free Software
|
|
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
//
|
|
#endregion
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Text;
|
|
|
|
namespace Duplicati.Library.SharpRSync
|
|
{
|
|
/// <summary>
|
|
/// This class contains operations on a RDiff compatible delta file
|
|
/// </summary>
|
|
public class DeltaFile
|
|
{
|
|
/// <summary>
|
|
/// The size of the internal buffer used to read in data
|
|
/// </summary>
|
|
private const int BUFFER_SIZE = 100 * 1024;
|
|
|
|
/// <summary>
|
|
/// The ChecksumFileReader used to perform signature lookups
|
|
/// </summary>
|
|
ChecksumFileReader m_checksum;
|
|
|
|
/// <summary>
|
|
/// The possibly modified input data
|
|
/// </summary>
|
|
System.IO.Stream m_inputStream;
|
|
|
|
/// <summary>
|
|
/// Constructs a new DeltaFile based on a signature file.
|
|
/// This instance can be used to create a new DeltaFile
|
|
/// </summary>
|
|
/// <param name="checksum">The checksum to use</param>
|
|
public DeltaFile(ChecksumFileReader checksum)
|
|
{
|
|
m_checksum = checksum;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Constructs a new DeltaFile by reading the stream.
|
|
/// This instance can be used to patch a file.
|
|
/// </summary>
|
|
/// <param name="inputStream">The stream containing the delta information</param>
|
|
public DeltaFile(System.IO.Stream inputStream)
|
|
{
|
|
m_inputStream = inputStream;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a new file based on the basefile and the delta information.
|
|
/// The basefile and output stream cannot point to the same resource (ea. file).
|
|
/// The base file MUST be seekable.
|
|
/// </summary>
|
|
/// <param name="basefile">A seekable stream with the baseinformation</param>
|
|
/// <param name="output">The stream to write the patched data to. Must not point to the same resource as the basefile.</param>
|
|
public void PatchFile(System.IO.Stream basefile, System.IO.Stream output)
|
|
{
|
|
//Validate the file header
|
|
byte[] sig = new byte[4];
|
|
if (Utility.ForceStreamRead(m_inputStream, sig, 4) != 4)
|
|
throw new Exception(Strings.DeltaFile.EndofstreamBeforeSignatureError);
|
|
for (int i = 0; i < sig.Length; i++)
|
|
if (RDiffBinary.DELTA_MAGIC[i] != sig[i])
|
|
throw new Exception(Strings.DeltaFile.InvalidSignatureError);
|
|
|
|
//Keep reading until we hit the end command
|
|
while (true)
|
|
{
|
|
int command = m_inputStream.ReadByte();
|
|
if (command == RDiffBinary.EndCommand)
|
|
break;
|
|
|
|
//It is an error to omit the end command
|
|
if (command < 0)
|
|
throw new Exception(Strings.DeltaFile.EndofstreamWithoutMarkerError);
|
|
|
|
if (Enum.IsDefined(typeof(RDiffBinary.LiteralDeltaCommand), (RDiffBinary.LiteralDeltaCommand)command))
|
|
{
|
|
//Find out how many bytes of literal data there is
|
|
int len = RDiffBinary.GetLiteralLength((RDiffBinary.LiteralDeltaCommand)command);
|
|
byte[] tmp = new byte[len];
|
|
if (Utility.ForceStreamRead(m_inputStream, tmp, tmp.Length) != tmp.Length)
|
|
throw new Exception(Strings.DeltaFile.UnexpectedEndofstreamError);
|
|
long size = RDiffBinary.DecodeLength(tmp);
|
|
if (size < 0)
|
|
throw new Exception(Strings.DeltaFile.InvalidLitteralSizeError);
|
|
|
|
//Copy the literal data from the patch to the output
|
|
Utility.StreamCopy(m_inputStream, output, size);
|
|
}
|
|
else if (Enum.IsDefined(typeof(RDiffBinary.CopyDeltaCommand), (RDiffBinary.CopyDeltaCommand)command))
|
|
{
|
|
//Find the offset of the data in the base file
|
|
int len = RDiffBinary.GetCopyOffsetSize((RDiffBinary.CopyDeltaCommand)command);
|
|
byte[] tmp = new byte[len];
|
|
if (Utility.ForceStreamRead(m_inputStream, tmp, tmp.Length) != tmp.Length)
|
|
throw new Exception(Strings.DeltaFile.UnexpectedEndofstreamError);
|
|
long offset = RDiffBinary.DecodeLength(tmp);
|
|
if (offset < 0)
|
|
throw new Exception(Strings.DeltaFile.InvalidCopyOffsetError);
|
|
|
|
//Find the length of the data to copy from the basefile
|
|
len = RDiffBinary.GetCopyLengthSize((RDiffBinary.CopyDeltaCommand)command);
|
|
tmp = new byte[len];
|
|
if (Utility.ForceStreamRead(m_inputStream, tmp, tmp.Length) != tmp.Length)
|
|
throw new Exception(Strings.DeltaFile.UnexpectedEndofstreamError);
|
|
long length = RDiffBinary.DecodeLength(tmp);
|
|
if (length < 0)
|
|
throw new Exception(Strings.DeltaFile.InvalidCopyLengthError);
|
|
|
|
//Seek to the begining, and copy
|
|
basefile.Position = offset;
|
|
Utility.StreamCopy(basefile, output, length);
|
|
}
|
|
else if (command <= RDiffBinary.LiteralLimit)
|
|
{
|
|
//Literal data less than 64 bytes are found, copy it
|
|
Utility.StreamCopy(m_inputStream, output, command);
|
|
}
|
|
else
|
|
throw new Exception(Strings.DeltaFile.UnknownCommandError);
|
|
}
|
|
|
|
output.Flush();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Generates a delta file from input, and writes it to output
|
|
/// </summary>
|
|
/// <param name="input">The stream to generate the delta from</param>
|
|
/// <param name="output">The stream to write the delta to</param>
|
|
public void GenerateDeltaFile(System.IO.Stream input, System.IO.Stream output)
|
|
{
|
|
output.Write(RDiffBinary.DELTA_MAGIC, 0, RDiffBinary.DELTA_MAGIC.Length);
|
|
|
|
int blocklength = m_checksum.BlockLength;
|
|
int buffersize = BUFFER_SIZE;
|
|
|
|
//If we have some insanely large blocks, try to handle them nicely anyway
|
|
if (blocklength > BUFFER_SIZE / 2)
|
|
buffersize = blocklength * 4;
|
|
|
|
//The number of matched bytes
|
|
long matched = 0;
|
|
//The first matched byte
|
|
long matched_offset = 0;
|
|
//The index of the next expected block
|
|
long next_match_key = 0;
|
|
|
|
//The number of unmatched bytes
|
|
int unmatched = 0;
|
|
//The index of the first unmatched byte
|
|
int unmatched_offset = 0;
|
|
|
|
//Keep a local copy of the lookup table
|
|
bool[] weakLookup = m_checksum.WeakLookup;
|
|
|
|
//We use statically allocated buffers, and we need two buffers
|
|
// to prevent Array.Copy from allocating a temp buffer
|
|
byte[] working_data = new byte[BUFFER_SIZE];
|
|
byte[] temp_work = new byte[BUFFER_SIZE];
|
|
byte[] md4buf = new byte[blocklength];
|
|
|
|
//Read the initial buffer block
|
|
int buffer_len = Utility.ForceStreamRead(input, working_data);
|
|
int buffer_index = 0;
|
|
blocklength = Math.Min(blocklength, buffer_len);
|
|
|
|
//Setup the initial checksum
|
|
uint weakChecksum = Adler32Checksum.Calculate(working_data, 0, blocklength);
|
|
|
|
long indexMatched;
|
|
bool force_buffer_refill = false;
|
|
|
|
while (blocklength > 0)
|
|
{
|
|
//Check if the block matches somewhere
|
|
indexMatched = m_checksum.LookupChunck(weakChecksum, working_data, buffer_index, blocklength, next_match_key);
|
|
|
|
if (indexMatched >= 0)
|
|
{
|
|
//We have a match, flush unmatched
|
|
if (unmatched > 0)
|
|
{
|
|
WriteLiteral(working_data, unmatched_offset, unmatched, output);
|
|
unmatched = 0;
|
|
}
|
|
|
|
//First match
|
|
if (matched == 0)
|
|
{
|
|
matched_offset = indexMatched * blocklength;
|
|
}
|
|
else if (indexMatched != next_match_key)
|
|
{
|
|
//Subsequent match, but the sequence does not fit
|
|
WriteCopy(matched_offset, matched, output);
|
|
|
|
//Pretend this was the fist
|
|
matched = 0;
|
|
matched_offset = indexMatched * blocklength;
|
|
}
|
|
|
|
//If the next block matches this signature, we can write larger
|
|
// copy instructions and thus safe space
|
|
next_match_key = indexMatched + 1;
|
|
|
|
//Adjust the counters
|
|
matched += blocklength;
|
|
buffer_index += blocklength;
|
|
blocklength = Math.Min(blocklength, buffer_len - buffer_index);
|
|
|
|
//Reset the checksum to fit the new block
|
|
weakChecksum = Adler32Checksum.Calculate(working_data, buffer_index, blocklength);
|
|
}
|
|
else
|
|
{
|
|
//No match, flush accumulated matches, if any
|
|
if (matched > 0)
|
|
{
|
|
//Send the matching bytes as a copy
|
|
WriteCopy(matched_offset, matched, output);
|
|
matched = 0;
|
|
matched_offset = 0;
|
|
|
|
//We do not immediately start tapping the unmatched bytes,
|
|
// because the buffer may be nearly empty, and we
|
|
// want to gather as many unmatched bytes as possible
|
|
// to avoid the instruction overhead in the file
|
|
force_buffer_refill = true;
|
|
}
|
|
else
|
|
{
|
|
int lastPossible = buffer_len - blocklength;
|
|
if (unmatched == 0)
|
|
unmatched_offset = buffer_index;
|
|
|
|
//Local speedup for long non-matching regions
|
|
for (/* buffer_index = buffer_index */; buffer_index < lastPossible; buffer_index++)
|
|
{
|
|
weakChecksum = Adler32Checksum.Roll(working_data[buffer_index], working_data[buffer_index + blocklength], weakChecksum, blocklength);
|
|
if (weakLookup[weakChecksum >> 16])
|
|
break;
|
|
}
|
|
|
|
unmatched = buffer_index - unmatched_offset;
|
|
|
|
//If this is the last block, claim the remaining bytes as unmatched
|
|
if (temp_work == null)
|
|
{
|
|
unmatched += blocklength;
|
|
blocklength = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
//If we are out of buffer, try to load some more
|
|
if (force_buffer_refill || buffer_len - buffer_index <= m_checksum.BlockLength)
|
|
{
|
|
//Prevent continous refill
|
|
force_buffer_refill = false;
|
|
|
|
//If we have read the last bytes already, then just skip this
|
|
if (temp_work != null)
|
|
{
|
|
int remaining_bytes = buffer_len - buffer_index;
|
|
Array.Copy(working_data, buffer_index, temp_work, 0, remaining_bytes);
|
|
|
|
int tempread = Utility.ForceStreamRead(input, temp_work, remaining_bytes, temp_work.Length - remaining_bytes);
|
|
|
|
if (tempread > 0)
|
|
{
|
|
//We are about to discard some data, if it is unmatched, write it to stream
|
|
if (unmatched > 0)
|
|
{
|
|
WriteLiteral(working_data, unmatched_offset, unmatched, output);
|
|
unmatched = 0;
|
|
}
|
|
|
|
//Now swap the arrays
|
|
byte[] tmp = working_data;
|
|
working_data = temp_work;
|
|
temp_work = tmp;
|
|
|
|
buffer_index = 0;
|
|
buffer_len = remaining_bytes + tempread;
|
|
}
|
|
else
|
|
{
|
|
//Mark as done
|
|
temp_work = null;
|
|
|
|
//The last round has a smaller block length
|
|
blocklength = remaining_bytes;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//There cannot be both matched and unmatched bytes written
|
|
if (matched > 0 && unmatched > 0)
|
|
throw new Exception(Strings.DeltaFile.InternalBufferError);
|
|
|
|
if (matched > 0)
|
|
WriteCopy(matched_offset, matched, output);
|
|
|
|
if (unmatched > 0)
|
|
WriteLiteral(working_data, unmatched_offset, unmatched, output);
|
|
|
|
output.WriteByte((byte)RDiffBinary.EndCommand);
|
|
output.Flush();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Writes a literal command to a delta stream
|
|
/// </summary>
|
|
/// <param name="data">The literal data to write</param>
|
|
/// <param name="output">The output delta stream</param>
|
|
private void WriteLiteral(byte[] data, int offset, int count, System.IO.Stream output)
|
|
{
|
|
output.WriteByte((byte)RDiffBinary.FindLiteralDeltaCommand(count));
|
|
byte[] len = RDiffBinary.EncodeLength(count);
|
|
output.Write(len, 0, len.Length);
|
|
output.Write(data, offset, count);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Write a copy command to a delta stream
|
|
/// </summary>
|
|
/// <param name="offset">The offset in the basefile where the data is located</param>
|
|
/// <param name="length">The length of the data to copy</param>
|
|
/// <param name="output">The output delta stream</param>
|
|
private void WriteCopy(long offset, long length, System.IO.Stream output)
|
|
{
|
|
output.WriteByte((byte)RDiffBinary.FindCopyDeltaCommand(offset, length));
|
|
byte[] len = RDiffBinary.EncodeLength(offset);
|
|
output.Write(len, 0, len.Length);
|
|
len = RDiffBinary.EncodeLength(length);
|
|
output.Write(len, 0, len.Length);
|
|
}
|
|
}
|
|
}
|