diff --git a/Duplicati/SharpRSync/Adler32Checksum.cs b/Duplicati/SharpRSync/Adler32Checksum.cs
index a295a1124..1a00d15ac 100644
--- a/Duplicati/SharpRSync/Adler32Checksum.cs
+++ b/Duplicati/SharpRSync/Adler32Checksum.cs
@@ -44,6 +44,11 @@ namespace Duplicati.SharpRSync
private long m_count;
private long m_blockSize;
+ ///
+ /// Constructs a new adler32 instance, reading from a rolling buffer
+ ///
+ /// The buffer to read from
+ /// The size of a single block
public Adler32Checksum(RollingBuffer buffer, int blocksize)
{
m_buffer = buffer;
@@ -52,7 +57,12 @@ namespace Duplicati.SharpRSync
m_blockSize = blocksize;
Reset();
}
-
+
+ ///
+ /// Constructs a new adler32 instance, reading from a stream
+ ///
+ /// The buffer to read from
+ /// The size of a single block
public Adler32Checksum(System.IO.Stream source, int blocksize)
{
m_source = source;
@@ -62,21 +72,40 @@ namespace Duplicati.SharpRSync
Reset();
}
+ ///
+ /// Constructs a new adler32 instance, reading from a stream
+ ///
+ /// The buffer to read from
public Adler32Checksum(System.IO.Stream source)
: this(source, DEFAULT_BLOCK_SIZE)
{
}
+ ///
+ /// Gets a value indicating if the last data has been read from the source
+ ///
public bool Done { get { return m_done; } }
+
+ ///
+ /// Gets the current checksum
+ ///
public uint Checksum { get { return (uint)((m_s1 & 0xffff) + (m_s2 << 16)); } }
- public void Reset()
+ ///
+ /// Resets the checksum and calculates the checksum for a new block
+ ///
+ /// False if there were no new data
+ public bool Reset()
{
int i;
byte[] buf = new byte[4];
int a;
+ if (m_done)
+ return false;
+
m_s1 = m_s2 = 0;
+ m_count = 0;
for (i = 0; i < m_blockSize - (m_blockSize % 4); i += 4)
{
@@ -85,7 +114,7 @@ namespace Duplicati.SharpRSync
else
{
a = m_buffer.Advance(4);
- m_buffer.GetHead(buf, 0, 4);
+ m_buffer.GetHead(buf, 0, a);
}
if (a == 4)
@@ -106,18 +135,45 @@ namespace Duplicati.SharpRSync
UpdateChecksum(buf[j]);
m_count += a;
m_done = true;
- return;
+ return m_count != 0;
}
}
+ if (m_source != null)
+ a = Utility.ForceStreamRead(m_source, buf, (int)m_blockSize % 4);
+ else
+ {
+ a = m_buffer.Advance((int)m_blockSize % 4);
+ m_buffer.GetHead(buf, 0, a);
+ }
- a = Utility.ForceStreamRead(m_source, buf, (int)m_blockSize % 4);
for (i = 0; i < a; i++)
UpdateChecksum(buf[i]);
m_count += a;
-
+ return true;
}
+ ///
+ /// Rolls the checksum a number of bytes
+ ///
+ /// The number of bytes to roll
+ /// True if there is new data, false otherwise
+ public bool AdvanceChecksum(int count)
+ {
+ if (m_done)
+ return false;
+
+ for (int i = 0; i < count; i++)
+ if (!AdvanceChecksum())
+ break;
+
+ return true;
+ }
+
+ ///
+ /// Rolls the checksum a single byte
+ ///
+ /// True if a new byte was checksummed, false if there was no more data
public bool AdvanceChecksum()
{
if (m_done)
@@ -125,7 +181,14 @@ namespace Duplicati.SharpRSync
int b;
if (m_source != null)
+ {
b = m_source.ReadByte();
+ if (b == -1)
+ {
+ m_done = true;
+ return false;
+ }
+ }
else
{
if (m_buffer.Advance(1) != 1)
@@ -133,19 +196,19 @@ namespace Duplicati.SharpRSync
m_done = true;
return false;
}
- b = m_buffer.GetHead(1)[0];
+ b = m_buffer.GetByteAt(m_buffer.Count - 1);
}
- if (b == -1)
- {
- m_done = true;
- return false;
- }
+ m_count += 1;
UpdateChecksum((byte)b);
return true;
}
+ ///
+ /// Updates the checksum with the read byte
+ ///
+ /// The byte read
private void UpdateChecksum(byte value)
{
m_s1 += value + CHAR_OFFSET;
@@ -155,5 +218,51 @@ namespace Duplicati.SharpRSync
m_s2 %= ushort.MaxValue + 1;
}
+
+ ///
+ /// Rolls the buffer, so the checksum now fits a block that is shifted one byte from the previous one.
+ ///
+ /// True if new data was included, false otherwise
+ public bool Rollbuffer()
+ {
+ if (m_buffer == null)
+ throw new Exception("Cannot roll buffer, when the source is not a rolling buffer instance");
+
+ if (m_done)
+ return false;
+
+ if (m_buffer.Advance(1) != 1)
+ {
+ m_done = true;
+ return false;
+ }
+
+ byte out_byte = m_buffer.GetByteAt(m_buffer.Count - m_count - 1);
+ byte in_byte = m_buffer.GetByteAt(m_buffer.Count - 1);
+
+ //TODO: This can be done much nicer, but the .Net overflow detection
+ //prevents it.
+
+ /*
+ * m_s1 += in - out;
+ * m_s2 += m_s1 - (m_count *(out + CHAR_OFFSET));
+ */
+
+ int diff = in_byte - out_byte;
+ if (diff < 0)
+ diff += ushort.MaxValue + 1;
+
+ m_s1 = (uint)((diff + m_s1) % (ushort.MaxValue + 1));
+
+ diff = (int)((m_count * (out_byte + CHAR_OFFSET)) % (ushort.MaxValue + 1));
+ diff = (int)m_s1 - diff;
+ if (diff < 0)
+ diff += ushort.MaxValue + 1;
+
+ m_s2 = (uint)((m_s2 + diff) % (ushort.MaxValue + 1));
+
+ return true;
+ }
+
}
}
diff --git a/Duplicati/SharpRSync/DeltaFile.cs b/Duplicati/SharpRSync/DeltaFile.cs
index 1d8b4e7c5..91e83c668 100644
--- a/Duplicati/SharpRSync/DeltaFile.cs
+++ b/Duplicati/SharpRSync/DeltaFile.cs
@@ -79,7 +79,7 @@ namespace Duplicati.SharpRSync
if (command < 0)
throw new Exception("Stream ended but had no end marker");
- if (Enum.IsDefined(typeof(RDiffBinary.LiteralDeltaCommand), command))
+ if (Enum.IsDefined(typeof(RDiffBinary.LiteralDeltaCommand), (RDiffBinary.LiteralDeltaCommand)command))
{
//Find out how many bytes of literal data there is
int len = RDiffBinary.GetLiteralLength((RDiffBinary.LiteralDeltaCommand)command);
@@ -93,7 +93,7 @@ namespace Duplicati.SharpRSync
//Copy the literal data from the patch to the output
Utility.StreamCopy(m_inputStream, output, size);
}
- else if (Enum.IsDefined(typeof(RDiffBinary.CopyDeltaCommand), command))
+ else if (Enum.IsDefined(typeof(RDiffBinary.CopyDeltaCommand), (RDiffBinary.CopyDeltaCommand)command))
{
//Find the offset of the data in the base file
int len = RDiffBinary.GetCopyOffsetSize((RDiffBinary.CopyDeltaCommand)command);
@@ -144,14 +144,17 @@ namespace Duplicati.SharpRSync
byte[] md4buffer = new byte[m_checksum.BlockLength];
- int unmatched = 0;
+ long unmatched = 0;
+ long matched = 0;
+ long matched_offset = 0;
do
{
+ bool foundMatch = false;
List> strong = m_checksum.FindChunk(adler.Checksum);
//No weak matches :(
- if (strong != null && strong.Count == 0)
+ if (strong == null || strong.Count == 0)
unmatched++;
else
{
@@ -186,36 +189,58 @@ namespace Duplicati.SharpRSync
if (unmatched > 0)
{
WriteLiteral(buffer.GetTail(unmatched), output);
+ buffer.DropTail(unmatched);
unmatched = 0;
}
+ if (matched == 0)
+ matched_offset = k.Key * m_checksum.BlockLength;
- //Send the matching bytes as a copy
- WriteCopy(k.Key * m_checksum.BlockLength, m_checksum.BlockLength, output);
- adler.Reset();
+ matched += buffer.Count;
+ foundMatch = true;
buffer.DropTail(buffer.Count);
break;
}
}
-
- //Avoid keeping too large blocks in memory.
- //This gives an overhead of app. 3 bytes pr. 2Kb data
- if (unmatched > m_checksum.BlockLength)
- {
- WriteLiteral(buffer.GetTail(unmatched), output);
- buffer.DropTail(unmatched);
- unmatched = 0;
- adler.Reset();
- }
}
- } while (adler.AdvanceChecksum());
+
+ //If this byte was not matched, and we have queued up matches,
+ // flush them now
+ if (!foundMatch && matched > 0)
+ {
+ //Send the matching bytes as a copy
+ WriteCopy(matched_offset, matched, output);
+ matched = 0;
+ matched_offset = 0;
+ }
+
+ //Avoid keeping too large blocks in memory.
+ //This gives an overhead of app. 3 bytes pr. 2Kb data
+ if (unmatched >= m_checksum.BlockLength + 1)
+ {
+ WriteLiteral(buffer.GetTail(unmatched - 1), output);
+ buffer.DropTail(unmatched - 1);
+ unmatched = 1;
+ }
+
+ } while ( buffer.Count == 0 ? adler.Reset() : adler.Rollbuffer());
+
+ //If it is still in the buffer, it was not matched
+ unmatched = buffer.Count;
+
+ if (matched > 0 && unmatched > 0)
+ throw new Exception("Internal error, had buffered both matched and unmatched blocks!");
+
+ if (matched > 0)
+ {
+ //Send the matching bytes as a copy
+ WriteCopy(matched_offset, matched, output);
+ }
//Any trailing bytes are treated as a literal
if (unmatched > 0)
{
WriteLiteral(buffer.GetTail(unmatched), output);
buffer.DropTail(unmatched);
- unmatched = 0;
- adler.Reset();
}
output.WriteByte((byte)RDiffBinary.EndCommand);
@@ -242,7 +267,7 @@ namespace Duplicati.SharpRSync
/// The offset in the basefile where the data is located
/// The length of the data to copy
/// The output delta stream
- private void WriteCopy(long offset, int length, System.IO.Stream output)
+ private void WriteCopy(long offset, long length, System.IO.Stream output)
{
output.WriteByte((byte)RDiffBinary.FindCopyDeltaCommand(offset, length));
byte[] len = RDiffBinary.EncodeLength(offset);
diff --git a/Duplicati/SharpRSync/MD4.cs b/Duplicati/SharpRSync/MD4.cs
index d6f87c4e7..3b7d24203 100644
--- a/Duplicati/SharpRSync/MD4.cs
+++ b/Duplicati/SharpRSync/MD4.cs
@@ -1,22 +1,3 @@
-#region Disclaimer / License
-// Copyright (C) 2008, Kenneth Skovhede
-// http://www.hexad.dk, opensource@hexad.dk
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public
-// License along with this library; if not, write to the Free Software
-// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-//
-#endregion
//
// MD4.cs - Message Digest 4 Abstract class
//
diff --git a/Duplicati/SharpRSync/RDiffBinary.cs b/Duplicati/SharpRSync/RDiffBinary.cs
index a6775242b..f69c28aec 100644
--- a/Duplicati/SharpRSync/RDiffBinary.cs
+++ b/Duplicati/SharpRSync/RDiffBinary.cs
@@ -72,13 +72,12 @@ namespace Duplicati.SharpRSync
/// The number of bytes required
public static int FindLength(long size)
{
- //These limits should be unsigned, but they are read as signed in RDiff?
int count = 1;
if (size > byte.MaxValue)
count++;
- if (size > short.MaxValue)
+ if (size > ushort.MaxValue)
count += 2;
- if (size > int.MaxValue)
+ if (size > uint.MaxValue)
count += 4;
if (size > long.MaxValue)
throw new Exception("Value must be less than " + long.MaxValue);
@@ -95,11 +94,16 @@ namespace Duplicati.SharpRSync
if (data.Length == 1)
return (long)data[0];
else if (data.Length == 2)
- return (long)BitConverter.ToInt16(FixEndian(data), 0);
+ return (long)BitConverter.ToUInt16(FixEndian(data), 0);
else if (data.Length == 4)
- return (long)BitConverter.ToInt32(FixEndian(data), 0);
+ return (long)BitConverter.ToUInt32(FixEndian(data), 0);
else if (data.Length == 8)
- return BitConverter.ToInt64(FixEndian(data), 0);
+ {
+ long tmp = BitConverter.ToInt64(FixEndian(data), 0);
+ if (tmp < 0)
+ throw new Exception("Unable to size item, because it is larger than " + long.MaxValue.ToString());
+ return tmp;
+ }
else
throw new Exception("Invalid data length");
}
@@ -111,11 +115,11 @@ namespace Duplicati.SharpRSync
///
/// The value to write
/// The written bytes
- public static byte[] EncodeLength(int size)
+ public static byte[] EncodeLength(long size)
{
int len = FindLength(size);
if (len == 1)
- return FixEndian(BitConverter.GetBytes((byte)size));
+ return new byte[] { (byte)size };
else if (len == 2)
return FixEndian(BitConverter.GetBytes((short)size));
else if (len == 4)
@@ -132,11 +136,12 @@ namespace Duplicati.SharpRSync
/// The offset in the file where the copy begins
/// The size of the copy
/// The delta copy command
- public static CopyDeltaCommand FindCopyDeltaCommand(int offset, int size)
+ public static CopyDeltaCommand FindCopyDeltaCommand(long offset, long size)
{
int i1 = FindLength(offset);
int i2 = FindLength(size);
- return CopyCommand[i1][i2];
+ //TODO: Not good, long = 8
+ return CopyCommand[i2 - 1][i1 - 1];
}
///
diff --git a/Duplicati/SharpRSync/RollingBuffer.cs b/Duplicati/SharpRSync/RollingBuffer.cs
index 765f80468..1ac576d53 100644
--- a/Duplicati/SharpRSync/RollingBuffer.cs
+++ b/Duplicati/SharpRSync/RollingBuffer.cs
@@ -38,11 +38,12 @@ namespace Duplicati.SharpRSync
{
m_stream = stream;
m_buffers = new List();
+ m_buffers.Add(new byte[BUFFER_SIZE]);
m_headIndex = 0;
m_tailIndex = 0;
}
- public int Count { get { return m_buffers.Count * BUFFER_SIZE - m_tailIndex - (BUFFER_SIZE - m_headIndex); } }
+ public long Count { get { return m_buffers.Count * BUFFER_SIZE - m_tailIndex - (BUFFER_SIZE - m_headIndex); } }
///
/// Reads the supplied amount of bytes
@@ -76,7 +77,7 @@ namespace Duplicati.SharpRSync
///
/// The number of bytes to return
/// The bytes requested
- public byte[] GetHead(int count)
+ public byte[] GetHead(long count)
{
byte[] tmp = new byte[count];
GetHead(tmp, 0, count);
@@ -108,10 +109,10 @@ namespace Duplicati.SharpRSync
/// The buffer to write the bytes into
/// The offset into the buffer
/// The bytes requested
- public void GetHead(byte[] buf, int offset, int count)
+ public void GetHead(byte[] buf, long offset, long count)
{
int ix = 0;
- int total = this.Count;
+ long total = this.Count;
if (total < count)
throw new Exception("Buffer has too few bytes");
@@ -120,21 +121,23 @@ namespace Duplicati.SharpRSync
while (total > count)
{
int availible = GetAvalible(ix);
- int diff = Math.Min(availible, total - count);
+ int diff = (int)Math.Min(availible, total - count);
total -= diff;
- availible -= diff;
b += diff;
- if (availible == 0)
+ if (b >= BUFFER_SIZE)
{
ix++;
b = 0;
}
}
+ if (ix == 0)
+ b -= m_tailIndex;
+
while (count > 0)
{
- int avalible = Math.Min(GetAvalible(ix), count);
+ int avalible = (int)Math.Min(GetAvalible(ix) - b, count);
Array.Copy(m_buffers[ix], b, buf, offset, avalible);
count -= avalible;
@@ -149,9 +152,9 @@ namespace Duplicati.SharpRSync
///
/// The number of bytes to return
/// The bytes requested
- public byte[] GetTail(int count)
+ public byte[] GetTail(long count)
{
- byte[] tmp = new byte[BUFFER_SIZE];
+ byte[] tmp = new byte[count];
GetTail(tmp, 0, count);
return tmp;
}
@@ -162,7 +165,7 @@ namespace Duplicati.SharpRSync
/// The number of bytes to return
/// The buffer to write the bytes into
/// The offset into the buffer
- public void GetTail(byte[] buf, int offset, int count)
+ public void GetTail(byte[] buf, long offset, long count)
{
int ix = 0;
if (this.Count < count)
@@ -171,7 +174,7 @@ namespace Duplicati.SharpRSync
int b = m_tailIndex;
while (count > 0)
{
- int avalible = Math.Min(GetAvalible(ix), count);
+ int avalible = (int)Math.Min(GetAvalible(ix), count);
Array.Copy(m_buffers[ix], b, buf, offset, avalible);
count -= avalible;
@@ -185,11 +188,11 @@ namespace Duplicati.SharpRSync
/// Removes the bytes from memory
///
/// The number of bytes to drop
- public void DropTail(int count)
+ public void DropTail(long count)
{
if (count < (BUFFER_SIZE - m_tailIndex))
{
- m_tailIndex += count;
+ m_tailIndex += (int)count;
return;
}
@@ -197,13 +200,42 @@ namespace Duplicati.SharpRSync
count -= (BUFFER_SIZE - m_tailIndex);
m_tailIndex = 0;
- while (count > BUFFER_SIZE)
+ while (count >= BUFFER_SIZE)
{
m_buffers.RemoveAt(0);
count -= BUFFER_SIZE;
}
- m_tailIndex += count;
+ m_tailIndex += (int)count;
+ if (m_buffers.Count == 0)
+ {
+ m_headIndex = 0;
+ m_buffers.Add(new byte[BUFFER_SIZE]);
+ }
+ }
+
+ ///
+ /// Returns the byte at the given position.
+ ///
+ /// The index of the byte
+ /// The byte value
+ public byte GetByteAt(long index)
+ {
+ int ix = 0;
+ if (index > this.Count - 1)
+ throw new Exception("Buffer has too few bytes");
+ if (index < 0)
+ throw new Exception("Index cannot be negative");
+
+ while (GetAvalible(ix) <= index)
+ {
+ index -= GetAvalible(ix);
+ ix++;
+ }
+
+ long offset = ix == 0 ? m_tailIndex : 0;
+ offset += index;
+ return m_buffers[ix][offset];
}
}
}