haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Various fixes to SharpRSync. It mostly works, but has some trouble recognizing blocks with arbitrary offsets.

Browse Source 
git-svn-id: https://duplicati.googlecode.com/svn/trunk@41 59da171f-624f-0410-aa54-27559c288bec
This commit is contained in:
kenneth.skovhede@gmail.com
2008-09-26 15:42:51 +00:00
parent aeb8d02a51
commit 03c1e41acb
5 changed files with 230 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/SharpRSync/Adler32Checksum.cs
+121 -12
View File
@@ -44,6 +44,11 @@ namespace Duplicati.SharpRSync
private long m_count;
private long m_blockSize;
/// <summary>
/// Constructs a new adler32 instance, reading from a rolling buffer
/// </summary>
/// <param name="buffer">The buffer to read from</param>
/// <param name="blocksize">The size of a single block</param>
public Adler32Checksum(RollingBuffer buffer, int blocksize)
{
m_buffer = buffer;
@@ -52,7 +57,12 @@ namespace Duplicati.SharpRSync
m_blockSize = blocksize;
Reset();
}
/// <summary>
/// Constructs a new adler32 instance, reading from a stream
/// </summary>
/// <param name="buffer">The buffer to read from</param>
/// <param name="blocksize">The size of a single block</param>
public Adler32Checksum(System.IO.Stream source, int blocksize)
{
m_source = source;
@@ -62,21 +72,40 @@ namespace Duplicati.SharpRSync
Reset();
}
/// <summary>
/// Constructs a new adler32 instance, reading from a stream
/// </summary>
/// <param name="sourc">The buffer to read from</param>
public Adler32Checksum(System.IO.Stream source)
: this(source, DEFAULT_BLOCK_SIZE)
{
}
/// <summary>
/// Gets a value indicating if the last data has been read from the source
/// </summary>
public bool Done { get { return m_done; } }
/// <summary>
/// Gets the current checksum
/// </summary>
public uint Checksum { get { return (uint)((m_s1 & 0xffff) + (m_s2 << 16)); } }
public void Reset()
/// <summary>
/// Resets the checksum and calculates the checksum for a new block
/// </summary>
/// <returns>False if there were no new data</returns>
public bool Reset()
{
int i;
byte[] buf = new byte[4];
int a;
if (m_done)
return false;
m_s1 = m_s2 = 0;
m_count = 0;
for (i = 0; i < m_blockSize - (m_blockSize % 4); i += 4)
{
@@ -85,7 +114,7 @@ namespace Duplicati.SharpRSync
else
{
a = m_buffer.Advance(4);
m_buffer.GetHead(buf, 0, 4);
m_buffer.GetHead(buf, 0, a);
}
if (a == 4)
@@ -106,18 +135,45 @@ namespace Duplicati.SharpRSync
UpdateChecksum(buf[j]);
m_count += a;
m_done = true;
return;
return m_count != 0;
}
}
if (m_source != null)
a = Utility.ForceStreamRead(m_source, buf, (int)m_blockSize % 4);
else
{
a = m_buffer.Advance((int)m_blockSize % 4);
m_buffer.GetHead(buf, 0, a);
}
a = Utility.ForceStreamRead(m_source, buf, (int)m_blockSize % 4);
for (i = 0; i < a; i++)
UpdateChecksum(buf[i]);
m_count += a;
return true;
}
/// <summary>
/// Rolls the checksum a number of bytes
/// </summary>
/// <param name="count">The number of bytes to roll</param>
/// <returns>True if there is new data, false otherwise</returns>
public bool AdvanceChecksum(int count)
{
if (m_done)
return false;
for (int i = 0; i < count; i++)
if (!AdvanceChecksum())
break;
return true;
}
/// <summary>
/// Rolls the checksum a single byte
/// </summary>
/// <returns>True if a new byte was checksummed, false if there was no more data</returns>
public bool AdvanceChecksum()
{
if (m_done)
@@ -125,7 +181,14 @@ namespace Duplicati.SharpRSync
int b;
if (m_source != null)
{
b = m_source.ReadByte();
if (b == -1)
{
m_done = true;
return false;
}
}
else
{
if (m_buffer.Advance(1) != 1)
@@ -133,19 +196,19 @@ namespace Duplicati.SharpRSync
m_done = true;
return false;
}
b = m_buffer.GetHead(1)[0];
b = m_buffer.GetByteAt(m_buffer.Count - 1);
}
if (b == -1)
{
m_done = true;
return false;
}
m_count += 1;
UpdateChecksum((byte)b);
return true;
}
/// <summary>
/// Updates the checksum with the read byte
/// </summary>
/// <param name="value">The byte read</param>
private void UpdateChecksum(byte value)
{
m_s1 += value + CHAR_OFFSET;
@@ -155,5 +218,51 @@ namespace Duplicati.SharpRSync
m_s2 %= ushort.MaxValue + 1;
}
/// <summary>
/// Rolls the buffer, so the checksum now fits a block that is shifted one byte from the previous one.
/// </summary>
/// <returns>True if new data was included, false otherwise</returns>
public bool Rollbuffer()
{
if (m_buffer == null)
throw new Exception("Cannot roll buffer, when the source is not a rolling buffer instance");
if (m_done)
return false;
if (m_buffer.Advance(1) != 1)
{
m_done = true;
return false;
}
byte out_byte = m_buffer.GetByteAt(m_buffer.Count - m_count - 1);
byte in_byte = m_buffer.GetByteAt(m_buffer.Count - 1);
//TODO: This can be done much nicer, but the .Net overflow detection
//prevents it.
/*
* m_s1 += in - out;
* m_s2 += m_s1 - (m_count *(out + CHAR_OFFSET));
*/
int diff = in_byte - out_byte;
if (diff < 0)
diff += ushort.MaxValue + 1;
m_s1 = (uint)((diff + m_s1) % (ushort.MaxValue + 1));
diff = (int)((m_count * (out_byte + CHAR_OFFSET)) % (ushort.MaxValue + 1));
diff = (int)m_s1 - diff;
if (diff < 0)
diff += ushort.MaxValue + 1;
m_s2 = (uint)((m_s2 + diff) % (ushort.MaxValue + 1));
return true;
}
}
}
Duplicati/SharpRSync/DeltaFile.cs
+46 -21
View File
@@ -79,7 +79,7 @@ namespace Duplicati.SharpRSync
if (command < 0)
throw new Exception("Stream ended but had no end marker");
if (Enum.IsDefined(typeof(RDiffBinary.LiteralDeltaCommand), command))
if (Enum.IsDefined(typeof(RDiffBinary.LiteralDeltaCommand), (RDiffBinary.LiteralDeltaCommand)command))
{
//Find out how many bytes of literal data there is
int len = RDiffBinary.GetLiteralLength((RDiffBinary.LiteralDeltaCommand)command);
@@ -93,7 +93,7 @@ namespace Duplicati.SharpRSync
//Copy the literal data from the patch to the output
Utility.StreamCopy(m_inputStream, output, size);
}
else if (Enum.IsDefined(typeof(RDiffBinary.CopyDeltaCommand), command))
else if (Enum.IsDefined(typeof(RDiffBinary.CopyDeltaCommand), (RDiffBinary.CopyDeltaCommand)command))
{
//Find the offset of the data in the base file
int len = RDiffBinary.GetCopyOffsetSize((RDiffBinary.CopyDeltaCommand)command);
@@ -144,14 +144,17 @@ namespace Duplicati.SharpRSync
byte[] md4buffer = new byte[m_checksum.BlockLength];
int unmatched = 0;
long unmatched = 0;
long matched = 0;
long matched_offset = 0;
do
{
bool foundMatch = false;
List<KeyValuePair<int, byte[]>> strong = m_checksum.FindChunk(adler.Checksum);
//No weak matches :(
if (strong != null && strong.Count == 0)
if (strong == null || strong.Count == 0)
unmatched++;
else
{
@@ -186,36 +189,58 @@ namespace Duplicati.SharpRSync
if (unmatched > 0)
{
WriteLiteral(buffer.GetTail(unmatched), output);
buffer.DropTail(unmatched);
unmatched = 0;
}
if (matched == 0)
matched_offset = k.Key * m_checksum.BlockLength;
//Send the matching bytes as a copy
WriteCopy(k.Key * m_checksum.BlockLength, m_checksum.BlockLength, output);
adler.Reset();
matched += buffer.Count;
foundMatch = true;
buffer.DropTail(buffer.Count);
break;
}
}
//Avoid keeping too large blocks in memory.
//This gives an overhead of app. 3 bytes pr. 2Kb data
if (unmatched > m_checksum.BlockLength)
{
WriteLiteral(buffer.GetTail(unmatched), output);
buffer.DropTail(unmatched);
unmatched = 0;
adler.Reset();
}
}
} while (adler.AdvanceChecksum());
//If this byte was not matched, and we have queued up matches,
// flush them now
if (!foundMatch && matched > 0)
{
//Send the matching bytes as a copy
WriteCopy(matched_offset, matched, output);
matched = 0;
matched_offset = 0;
}
//Avoid keeping too large blocks in memory.
//This gives an overhead of app. 3 bytes pr. 2Kb data
if (unmatched >= m_checksum.BlockLength + 1)
{
WriteLiteral(buffer.GetTail(unmatched - 1), output);
buffer.DropTail(unmatched - 1);
unmatched = 1;
}
} while ( buffer.Count == 0 ? adler.Reset() : adler.Rollbuffer());
//If it is still in the buffer, it was not matched
unmatched = buffer.Count;
if (matched > 0 && unmatched > 0)
throw new Exception("Internal error, had buffered both matched and unmatched blocks!");
if (matched > 0)
{
//Send the matching bytes as a copy
WriteCopy(matched_offset, matched, output);
}
//Any trailing bytes are treated as a literal
if (unmatched > 0)
{
WriteLiteral(buffer.GetTail(unmatched), output);
buffer.DropTail(unmatched);
unmatched = 0;
adler.Reset();
}
output.WriteByte((byte)RDiffBinary.EndCommand);
@@ -242,7 +267,7 @@ namespace Duplicati.SharpRSync
/// <param name="offset">The offset in the basefile where the data is located</param>
/// <param name="length">The length of the data to copy</param>
/// <param name="output">The output delta stream</param>
private void WriteCopy(long offset, int length, System.IO.Stream output)
private void WriteCopy(long offset, long length, System.IO.Stream output)
{
output.WriteByte((byte)RDiffBinary.FindCopyDeltaCommand(offset, length));
byte[] len = RDiffBinary.EncodeLength(offset);
Duplicati/SharpRSync/MD4.cs
-19
View File
@@ -1,22 +1,3 @@
#region Disclaimer / License
// Copyright (C) 2008, Kenneth Skovhede
// http://www.hexad.dk, opensource@hexad.dk
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
#endregion
//
// MD4.cs - Message Digest 4 Abstract class
//
Duplicati/SharpRSync/RDiffBinary.cs
+15 -10
View File
@@ -72,13 +72,12 @@ namespace Duplicati.SharpRSync
/// <returns>The number of bytes required</returns>
public static int FindLength(long size)
{
//These limits should be unsigned, but they are read as signed in RDiff?
int count = 1;
if (size > byte.MaxValue)
count++;
if (size > short.MaxValue)
if (size > ushort.MaxValue)
count += 2;
if (size > int.MaxValue)
if (size > uint.MaxValue)
count += 4;
if (size > long.MaxValue)
throw new Exception("Value must be less than " + long.MaxValue);
@@ -95,11 +94,16 @@ namespace Duplicati.SharpRSync
if (data.Length == 1)
return (long)data[0];
else if (data.Length == 2)
return (long)BitConverter.ToInt16(FixEndian(data), 0);
return (long)BitConverter.ToUInt16(FixEndian(data), 0);
else if (data.Length == 4)
return (long)BitConverter.ToInt32(FixEndian(data), 0);
return (long)BitConverter.ToUInt32(FixEndian(data), 0);
else if (data.Length == 8)
return BitConverter.ToInt64(FixEndian(data), 0);
{
long tmp = BitConverter.ToInt64(FixEndian(data), 0);
if (tmp < 0)
throw new Exception("Unable to size item, because it is larger than " + long.MaxValue.ToString());
return tmp;
}
else
throw new Exception("Invalid data length");
}
@@ -111,11 +115,11 @@ namespace Duplicati.SharpRSync
/// </summary>
/// <param name="size">The value to write</param>
/// <returns>The written bytes</returns>
public static byte[] EncodeLength(int size)
public static byte[] EncodeLength(long size)
{
int len = FindLength(size);
if (len == 1)
return FixEndian(BitConverter.GetBytes((byte)size));
return new byte[] { (byte)size };
else if (len == 2)
return FixEndian(BitConverter.GetBytes((short)size));
else if (len == 4)
@@ -132,11 +136,12 @@ namespace Duplicati.SharpRSync
/// <param name="offset">The offset in the file where the copy begins</param>
/// <param name="size">The size of the copy</param>
/// <returns>The delta copy command</returns>
public static CopyDeltaCommand FindCopyDeltaCommand(int offset, int size)
public static CopyDeltaCommand FindCopyDeltaCommand(long offset, long size)
{
int i1 = FindLength(offset);
int i2 = FindLength(size);
return CopyCommand[i1][i2];
//TODO: Not good, long = 8
return CopyCommand[i2 - 1][i1 - 1];
}
/// <summary>
Duplicati/SharpRSync/RollingBuffer.cs
+48 -16
View File
@@ -38,11 +38,12 @@ namespace Duplicati.SharpRSync
{
m_stream = stream;
m_buffers = new List<byte[]>();
m_buffers.Add(new byte[BUFFER_SIZE]);
m_headIndex = 0;
m_tailIndex = 0;
}
public int Count { get { return m_buffers.Count * BUFFER_SIZE - m_tailIndex - (BUFFER_SIZE - m_headIndex); } }
public long Count { get { return m_buffers.Count * BUFFER_SIZE - m_tailIndex - (BUFFER_SIZE - m_headIndex); } }
/// <summary>
/// Reads the supplied amount of bytes
@@ -76,7 +77,7 @@ namespace Duplicati.SharpRSync
/// </summary>
/// <param name="count">The number of bytes to return</param>
/// <returns>The bytes requested</returns>
public byte[] GetHead(int count)
public byte[] GetHead(long count)
{
byte[] tmp = new byte[count];
GetHead(tmp, 0, count);
@@ -108,10 +109,10 @@ namespace Duplicati.SharpRSync
/// <param name="buf">The buffer to write the bytes into</param>
/// <param name="offset">The offset into the buffer</param>
/// <returns>The bytes requested</returns>
public void GetHead(byte[] buf, int offset, int count)
public void GetHead(byte[] buf, long offset, long count)
{
int ix = 0;
int total = this.Count;
long total = this.Count;
if (total < count)
throw new Exception("Buffer has too few bytes");
@@ -120,21 +121,23 @@ namespace Duplicati.SharpRSync
while (total > count)
{
int availible = GetAvalible(ix);
int diff = Math.Min(availible, total - count);
int diff = (int)Math.Min(availible, total - count);
total -= diff;
availible -= diff;
b += diff;
if (availible == 0)
if (b >= BUFFER_SIZE)
{
ix++;
b = 0;
}
}
if (ix == 0)
b -= m_tailIndex;
while (count > 0)
{
int avalible = Math.Min(GetAvalible(ix), count);
int avalible = (int)Math.Min(GetAvalible(ix) - b, count);
Array.Copy(m_buffers[ix], b, buf, offset, avalible);
count -= avalible;
@@ -149,9 +152,9 @@ namespace Duplicati.SharpRSync
/// </summary>
/// <param name="count">The number of bytes to return</param>
/// <returns>The bytes requested</returns>
public byte[] GetTail(int count)
public byte[] GetTail(long count)
{
byte[] tmp = new byte[BUFFER_SIZE];
byte[] tmp = new byte[count];
GetTail(tmp, 0, count);
return tmp;
}
@@ -162,7 +165,7 @@ namespace Duplicati.SharpRSync
/// <param name="count">The number of bytes to return</param>
/// <param name="buf">The buffer to write the bytes into</param>
/// <param name="offset">The offset into the buffer</param>
public void GetTail(byte[] buf, int offset, int count)
public void GetTail(byte[] buf, long offset, long count)
{
int ix = 0;
if (this.Count < count)
@@ -171,7 +174,7 @@ namespace Duplicati.SharpRSync
int b = m_tailIndex;
while (count > 0)
{
int avalible = Math.Min(GetAvalible(ix), count);
int avalible = (int)Math.Min(GetAvalible(ix), count);
Array.Copy(m_buffers[ix], b, buf, offset, avalible);
count -= avalible;
@@ -185,11 +188,11 @@ namespace Duplicati.SharpRSync
/// Removes the bytes from memory
/// </summary>
/// <param name="count">The number of bytes to drop</param>
public void DropTail(int count)
public void DropTail(long count)
{
if (count < (BUFFER_SIZE - m_tailIndex))
{
m_tailIndex += count;
m_tailIndex += (int)count;
return;
}
@@ -197,13 +200,42 @@ namespace Duplicati.SharpRSync
count -= (BUFFER_SIZE - m_tailIndex);
m_tailIndex = 0;
while (count > BUFFER_SIZE)
while (count >= BUFFER_SIZE)
{
m_buffers.RemoveAt(0);
count -= BUFFER_SIZE;
}
m_tailIndex += count;
m_tailIndex += (int)count;
if (m_buffers.Count == 0)
{
m_headIndex = 0;
m_buffers.Add(new byte[BUFFER_SIZE]);
}
}
/// <summary>
/// Returns the byte at the given position.
/// </summary>
/// <param name="index">The index of the byte</param>
/// <returns>The byte value</returns>
public byte GetByteAt(long index)
{
int ix = 0;
if (index > this.Count - 1)
throw new Exception("Buffer has too few bytes");
if (index < 0)
throw new Exception("Index cannot be negative");
while (GetAvalible(ix) <= index)
{
index -= GetAvalible(ix);
ix++;
}
long offset = ix == 0 ? m_tailIndex : 0;
offset += index;
return m_buffers[ix][offset];
}
}
}