diff --git a/Duplicati/SharpRSync/Adler32Checksum.cs b/Duplicati/SharpRSync/Adler32Checksum.cs index a295a1124..1a00d15ac 100644 --- a/Duplicati/SharpRSync/Adler32Checksum.cs +++ b/Duplicati/SharpRSync/Adler32Checksum.cs @@ -44,6 +44,11 @@ namespace Duplicati.SharpRSync private long m_count; private long m_blockSize; + /// + /// Constructs a new adler32 instance, reading from a rolling buffer + /// + /// The buffer to read from + /// The size of a single block public Adler32Checksum(RollingBuffer buffer, int blocksize) { m_buffer = buffer; @@ -52,7 +57,12 @@ namespace Duplicati.SharpRSync m_blockSize = blocksize; Reset(); } - + + /// + /// Constructs a new adler32 instance, reading from a stream + /// + /// The buffer to read from + /// The size of a single block public Adler32Checksum(System.IO.Stream source, int blocksize) { m_source = source; @@ -62,21 +72,40 @@ namespace Duplicati.SharpRSync Reset(); } + /// + /// Constructs a new adler32 instance, reading from a stream + /// + /// The buffer to read from public Adler32Checksum(System.IO.Stream source) : this(source, DEFAULT_BLOCK_SIZE) { } + /// + /// Gets a value indicating if the last data has been read from the source + /// public bool Done { get { return m_done; } } + + /// + /// Gets the current checksum + /// public uint Checksum { get { return (uint)((m_s1 & 0xffff) + (m_s2 << 16)); } } - public void Reset() + /// + /// Resets the checksum and calculates the checksum for a new block + /// + /// False if there were no new data + public bool Reset() { int i; byte[] buf = new byte[4]; int a; + if (m_done) + return false; + m_s1 = m_s2 = 0; + m_count = 0; for (i = 0; i < m_blockSize - (m_blockSize % 4); i += 4) { @@ -85,7 +114,7 @@ namespace Duplicati.SharpRSync else { a = m_buffer.Advance(4); - m_buffer.GetHead(buf, 0, 4); + m_buffer.GetHead(buf, 0, a); } if (a == 4) @@ -106,18 +135,45 @@ namespace Duplicati.SharpRSync UpdateChecksum(buf[j]); m_count += a; m_done = true; - return; + return m_count != 0; } } + if (m_source != null) + a = Utility.ForceStreamRead(m_source, buf, (int)m_blockSize % 4); + else + { + a = m_buffer.Advance((int)m_blockSize % 4); + m_buffer.GetHead(buf, 0, a); + } - a = Utility.ForceStreamRead(m_source, buf, (int)m_blockSize % 4); for (i = 0; i < a; i++) UpdateChecksum(buf[i]); m_count += a; - + return true; } + /// + /// Rolls the checksum a number of bytes + /// + /// The number of bytes to roll + /// True if there is new data, false otherwise + public bool AdvanceChecksum(int count) + { + if (m_done) + return false; + + for (int i = 0; i < count; i++) + if (!AdvanceChecksum()) + break; + + return true; + } + + /// + /// Rolls the checksum a single byte + /// + /// True if a new byte was checksummed, false if there was no more data public bool AdvanceChecksum() { if (m_done) @@ -125,7 +181,14 @@ namespace Duplicati.SharpRSync int b; if (m_source != null) + { b = m_source.ReadByte(); + if (b == -1) + { + m_done = true; + return false; + } + } else { if (m_buffer.Advance(1) != 1) @@ -133,19 +196,19 @@ namespace Duplicati.SharpRSync m_done = true; return false; } - b = m_buffer.GetHead(1)[0]; + b = m_buffer.GetByteAt(m_buffer.Count - 1); } - if (b == -1) - { - m_done = true; - return false; - } + m_count += 1; UpdateChecksum((byte)b); return true; } + /// + /// Updates the checksum with the read byte + /// + /// The byte read private void UpdateChecksum(byte value) { m_s1 += value + CHAR_OFFSET; @@ -155,5 +218,51 @@ namespace Duplicati.SharpRSync m_s2 %= ushort.MaxValue + 1; } + + /// + /// Rolls the buffer, so the checksum now fits a block that is shifted one byte from the previous one. + /// + /// True if new data was included, false otherwise + public bool Rollbuffer() + { + if (m_buffer == null) + throw new Exception("Cannot roll buffer, when the source is not a rolling buffer instance"); + + if (m_done) + return false; + + if (m_buffer.Advance(1) != 1) + { + m_done = true; + return false; + } + + byte out_byte = m_buffer.GetByteAt(m_buffer.Count - m_count - 1); + byte in_byte = m_buffer.GetByteAt(m_buffer.Count - 1); + + //TODO: This can be done much nicer, but the .Net overflow detection + //prevents it. + + /* + * m_s1 += in - out; + * m_s2 += m_s1 - (m_count *(out + CHAR_OFFSET)); + */ + + int diff = in_byte - out_byte; + if (diff < 0) + diff += ushort.MaxValue + 1; + + m_s1 = (uint)((diff + m_s1) % (ushort.MaxValue + 1)); + + diff = (int)((m_count * (out_byte + CHAR_OFFSET)) % (ushort.MaxValue + 1)); + diff = (int)m_s1 - diff; + if (diff < 0) + diff += ushort.MaxValue + 1; + + m_s2 = (uint)((m_s2 + diff) % (ushort.MaxValue + 1)); + + return true; + } + } } diff --git a/Duplicati/SharpRSync/DeltaFile.cs b/Duplicati/SharpRSync/DeltaFile.cs index 1d8b4e7c5..91e83c668 100644 --- a/Duplicati/SharpRSync/DeltaFile.cs +++ b/Duplicati/SharpRSync/DeltaFile.cs @@ -79,7 +79,7 @@ namespace Duplicati.SharpRSync if (command < 0) throw new Exception("Stream ended but had no end marker"); - if (Enum.IsDefined(typeof(RDiffBinary.LiteralDeltaCommand), command)) + if (Enum.IsDefined(typeof(RDiffBinary.LiteralDeltaCommand), (RDiffBinary.LiteralDeltaCommand)command)) { //Find out how many bytes of literal data there is int len = RDiffBinary.GetLiteralLength((RDiffBinary.LiteralDeltaCommand)command); @@ -93,7 +93,7 @@ namespace Duplicati.SharpRSync //Copy the literal data from the patch to the output Utility.StreamCopy(m_inputStream, output, size); } - else if (Enum.IsDefined(typeof(RDiffBinary.CopyDeltaCommand), command)) + else if (Enum.IsDefined(typeof(RDiffBinary.CopyDeltaCommand), (RDiffBinary.CopyDeltaCommand)command)) { //Find the offset of the data in the base file int len = RDiffBinary.GetCopyOffsetSize((RDiffBinary.CopyDeltaCommand)command); @@ -144,14 +144,17 @@ namespace Duplicati.SharpRSync byte[] md4buffer = new byte[m_checksum.BlockLength]; - int unmatched = 0; + long unmatched = 0; + long matched = 0; + long matched_offset = 0; do { + bool foundMatch = false; List> strong = m_checksum.FindChunk(adler.Checksum); //No weak matches :( - if (strong != null && strong.Count == 0) + if (strong == null || strong.Count == 0) unmatched++; else { @@ -186,36 +189,58 @@ namespace Duplicati.SharpRSync if (unmatched > 0) { WriteLiteral(buffer.GetTail(unmatched), output); + buffer.DropTail(unmatched); unmatched = 0; } + if (matched == 0) + matched_offset = k.Key * m_checksum.BlockLength; - //Send the matching bytes as a copy - WriteCopy(k.Key * m_checksum.BlockLength, m_checksum.BlockLength, output); - adler.Reset(); + matched += buffer.Count; + foundMatch = true; buffer.DropTail(buffer.Count); break; } } - - //Avoid keeping too large blocks in memory. - //This gives an overhead of app. 3 bytes pr. 2Kb data - if (unmatched > m_checksum.BlockLength) - { - WriteLiteral(buffer.GetTail(unmatched), output); - buffer.DropTail(unmatched); - unmatched = 0; - adler.Reset(); - } } - } while (adler.AdvanceChecksum()); + + //If this byte was not matched, and we have queued up matches, + // flush them now + if (!foundMatch && matched > 0) + { + //Send the matching bytes as a copy + WriteCopy(matched_offset, matched, output); + matched = 0; + matched_offset = 0; + } + + //Avoid keeping too large blocks in memory. + //This gives an overhead of app. 3 bytes pr. 2Kb data + if (unmatched >= m_checksum.BlockLength + 1) + { + WriteLiteral(buffer.GetTail(unmatched - 1), output); + buffer.DropTail(unmatched - 1); + unmatched = 1; + } + + } while ( buffer.Count == 0 ? adler.Reset() : adler.Rollbuffer()); + + //If it is still in the buffer, it was not matched + unmatched = buffer.Count; + + if (matched > 0 && unmatched > 0) + throw new Exception("Internal error, had buffered both matched and unmatched blocks!"); + + if (matched > 0) + { + //Send the matching bytes as a copy + WriteCopy(matched_offset, matched, output); + } //Any trailing bytes are treated as a literal if (unmatched > 0) { WriteLiteral(buffer.GetTail(unmatched), output); buffer.DropTail(unmatched); - unmatched = 0; - adler.Reset(); } output.WriteByte((byte)RDiffBinary.EndCommand); @@ -242,7 +267,7 @@ namespace Duplicati.SharpRSync /// The offset in the basefile where the data is located /// The length of the data to copy /// The output delta stream - private void WriteCopy(long offset, int length, System.IO.Stream output) + private void WriteCopy(long offset, long length, System.IO.Stream output) { output.WriteByte((byte)RDiffBinary.FindCopyDeltaCommand(offset, length)); byte[] len = RDiffBinary.EncodeLength(offset); diff --git a/Duplicati/SharpRSync/MD4.cs b/Duplicati/SharpRSync/MD4.cs index d6f87c4e7..3b7d24203 100644 --- a/Duplicati/SharpRSync/MD4.cs +++ b/Duplicati/SharpRSync/MD4.cs @@ -1,22 +1,3 @@ -#region Disclaimer / License -// Copyright (C) 2008, Kenneth Skovhede -// http://www.hexad.dk, opensource@hexad.dk -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// -#endregion // // MD4.cs - Message Digest 4 Abstract class // diff --git a/Duplicati/SharpRSync/RDiffBinary.cs b/Duplicati/SharpRSync/RDiffBinary.cs index a6775242b..f69c28aec 100644 --- a/Duplicati/SharpRSync/RDiffBinary.cs +++ b/Duplicati/SharpRSync/RDiffBinary.cs @@ -72,13 +72,12 @@ namespace Duplicati.SharpRSync /// The number of bytes required public static int FindLength(long size) { - //These limits should be unsigned, but they are read as signed in RDiff? int count = 1; if (size > byte.MaxValue) count++; - if (size > short.MaxValue) + if (size > ushort.MaxValue) count += 2; - if (size > int.MaxValue) + if (size > uint.MaxValue) count += 4; if (size > long.MaxValue) throw new Exception("Value must be less than " + long.MaxValue); @@ -95,11 +94,16 @@ namespace Duplicati.SharpRSync if (data.Length == 1) return (long)data[0]; else if (data.Length == 2) - return (long)BitConverter.ToInt16(FixEndian(data), 0); + return (long)BitConverter.ToUInt16(FixEndian(data), 0); else if (data.Length == 4) - return (long)BitConverter.ToInt32(FixEndian(data), 0); + return (long)BitConverter.ToUInt32(FixEndian(data), 0); else if (data.Length == 8) - return BitConverter.ToInt64(FixEndian(data), 0); + { + long tmp = BitConverter.ToInt64(FixEndian(data), 0); + if (tmp < 0) + throw new Exception("Unable to size item, because it is larger than " + long.MaxValue.ToString()); + return tmp; + } else throw new Exception("Invalid data length"); } @@ -111,11 +115,11 @@ namespace Duplicati.SharpRSync /// /// The value to write /// The written bytes - public static byte[] EncodeLength(int size) + public static byte[] EncodeLength(long size) { int len = FindLength(size); if (len == 1) - return FixEndian(BitConverter.GetBytes((byte)size)); + return new byte[] { (byte)size }; else if (len == 2) return FixEndian(BitConverter.GetBytes((short)size)); else if (len == 4) @@ -132,11 +136,12 @@ namespace Duplicati.SharpRSync /// The offset in the file where the copy begins /// The size of the copy /// The delta copy command - public static CopyDeltaCommand FindCopyDeltaCommand(int offset, int size) + public static CopyDeltaCommand FindCopyDeltaCommand(long offset, long size) { int i1 = FindLength(offset); int i2 = FindLength(size); - return CopyCommand[i1][i2]; + //TODO: Not good, long = 8 + return CopyCommand[i2 - 1][i1 - 1]; } /// diff --git a/Duplicati/SharpRSync/RollingBuffer.cs b/Duplicati/SharpRSync/RollingBuffer.cs index 765f80468..1ac576d53 100644 --- a/Duplicati/SharpRSync/RollingBuffer.cs +++ b/Duplicati/SharpRSync/RollingBuffer.cs @@ -38,11 +38,12 @@ namespace Duplicati.SharpRSync { m_stream = stream; m_buffers = new List(); + m_buffers.Add(new byte[BUFFER_SIZE]); m_headIndex = 0; m_tailIndex = 0; } - public int Count { get { return m_buffers.Count * BUFFER_SIZE - m_tailIndex - (BUFFER_SIZE - m_headIndex); } } + public long Count { get { return m_buffers.Count * BUFFER_SIZE - m_tailIndex - (BUFFER_SIZE - m_headIndex); } } /// /// Reads the supplied amount of bytes @@ -76,7 +77,7 @@ namespace Duplicati.SharpRSync /// /// The number of bytes to return /// The bytes requested - public byte[] GetHead(int count) + public byte[] GetHead(long count) { byte[] tmp = new byte[count]; GetHead(tmp, 0, count); @@ -108,10 +109,10 @@ namespace Duplicati.SharpRSync /// The buffer to write the bytes into /// The offset into the buffer /// The bytes requested - public void GetHead(byte[] buf, int offset, int count) + public void GetHead(byte[] buf, long offset, long count) { int ix = 0; - int total = this.Count; + long total = this.Count; if (total < count) throw new Exception("Buffer has too few bytes"); @@ -120,21 +121,23 @@ namespace Duplicati.SharpRSync while (total > count) { int availible = GetAvalible(ix); - int diff = Math.Min(availible, total - count); + int diff = (int)Math.Min(availible, total - count); total -= diff; - availible -= diff; b += diff; - if (availible == 0) + if (b >= BUFFER_SIZE) { ix++; b = 0; } } + if (ix == 0) + b -= m_tailIndex; + while (count > 0) { - int avalible = Math.Min(GetAvalible(ix), count); + int avalible = (int)Math.Min(GetAvalible(ix) - b, count); Array.Copy(m_buffers[ix], b, buf, offset, avalible); count -= avalible; @@ -149,9 +152,9 @@ namespace Duplicati.SharpRSync /// /// The number of bytes to return /// The bytes requested - public byte[] GetTail(int count) + public byte[] GetTail(long count) { - byte[] tmp = new byte[BUFFER_SIZE]; + byte[] tmp = new byte[count]; GetTail(tmp, 0, count); return tmp; } @@ -162,7 +165,7 @@ namespace Duplicati.SharpRSync /// The number of bytes to return /// The buffer to write the bytes into /// The offset into the buffer - public void GetTail(byte[] buf, int offset, int count) + public void GetTail(byte[] buf, long offset, long count) { int ix = 0; if (this.Count < count) @@ -171,7 +174,7 @@ namespace Duplicati.SharpRSync int b = m_tailIndex; while (count > 0) { - int avalible = Math.Min(GetAvalible(ix), count); + int avalible = (int)Math.Min(GetAvalible(ix), count); Array.Copy(m_buffers[ix], b, buf, offset, avalible); count -= avalible; @@ -185,11 +188,11 @@ namespace Duplicati.SharpRSync /// Removes the bytes from memory /// /// The number of bytes to drop - public void DropTail(int count) + public void DropTail(long count) { if (count < (BUFFER_SIZE - m_tailIndex)) { - m_tailIndex += count; + m_tailIndex += (int)count; return; } @@ -197,13 +200,42 @@ namespace Duplicati.SharpRSync count -= (BUFFER_SIZE - m_tailIndex); m_tailIndex = 0; - while (count > BUFFER_SIZE) + while (count >= BUFFER_SIZE) { m_buffers.RemoveAt(0); count -= BUFFER_SIZE; } - m_tailIndex += count; + m_tailIndex += (int)count; + if (m_buffers.Count == 0) + { + m_headIndex = 0; + m_buffers.Add(new byte[BUFFER_SIZE]); + } + } + + /// + /// Returns the byte at the given position. + /// + /// The index of the byte + /// The byte value + public byte GetByteAt(long index) + { + int ix = 0; + if (index > this.Count - 1) + throw new Exception("Buffer has too few bytes"); + if (index < 0) + throw new Exception("Index cannot be negative"); + + while (GetAvalible(ix) <= index) + { + index -= GetAvalible(ix); + ix++; + } + + long offset = ix == 0 ? m_tailIndex : 0; + offset += index; + return m_buffers[ix][offset]; } } }