#region Disclaimer / License // Copyright (C) 2009, Kenneth Skovhede // http://www.hexad.dk, opensource@hexad.dk // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public // License as published by the Free Software Foundation; either // version 2.1 of the License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // #endregion using System; using System.Collections.Generic; using System.Text; namespace Duplicati.Library.SharpRSync { /// /// This class reads and writes signature files in the same format as RDiff /// public class ChecksumFile { private const int MD4_LENGTH = 16; private int m_blocklen; private int m_stronglen; //This dictionary has the weak checksum as the key, and an index into the stronglist private Dictionary> m_checksums; //This list is ordered and each entry has the weak and strong checksum private List> m_strongList; private Adler32Checksum m_adler; private System.Security.Cryptography.HashAlgorithm m_md4; private System.Security.Cryptography.HashAlgorithm m_sha; /// /// Reads a ChecksumFile from a stream /// /// The stream to read from public ChecksumFile(System.IO.Stream input) { m_checksums = new Dictionary>(); m_strongList = new List>(); byte[] sig = new byte[4]; if (Utility.ForceStreamRead(input, sig, 4) != 4) throw new Exception("End of stream occured while reading initial 4 bytes of signature file"); for (int i = 0; i < sig.Length; i++) if (RDiffBinary.SIGNATURE_MAGIC[i] != sig[i]) throw new Exception("Signature stream did not have the correct start marker"); if (Utility.ForceStreamRead(input, sig, 4) != 4) throw new Exception("End of stream occured while reading blocksize in signature file"); m_blocklen = BitConverter.ToInt32(RDiffBinary.FixEndian(sig), 0); if (m_blocklen < 1 || m_blocklen > int.MaxValue / 2) throw new Exception("Blocklen in signature file was: " + m_blocklen + ", which does not seem correct"); if (Utility.ForceStreamRead(input, sig, 4) != 4) throw new Exception("End of stream occured while reading stronglen in signature file"); m_stronglen = BitConverter.ToInt32(RDiffBinary.FixEndian(sig), 0); if (m_stronglen < 1 || m_stronglen > MD4_LENGTH) throw new Exception("Stronglen in signature file was: " + m_stronglen + ", which does not seem correct"); while(true) { if (Utility.ForceStreamRead(input, sig, 4) != 4) break; uint weak = BitConverter.ToUInt32(RDiffBinary.FixEndian(sig), 0); ; byte[] strongbuf = new byte[m_stronglen]; if (Utility.ForceStreamRead(input, strongbuf, strongbuf.Length) != strongbuf.Length) throw new Exception("End of stream occured while reading strong signature from signature file"); if (!m_checksums.ContainsKey(weak)) m_checksums[weak] = new List(); m_checksums[weak].Add(m_strongList.Count); m_strongList.Add(new KeyValuePair(weak, strongbuf)); } } /// /// Constructs a new CheckSum file with default sizes /// public ChecksumFile() : this(Adler32Checksum.DEFAULT_BLOCK_SIZE, 8) { } /// /// Constructs a new CheckSum file /// The length of a single block /// The number of bytes in the MD4 checksum /// public ChecksumFile(int blocklength, int stronglength) { m_blocklen = blocklength; m_stronglen = stronglength; m_checksums = new Dictionary>(); m_strongList = new List>(); m_md4 = System.Security.Cryptography.MD4.Create("MD4"); m_md4.Initialize(); m_sha = System.Security.Cryptography.HashAlgorithm.Create("SHA256"); m_sha.Initialize(); } /// /// Adds all checksums in an entire stream /// /// The stream to read checksums from public void AddStream(System.IO.Stream stream) { byte[] buffer = new byte[m_blocklen]; int a; while ((a = Utility.ForceStreamRead(stream, buffer, buffer.Length)) != 0) { if (a != buffer.Length) { byte[] tmp = new byte[a]; Array.Copy(buffer, tmp, a); AddChunk(tmp); } else AddChunk(buffer); } } /// /// Adds a chunck of data to checksum list /// /// The data to add a checksum entry for public void AddChunk(byte[] buffer) { m_adler = new Adler32Checksum(new System.IO.MemoryStream(buffer), m_blocklen); if (!m_md4.CanReuseTransform) m_md4 = System.Security.Cryptography.MD4.Create("MD4"); m_md4.Initialize(); byte[] strong = m_md4.ComputeHash(buffer); if (strong.Length > m_stronglen) { byte[] tmp = new byte[m_stronglen]; Array.Copy(strong, tmp, tmp.Length); strong = tmp; } if (!m_checksums.ContainsKey(m_adler.Checksum)) m_checksums[m_adler.Checksum] = new List(); m_checksums[m_adler.Checksum].Add(m_strongList.Count); m_strongList.Add(new KeyValuePair(m_adler.Checksum, strong)); //TODO: Figure out if we can get a complete file signature this way //m_sha.ComputeHash(buffer); } /// /// Writes a RDiff compatible file with the signatures /// /// public void Save(System.IO.Stream output) { output.Write(RDiffBinary.SIGNATURE_MAGIC, 0, RDiffBinary.SIGNATURE_MAGIC.Length); output.Write(RDiffBinary.FixEndian(BitConverter.GetBytes(m_blocklen)), 0, 4); output.Write(RDiffBinary.FixEndian(BitConverter.GetBytes(m_stronglen)), 0, 4); foreach(KeyValuePair k in m_strongList) { output.Write(RDiffBinary.FixEndian(BitConverter.GetBytes(k.Key)), 0, 4); output.Write(k.Value, 0, m_stronglen); } output.Flush(); } /// /// Looks in the checksum table for a match to the weak entry /// /// The entry to find a match for /// A list of possible MD4 matches, and their indexes public List> FindChunk(uint weak) { if (!m_checksums.ContainsKey(weak)) return null; else { List> lst = new List>(); foreach (int i in m_checksums[weak]) lst.Add(new KeyValuePair(i, m_strongList[i].Value)); return lst; } } /// /// Gets the number of bytes in each hashed block /// public int BlockLength { get { return m_blocklen; } } /// /// Gets the number of bytes in a signature file /// public int StrongLength { get { return m_stronglen; } } } }