diff --git a/proprietary/DiskImage/Filesystem/Ntfs/NtfsBitmap.cs b/proprietary/DiskImage/Filesystem/Ntfs/NtfsBitmap.cs new file mode 100644 index 000000000..c6892512d --- /dev/null +++ b/proprietary/DiskImage/Filesystem/Ntfs/NtfsBitmap.cs @@ -0,0 +1,536 @@ +// Copyright (c) 2026 Duplicati Inc. All rights reserved. + +using System; +using System.Buffers.Binary; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using Duplicati.Proprietary.DiskImage.Partition; + +namespace Duplicati.Proprietary.DiskImage.Filesystem.Ntfs; + +/// +/// Reads and provides access to the NTFS $Bitmap file, which tracks cluster allocation. +/// Each bit in the bitmap represents one cluster: 1 = allocated, 0 = free. +/// +internal class NtfsBitmap +{ + /// + /// Magic number "FILE" at the start of an MFT record (little-endian: 0x454C4946). + /// + private const uint MFT_RECORD_MAGIC = 0x454C4946; + + /// + /// MFT record flag indicating the record is in use. + /// + private const ushort MFT_RECORD_IN_USE = 0x0001; + + /// + /// Attribute type code for $STANDARD_INFORMATION. + /// + private const uint ATTR_STANDARD_INFORMATION = 0x10; + + /// + /// Attribute type code for $ATTRIBUTE_LIST. + /// + private const uint ATTR_ATTRIBUTE_LIST = 0x20; + + /// + /// Attribute type code for $FILE_NAME. + /// + private const uint ATTR_FILE_NAME = 0x30; + + /// + /// Attribute type code for $DATA. + /// + private const uint ATTR_DATA = 0x80; + + /// + /// Flag indicating a non-resident attribute. + /// + private const byte ATTR_NON_RESIDENT_FLAG = 0x01; + + /// + /// The boot sector containing NTFS geometry information. + /// + private readonly NtfsBootSector m_bootSector; + + /// + /// The raw bitmap data. + /// + private readonly byte[] m_bitmapData; + + /// + /// BitArray for fast cluster allocation lookup. + /// + private readonly BitArray m_allocationBitmap; + + /// + /// Total number of clusters in the volume. + /// + private readonly long m_totalClusters; + + /// + /// Initializes a new instance of the class by reading the $Bitmap from disk. + /// + /// The partition containing the NTFS filesystem. + /// The parsed boot sector containing NTFS geometry. + /// Cancellation token. + /// Thrown if partition or bootSector is null. + /// Thrown if the $Bitmap record cannot be parsed. + public NtfsBitmap(IPartition partition, NtfsBootSector bootSector, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(partition); + m_bootSector = bootSector; + m_totalClusters = bootSector.TotalClusters; + + // Calculate the size of the bitmap in bytes (1 bit per cluster, rounded up) + var bitmapSizeInBytes = (int)((m_totalClusters + 7) / 8); + + // Read the $Bitmap MFT record (record 6) + var bitmapRecordData = ReadMftRecord(partition, 6, cancellationToken); + + // Parse the $DATA attribute to get the bitmap data + m_bitmapData = ExtractBitmapData(partition, bitmapRecordData, bitmapSizeInBytes, cancellationToken); + + // Build the allocation bitmap + m_allocationBitmap = BuildAllocationBitmap(); + } + + /// + /// Initializes a new instance of the class from raw bitmap data. + /// This constructor is primarily used for testing. + /// + /// The parsed boot sector containing NTFS geometry. + /// The raw bitmap data. + public NtfsBitmap(NtfsBootSector bootSector, byte[] bitmapData) + { + ArgumentNullException.ThrowIfNull(bitmapData); + m_bootSector = bootSector; + m_bitmapData = bitmapData; + m_totalClusters = bootSector.TotalClusters; + m_allocationBitmap = BuildAllocationBitmap(); + } + + /// + /// Gets the total number of clusters in the volume. + /// + public long TotalClusters => m_totalClusters; + + /// + /// Gets the number of allocated (in-use) clusters. + /// + public long AllocatedClusters + { + get + { + var count = 0; + foreach (bool bit in m_allocationBitmap) + if (bit) + count++; + + return count; + } + } + + /// + /// Gets the number of free clusters. + /// + public long FreeClusters => m_totalClusters - AllocatedClusters; + + /// + /// Checks if a cluster is allocated (in use). + /// + /// The cluster number to check. + /// True if the cluster is allocated; otherwise, false. + /// Thrown if cluster number is out of range. + public bool IsClusterAllocated(long clusterNumber) + { + if (clusterNumber < 0 || clusterNumber >= m_totalClusters) + throw new ArgumentOutOfRangeException(nameof(clusterNumber), $"Cluster number {clusterNumber} is out of range. Valid range is 0 to {m_totalClusters - 1}."); + + return m_allocationBitmap[(int)clusterNumber]; + } + + /// + /// Reads an MFT record from disk. + /// + /// The partition to read from. + /// The MFT record number to read. + /// Cancellation token. + /// The raw MFT record data with fixup array applied. + private byte[] ReadMftRecord(IPartition partition, long recordNumber, CancellationToken cancellationToken) + { + var recordSize = m_bootSector.MftRecordSize; + var recordOffset = m_bootSector.MftByteOffset + (recordNumber * recordSize); + + var recordData = new byte[recordSize]; + + using var stream = partition.OpenReadAsync(cancellationToken).Result; + stream.Seek(recordOffset, SeekOrigin.Begin); + stream.ReadExactly(recordData); + + // Apply the fixup array to restore the original sector data + ApplyFixupArray(recordData); + + return recordData; + } + + /// + /// Extracts the bitmap data from the $Bitmap MFT record. + /// + /// The partition to read from. + /// The MFT record data. + /// The expected size of the bitmap data. + /// Cancellation token. + /// The bitmap data. + private byte[] ExtractBitmapData(IPartition partition, byte[] recordData, int expectedSize, CancellationToken cancellationToken) + { + // Validate the FILE magic + var magic = BinaryPrimitives.ReadUInt32LittleEndian(recordData.AsSpan(0, 4)); + if (magic != MFT_RECORD_MAGIC) + throw new InvalidDataException($"Invalid MFT record magic: expected 0x{MFT_RECORD_MAGIC:X8}, got 0x{magic:X8}."); + + // Get the offset to the first attribute + var firstAttributeOffset = BinaryPrimitives.ReadUInt16LittleEndian(recordData.AsSpan(0x14, 2)); + + // Find the $DATA attribute (type 0x80) + int attributeOffset = firstAttributeOffset; + while (attributeOffset < recordData.Length) + { + var attrType = BinaryPrimitives.ReadUInt32LittleEndian(recordData.AsSpan(attributeOffset, 4)); + + // End of attributes marker + if (attrType == 0xFFFFFFFF) + break; + + // Found $DATA attribute + if (attrType == ATTR_DATA) + { + var isNonResident = (recordData[attributeOffset + 8] & ATTR_NON_RESIDENT_FLAG) != 0; + + if (isNonResident) + return ReadNonResidentAttributeData(partition, recordData, attributeOffset, expectedSize, cancellationToken); + else + return ReadResidentAttributeData(recordData, attributeOffset); + } + + // Move to the next attribute + var attrLength = BinaryPrimitives.ReadUInt32LittleEndian(recordData.AsSpan(attributeOffset + 4, 4)); + if (attrLength == 0) + break; + + attributeOffset += (int)attrLength; + } + + throw new InvalidDataException("$DATA attribute not found in $Bitmap MFT record."); + } + + /// + /// Reads data from a resident attribute. + /// + /// The MFT record data. + /// The offset to the attribute. + /// The attribute data. + private static byte[] ReadResidentAttributeData(byte[] recordData, int attributeOffset) + { + // Resident attribute header: + // 0x00: Attribute type (4 bytes) + // 0x04: Attribute length (4 bytes) + // 0x08: Non-resident flag (1 byte) + // 0x09: Name length (1 byte) + // 0x0A: Name offset (2 bytes) + // 0x0C: Flags (2 bytes) + // 0x0E: Attribute ID (2 bytes) + // 0x10: Data length (4 bytes) + // 0x14: Data offset (2 bytes) + // 0x16: Reserved (1 byte) + // 0x17: Reserved (1 byte) + // Data starts at offset specified by data offset field + + var dataLength = BinaryPrimitives.ReadUInt32LittleEndian(recordData.AsSpan(attributeOffset + 0x10, 4)); + var dataOffset = BinaryPrimitives.ReadUInt16LittleEndian(recordData.AsSpan(attributeOffset + 0x14, 2)); + + var data = new byte[dataLength]; + Buffer.BlockCopy(recordData, attributeOffset + dataOffset, data, 0, (int)dataLength); + return data; + } + + /// + /// Reads data from a non-resident attribute by following data runs. + /// + /// The partition to read from. + /// The MFT record data. + /// The offset to the attribute. + /// The expected size of the data. + /// Cancellation token. + /// The attribute data. + private byte[] ReadNonResidentAttributeData(IPartition partition, byte[] recordData, int attributeOffset, int expectedSize, CancellationToken cancellationToken) + { + // Non-resident attribute header: + // 0x00: Attribute type (4 bytes) + // 0x04: Attribute length (4 bytes) + // 0x08: Non-resident flag (1 byte) + // 0x09: Name length (1 byte) + // 0x0A: Name offset (2 bytes) + // 0x0C: Starting VCN (8 bytes) + // 0x14: Ending VCN (8 bytes) + // 0x1C: Run offset (2 bytes) - offset to data run list from start of attribute + // 0x1E: Compression unit size (2 bytes) + // 0x20: Padding (4 bytes) + // 0x24: Allocated size (8 bytes) + // 0x2C: Data size (8 bytes) + // 0x34: Initialized size (8 bytes) + // 0x3C: Compressed size (8 bytes) - only if compressed + + var runOffset = BinaryPrimitives.ReadUInt16LittleEndian(recordData.AsSpan(attributeOffset + 0x1C, 2)); + var dataSize = BinaryPrimitives.ReadUInt64LittleEndian(recordData.AsSpan(attributeOffset + 0x30, 8)); + + // Parse the data runs + var dataRuns = ParseDataRuns(recordData, attributeOffset + runOffset); + + // Read the data from the clusters specified by the data runs + var result = new byte[Math.Min(expectedSize, (long)dataSize)]; + var resultOffset = 0; + + using var stream = partition.OpenReadAsync(cancellationToken).Result; + + foreach (var (startCluster, clusterCount) in dataRuns) + { + if (startCluster == 0) + { + // Sparse run (unallocated extent) - fill with zeros + var sparseBytes = (int)Math.Min(clusterCount * m_bootSector.ClusterSize, result.Length - resultOffset); + Array.Clear(result, resultOffset, sparseBytes); + resultOffset += sparseBytes; + } + else + { + var bytesToRead = (int)Math.Min(clusterCount * m_bootSector.ClusterSize, result.Length - resultOffset); + var byteOffset = startCluster * m_bootSector.ClusterSize; + + stream.Seek(byteOffset, SeekOrigin.Begin); + stream.ReadExactly(result, resultOffset, bytesToRead); + resultOffset += bytesToRead; + } + + if (resultOffset >= result.Length) + break; + } + + return result; + } + + /// + /// Parses the data run list from an attribute. + /// + /// The MFT record data containing the data runs. + /// The offset to the start of the data run list. + /// A list of (startCluster, clusterCount) tuples. + internal static List<(long startCluster, long clusterCount)> ParseDataRuns(byte[] recordData, int runsOffset) + { + var runs = new List<(long startCluster, long clusterCount)>(); + long currentCluster = 0; + var offset = runsOffset; + + while (offset < recordData.Length) + { + var header = recordData[offset++]; + + // End of data runs + if (header == 0) + break; + + // Extract length and offset byte counts from header + var lengthBytes = header & 0x0F; + var offsetBytes = (header >> 4) & 0x0F; + + if (lengthBytes == 0 || lengthBytes > 8 || offsetBytes > 8) + throw new InvalidDataException($"Invalid data run header: 0x{header:X2}"); + + // Read the run length (number of clusters) + long runLength = 0; + for (var i = 0; i < lengthBytes; i++) + runLength |= (long)recordData[offset++] << (i * 8); + + // Read the run offset (signed, relative to previous run) + long runOffsetValue = 0; + if (offsetBytes > 0) + { + // Check if the offset is negative (sign bit is set in the last byte) + var isNegative = (recordData[offset + offsetBytes - 1] & 0x80) != 0; + + for (var i = 0; i < offsetBytes; i++) + runOffsetValue |= (long)recordData[offset++] << (i * 8); + + // Sign-extend if negative + if (isNegative) + runOffsetValue |= -1L << (offsetBytes * 8); + } + + // Calculate absolute cluster number + currentCluster += runOffsetValue; + + // A zero offset indicates a sparse run (unallocated extent) + if (runOffsetValue == 0) + runs.Add((0, runLength)); // Sparse run + else + runs.Add((currentCluster, runLength)); + } + + return runs; + } + + /// + /// Parses the data run list from a ReadOnlySpan. + /// + /// The data containing the data runs. + /// The offset to the start of the data run list. + /// A list of (startCluster, clusterCount) tuples. + internal static List<(long startCluster, long clusterCount)> ParseDataRuns(ReadOnlySpan data, int runsOffset) + { + var runs = new List<(long startCluster, long clusterCount)>(); + long currentCluster = 0; + var offset = runsOffset; + + while (offset < data.Length) + { + var header = data[offset++]; + + // End of data runs + if (header == 0) + break; + + // Extract length and offset byte counts from header + var lengthBytes = header & 0x0F; + var offsetBytes = (header >> 4) & 0x0F; + + if (lengthBytes == 0 || lengthBytes > 8 || offsetBytes > 8) + throw new InvalidDataException($"Invalid data run header: 0x{header:X2}"); + + // Read the run length (number of clusters) + long runLength = 0; + for (var i = 0; i < lengthBytes; i++) + runLength |= (long)data[offset++] << (i * 8); + + // Read the run offset (signed, relative to previous run) + long runOffsetValue = 0; + if (offsetBytes > 0) + { + // Check if the offset is negative (sign bit is set in the last byte) + var isNegative = (data[offset + offsetBytes - 1] & 0x80) != 0; + + for (var i = 0; i < offsetBytes; i++) + runOffsetValue |= (long)data[offset++] << (i * 8); + + // Sign-extend if negative + if (isNegative) + runOffsetValue |= -1L << (offsetBytes * 8); + } + + // Calculate absolute cluster number + currentCluster += runOffsetValue; + + // A zero offset indicates a sparse run (unallocated extent) + if (runOffsetValue == 0) + runs.Add((0, runLength)); // Sparse run + else + runs.Add((currentCluster, runLength)); + } + + return runs; + } + + /// + /// Applies the fixup array to restore the original sector data in an MFT record. + /// Every NTFS multi-sector structure uses a fixup array for integrity checking. + /// The last two bytes of each 512-byte sector are replaced with a sequence number during write; + /// the original values are stored in the fixup array. + /// + /// The MFT record data to fix up. + /// Thrown if fixup validation fails. + internal static void ApplyFixupArray(byte[] recordData) + { + // Fixup array is at offset 0x04 in the MFT record header: + // 0x04: Fixup offset (2 bytes) - offset to fixup array + // 0x06: Fixup count (2 bytes) - number of entries in fixup array (also number of sectors) + + var fixupOffset = BinaryPrimitives.ReadUInt16LittleEndian(recordData.AsSpan(0x04, 2)); + var fixupCount = BinaryPrimitives.ReadUInt16LittleEndian(recordData.AsSpan(0x06, 2)); + + if (fixupCount == 0) + return; // No fixup needed + + // The fixup array starts with the sequence number (2 bytes), followed by the original values + var sequenceNumber = BinaryPrimitives.ReadUInt16LittleEndian(recordData.AsSpan(fixupOffset, 2)); + + // Apply fixup to each sector + for (var i = 1; i < fixupCount; i++) + { + var sectorOffset = i * 512 - 2; // Last 2 bytes of each 512-byte sector + + // Verify the sequence number matches + var sectorSequence = BinaryPrimitives.ReadUInt16LittleEndian(recordData.AsSpan(sectorOffset, 2)); + if (sectorSequence != sequenceNumber) + throw new InvalidDataException( + $"Fixup sequence number mismatch at sector {i}: expected 0x{sequenceNumber:X4}, got 0x{sectorSequence:X4}."); + + // Restore the original value from the fixup array + var originalValue = BinaryPrimitives.ReadUInt16LittleEndian(recordData.AsSpan(fixupOffset + i * 2, 2)); + BinaryPrimitives.WriteUInt16LittleEndian(recordData.AsSpan(sectorOffset, 2), originalValue); + } + } + + /// + /// Applies the fixup array to restore the original sector data in an MFT record (span version). + /// + /// The MFT record data to fix up. + /// Thrown if fixup validation fails. + internal static void ApplyFixupArray(Span recordData) + { + var fixupOffset = BinaryPrimitives.ReadUInt16LittleEndian(recordData.Slice(0x04, 2)); + var fixupCount = BinaryPrimitives.ReadUInt16LittleEndian(recordData.Slice(0x06, 2)); + + if (fixupCount == 0) + return; + + var sequenceNumber = BinaryPrimitives.ReadUInt16LittleEndian(recordData.Slice(fixupOffset, 2)); + + for (var i = 1; i < fixupCount; i++) + { + var sectorOffset = i * 512 - 2; + + var sectorSequence = BinaryPrimitives.ReadUInt16LittleEndian(recordData.Slice(sectorOffset, 2)); + if (sectorSequence != sequenceNumber) + throw new InvalidDataException( + $"Fixup sequence number mismatch at sector {i}: expected 0x{sequenceNumber:X4}, got 0x{sectorSequence:X4}."); + + var originalValue = BinaryPrimitives.ReadUInt16LittleEndian(recordData.Slice(fixupOffset + i * 2, 2)); + BinaryPrimitives.WriteUInt16LittleEndian(recordData.Slice(sectorOffset, 2), originalValue); + } + } + + /// + /// Builds the cluster allocation bitmap from the bitmap data. + /// + /// A BitArray where each bit indicates whether the corresponding cluster is allocated. + private BitArray BuildAllocationBitmap() + { + var bitmap = new BitArray((int)m_totalClusters); + + for (var i = 0; i < m_totalClusters; i++) + { + var byteIndex = i / 8; + var bitIndex = i % 8; + + if (byteIndex < m_bitmapData.Length) + // NTFS uses LSB-first ordering: bit 0 of byte 0 is cluster 0 + bitmap[i] = (m_bitmapData[byteIndex] & (1 << bitIndex)) != 0; + else + bitmap[i] = false; // Beyond bitmap size, treat as free + } + + return bitmap; + } +} \ No newline at end of file