mstar: add sparse_write and LZO compression support + some fixes

src/formats/mstar.rs

@@ -5,6 +5,13 @@ use lz4::block::decompress;
 use lzma_rs::lzma_decompress;
 
 use crate::common;
+use crate::utils::lzop::{decompress_lzop};
+use crate::utils::sparse::{unsparse_to_file};
+
+//change whether the "userdata" partition is skipped
+// this is because the userdata partition is sometimes enourmous sizes like 27gb, and it will fail to allocate memory on most computers
+// if you want to attempt to extract "userdata" you can enable the option
+static CONFIG_SKIP_USERDATA: bool = true;
 
 pub fn is_mstar_file(file: &File) -> bool {
     let header = common::read_file(&file, 0, 32768).expect("Failed to read from file.");
@@ -80,11 +87,12 @@ pub fn extract_mstar(file: &File, output_folder: &str) -> Result<(), Box<dyn std
                 let size = parse_number(parts[4]).unwrap_or(0);
 
                 //try to get partname from comment
-                let mut partname = if let Some(idx) = line.find('#') {
-                    line[idx + 1..].trim()
-                } else {
-                    "unknown"
-                };
+                //let mut partname = if let Some(idx) = line.find('#') {
+                //    line[idx + 1..].trim()
+                //} else {
+                //    "unknown"
+                //};
+                let mut partname = "unknown";
 
                 let mut compression = "none";
                 let mut lz4_expect_size = 0;
@@ -114,6 +122,14 @@ pub fn extract_mstar(file: &File, output_folder: &str) -> Result<(), Box<dyn std
                             partname = parts[4]
                         }
                     }
+                    if lines[j].starts_with("sparse_write"){
+                        compression = "sparse"; //its not really compression but anyway
+                        let parts: Vec<&str> = lines[j].split_whitespace().collect();
+                        // get part name from sparse_write
+                        if partname == "unknown" {
+                            partname = parts[3]
+                        }
+                    }
 
                     // check if its boot partition
                     if lines[j].starts_with("mmc write.boot") {
@@ -133,13 +149,16 @@ pub fn extract_mstar(file: &File, output_folder: &str) -> Result<(), Box<dyn std
                     j += 1;
                 }
 
-                println!("\nPart: Offset: {}, Size: {} --> {}", offset, size, partname);
+                println!("\nPart - Offset: {}, Size: {} --> {}", offset, size, partname);
 
-                if partname == "unknown"{
+                if partname == "unknown" {
                     println!("- Unknown destination, skipping!");
+                } else if partname == "userdata" && CONFIG_SKIP_USERDATA {
+                    println!("- Skipping userdata according to config!")
                 } else {
                     let data = common::read_file(&file, offset, size.try_into().unwrap())?;
-                    let out_data;
+                    let out_data; 
+                    let output_path = Path::new(&output_folder).join(format!("{}.bin", partname));
 
                     if compression == "lzma" {
                         println!("- Decompressing LZMA...");
@@ -153,15 +172,19 @@ pub fn extract_mstar(file: &File, output_folder: &str) -> Result<(), Box<dyn std
                         println!("- Decompressing lz4, expected size: {}", lz4_expect_size);
                         out_data = decompress_lz4(&data, lz4_expect_size.try_into().unwrap())?;
                     } else if compression == "lzo" {
-                        // nothing in rust to parse lzo file with header
-                        println!("- lzo compression is not supported yet!");
-                        out_data = data;
-                    }else {
+                        println!("- Decompessing LZO..");
+                        out_data = decompress_lzop(&data)?;
+                    } else if compression == "sparse" {
+                        println!("- Unsparsing...");
+                        unsparse_to_file(&data, output_path)?;
+                        println!("-- Saved file!");
+                        i += 1;
+                        continue
+                        //out_data = unsparse(&data)?;
+                    } else {
                         out_data = data;
                     }
 
-                    let output_path = Path::new(&output_folder).join(partname.to_owned() + ".bin");
-
                     fs::create_dir_all(&output_folder)?;
                     let mut out_file = OpenOptions::new()
                         .append(true)

src/utils.rs

@@ -1,3 +1,5 @@
 pub mod aes;
 pub mod pana_dvd_crypto;
-pub mod lzss;
+pub mod lzss;
+pub mod lzop;
+pub mod sparse;

src/utils/lzop.rs

@@ -0,0 +1,73 @@
+use binrw::{BinRead, BinReaderExt};
+use std::io::{Cursor};
+
+use simd_adler32::adler32;
+use crate::common;
+
+#[derive(BinRead)]
+struct LzopHeader {
+    #[br(count = 9)] magic_bytes: Vec<u8>,
+    _version: u16,
+    _lib_version: u16,
+    _version_needed_to_extract: u16,
+    method: u8,
+    _level: u8,
+    _flags: u32,
+    _mode: u32,
+    _mtime_low: u32,
+    _mtime_high: u32,
+    _name_len: u8,
+    #[br(count = _name_len)] _name_bytes: Vec<u8>,
+    _header_checksum: u32,
+}
+
+#[derive(BinRead, Debug)]
+struct SegmentHeader {
+    uncompressed_size: u32,
+    compressed_size: u32,
+    checksum: u32,
+}
+
+pub fn decompress_lzop(data: &[u8]) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
+    let mut data_reader = Cursor::new(data);
+    let header: LzopHeader = data_reader.read_be()?;
+    if header.magic_bytes != b"\x89LZO\x00\x0D\x0A\x1A\x0A" {
+        return Err("Invalid magic!".into());
+    }
+    if ![1, 2, 3].contains(&header.method) {
+        return Err("Unsupported compression method!".into());
+    }
+
+    let lzo = minilzo_rs::LZO::init()?;
+    let mut decompressed_output = Vec::new();
+    loop {
+        if (data.len() as u64 - data_reader.position()) < 12 { //check if there are enough bytes to read a segment header
+            break;
+        }
+        let segment_header: SegmentHeader = data_reader.read_be()?;
+        if segment_header.compressed_size > segment_header.uncompressed_size {
+            println!("{:?}", segment_header);
+        }
+        //println!("{:?}", segment_header);
+        if segment_header.uncompressed_size == 0 {
+            break
+        }
+
+        let stored_data = common::read_exact(&mut data_reader, segment_header.compressed_size as usize)?;
+        let out_data = 
+        if segment_header.uncompressed_size == segment_header.compressed_size { //if uncomp size = comp size, this segment is not compressed
+            stored_data
+        } else {
+            lzo.decompress(&stored_data, segment_header.uncompressed_size as usize)?
+        };
+
+        let calc_checksum = adler32(&out_data.as_slice());
+        if calc_checksum != segment_header.checksum {
+            return Err("Invalid segment checksum! Data corrupted?".into());
+        };
+
+        decompressed_output.extend_from_slice(&out_data);
+    }
+
+    Ok(decompressed_output)
+}

src/utils/sparse.rs

@@ -0,0 +1,61 @@
+use binrw::{BinRead, BinReaderExt};
+use std::fs::OpenOptions;
+use std::path::{PathBuf};
+use std::io::{Cursor, Seek, SeekFrom, Write};
+
+use crate::common;
+
+#[derive(BinRead)]
+struct SparseHeader {
+    #[br(count = 4)] magic_bytes: Vec<u8>,
+    _major_version: u16,
+    _minor_version: u16,
+    _file_header_size: u16,
+    _chunk_header_size: u16,
+    block_size: u32,
+    _total_blocks: u32,
+    total_chunks: u32,
+    _image_checksum: u32
+}
+
+#[derive(BinRead, Debug)]
+struct ChunkHeader {
+    chunk_type: u16,
+    _reserved1: u16,
+    chunk_size: u32,
+    total_size: u32,
+}
+
+pub fn unsparse_to_file(data: &[u8], file_path: PathBuf) -> Result<(), Box<dyn std::error::Error>> {
+    let mut data_reader = Cursor::new(data);
+    let file_header: SparseHeader = data_reader.read_le()?;
+    if file_header.magic_bytes != b"\x3A\xFF\x26\xED" {
+        return Err("Invalid magic!".into());
+    }
+
+    let mut out_file = OpenOptions::new().create(true).read(true).write(true).open(file_path)?;
+
+    for _i in 0..file_header.total_chunks{
+        let chunk_header: ChunkHeader = data_reader.read_le()?;
+        let chunk_data = common::read_exact(&mut data_reader, chunk_header.total_size as usize - 12)?;
+
+        if chunk_header.chunk_type == 0xCAC1 { //"raw" type chunk (actual data)
+            out_file.write_all(&chunk_data)?; 
+        } else if chunk_header.chunk_type == 0xCAC3 { // "dont care" type chunk (skip over)
+            let skip_size = file_header.block_size as u64 * chunk_header.chunk_size as u64;
+            let current_pos = out_file.stream_position()?;
+            let new_pos = current_pos + skip_size;
+
+            //enlarge file with zeros if the seek is larger than file
+            let current_file_size = out_file.metadata()?.len();
+            if new_pos > current_file_size {
+                out_file.set_len(new_pos)?;
+            }
+            
+            out_file.seek(SeekFrom::Start(new_pos))?;
+        }
+    }
+
+    Ok(())
+
+}