pana_dvd: improve some struct definitions and logic

README.md

@@ -3,6 +3,8 @@ Extractor for various file formats.
 This is a tool that is able to extract the contents of various firmware package formats, mostly from TVs and AV devices.   
 Built in Rust, and made to not depend on any external dependancies, only Rust crates. This way it can work on Windows, Linux and MacOS and even Android.   
 Please note that this project is still under active development and errors may occur. Feel free to make an issue in that case, or if you have any feature request.   
+   
+**PLEASE NOTE** - this program is NOT, and will never be designed for re-packing the extracted files!
 
 # Installation
 You can download the latest auto build for Windows and Linux x86-64 from [here](https://nightly.link/theubusu/unixtract/workflows/rust/main).   

src/formats/pana_dvd/include.rs

@@ -1,8 +1,10 @@
 use crate::utils::common;
-use binrw::BinRead;
+use binrw::{BinRead, BinWrite};
 use super::pana_dvd_crypto::{decrypt_data};
 use crate::utils::aes::{decrypt_aes128_cbc_nopad};
 
+//find key
+
 pub fn find_key<'a>(key_array: &'a [&'a str], data: &[u8], expected_magic: &[u8], magic_offset: usize) -> Result<Option<[u8; 8]>, Box<dyn std::error::Error>> {
     for key_hex in key_array {
         let key_bytes = hex::decode(key_hex)?;
@@ -33,7 +35,7 @@ pub fn find_aes_key_pair<'a>(key_array: &'a [(&'a str, &'a str, &'a str)], data:
     Ok(None)
 }
 
-pub static MAX_HEADER_SIZE: usize = 0x2000;
+// --
 
 #[derive(BinRead)]
 pub struct AesHeaderFileEntry {
@@ -43,24 +45,28 @@ pub struct AesHeaderFileEntry {
 
 pub struct FileEntry {
     pub offset: u32,
-    pub base_offset: u32,
+    pub size: u32,
+    pub header_size: u32,
 }
 
 //checksums are mostly Adler32, but some very old files use Checksum32 instead.
 
-#[derive(BinRead)]
+pub static LIST_SIZE: usize = 0x1800;
+
+#[derive(BinRead, BinWrite)]
 pub struct ModuleEntry {
-    #[br(count = 4)] pub name_bytes: Vec<u8>,
+    name_bytes: [u8; 4],
     version_bytes: [u8; 4],
     _unk: u32,
     pub offset: u32,
-    platform_bytes: [u8; 8],
-    _unk1: u16,
+    model_id_bytes: [u8; 8],
+    pub _oem_id: u8,                //0x4D = Panasonic
+    pub _force_update_flag: u8,     //if not 0x00, force the update and also "Scheduling Format HDD"
     id_bytes: [u8; 6],
     pub size: u32,
-    pub data_checksum: u32, //checksum of the entrys' DATA
-    _unk2: u32,
-    _entry_checksum: u32, //checksum of THIS header entry (all previous 44 bytes)
+    pub data_checksum: u32,         //checksum of the entrys' DATA
+    pub _flags: [u8; 4],            //last byte denotes compression type in some firmwares, see CompressionType
+    pub _entry_checksum: u32,       //checksum of THIS header entry (all previous 44 bytes)
 }
 impl ModuleEntry {
     pub fn name(&self) -> String {
@@ -69,24 +75,24 @@ impl ModuleEntry {
     pub fn version(&self) -> String {
         common::string_from_bytes(&self.version_bytes)
     }
-    pub fn platform(&self) -> String {
-        common::string_from_bytes(&self.platform_bytes)
+    pub fn model_id(&self) -> String {
+        common::string_from_bytes(&self.model_id_bytes)
     }
     pub fn id(&self) -> String {
         common::string_from_bytes(&self.id_bytes)
     }
     pub fn is_valid(&self) -> bool {
-        self.name().is_ascii() && self.platform().is_ascii()
+        self.name().is_ascii() && self.model_id().is_ascii()
     }
 }
 
 #[derive(BinRead)]
 pub struct MainListHeader {
-    _checksum: u32, //checksum of the MAIN LIST
-    _unk: u32, //seems to be always 1?
-    pub list_size: u32,
-    pub decompressed_part_size: u32,
-    _unk2: u32,
+    pub _checksum: u32,                 //checksum of whole list excluding this field
+    _unk: u32,                          //seems to be always 1
+    pub list_size: u32,                 //size of the entire list, including the entries AND this header
+    pub decompressed_part_size: u32,    
+    pub _compression_type: u32,         //see CompressionType
 }
 impl MainListHeader {
     pub fn entry_count(&self) -> u32 {
@@ -105,30 +111,55 @@ pub const COMPRESSED_FILE_MAGIC: &[u8; 8] = b"EXTRHEAD";
 #[derive(BinRead)]
 pub struct CompressedFileHeader {
     _magic_bytes: [u8; 8],      // EXTRHEAD
-    _unk_string: [u8; 4],       // DRV\x20 ?
-    _compressed_flag: u16,      // checks for 1 here, else -> "Error! Not compress"
-    pub compression_type: u16,  // 0 - not compressed, 1 - GZIP , 2 - LZSS
+    _unk_string: [u8; 4],       // "DRV " ?
+    pub _compressed_flag: u16,  // checks for 1 here, else -> "Error! Not compress"
+    pub compression_type: u16,  // see CompressionType. GzipAndLzss is not used in this context.
     pub dest_size: u32,         // decompressed size
-    _dest_address: u32,
+    pub _dest_address: u32,
     pub src_size: u32,          // compressed size
-    _src_address: u32,
-    _footer_offset: u32,        // offset to EXTRFOOT
+    pub _src_address: u32,
+    pub _footer_offset: u32,    // offset to footer, with magic bytes "EXTRFOOT". it aligns data to 8 bytes for encryption.
     _unk: u32,
-    _checksum: u32,             // adler32 calculated every checksum_skip bytes of decompressed data
-    _checksum_skip: u32,
+    pub _checksum: u32,         // adler32 calculated every checksum_skip bytes of decompressed data
+    pub _checksum_skip: u32,
     _unused: [u8; 16],
 }
-impl CompressedFileHeader {
-    pub fn compression_type_str(&self) -> &str {
-        if self.compression_type == 0 {
-            return "Uncompressed"
-        } else if self.compression_type == 1 {
-            return "GZIP"
-        } else if self.compression_type == 2 {
-            return "LZSS"
-        } else {
-            return "Unknown"
+
+#[derive(Debug, PartialEq)]
+pub enum CompressionType {
+    None,
+    Gzip,
+    Lzss,
+    GzipAndLzss,
+    Unknown,
+}
+impl From<u16> for CompressionType {
+    fn from(value: u16) -> Self {
+        match value {
+            0 => CompressionType::None,
+            1 => CompressionType::Gzip,
+            2 => CompressionType::Lzss,
+            3 => CompressionType::GzipAndLzss,
+            _ => CompressionType::Unknown
         }
     }
 }
 
+#[derive(BinRead)]
+pub struct DriveHeader {
+    manufacturer_bytes: [u8; 8],
+    model_bytes: [u8; 16],
+    version_bytes: [u8; 7],
+    _unk: u8,
+}
+impl DriveHeader {
+    pub fn manufacturer(&self) -> String {
+        common::string_from_bytes(&self.manufacturer_bytes)
+    }
+    pub fn model(&self) -> String {
+        common::string_from_bytes(&self.model_bytes)
+    }
+    pub fn version(&self) -> String {
+        common::string_from_bytes(&self.version_bytes)
+    }
+}

src/formats/pana_dvd/mod.rs

@@ -7,7 +7,7 @@ use crate::AppContext;
 
 use std::path::{Path, PathBuf};
 use std::fs::{self, OpenOptions};
-use std::io::{Write, Read, Cursor, Seek, SeekFrom};
+use std::io::{Write, Cursor, Seek, SeekFrom};
 use binrw::BinReaderExt;
 
 use crate::keys;
@@ -64,78 +64,82 @@ pub fn extract_pana_dvd(app_ctx: &AppContext, ctx: Box<dyn Any>) -> Result<(), B
     let mut file = app_ctx.file().ok_or("Extractor expected file")?;
     let context = ctx.downcast::<PanaDvdContext>().expect("Missing context");
 
-    let mut data = Vec::new();  // we need to load the entire file into memory so we can swap it with AES decrypted data if its AES encrypted
-    file.read_to_end(&mut data)?;
-    let mut file_reader = Cursor::new(data);
-
     let matching_key = context.matching_key;
     let mut file_entries: Vec<FileEntry> = Vec::new();
 
+    //AES files can contain multiple firmwares inside of itself
     if context.is_aes {
         let (aes_key, aes_iv) = (context.aes_key.unwrap(), context.aes_iv.unwrap());
         println!("Using key: {} + AES key: {}, IV: {}", hex::encode_upper(matching_key), hex::encode_upper(aes_key), hex::encode_upper(aes_iv));
-        println!("Decrypting AES...\n");
-        let aes_decrypted = decrypt_aes128_cbc_nopad(&file_reader.get_ref(), &aes_key, &aes_iv)?;
-        file_reader = Cursor::new(aes_decrypted); //set the file reader to use AES decrypted stream
 
-        //read file entries in extra header
-        let file_table = common::read_exact(&mut file_reader, 48)?;
-        let mut file_table_reader = Cursor::new(decrypt_data(&file_table, &matching_key));
+        //read inner file table
+        let file_table = common::read_exact(&mut file, 48)?;
+        let file_table_dec_aes = decrypt_aes128_cbc_nopad(&file_table, &aes_key, &aes_iv)?;
+        let mut file_table_reader = Cursor::new(decrypt_data(&file_table_dec_aes, &matching_key));
+        
         for _i in 0..4 {
             let file_entry: AesHeaderFileEntry = file_table_reader.read_le()?;
             if file_entry.size == 0 && file_entry.offset == 0 {
                 break
             }
+            //ignore duplicate entries
             if !file_entries.iter().any(|f| f.offset == file_entry.offset ){
-                file_entries.push(FileEntry { offset: file_entry.offset, base_offset: context.base_hdr_size });
+                file_entries.push(FileEntry { offset: file_entry.offset, size: file_entry.size, header_size: context.base_hdr_size });
             } 
         }
 
     } else {
         println!("Using key: {}", hex::encode_upper(matching_key));
-        file_entries.push(FileEntry { offset: 0, base_offset: context.base_hdr_size });
+        file_entries.push(FileEntry { offset: 0, size: file.metadata()?.len() as u32, header_size: context.base_hdr_size });
     }
 
-    if file_entries.len() == 1 {
-        //only one file, standard extraction
-        println!("File contains no extra sub-files...\n");
-        extract_file(app_ctx, &mut file_reader, file_entries[0].offset as u64, file_entries[0].base_offset as u64, matching_key, &app_ctx.output_dir)?;
-    } else {
-        println!("File contains {} sub-files...", file_entries.len());
-        for (i, file_entry ) in file_entries.iter().enumerate() {
-            println!("\nExtracting file {}/{} - Offset: {}, base: {}", i + 1, file_entries.len(), file_entry.offset, file_entry.base_offset);
-            extract_file(app_ctx, &mut file_reader, file_entry.offset as u64, file_entry.base_offset as u64, matching_key, &app_ctx.output_dir.join(format!("file_{}", i + 1)))?;
-        }
+    println!("File contains {} sub-files...", file_entries.len());
+    for (i, file_entry ) in file_entries.iter().enumerate() {
+        let data = common::read_file(&mut file, file_entries[0].offset as u64, file_entries[0].size as usize)?;
+        let dec_data = if context.is_aes {
+            let (aes_key, aes_iv) = (context.aes_key.unwrap(), context.aes_iv.unwrap());
+            decrypt_aes128_cbc_nopad(&data, &aes_key, &aes_iv)?
+        } else {
+            data
+        };
+
+        let mut file_reader = Cursor::new(dec_data);
+
+        let output_dir = if file_entries.len() == 1 {
+            &app_ctx.output_dir
+        } else {
+            &app_ctx.output_dir.join(format!("file_{}", i + 1))
+        };
+
+        println!("\nExtracting file {}/{} - Offset: {}, Size: {}, Header size: {}", 
+                i + 1, file_entries.len(), file_entry.offset, file_entry.size, file_entry.header_size);
+        
+        extract_file(app_ctx, &mut file_reader, file_entry.header_size as u64, matching_key, output_dir)?;
     }
 
     Ok(())
 }
 
-fn extract_file(app_ctx: &AppContext, file_reader: &mut Cursor<Vec<u8>>, offset: u64, base_offset: u64, key: [u8; 8], output_folder: &PathBuf) -> Result<(), Box<dyn std::error::Error>> {
-    file_reader.seek(SeekFrom::Start(offset + base_offset))?;
- 
-    let enc_header = common::read_exact(file_reader, MAX_HEADER_SIZE)?;
-    let dec_header = decrypt_data(&enc_header, &key);
-    opt_dump_dec_hdr(app_ctx, &dec_header, "header")?;
+fn extract_file(app_ctx: &AppContext, file_reader: &mut Cursor<Vec<u8>>, header_size: u64, key: [u8; 8], output_folder: &PathBuf) -> Result<(), Box<dyn std::error::Error>> { 
+    let enc_list = common::read_exact(file_reader, LIST_SIZE)?;
+    let dec_list = decrypt_data(&enc_list, &key);
+    opt_dump_dec_hdr(app_ctx, &dec_list, "module_list")?;
+
+    let mut list_reader = Cursor::new(dec_list);
+    list_reader.seek(SeekFrom::Start(header_size))?;
 
-    let mut hdr_reader = Cursor::new(dec_header);
     let mut modules: Vec<ModuleEntry> = Vec::new();
 
     for i in 0..100 {
-        let mut entry: ModuleEntry = hdr_reader.read_le()?;
+        let entry: ModuleEntry = list_reader.read_le()?;
         if !entry.is_valid() {break};
-        println!("Module {} - Name: {}, Version: {}, Platform: {}, ID: {}, Offset: {}, Size: {}",
-                i + 1, entry.name(), entry.version(), entry.platform(), entry.id(), entry.offset, entry.size);
+        println!("Module {} - Name: {}, Version: {}, Model ID: {}, ID: {}, Offset: {}, Size: {}",
+                i + 1, entry.name(), entry.version(), entry.model_id(), entry.id(), entry.offset, entry.size);
         if modules.iter().any(|m| m.offset == entry.offset ){
             println!("- Duplicate module, skipping!");
             continue
         }
 
-        //prevent collision of modules with the same name
-        if modules.iter().any(|m| m.name() == entry.name() ){
-            entry.name_bytes = format!("{}({})", entry.name(), i + 1).as_bytes().to_vec();
-        }
-
         modules.push(entry);
     }
 
@@ -145,10 +149,14 @@ fn extract_file(app_ctx: &AppContext, file_reader: &mut Cursor<Vec<u8>>, offset:
         println!("\n({}/{}) - {}, Offset: {}, Size: {}, Checksum: {:#010x}",
                 mod_i, modules.len(), module.name(), module.offset, module.size, module.data_checksum);
 
-        let output_path = Path::new(&output_folder).join(format!("{}.bin", module.name()));
-
-        let rel_offset: u64 = offset + module.offset as u64;
-        file_reader.seek(SeekFrom::Start(rel_offset))?;
+        //if there is multiple modules with the same name, add the module ID to the outptut file to prevent collision
+        let output_path = if modules.iter().filter(|m| m.name() == module.name()).nth(1).is_some() {
+            Path::new(&output_folder).join(format!("{}_{}.bin", mod_i, module.name()))
+        } else {
+            Path::new(&output_folder).join(format!("{}.bin", module.name()))
+        };
+        
+        file_reader.seek(SeekFrom::Start(module.offset as u64))?;
 
         if module.name() == "MAIN" {
             println!("- Extracting MAIN...");
@@ -228,38 +236,43 @@ fn extract_main(file_reader: &mut Cursor<Vec<u8>>, key: [u8; 8], output_path: &P
 fn decompress_data(data: &[u8]) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
     let mut data_reader = Cursor::new(data);
     let header: CompressedFileHeader = data_reader.read_le()?;
-    println!("Compressed size: {}, Decompressed size: {}, Compression type: {}({})", 
-            header.src_size, header.dest_size, header.compression_type, header.compression_type_str());
+    let compression_type = CompressionType::from(header.compression_type);
+
+    println!("Compressed size: {}, Decompressed size: {}, Compression type: {:?}({})", 
+            header.src_size, header.dest_size, compression_type, header.compression_type);
 
     let compressed_data = common::read_exact(&mut data_reader, header.src_size as usize)?;
-    let decompressed_data;
+    let mut decompressed_data;
 
-    if header.compression_type == 1 { //gzip + optionally lzss
+    if compression_type == CompressionType::Gzip {
         println!("- Decompressing GZIP...");
         let (decompressed_gzip, gzip_filename) = decompress_gzip_get_filename(&compressed_data)?;
         if let Some(gzip_filename) = gzip_filename {
             println!("- GZIP filename: {}", gzip_filename);
-        }
-        // the decompressed data can have another header
-        if decompressed_gzip.starts_with(COMPRESSED_FILE_MAGIC) {
-            decompressed_data = decompress_data(&decompressed_gzip)?;
-        } else {
-            decompressed_data = decompressed_gzip;
-        }
-    } else if header.compression_type == 2 { //only lzss
+        }    
+        decompressed_data = decompressed_gzip;
+    
+    } else if compression_type == CompressionType::Lzss {
         println!("- Decompressing LZSS...");
         decompressed_data = decompress_lzss(&compressed_data);
         if decompressed_data.len() != header.dest_size as usize {
             return Err("Decompressed size does not match size in header, decompression failed!".into());
         }
-    } else if header.compression_type == 0 { //no compression. havent encountered one yet
+
+    } else if compression_type == CompressionType::None {
         decompressed_data = compressed_data;
             
+    //GzipAndLzss is not used in this context.
     } else {
         println!("- Unknown compression method!");
         decompressed_data = compressed_data;
     }
 
+    // the decompressed data can have another header
+    if decompressed_data.starts_with(COMPRESSED_FILE_MAGIC) {
+        decompressed_data = decompress_data(&decompressed_data)?;
+    }
+
     Ok(decompressed_data)
 }
 
@@ -275,12 +288,16 @@ fn extract_drv(mut data: Vec<u8>, key: &[u8; 8]) -> Result<Vec<u8>, Box<dyn std:
     let last_decrypted = decrypt_data(&data[data_size as usize - decrypt_size - 48..data_size - 48], &key);
     data[data_size as usize - decrypt_size - 48..data_size - 48].copy_from_slice(&last_decrypted);
 
+    let mut reader = Cursor::new(&data);
+    let header: DriveHeader = reader.read_le()?;
+    println!("- DRIVE info:\n-- Manufacturer ID: {}\n-- Model: {}\n-- Version: {}", header.manufacturer(), header.model(), header.version());
+
     //can be compressed
-    if data[header_size..].starts_with(COMPRESSED_FILE_MAGIC) {
-        let decompressed = decompress_data(&data[header_size..])?;
-        data.truncate(header_size);
-        data.extend_from_slice(&decompressed);
-    }
+    let out_data = if data[header_size..].starts_with(COMPRESSED_FILE_MAGIC) {
+        decompress_data(&data[header_size..])?
+    } else {
+        data[header_size..].to_vec()
+    };
     
-    Ok(data)
+    Ok(out_data)
 }