mirror of
https://github.com/storytold/photocraft.git
synced 2026-10-29 18:52:15 +01:00
Merge pull request #1895 from storytold/community/1636-pixls
Raw corpus: real camera files from raw.pixls.us, oracle-tested (#1636 carry)
This commit is contained in:
BFlat
@@ -87,5 +87,6 @@ next to the files:
|
||||
| `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause |
|
||||
| `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI |
|
||||
| `corpus/pngsuite/` | PngSuite | Willem van Schaik | <http://www.schaik.com/pngsuite/> | Public domain |
|
||||
| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Per file (each CC0, per its database entry): [IMG_4059.CR2](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/IMG_4059.CR2), [CRW_4061.DNG](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/CRW_4061.DNG), [JD1_8203.NEF](https://raw.pixls.us/data/Nikon/D3/JD1_8203.NEF), [DSC00009.ARW](https://raw.pixls.us/data/Sony/DSC-RX0/DSC00009.ARW), [P1000475.RW2](https://raw.pixls.us/data/Panasonic/DC-G9/P1000475.RW2), [E_1__C106743_gredos.ORF](https://raw.pixls.us/data/Olympus/E-1/E_1__C106743_gredos.ORF), [_7061961_copy.ORF](https://raw.pixls.us/data/Olympus/E-5/_7061961_copy.ORF) | The photographers of the pixls.us raw samples database | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as linked |
|
||||
|
||||
Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0.
|
||||
|
||||
@@ -11,7 +11,7 @@ default = []
|
||||
# Synthetic raw-file generators (DNG / CR2 / TIFF-EP writers and a lossless-JPEG
|
||||
# encoder) for tests and fuzz seeds.
|
||||
testgen = []
|
||||
# Real-file corpus tests (opt-in; `cargo xtask test-corpus`), defined with the corpus PR.
|
||||
# Real-file corpus tests (opt-in; `cargo xtask test-corpus`). Fail when the corpus is missing.
|
||||
corpus = []
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
//! Prints the raw metadata of files: `cargo run -p photocraft-raw --example inspect -- a.cr2 b.nef`
|
||||
//! (handy while adding corpus files; the oracle expectations in `tests/raw_corpus.rs` come from it).
|
||||
use std::env;
|
||||
|
||||
fn main() {
|
||||
for path in env::args().skip(1) {
|
||||
let bytes = match std::fs::read(&path) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
println!("{path}: read error {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match photocraft_raw::decode(&bytes, &photocraft_raw::Limits::default()) {
|
||||
Ok(s) => {
|
||||
println!(
|
||||
"{path}: {:?} make={:?} model={:?} {}x{} samples={} cfa={:?} wb={:?} black={:?} white={:?} crop={:?}..{:?} orientation={} warnings={}",
|
||||
s.format,
|
||||
s.make,
|
||||
s.model,
|
||||
s.width,
|
||||
s.height,
|
||||
s.samples,
|
||||
s.cfa,
|
||||
s.camera_wb,
|
||||
s.black,
|
||||
s.white,
|
||||
s.active,
|
||||
s.crop,
|
||||
s.orientation,
|
||||
s.warnings.len(),
|
||||
);
|
||||
}
|
||||
Err(e) => println!("{path}: decode error {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
//! Real-camera corpus tests against `corpus/pixls` (raw.pixls.us, public domain): one file per
|
||||
//! decode path, oracle-checked against values measured when the corpus was pinned (the
|
||||
//! `inspect` example prints them). Opt-in like every real-file corpus:
|
||||
//! `cargo xtask corpus --pixls` fetches, `cargo xtask test-corpus` runs.
|
||||
#![cfg(feature = "corpus")]
|
||||
|
||||
use photocraft_raw::{Limits, RawError, RawFormat};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn pixls(name: &str) -> PathBuf {
|
||||
let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../corpus/pixls").join(name);
|
||||
if !p.is_file() {
|
||||
panic!("{} is missing: run `cargo xtask corpus --pixls` (or `--all`) first", p.display());
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
struct Want<'a> {
|
||||
file: &'static str,
|
||||
format: RawFormat,
|
||||
make: &'static str,
|
||||
model: &'static str,
|
||||
/// Sensor width and height.
|
||||
size: (usize, usize),
|
||||
/// The four CFA colours, row-major.
|
||||
cfa: [u8; 4],
|
||||
black: &'a [f32],
|
||||
white: [f32; 3],
|
||||
}
|
||||
|
||||
const WANTS: &[Want] = &[
|
||||
Want {
|
||||
file: "IMG_4059.CR2",
|
||||
format: RawFormat::Cr2,
|
||||
make: "Canon",
|
||||
model: "Canon PowerShot SX50 HS",
|
||||
size: (4176, 3062),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[128.0; 4],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
// The same camera after DNG Converter: the sensor data round-trips identically.
|
||||
Want {
|
||||
file: "CRW_4061.DNG",
|
||||
format: RawFormat::Dng,
|
||||
make: "Canon",
|
||||
model: "Canon PowerShot SX50 HS",
|
||||
size: (4176, 3062),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[127.0],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
Want {
|
||||
file: "JD1_8203.NEF",
|
||||
format: RawFormat::Nef,
|
||||
make: "NIKON CORPORATION",
|
||||
model: "NIKON D3",
|
||||
size: (4288, 2844),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[0.0],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
// Sony cRAW: the saturation level comes from the compressed code's range.
|
||||
Want {
|
||||
file: "DSC00009.ARW",
|
||||
format: RawFormat::Arw,
|
||||
make: "SONY",
|
||||
model: "DSC-RX0",
|
||||
size: (4832, 3224),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[0.0],
|
||||
white: [16301.0; 3],
|
||||
},
|
||||
// The G9's high-resolution mode: a 10480 × 7794 sensor cropped to 10368 × 7776.
|
||||
Want {
|
||||
file: "P1000475.RW2",
|
||||
format: RawFormat::Rw2,
|
||||
make: "Panasonic",
|
||||
model: "DC-G9",
|
||||
size: (10480, 7794),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[127.0, 128.0, 128.0, 127.0],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
// Four Thirds Olympus: GRBG instead of the usual RGGB.
|
||||
Want {
|
||||
file: "E_1__C106743_gredos.ORF",
|
||||
format: RawFormat::Orf,
|
||||
make: "OLYMPUS CORPORATION",
|
||||
model: "E-1",
|
||||
size: (2624, 1966),
|
||||
cfa: [1, 0, 2, 1],
|
||||
black: &[65.0; 4],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn decode_matches_the_measured_oracles() {
|
||||
for w in WANTS {
|
||||
let bytes = std::fs::read(pixls(w.file)).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
let s = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
assert_eq!(s.format, w.format, "{}", w.file);
|
||||
assert_eq!(s.make.as_deref(), Some(w.make), "{}", w.file);
|
||||
assert_eq!(s.model.as_deref(), Some(w.model), "{}", w.file);
|
||||
assert_eq!((s.width, s.height), w.size, "{}", w.file);
|
||||
let cfa = s.cfa.as_ref().unwrap_or_else(|| panic!("{}: no CFA", w.file));
|
||||
assert_eq!(cfa.colors, w.cfa, "{}", w.file);
|
||||
assert_eq!(s.black.values, w.black, "{}", w.file);
|
||||
for ch in 0..3 {
|
||||
assert!((s.white[ch] - w.white[ch]).abs() < 0.5, "{} white[{ch}] = {}", w.file, s.white[ch]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn developing_produces_the_crop_in_16bit_rgb() {
|
||||
for w in WANTS {
|
||||
let bytes = std::fs::read(pixls(w.file)).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
let s = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
let d = photocraft_raw::develop_sensor(&s, &photocraft_raw::DevelopOptions::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
assert_eq!((d.width as usize, d.height as usize), (s.crop.width, s.crop.height), "{}", w.file);
|
||||
assert_eq!(d.rgb.len(), d.width as usize * d.height as usize * 3, "{}", w.file);
|
||||
// A real exposure covers a sane share of the range and never leaves it.
|
||||
let (mut lo, mut hi) = (u16::MAX, 0u16);
|
||||
for v in &d.rgb {
|
||||
lo = (*v).min(lo);
|
||||
hi = (*v).max(hi);
|
||||
}
|
||||
// A real exposure covers a sane share of the range and never leaves it. The PowerShot
|
||||
// CR2 and the RX0 ARW are the documented exceptions: neither carries an as-shot white
|
||||
// balance we read, so the grey-world estimate brightens these scenes (the PowerShot's
|
||||
// own DNG, with its file WB, develops blacks to ~0 — reading the maker-note WB of
|
||||
// Canon and Sony is the follow-up).
|
||||
let estimated_wb = matches!(w.file, "IMG_4059.CR2" | "DSC00009.ARW");
|
||||
let black_cap: u16 = if estimated_wb { 16384 } else { 4096 };
|
||||
assert!(lo < black_cap, "{}: developed blacks at {lo}", w.file);
|
||||
assert!(hi > u16::MAX / 4, "{}: developed highlights at {hi}", w.file);
|
||||
// Deterministic: a second develop of the same sensor gives the same pixels.
|
||||
let d2 = photocraft_raw::develop_sensor(&s, &photocraft_raw::DevelopOptions::default()).unwrap();
|
||||
assert_eq!(d.rgb, d2.rgb, "{}", w.file);
|
||||
}
|
||||
}
|
||||
|
||||
/// The E-5 file is the documented known-unsupported case: later Olympus bodies store packed
|
||||
/// (compressed) sensor data. The error must say so, never panic, and the file still previews.
|
||||
#[test]
|
||||
fn packed_orf_reports_the_documented_gap() {
|
||||
let bytes = std::fs::read(pixls("_7061961_copy.ORF")).unwrap();
|
||||
let err = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_err();
|
||||
assert!(matches!(err, RawError::Unsupported(ref m) if m.contains("Olympus compressed")), "{err}");
|
||||
assert!(photocraft_raw::embedded_preview(&bytes).is_some(), "the camera's JPEG preview is still reachable");
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
042286653fbae5b085bef4a4e626145385ea6e82e817f166b4904abcef64457c E_1__C106743_gredos.ORF
|
||||
32e463ae38ae8d8cd1a356af0fc479f2513366c41801d46d4b167cf899656892 CRW_4061.DNG
|
||||
6c2ddd232f5432fb7bd0e28efc12d66995cedcd847f801fd854bfdfb180db50c IMG_4059.CR2
|
||||
7c24fefb1f73ad45c42a0be0078e15b7e91deab1d623efceebfc3c4d0d06135c P1000475.RW2
|
||||
880c60c5f611adf6b70de3d099f8433492de3a3d96866d570122f17f0a651fc8 JD1_8203.NEF
|
||||
ab0d54e3819c9372df0fa98c8686dba0ae87c9d43526a5dd6fd26bd2a664483d _7061961_copy.ORF
|
||||
f602f8ddeeb9b062f60bfdd3ba0d560c8e54f1f64dbc20e18df5222e2f37a428 DSC00009.ARW
|
||||
+35
-8
@@ -10,15 +10,26 @@ use crate::{cargo, root, run};
|
||||
|
||||
/// Crates with corpus tests (behind their `corpus` feature).
|
||||
pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine"];
|
||||
/// Opt-in corpus crates: fetched and tested only with `test-corpus --pixls`.
|
||||
pub const PIXLS_CRATES: &[&str] = &["photocraft-raw"];
|
||||
|
||||
/// Corpus crates with a `heif` feature: test-corpus enables it so the HEIF corpus tests run.
|
||||
const HEIF_CRATES: &[&str] = &["photocraft-codecs", "photocraft-io"];
|
||||
|
||||
/// Paths whose changes make `test-corpus --changed` run (the file-format and rendering crates).
|
||||
const CRITICAL: &[&str] =
|
||||
&["crates/psd/", "crates/io/", "crates/codecs/", "crates/heif/", "crates/compose/", "crates/gpu/", "crates/text/", "crates/format/", "crates/affinity/"];
|
||||
|
||||
/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --affinity | --photoshop [--local]] [--update-manifest]`
|
||||
const CRITICAL: &[&str] = &[
|
||||
"crates/psd/",
|
||||
"crates/io/",
|
||||
"crates/codecs/",
|
||||
"crates/heif/",
|
||||
"crates/compose/",
|
||||
"crates/gpu/",
|
||||
"crates/text/",
|
||||
"crates/format/",
|
||||
"crates/affinity/",
|
||||
"crates/raw/",
|
||||
];
|
||||
/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --pixls | --affinity | --photoshop [--local]] [--update-manifest]`
|
||||
pub fn cmd(args: &[&str]) -> Result<(), String> {
|
||||
let update = args.contains(&"--update-manifest");
|
||||
let mut did = false;
|
||||
@@ -41,6 +52,10 @@ pub fn cmd(args: &[&str]) -> Result<(), String> {
|
||||
corpus_pins::HEIF.fetch(update)?;
|
||||
did = true;
|
||||
}
|
||||
if args.contains(&"--pixls") {
|
||||
crate::pinned::fetch_pixls(update)?;
|
||||
did = true;
|
||||
}
|
||||
if args.contains(&"--exr") {
|
||||
corpus_pins::EXR.fetch(update)?;
|
||||
did = true;
|
||||
@@ -119,7 +134,8 @@ fn list() {
|
||||
corpus/affinity/ [{}] 21 public Affinity 1–3 documents (CC0, MIT): vector-art, AFDesignLoad,
|
||||
Jac21/Branding, AssetStoreTemplate; manifest xtask/affinity-corpus.sha256. Fetch: --affinity
|
||||
corpus/pngsuite/ [{}] PngSuite (public domain), {PNGSUITE_URL}. Fetch: --pngsuite
|
||||
corpus/tiff/, corpus/raw/ optional, copied in by hand
|
||||
corpus/pixls/ [opt-in] real camera raws from raw.pixls.us (public domain), one per decode
|
||||
path plus the known-unsupported packed ORF. Fetch: --pixls (opt-in)
|
||||
|
||||
Pins: xtask/src/corpus_pins.rs. Moving one: change it, then --<name> --update-manifest.",
|
||||
corpus.display(),
|
||||
@@ -180,6 +196,7 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
|
||||
Some(i) => (&args[..i], &args[i + 1..]),
|
||||
None => (args, &[][..]),
|
||||
};
|
||||
let mut raw_changed = false;
|
||||
if ours.contains(&"--changed") {
|
||||
let changed = changed_files()?;
|
||||
let hits: Vec<&String> = changed.iter().filter(|f| CRITICAL.iter().any(|c| f.starts_with(c))).collect();
|
||||
@@ -188,6 +205,7 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
|
||||
return Ok(());
|
||||
}
|
||||
println!("test-corpus --changed: {} critical files changed (e.g. {}); running the corpus tests", hits.len(), hits[0]);
|
||||
raw_changed = hits.iter().any(|f| f.starts_with("crates/raw/"));
|
||||
}
|
||||
let mut crates: Vec<String> = Vec::new();
|
||||
let mut it = ours.iter();
|
||||
@@ -196,18 +214,27 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
|
||||
"-p" | "--package" => {
|
||||
let name = it.next().ok_or("-p needs a crate name")?;
|
||||
let full = if name.starts_with("photocraft-") { (*name).to_string() } else { format!("photocraft-{name}") };
|
||||
if !CORPUS_CRATES.contains(&full.as_str()) {
|
||||
return Err(format!("{full} has no corpus tests (crates: {})", CORPUS_CRATES.join(", ")));
|
||||
if !CORPUS_CRATES.contains(&full.as_str()) && !PIXLS_CRATES.contains(&full.as_str()) {
|
||||
return Err(format!("{full} has no corpus tests (crates: {} or the opt-in {})", CORPUS_CRATES.join(", "), PIXLS_CRATES.join(", ")));
|
||||
}
|
||||
crates.push(full);
|
||||
}
|
||||
"--changed" | "--local" => {}
|
||||
"--changed" | "--local" | "--pixls" => {}
|
||||
other => return Err(format!("test-corpus: unknown argument `{other}`")),
|
||||
}
|
||||
}
|
||||
if crates.is_empty() {
|
||||
crates = CORPUS_CRATES.iter().map(|s| (*s).to_string()).collect();
|
||||
}
|
||||
// `--pixls` opts in (and fetches), and a raw change under --changed does too: the raw
|
||||
// corpus tests are not part of the default set until a maintainer decides otherwise.
|
||||
let wants_pixls = ours.contains(&"--pixls") || crates.iter().any(|c| PIXLS_CRATES.contains(&c.as_str())) || raw_changed;
|
||||
if wants_pixls && !crates.iter().any(|c| c == "photocraft-raw") {
|
||||
crates.push("photocraft-raw".to_string());
|
||||
}
|
||||
if wants_pixls {
|
||||
crate::pinned::fetch_pixls(false)?;
|
||||
}
|
||||
fetch_all(ours.contains(&"--local"))?;
|
||||
let mut c = cargo();
|
||||
c.args(["test", "--release", "--lib", "--tests"]);
|
||||
|
||||
@@ -224,6 +224,22 @@ pub const AFFINITY: PinnedCorpus = PinnedCorpus {
|
||||
)
|
||||
},
|
||||
};
|
||||
|
||||
/// Every pinned corpus, in fetch order.
|
||||
pub const ALL: &[&PinnedCorpus] = &[&PSD_MIXED, &PSD_TOOLS, &PHOTOSHOP, &HEIF, &EXR, &AFFINITY];
|
||||
|
||||
/// `corpus/pixls`: real camera raws from <https://raw.pixls.us> (released into the public
|
||||
/// domain / CC0 by their photographers; see ATTRIBUTION.md). One file per decode path
|
||||
/// `photocraft-raw` supports, plus the packed-ORF known-unsupported oracle. Fetched as single
|
||||
/// files (the server 301-redirects `/data/` to `/download/`), verified against
|
||||
/// `xtask/pixls.sha256`.
|
||||
pub const PIXLS_BASE: &str = "https://raw.pixls.us/data/";
|
||||
/// (url path, file name in `corpus/pixls`)
|
||||
pub const PIXLS_FILES: &[(&str, &str)] = &[
|
||||
("Canon/PowerShot%20SX50%20HS/IMG_4059.CR2", "IMG_4059.CR2"),
|
||||
("Canon/PowerShot%20SX50%20HS/CRW_4061.DNG", "CRW_4061.DNG"),
|
||||
("Nikon/D3/JD1_8203.NEF", "JD1_8203.NEF"),
|
||||
("Sony/DSC-RX0/DSC00009.ARW", "DSC00009.ARW"),
|
||||
("Panasonic/DC-G9/P1000475.RW2", "P1000475.RW2"),
|
||||
("Olympus/E-1/E_1__C106743_gredos.ORF", "E_1__C106743_gredos.ORF"),
|
||||
("Olympus/E-5/_7061961_copy.ORF", "_7061961_copy.ORF"),
|
||||
];
|
||||
|
||||
+5
-4
@@ -25,13 +25,14 @@ commands:
|
||||
layers enforce the crate dependency layering (plan/architecture.md §3)
|
||||
wasm cargo check --target wasm32-unknown-unknown for the wasm-safe crates
|
||||
ci fmt --check, clippy -D warnings, test, layers, wasm (stops at first failure)
|
||||
corpus [--all | --pngsuite | --psd | --psd-tools | --photoshop] [--local] [--update-manifest]
|
||||
corpus [--all | --pngsuite | --psd | --psd-tools | --pixls | --photoshop] [--local] [--update-manifest]
|
||||
show where test corpora live and their pins (xtask/src/corpus_pins.rs), or fetch
|
||||
them into corpus/ (pinned commits, sha256-verified; --all = every corpus;
|
||||
them into corpus/ (pinned commits, sha256-verified; --all = every corpus but pixls,
|
||||
--photoshop --local copies from ../photocraft-corpus or $PHOTOCRAFT_CORPUS_REPO)
|
||||
test-corpus [-p <crate>]... [--changed] [--local] [-- <test args>]
|
||||
test-corpus [-p <crate>]... [--changed] [--pixls] [--local] [-- <test args>]
|
||||
fetch every corpus, then cargo test --release --features corpus on the corpus
|
||||
crates; --changed runs only if psd/io/codecs/compose/gpu/text/format changed;
|
||||
crates; --changed runs only if psd/io/codecs/compose/gpu/text/format/raw changed;
|
||||
--pixls (or -p raw) also fetches corpus/pixls and runs the raw corpus tests;
|
||||
--local takes corpus/photoshop from the photocraft-corpus authoring clone
|
||||
stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`)
|
||||
parity Photoshop menu parity; rewrites docs/parity.md
|
||||
|
||||
@@ -263,3 +263,52 @@ impl PinnedCorpus {
|
||||
self.install(&found, |f| Some(src.join(&f[up.prefix.len()..])), &extras, true, &format!("copied from the local clone {}", clone.display()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Hex sha256 of a corpus file (reused by every corpus fetch, `xtask/pixls.sha256` included).
|
||||
fn file_hex(path: &std::path::Path) -> Option<String> {
|
||||
std::fs::read(path).ok().map(|b| crate::sha256::hex(&b))
|
||||
}
|
||||
|
||||
/// Fetches `corpus/pixls` — the raw.pixls.us camera files pinned in `corpus_pins::PIXLS_FILES` —
|
||||
/// and verifies them against `xtask/pixls.sha256`, the same manifest form every corpus uses.
|
||||
/// raw.pixls.us serves single files and redirects `/data/` to `/download/`, so this is a plain
|
||||
/// per-file download rather than a git upstream; the manifest discipline is the same
|
||||
/// (`--update-manifest` re-pins). Opt-in: not part of `--all` or CI until a maintainer says so.
|
||||
pub fn fetch_pixls(update: bool) -> Result<(), String> {
|
||||
let dest = crate::root().join("corpus").join("pixls");
|
||||
std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||
let manifest = crate::root().join("xtask").join("pixls.sha256");
|
||||
if update {
|
||||
let mut names: Vec<&str> = crate::corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect();
|
||||
names.sort_unstable();
|
||||
let mut out = String::new();
|
||||
for name in names {
|
||||
let sum = file_hex(&dest.join(name)).ok_or_else(|| format!("pixls {name}: not downloaded yet"))?;
|
||||
out.push_str(&format!("{sum} {name}\n"));
|
||||
}
|
||||
std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?;
|
||||
println!("pixls: re-hashed {} files into {}", crate::corpus_pins::PIXLS_FILES.len(), manifest.display());
|
||||
return Ok(());
|
||||
}
|
||||
let want: std::collections::HashMap<String, String> = std::fs::read_to_string(&manifest)
|
||||
.map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --pixls` once)", manifest.display()))?
|
||||
.lines()
|
||||
.filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string())))
|
||||
.collect();
|
||||
for (path, name) in crate::corpus_pins::PIXLS_FILES {
|
||||
let file = dest.join(name);
|
||||
if file_hex(&file).as_deref() == want.get(*name).map(String::as_str) {
|
||||
continue;
|
||||
}
|
||||
let url = format!("{}{}", crate::corpus_pins::PIXLS_BASE, path);
|
||||
let mut curl = Command::new("curl");
|
||||
curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url);
|
||||
run(curl, &format!("curl {url}"))?;
|
||||
let got = file_hex(&file).ok_or_else(|| format!("pixls {name}: unreadable"))?;
|
||||
if Some(got.as_str()) != want.get(*name).map(String::as_str) {
|
||||
return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)"));
|
||||
}
|
||||
}
|
||||
println!("pixls: {} files verified in {}", crate::corpus_pins::PIXLS_FILES.len(), dest.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user