Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b764320e06 | ||
|
|
a84b375f5d | ||
|
|
52358c5bf9 | ||
|
|
c1e68c27c5 | ||
|
|
d4edff9059 | ||
|
|
80febc9993 | ||
|
|
c500c2fae7 | ||
|
|
0a1b4ec173 | ||
|
|
b8d8f028c9 | ||
|
|
cd56367295 | ||
|
|
40a2d2844f | ||
|
|
f79968f336 | ||
|
|
df43e09eca | ||
|
|
cb0f335e8a | ||
|
|
38d06d3104 | ||
|
|
fe33378333 | ||
|
|
5944cd106b | ||
|
|
fd81bf4cf8 | ||
|
|
d23318cfa6 | ||
|
|
142544c9af | ||
|
|
99a5f2a1bc | ||
|
|
863cad22bd | ||
|
|
c06657c8f9 | ||
|
|
532a80211f | ||
|
|
d06a367b87 | ||
|
|
ce6abe6e23 | ||
|
|
ece3562dc9 | ||
|
|
84d4455682 | ||
|
|
b9a7f2083b | ||
|
|
b36f3e0875 | ||
|
|
e4379184b5 | ||
|
|
5ccb56da2d | ||
|
|
16f589448d | ||
|
|
75ea3c9a1f | ||
|
|
7529190587 | ||
|
|
3feb757821 | ||
|
|
b9f9f84907 | ||
|
|
68ec176719 | ||
|
|
d17c3f4fec | ||
|
|
783a51950f | ||
|
|
51d3d27fd3 | ||
|
|
ccfd22b2a9 | ||
|
|
a5ee329c36 | ||
|
|
2091874050 | ||
|
|
22e8a82fa1 | ||
|
|
01751bf2f0 | ||
|
|
119eb1f5ad | ||
|
|
d3638d786d | ||
|
|
e5a258a648 | ||
|
|
c64369e56c | ||
|
|
f29500c138 | ||
|
|
9d11918bd8 | ||
|
|
8d2bb14f99 | ||
|
|
b53aaa7d03 | ||
|
|
3fa0f30d43 | ||
|
|
9ea848570f | ||
|
|
d6061eb0aa | ||
|
|
1759e0bdd5 | ||
|
|
514b020f74 | ||
|
|
18b0b19a67 | ||
|
|
a8bda9240c | ||
|
|
0692058602 | ||
|
|
8150d16b6f | ||
|
|
c8f238ae60 | ||
|
|
8df49ac053 | ||
|
|
328cd8c664 | ||
|
|
f703a67817 | ||
|
|
105af51100 | ||
|
|
57bf17d348 | ||
|
|
11df30b914 | ||
|
|
43162c40ad | ||
|
|
be57f11747 | ||
|
|
8ba8f69252 | ||
|
|
be97268a7c | ||
|
|
7bd3826178 | ||
|
|
17aa71850c | ||
|
|
20204f0ddc | ||
|
|
1df6a1759c | ||
|
|
b4f7b1d8a9 | ||
|
|
f881828cd8 | ||
|
|
faca7486c1 | ||
|
|
16cfbc170e | ||
|
|
355d736487 | ||
|
|
f201aa5915 | ||
|
|
b69b787d63 | ||
|
|
e630d6697b | ||
|
|
a15e8227b4 | ||
|
|
12563050a6 | ||
|
|
1abd23cf94 | ||
|
|
11ba3814e5 | ||
|
|
bc0ad7d0c3 | ||
|
|
dbe25402c1 | ||
|
|
5400c02b81 | ||
|
|
835863ebe9 | ||
|
|
d0b4438c47 | ||
|
|
7217e6ac15 | ||
|
|
675afe9b71 | ||
|
|
e91d535c36 | ||
|
|
7eaa54d633 | ||
|
|
4e8dd18594 | ||
|
|
b88cbe6bfc | ||
|
|
18cc6dcdc8 |
@@ -0,0 +1,97 @@
|
||||
---
|
||||
name: feature-walkthrough
|
||||
description: >-
|
||||
Explain the full logic and process of the current branch end-to-end so someone
|
||||
with no prior knowledge of the task can understand, review, and reproduce it.
|
||||
Scopes the change from the branch diff, traces the flow across every layer it
|
||||
touches (frontend tool/hook/component, Java controller/service/endpoint, Python
|
||||
engine, config, i18n, tests), and produces a self-contained walkthrough document
|
||||
with Mermaid diagrams (sequence/flow/architecture), annotated file map with
|
||||
clickable references, before/after behavior, screenshots where a UI is involved,
|
||||
a "try it locally" section, and edge cases/risks. Use when asked for a feature or
|
||||
branch walkthrough, "explain what this branch does", a design/logic writeup, PR
|
||||
reviewer onboarding, or a hand-off doc. Pass --html to also emit a rendered HTML
|
||||
version; --no-screens to skip screenshots.
|
||||
argument-hint: "[branch-or-area] [--html] [--no-screens]"
|
||||
allowed-tools: Read, Write, Edit, Glob, Grep, Bash
|
||||
---
|
||||
|
||||
# Feature / Branch Walkthrough
|
||||
|
||||
Turn the current branch into a walkthrough a newcomer can follow. Audience:
|
||||
**someone who has never seen this task**. Explain the *why*, the *flow*, and *how to
|
||||
try it* - not just a diff summary.
|
||||
|
||||
`$ARGUMENTS` may name a branch or area to focus on; default is the current branch
|
||||
vs `main`. Flags: `--html` (also emit a rendered HTML twin), `--no-screens`.
|
||||
|
||||
## Process
|
||||
|
||||
### 1. Scope the change
|
||||
- `git log --oneline main..HEAD` and `git diff --stat main...HEAD` for the shape.
|
||||
- Read the PR description / commit messages for stated intent. Do **not** invent
|
||||
history or motivation that isn't evidenced (state current behavior in present tense).
|
||||
- Classify touched files by layer:
|
||||
- **Frontend**: tools (`frontend/editor/src/core/components/tools/*` or `.../core/tools/*`),
|
||||
hooks (`core/hooks/tools/*`, `useToolOperation`), contexts, routes, i18n
|
||||
(`public/locales/en-US`).
|
||||
- **Java backend**: controllers (`.../controller/api/...`), services, models, config.
|
||||
- **Engine**: `engine/src/stirling/{agents,contracts,api,services}`.
|
||||
- **Config / build / docker / tests.**
|
||||
|
||||
### 2. Trace the flow end-to-end
|
||||
Follow one real path from user action to result. For a typical PDF tool that's:
|
||||
UI control → `useToolOperation` hook → `POST /api/v1/...` → Spring controller →
|
||||
service (PDFBox / LibreOffice / engine call) → response → review panel → download.
|
||||
Read the actual files so the narrative is true to the code, and collect the exact
|
||||
file:line anchors you'll cite.
|
||||
|
||||
### 3. Draw the diagrams (Mermaid)
|
||||
Pick what fits; usually 2-3 of:
|
||||
- **Sequence diagram** - request/response across frontend → backend → engine.
|
||||
- **Flowchart** - the core decision/branching logic of the feature.
|
||||
- **Architecture/component** - new pieces and how they wire to existing ones.
|
||||
- **State** - if the feature has modes/steps.
|
||||
Keep nodes labeled in plain language. Validate the Mermaid parses before shipping.
|
||||
|
||||
### 4. Screenshots (unless --no-screens)
|
||||
If a UI is involved, capture key states with the stubbed Playwright harness
|
||||
(see the **ui-walkthrough** skill and `files-page-screenshots.spec.ts` for the
|
||||
pattern) or, for before/after, capture `main` then the branch. Drop PNGs in
|
||||
`walkthrough/<feature>/` and reference them from the doc. For backend-only
|
||||
changes, show request/response examples (curl + JSON) instead.
|
||||
|
||||
### 5. Write the walkthrough
|
||||
Create `walkthrough/<feature>/FEATURE-WALKTHROUGH.md` with:
|
||||
1. **TL;DR** - what the branch does and who it's for, in 3-4 sentences.
|
||||
2. **Problem & approach** - what wasn't possible before; the chosen solution.
|
||||
3. **Architecture diagram** + 1-paragraph orientation.
|
||||
4. **End-to-end flow** - the sequence diagram + a numbered walk of each step,
|
||||
each citing the real file (clickable `path:line`).
|
||||
5. **Key files** - annotated map (path → one line on its role).
|
||||
6. **Logic deep-dive** - the flowchart + prose for the non-obvious decisions.
|
||||
7. **Behavior** - before vs after; screenshots or request/response examples.
|
||||
8. **Try it locally** - exact steps (`task dev` / `task dev:all`, the route to
|
||||
open or the curl to run, any env like `DOCKER_ENABLE_SECURITY` or a test
|
||||
license key). Make it copy-pasteable.
|
||||
9. **Edge cases, risks, follow-ups** - what's untested, known limits, gotchas.
|
||||
|
||||
Markdown is the primary deliverable - it renders with diagrams in GitHub PRs and
|
||||
IDEs, no build step, ideal for review.
|
||||
|
||||
### 6. If `--html`
|
||||
Also emit `walkthrough/<feature>/walkthrough.html`: the same content with Mermaid
|
||||
rendered via `mermaid.initialize({startOnLoad:true})` (script from CDN; note in
|
||||
the file that rendering diagrams needs network, the `.md` is the offline copy) and
|
||||
screenshots inline. Keep it self-contained otherwise.
|
||||
|
||||
### 7. Deliver
|
||||
Give the doc path and a short chat summary. Offer to `SendUserFile` it.
|
||||
|
||||
## Principles
|
||||
- **True to the code.** Every claim traces to a file you read; cite `path:line`.
|
||||
No fabricated migration/version history.
|
||||
- **Newcomer-first.** Define repo-specific terms (FileContext, `useToolOperation`,
|
||||
the `@app/*` layer cascade, stubbed vs live tests) on first use.
|
||||
- **Show, don't assert.** Prefer a diagram + a real example over adjectives.
|
||||
- Don't commit the `walkthrough/` output unless asked.
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
name: ui-before-after
|
||||
description: >-
|
||||
Analyse a branch or PR and automatically capture before/after screenshots of
|
||||
every UI surface its changes touch, then pixel-diff the pairs to surface what
|
||||
actually changed and assemble PR-ready before/after montage images. Generic and
|
||||
diff-driven: it derives the capture targets from the diff (changed tools/routes →
|
||||
URLs) instead of hand-listing screens, captures "before" from the base branch and
|
||||
"after" from the head, then keeps only the views that visually differ. Each
|
||||
comparison is auto-cropped to the region that actually changed (the bounding box of
|
||||
differing pixels), falling back to the full page only when the change spans most of
|
||||
it. Use for before/after shots, a visual diff of a branch/PR, "screenshots for the
|
||||
PR description", "show what changed in the UI", or a side-by-side of UI changes.
|
||||
Takes a PR number/URL (resolved via gh) or a branch; defaults to the current branch
|
||||
vs its base. Flags: --scope <selector>, --base <ref|merge-base>, --theme
|
||||
light|dark|both, --all (capture every route, not just changed), --no-autocrop,
|
||||
--pagewide <n>, --threshold <n>.
|
||||
argument-hint: "[PR# | PR-url | branch] [--scope <sel>] [--base <ref>] [--theme both] [--all] [--no-autocrop]"
|
||||
allowed-tools: Read, Write, Edit, Glob, Grep, Bash
|
||||
---
|
||||
|
||||
# UI Before / After (generic visual diff)
|
||||
|
||||
Point it at a branch or PR; it figures out which UI changed, screenshots every
|
||||
affected surface **before** (base) and **after** (head), pixel-diffs the pairs, and
|
||||
montages the ones that actually changed into images for the PR description.
|
||||
|
||||
`$ARGUMENTS`: a PR number/URL, a branch, or nothing (current branch vs base).
|
||||
By default it captures the full viewport and auto-crops each comparison to the region
|
||||
that changed. Flags: `--scope <css>` (narrow the *capture* to a container, e.g.
|
||||
`[data-sidebar="tool-panel"]`, when you already know where the change is),
|
||||
`--no-autocrop` (keep full frames), `--pagewide <fraction>` (above this share of the
|
||||
page, skip cropping; default 0.6), `--base <ref|merge-base>`,
|
||||
`--theme light|dark|both`, `--all` (walk every route, not just changed),
|
||||
`--threshold <fraction>` (diff sensitivity, default 0.001).
|
||||
|
||||
Shares the capture harness with **ui-walkthrough** - read its SKILL.md for the
|
||||
stubbed-Playwright setup, worktree node_modules + `generate-icons`, the
|
||||
stale-`:5173` gotcha, and the dark-mode init-script. Bundled helpers:
|
||||
[capture-spec.template.ts](capture-spec.template.ts), [diff-shots.mjs](diff-shots.mjs),
|
||||
[montage-template.html](montage-template.html), [shoot-sections.mjs](shoot-sections.mjs).
|
||||
|
||||
## Process
|
||||
|
||||
### 1. Resolve target + base
|
||||
```
|
||||
gh pr view <pr> --json number,title,headRefName,baseRefName,url,files # PR
|
||||
# or branch: base = merge-base(main, HEAD); head = HEAD
|
||||
gh pr diff <pr> --name-only # or: git diff --name-only <base>...HEAD
|
||||
```
|
||||
|
||||
### 2. Derive capture targets from the diff (the "analyse" step - no hand-listing)
|
||||
Map changed frontend files to URLs generically:
|
||||
- **Tools**: a changed `components/tools/<toolDir>/…` or `hooks/tools/<tool>/…` →
|
||||
toolId → URL via the repo's own rule `getToolUrlPath` in
|
||||
[toolsTaxonomy.ts:200](frontend/editor/src/core/data/toolsTaxonomy.ts): `/` + the
|
||||
id kebab-cased (`addPageNumbers` → `/add-page-numbers`).
|
||||
- **Pages/routes**: changed `filesPage/*` → `/files`, etc.
|
||||
- `--all`: enumerate every tool in the registry instead of just changed ones.
|
||||
Write `frontend/editor/screenshots/ui-diff/targets.json` =
|
||||
`[{ "id":"compress", "url":"/compress", "name":"Compress" }]`. This is what makes
|
||||
it generic - the spec never names a tool.
|
||||
|
||||
### 3. Capture AFTER (head) then BEFORE (base)
|
||||
Copy [capture-spec.template.ts](capture-spec.template.ts) →
|
||||
`src/core/tests/stubbed/ui-before-after.spec.ts` (it loops `targets.json`, seeds a
|
||||
sample PDF so file-dependent panels render, navigates to each URL, and screenshots
|
||||
the full viewport - or the `--scope` container if given). Ensure the harness is ready
|
||||
(node_modules + icons).
|
||||
```
|
||||
# after = current head
|
||||
cd frontend/editor && PR_SHOT_SIDE=after PR_SHOT_THEME=light \
|
||||
npx playwright test --project=stubbed ui-before-after.spec.ts
|
||||
# before = base, in an isolated worktree (copy the spec + targets.json in)
|
||||
git worktree add ../ba-base origin/<baseRefName> # or the merge-base
|
||||
# set up its frontend, copy spec + screenshots/ui-diff/targets.json across, then:
|
||||
cd ../ba-base/frontend/editor && PR_SHOT_SIDE=before PR_SHOT_THEME=light \
|
||||
npx playwright test --project=stubbed ui-before-after.spec.ts
|
||||
# copy its screenshots/ui-diff/before/ back next to after/. Repeat with
|
||||
# PR_SHOT_THEME=dark if --theme includes dark. Remove worktree when done.
|
||||
```
|
||||
|
||||
### 4. Auto-diff (surface what changed)
|
||||
```
|
||||
cd frontend/editor && node <skill>/diff-shots.mjs \
|
||||
screenshots/ui-diff/before screenshots/ui-diff/after screenshots/ui-diff
|
||||
```
|
||||
Produces `diff-report.json` classifying each view `unchanged | changed | added |
|
||||
removed`. For each changed view it computes the bounding box of differing pixels and
|
||||
writes cropped `__before_crop.png` / `__after_crop.png` / `__diff.png` to that region
|
||||
(+ padding) - **unless** the change covers more than `--pagewide` of the frame, where
|
||||
it keeps the full frame (`pageWide:true`). Drop `unchanged` - that's the noise the
|
||||
user doesn't want.
|
||||
|
||||
### 5. Montage the changes
|
||||
Build the manifest from the non-unchanged entries (group by tab/tool; each becomes a
|
||||
state row with before/after). For changed views use the cropped `cropBefore` /
|
||||
`cropAfter` from `diff-report.json` (tight on the affected region; full frame when
|
||||
`pageWide`); `added`/`removed` render the "not present" placeholder. Fill
|
||||
[montage-template.html](montage-template.html) (replace the `window.__BA__` data
|
||||
block; base64-inline the PNGs for portability), then render one PNG per section with
|
||||
[shoot-sections.mjs](shoot-sections.mjs). Optionally include the `__diff.png` overlay
|
||||
as a third column.
|
||||
|
||||
### 6. Deliver
|
||||
Output the `montage_<tab>.png` files + a short summary (N changed / added / removed,
|
||||
M unchanged skipped) and a paste-ready Markdown block. GitHub has no PR-body image
|
||||
API, so tell the user to drag the PNGs into the description. Do **not** post to the
|
||||
PR.
|
||||
|
||||
## Gotchas
|
||||
- Two installs (base worktree + head); junction main's node_modules only if its deps
|
||||
match that ref, else `npm ci` (see ui-walkthrough's stale-dep note).
|
||||
- A view that errors on one side (refactored/removed) → that side is missing; the
|
||||
diff marks it added/removed rather than failing the run.
|
||||
- Pixel diff needs equal dimensions, so capture at a fixed viewport (the template
|
||||
does); a view whose size changed is reported as "changed (dimensions differ)",
|
||||
uncropped.
|
||||
- Auto-crop uses a single bounding box, so two far-apart changes give one large crop
|
||||
(or trip `--pagewide`); narrow with `--scope` if that happens.
|
||||
- `getToolUrlPath` is the source of truth for tool URLs - use it, don't guess slugs.
|
||||
- Don't commit `screenshots/`, the throwaway spec, or the base worktree.
|
||||
@@ -0,0 +1,67 @@
|
||||
// Generic before/after capturer. NOT app-specific: it walks a targets.json that
|
||||
// the ui-before-after skill generates from the branch/PR diff, so nothing here is
|
||||
// hand-listed. Copy to src/core/tests/stubbed/ui-before-after.spec.ts, then run
|
||||
// once per (side, theme):
|
||||
// PR_SHOT_SIDE=after PR_SHOT_THEME=light \
|
||||
// npx playwright test --project=stubbed ui-before-after.spec.ts
|
||||
//
|
||||
// targets.json shape: [{ "id":"compress", "url":"/compress", "name":"Compress",
|
||||
// "needsFile": true }]
|
||||
import { test } from "@app/tests/helpers/stub-test-base";
|
||||
import type { Page } from "@playwright/test";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const SIDE = process.env.PR_SHOT_SIDE ?? "after";
|
||||
const THEME = process.env.PR_SHOT_THEME ?? "light";
|
||||
// Capture the full viewport by default so the affected region is in frame
|
||||
// wherever it is; diff-shots.mjs crops each comparison to what actually changed.
|
||||
// Set PR_SHOT_SCOPE to a selector to narrow the capture to one container.
|
||||
const SCOPE = process.env.PR_SHOT_SCOPE ?? "";
|
||||
const ROOT = path.resolve(process.cwd(), "screenshots", "ui-diff");
|
||||
const OUT = path.join(ROOT, SIDE);
|
||||
// A tiny sample PDF so file-dependent tool panels render. Point at a real fixture.
|
||||
const SAMPLE_PDF = process.env.PR_SHOT_SAMPLE ?? "src/core/tests/test-fixtures/sample.pdf";
|
||||
|
||||
type Target = { id: string; url: string; name?: string; needsFile?: boolean };
|
||||
const targets: Target[] = JSON.parse(fs.readFileSync(path.join(ROOT, "targets.json"), "utf-8"));
|
||||
|
||||
test.use({ autoGoto: false, viewport: { width: 1600, height: 900 }, seedJwt: true });
|
||||
|
||||
async function applyTheme(page: Page): Promise<void> {
|
||||
if (THEME !== "dark") return;
|
||||
await page.addInitScript(() => {
|
||||
localStorage.setItem("mantine-color-scheme", "dark");
|
||||
localStorage.setItem("mantine-color-scheme-value", "dark");
|
||||
});
|
||||
await page.emulateMedia({ colorScheme: "dark" });
|
||||
}
|
||||
|
||||
async function seedFile(page: Page): Promise<void> {
|
||||
if (!fs.existsSync(SAMPLE_PDF)) return;
|
||||
await page.goto("/", { waitUntil: "domcontentloaded" });
|
||||
await page.getByTestId("files-button").click().catch(() => {});
|
||||
await page.locator('[data-testid="file-input"]').setInputFiles(SAMPLE_PDF).catch(() => {});
|
||||
await page.locator(".file-sidebar-file-item").first().isVisible({ timeout: 8_000 }).catch(() => {});
|
||||
}
|
||||
|
||||
for (const t of targets) {
|
||||
// One test per target so a single failure doesn't drop the rest.
|
||||
test(`${SIDE}/${THEME} ${t.id}`, async ({ page }) => {
|
||||
fs.mkdirSync(OUT, { recursive: true });
|
||||
await applyTheme(page);
|
||||
if (t.needsFile !== false) await seedFile(page);
|
||||
await page.goto(t.url, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForTimeout(400); // settle Mantine portals/transitions
|
||||
const shot = path.join(OUT, `${t.id}__${THEME}.png`);
|
||||
if (SCOPE) {
|
||||
const scope = page.locator(SCOPE).first();
|
||||
if (await scope.isVisible({ timeout: 8_000 }).catch(() => false)) {
|
||||
await scope.screenshot({ path: shot });
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Full viewport (fixed size → stable dimensions for pixel diffing).
|
||||
await page.screenshot({ path: shot });
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
// Auto-diff before/ vs after/ screenshots, classify each as
|
||||
// unchanged | changed | added | removed, and CROP each changed pair to the
|
||||
// affected region (bounding box of differing pixels + padding) - unless the
|
||||
// change spans most of the page, in which case the full frame is kept.
|
||||
// Run from frontend/editor (so deps resolve):
|
||||
// node <skill>/diff-shots.mjs <beforeDir> <afterDir> [outDir]
|
||||
// Env:
|
||||
// DIFF_THRESHOLD min fraction of differing pixels to count as changed (default 0.001)
|
||||
// DIFF_PAD padding px around the affected region (default 24)
|
||||
// DIFF_PAGEWIDE if affected bbox area / image area exceeds this, keep full frame (default 0.6)
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(path.join(process.cwd(), "noop.js"));
|
||||
const pm = require("pixelmatch");
|
||||
const pixelmatch = pm.default || pm;
|
||||
const { PNG } = require("pngjs");
|
||||
|
||||
const beforeDir = path.resolve(process.argv[2]);
|
||||
const afterDir = path.resolve(process.argv[3]);
|
||||
const outDir = path.resolve(process.argv[4] || afterDir);
|
||||
const THRESHOLD = Number(process.env.DIFF_THRESHOLD ?? "0.001");
|
||||
const PAD = Number(process.env.DIFF_PAD ?? "24");
|
||||
const PAGEWIDE = Number(process.env.DIFF_PAGEWIDE ?? "0.6");
|
||||
|
||||
const read = (p) => PNG.sync.read(fs.readFileSync(p));
|
||||
const isShot = (f) => f.endsWith(".png") && !/__(diff|before_crop|after_crop)\.png$/.test(f);
|
||||
const list = (d) => (fs.existsSync(d) ? fs.readdirSync(d).filter(isShot) : []);
|
||||
const names = [...new Set([...list(beforeDir), ...list(afterDir)])].sort();
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
function cropPNG(src, x, y, w, h) {
|
||||
const out = new PNG({ width: w, height: h });
|
||||
PNG.bitblt(src, out, x, y, w, h, 0, 0);
|
||||
return out;
|
||||
}
|
||||
const writePNG = (p, png) => fs.writeFileSync(p, PNG.sync.write(png));
|
||||
|
||||
// Bounding box of differing pixels using a diff mask (alpha>0 where changed).
|
||||
function changedBBox(before, after, w, h) {
|
||||
const mask = new PNG({ width: w, height: h });
|
||||
pixelmatch(before.data, after.data, mask.data, w, h, { threshold: 0.1, diffMask: true });
|
||||
let minX = w, minY = h, maxX = -1, maxY = -1, count = 0;
|
||||
for (let y = 0; y < h; y++) {
|
||||
for (let x = 0; x < w; x++) {
|
||||
if (mask.data[(y * w + x) * 4 + 3] > 0) {
|
||||
count++;
|
||||
if (x < minX) minX = x; if (x > maxX) maxX = x;
|
||||
if (y < minY) minY = y; if (y > maxY) maxY = y;
|
||||
}
|
||||
}
|
||||
}
|
||||
return maxX < 0 ? null : { minX, minY, maxX, maxY, count };
|
||||
}
|
||||
|
||||
const report = [];
|
||||
for (const name of names) {
|
||||
const id = name.replace(/\.png$/, "");
|
||||
const bp = path.join(beforeDir, name), ap = path.join(afterDir, name);
|
||||
const hasB = fs.existsSync(bp), hasA = fs.existsSync(ap);
|
||||
if (hasB && !hasA) { report.push({ id, status: "removed", before: bp }); continue; }
|
||||
if (!hasB && hasA) { report.push({ id, status: "added", after: ap }); continue; }
|
||||
|
||||
const before = read(bp), after = read(ap);
|
||||
if (before.width !== after.width || before.height !== after.height) {
|
||||
report.push({ id, status: "changed", note: "dimensions differ", before: bp, after: ap });
|
||||
continue;
|
||||
}
|
||||
const w = after.width, h = after.height;
|
||||
const overlay = new PNG({ width: w, height: h });
|
||||
const px = pixelmatch(before.data, after.data, overlay.data, w, h, { threshold: 0.1 });
|
||||
const ratio = px / (w * h);
|
||||
if (ratio <= THRESHOLD) { report.push({ id, status: "unchanged", ratio: Number(ratio.toFixed(5)), before: bp, after: ap }); continue; }
|
||||
|
||||
const box = changedBBox(before, after, w, h);
|
||||
// Pad + clamp the affected region.
|
||||
const x = Math.max(0, box.minX - PAD), y = Math.max(0, box.minY - PAD);
|
||||
const x2 = Math.min(w, box.maxX + 1 + PAD), y2 = Math.min(h, box.maxY + 1 + PAD);
|
||||
const bw = x2 - x, bh = y2 - y;
|
||||
const pageWide = (bw * bh) / (w * h) > PAGEWIDE;
|
||||
|
||||
const entry = { id, status: "changed", ratio: Number(ratio.toFixed(5)), before: bp, after: ap, pageWide };
|
||||
if (pageWide) {
|
||||
// Change spans most of the page - keep the full frame, full overlay.
|
||||
const dp = path.join(outDir, `${id}__diff.png`); writePNG(dp, overlay);
|
||||
entry.diff = dp;
|
||||
} else {
|
||||
entry.bbox = { x, y, w: bw, h: bh };
|
||||
const cb = path.join(outDir, `${id}__before_crop.png`); writePNG(cb, cropPNG(before, x, y, bw, bh));
|
||||
const ca = path.join(outDir, `${id}__after_crop.png`); writePNG(ca, cropPNG(after, x, y, bw, bh));
|
||||
const dp = path.join(outDir, `${id}__diff.png`); writePNG(dp, cropPNG(overlay, x, y, bw, bh));
|
||||
entry.cropBefore = cb; entry.cropAfter = ca; entry.diff = dp;
|
||||
}
|
||||
report.push(entry);
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(outDir, "diff-report.json"), JSON.stringify(report, null, 2));
|
||||
const changed = report.filter((r) => r.status !== "unchanged");
|
||||
console.log(`diffed ${report.length} view(s): ${changed.length} changed/added/removed, ${report.length - changed.length} unchanged`);
|
||||
for (const r of changed) {
|
||||
const tail = r.status !== "changed" ? ""
|
||||
: r.pageWide ? " (page-wide → full frame)"
|
||||
: ` (${(r.ratio * 100).toFixed(2)}%, cropped to ${r.bbox.w}×${r.bbox.h})`;
|
||||
console.log(` ${r.status.padEnd(9)} ${r.id}${tail}${r.note ? " - " + r.note : ""}`);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Build EXAMPLE.html from montage-template.html using REAL files-page shots as
|
||||
stand-in before/after pairs (layout demo, not an actual PR diff). Inlines PNGs as
|
||||
data URIs so the HTML is portable. Run: python make_example.py"""
|
||||
import base64
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
|
||||
HERE = pathlib.Path(__file__).parent
|
||||
SHOTS = pathlib.Path(
|
||||
r"C:\Users\systo\git\Stirling-PDFNew\.claude\worktrees\kind-faraday-522a30"
|
||||
r"\frontend\editor\screenshots\files-page"
|
||||
)
|
||||
|
||||
|
||||
def uri(fname):
|
||||
p = SHOTS / fname
|
||||
return "data:image/png;base64," + base64.b64encode(p.read_bytes()).decode() if p.exists() else None
|
||||
|
||||
|
||||
data = {
|
||||
"pr": "DEMO",
|
||||
"title": "EXAMPLE — before/after montage (layout demo, real Files-page shots; not a real PR diff)",
|
||||
"base": "main", "head": "demo-branch",
|
||||
"cropSelector": "[data-sidebar=\"tool-panel\"] (real runs crop to the side; these demo shots are full-page)",
|
||||
"tabs": [
|
||||
{"id": "files", "title": "Files page", "ctx": "Each row = one flow state; left = base branch, right = this PR.",
|
||||
"states": [
|
||||
{"name": "Empty folder", "before": uri("01_empty_state_ctas.png"), "after": uri("02_empty_state_storage_off.png")},
|
||||
{"name": "Files + details panel", "before": uri("03_subtoolbar_with_files.png"), "after": uri("06_details_panel_save_to_server.png")},
|
||||
{"name": "Delete folder confirm", "before": None, "after": uri("19_delete_folder_dialog.png"), "note": "New in this PR"},
|
||||
]},
|
||||
{"id": "move", "title": "Move-to-folder dialog",
|
||||
"states": [
|
||||
{"name": "Dialog opened", "before": uri("07_move_dialog_collapsed.png"), "after": uri("08_move_dialog_create_folder_expanded.png")},
|
||||
{"name": "After folder created", "before": None, "after": uri("08b_move_dialog_after_create_folder.png"), "note": "New flow"},
|
||||
]},
|
||||
],
|
||||
}
|
||||
|
||||
tpl = (HERE / "montage-template.html").read_text(encoding="utf-8")
|
||||
out = re.sub(
|
||||
r"/\*__DATA__\*/.*?/\*__END__\*/",
|
||||
lambda _m: "/*__DATA__*/" + json.dumps(data) + "/*__END__*/",
|
||||
tpl, count=1, flags=re.S,
|
||||
)
|
||||
(HERE / "EXAMPLE.html").write_text(out, encoding="utf-8")
|
||||
print("wrote", HERE / "EXAMPLE.html", "(", (HERE / "EXAMPLE.html").stat().st_size // 1024, "KB )")
|
||||
@@ -0,0 +1,106 @@
|
||||
<!doctype html>
|
||||
<!--
|
||||
Before/After montage for a PR description. The ui-before-after skill replaces
|
||||
the JSON in the window.__BA__ data block below with the captured manifest, then
|
||||
screenshots each .tab-section (id="section-<tabId>") into a PNG to drag into the
|
||||
PR description. Self-contained; images may be relative paths or data URIs.
|
||||
|
||||
Data shape:
|
||||
{
|
||||
"pr":"6552","title":"...","base":"main","head":"feat/x",
|
||||
"cropSelector":"[data-sidebar=\"tool-panel\"]",
|
||||
"tabs":[
|
||||
{ "id":"sign","title":"Sign tool","states":[
|
||||
{"name":"Initial","before":"before/sign__initial.png","after":"after/sign__initial.png"},
|
||||
{"name":"Cert selected","before":null,"after":"after/sign__cert.png","note":"New in this PR"}
|
||||
]}
|
||||
]
|
||||
}
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Before / After</title>
|
||||
<style>
|
||||
:root { --bg:#ffffff; --ink:#0b0c0e; --muted:#6b7280; --line:#e5e7eb;
|
||||
--before:#6b7280; --after:#1f883d; --frame:#f3f4f6; --note:#b45309; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--ink);
|
||||
font:14px/1.5 -apple-system,"Segoe UI",Roboto,system-ui,sans-serif; }
|
||||
.wrap { max-width:1100px; margin:0 auto; padding:24px; }
|
||||
.doc-head { margin-bottom:8px; }
|
||||
.doc-head h1 { font-size:18px; margin:0 0 2px; }
|
||||
.doc-head .sub { color:var(--muted); font-size:12.5px; }
|
||||
.legend { display:flex; gap:14px; align-items:center; margin:10px 0 4px; font-size:12px; color:var(--muted); }
|
||||
.chip { font-size:10px; font-weight:700; letter-spacing:.04em; text-transform:uppercase;
|
||||
padding:2px 8px; border-radius:999px; color:#fff; }
|
||||
.chip.before { background:var(--before); } .chip.after { background:var(--after); }
|
||||
|
||||
.tab-section { border:1px solid var(--line); border-radius:14px; padding:18px 18px 8px;
|
||||
margin:18px 0; background:var(--bg); }
|
||||
.tab-section > h2 { font-size:16px; margin:0 0 2px; }
|
||||
.tab-section > .ctx { color:var(--muted); font-size:12px; margin-bottom:14px; }
|
||||
.state { margin-bottom:18px; }
|
||||
.state .name { font-weight:600; font-size:13.5px; margin-bottom:8px; display:flex; gap:8px; align-items:center; }
|
||||
.state .name .note { font-weight:500; color:var(--note); font-size:12px; }
|
||||
.pair { display:grid; grid-template-columns:1fr 1fr; gap:14px; align-items:start; }
|
||||
.cell { border:1px solid var(--line); border-radius:10px; overflow:hidden; background:var(--frame); }
|
||||
.cell .cap { display:flex; align-items:center; gap:8px; padding:7px 10px; border-bottom:1px solid var(--line);
|
||||
background:var(--bg); }
|
||||
.cell .cap .meta { color:var(--muted); font-size:11px; }
|
||||
.cell img { display:block; width:100%; height:auto; background:#fff; }
|
||||
.cell.empty .ph { display:flex; align-items:center; justify-content:center; height:160px; color:var(--muted);
|
||||
font-size:12.5px; text-align:center; padding:0 16px; }
|
||||
.single .pair { grid-template-columns:1fr; }
|
||||
.empty-doc { color:var(--muted); padding:40px; text-align:center; }
|
||||
@media (max-width:760px){ .pair{ grid-template-columns:1fr; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap" id="root"></div>
|
||||
|
||||
<script id="data">
|
||||
window.__BA__ = /*__DATA__*/{"pr":"","title":"No data","base":"","head":"","cropSelector":"","tabs":[]}/*__END__*/;
|
||||
</script>
|
||||
<script>
|
||||
(function(){
|
||||
var D = window.__BA__ || { tabs: [] };
|
||||
var root = document.getElementById("root");
|
||||
function el(html){ var t=document.createElement("template"); t.innerHTML=html.trim(); return t.content.firstChild; }
|
||||
function esc(s){ return (s==null?"":String(s)).replace(/[&<>]/g, function(c){return {"&":"&","<":"<",">":">"}[c];}); }
|
||||
|
||||
function cell(kind, src){
|
||||
if (src) {
|
||||
return '<div class="cell"><div class="cap"><span class="chip '+kind+'">'+kind+'</span></div>'+
|
||||
'<img src="'+esc(src)+'" alt="'+kind+'"/></div>';
|
||||
}
|
||||
return '<div class="cell empty"><div class="cap"><span class="chip '+kind+'">'+kind+'</span>'+
|
||||
'<span class="meta">not present</span></div><div class="ph">No '+kind+' screenshot for this state</div></div>';
|
||||
}
|
||||
|
||||
var head = '<div class="doc-head"><h1>'+esc(D.title || ("PR #"+D.pr))+'</h1>'+
|
||||
'<div class="sub">Before / after · base <code>'+esc(D.base)+'</code> → head <code>'+esc(D.head)+'</code>'+
|
||||
(D.cropSelector ? ' · cropped to <code>'+esc(D.cropSelector)+'</code>' : '')+'</div></div>'+
|
||||
'<div class="legend"><span class="chip before">Before</span> base branch'+
|
||||
'<span class="chip after">After</span> this PR</div>';
|
||||
root.appendChild(el('<div>'+head+'</div>'));
|
||||
|
||||
if (!D.tabs || !D.tabs.length){ root.appendChild(el('<div class="empty-doc">No tabs captured yet.</div>')); return; }
|
||||
|
||||
D.tabs.forEach(function(tab){
|
||||
var states = (tab.states||[]).map(function(s){
|
||||
var onlyOne = (!s.before || !s.after);
|
||||
return '<div class="state'+(onlyOne?' ':'')+'">'+
|
||||
'<div class="name">'+esc(s.name)+(s.note?'<span class="note">'+esc(s.note)+'</span>':'')+'</div>'+
|
||||
'<div class="pair">'+cell("before", s.before)+cell("after", s.after)+'</div></div>';
|
||||
}).join("");
|
||||
var sec = '<section class="tab-section" id="section-'+esc(tab.id)+'">'+
|
||||
'<h2>'+esc(tab.title)+'</h2>'+
|
||||
(tab.ctx?'<div class="ctx">'+esc(tab.ctx)+'</div>':'')+
|
||||
states+'</section>';
|
||||
root.appendChild(el(sec));
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,25 @@
|
||||
// Render each .tab-section of a montage HTML into its own PNG (the PR-ready image).
|
||||
// Run from frontend/editor (so @playwright/test resolves):
|
||||
// node <skill>/shoot-sections.mjs <montage.html> <outDir>
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(path.join(process.cwd(), "noop.js"));
|
||||
const { chromium } = require("@playwright/test");
|
||||
|
||||
const htmlPath = path.resolve(process.argv[2]);
|
||||
const outDir = path.resolve(process.argv[3] || path.dirname(htmlPath));
|
||||
|
||||
const browser = await chromium.launch();
|
||||
const page = await browser.newPage({ viewport: { width: 1200, height: 1200 }, deviceScaleFactor: 2 });
|
||||
await page.goto(pathToFileURL(htmlPath).href, { waitUntil: "load" });
|
||||
await page.waitForTimeout(250); // let images/fonts paint
|
||||
const ids = await page.$$eval(".tab-section", (els) => els.map((e) => e.id));
|
||||
if (!ids.length) { console.error("no .tab-section found"); process.exit(1); }
|
||||
for (const id of ids) {
|
||||
const name = id.replace(/^section-/, "");
|
||||
await page.locator("#" + id).screenshot({ path: path.join(outDir, `montage_${name}.png`) });
|
||||
console.log("wrote montage_" + name + ".png");
|
||||
}
|
||||
await browser.close();
|
||||
@@ -0,0 +1,120 @@
|
||||
---
|
||||
name: ui-walkthrough
|
||||
description: >-
|
||||
Full UI investigation of the current branch's feature. Enumerates every view
|
||||
and state (empty, populated, loading, error, each dialog/menu/panel, responsive
|
||||
breakpoints, light + dark + RTL), captures them with the stubbed Playwright
|
||||
harness, assembles a single-image HTML walkthrough with a global light/dark
|
||||
toggle slider, then runs two review passes: visual/consistency (alignment,
|
||||
spacing, professionalism, dark/light parity, contrast, truncation) and
|
||||
UX/ease-of-use (flow, discoverability, affordances, empty/error states,
|
||||
expectations). Use when asked for a UI walkthrough, screenshot review, design
|
||||
or QA pass, "find anywhere to make it easier/better for users", or before
|
||||
merging frontend work. Pass --fix to auto-apply safe frontend fixes and
|
||||
re-capture; --theme to limit themes; --no-rtl to skip RTL.
|
||||
argument-hint: "[feature/area] [--fix] [--theme light|dark|both] [--no-rtl] [--breakpoints]"
|
||||
allowed-tools: Read, Write, Edit, Glob, Grep, Bash
|
||||
---
|
||||
|
||||
# UI Walkthrough
|
||||
|
||||
Produce a reviewable HTML walkthrough of a feature's UI in every state and theme,
|
||||
then critique it. Optionally auto-fix and re-capture.
|
||||
|
||||
`$ARGUMENTS` may name the feature/area to focus on. If empty, scope from the
|
||||
current branch diff. Flags: `--fix`, `--theme light|dark|both` (default both),
|
||||
`--no-rtl`, `--breakpoints` (also capture phone/narrow widths).
|
||||
|
||||
## What this repo gives you (use it, don't reinvent)
|
||||
|
||||
- **Stubbed Playwright project** = backend-free screenshots via `page.route()` mocks.
|
||||
Reference implementation: `frontend/editor/src/core/tests/stubbed/files-page-screenshots.spec.ts`.
|
||||
It already shows the light / **dark** / **RTL** passes, JWT seeding, IndexedDB
|
||||
seeding, and dumping PNGs to a `screenshots/<area>/` folder. Copy its shape.
|
||||
- Helpers: `frontend/editor/src/core/tests/helpers/ui-helpers.ts`
|
||||
(`uploadFiles`, `openSettings`, `waitForModalOpen`, `dismissTourTooltip`, …)
|
||||
and the `stub-test-base` fixtures (`autoGoto`, `seedJwt`, `viewport`).
|
||||
- Config: `frontend/editor/playwright.config.ts` (run from `frontend/editor/`).
|
||||
- Report template: [report-template.html](report-template.html) - self-contained,
|
||||
one big image at a time, a global light/dark slider that flips every shot,
|
||||
thumbnail rail, prev/next + arrow keys, and a Findings tab.
|
||||
|
||||
## Process
|
||||
|
||||
### 1. Scope the feature
|
||||
- If `$ARGUMENTS` is empty: `git diff --name-only main...HEAD` and read the PR/commits.
|
||||
Identify changed pages, tools (`core/components/tools/<tool>` or `core/tools/<tool>`),
|
||||
dialogs, panels, and routes.
|
||||
- Enumerate **every view and state** to capture, e.g.:
|
||||
empty / populated / loading / error / disabled; each dialog, menu, popover, tooltip;
|
||||
each tab or step; selection + multi-select; success/result panel; and (if relevant)
|
||||
permission/role variants. Write the list down before capturing - it's the report's spine.
|
||||
|
||||
### 2. Prepare the harness (worktree-safe)
|
||||
Worktrees have no `node_modules` and no generated icons. From repo root:
|
||||
```
|
||||
cd frontend && npm ci # or junction main's node_modules (see memory)
|
||||
cd frontend/editor && node scripts/generate-icons.js
|
||||
```
|
||||
Kill any stale dev server first (it serves old modules):
|
||||
`Get-NetTCPConnection -LocalPort 5173 -State Listen | %{ Stop-Process -Id $_.OwningProcess -Force }`
|
||||
|
||||
### 3. Write the capture spec
|
||||
Create `frontend/editor/src/core/tests/stubbed/<feature>-walkthrough.spec.ts`,
|
||||
modeled on `files-page-screenshots.spec.ts`. For each enumerated view:
|
||||
- stub the APIs it needs, drive the UI to that state, wait on a real locator
|
||||
(not a fixed sleep), `await settle(page)` for Mantine portals, then
|
||||
`page.screenshot({ path: shotPath("NN_name_<theme>") })`.
|
||||
- Capture each view in **light and dark** (and RTL unless `--no-rtl`). Reuse the
|
||||
`enableDarkMode` / `enableRtl` init-script pattern from the reference spec
|
||||
(`localStorage["mantine-color-scheme"]="dark"` + `emulateMedia({colorScheme:"dark"})`).
|
||||
- Name shots `NN_<view>_<theme>.png` so light/dark pair up by suffix.
|
||||
- Prefer **stable test-ids** over translated accessible names (RTL/i18n breaks text locators).
|
||||
|
||||
Run it: `cd frontend/editor && npx playwright test --project=stubbed <feature>-walkthrough.spec.ts`.
|
||||
Add `--project=stubbed-firefox`/`-webkit` only if cross-browser layout matters.
|
||||
|
||||
### 4. Build the report
|
||||
- Copy `report-template.html` to `screenshots/<feature>/walkthrough.html` (so the
|
||||
relative `screenshots/...` image paths resolve, or rewrite paths to sit beside it).
|
||||
- Build the manifest and inject it: replace the JSON between the
|
||||
`/*__DATA__*/` … `/*__END__*/` markers with one `views[]` entry per view
|
||||
(`{id,title,light,dark,viewport,notes}`) and an empty `findings` object you'll
|
||||
fill in step 5. Keep `light`/`dark` as relative paths.
|
||||
- The toggle slider answers the "one big image + flip light/dark for all" request:
|
||||
it shows a single large screenshot, and switching the slider re-themes every view.
|
||||
|
||||
### 5. Review pass 1 - visual & consistency
|
||||
Open each screenshot (Read the PNG) and judge against the others:
|
||||
alignment & spacing rhythm, control placement, button hierarchy, typography,
|
||||
**light/dark parity** (contrast, invisible borders, washed-out text, wrong tokens),
|
||||
truncation/overflow, RTL mirroring, focus states, icon consistency, professional polish.
|
||||
Record each issue as a finding `{severity:high|med|low, view, title, detail, fix}`.
|
||||
|
||||
### 6. Review pass 2 - UX & ease of use
|
||||
Walk the flow as a first-time user: discoverability, number of steps, affordance
|
||||
clarity, empty-state guidance, error recovery, destructive-action confirmation,
|
||||
defaults, loading feedback, mobile reachability, accessible names, and whether the
|
||||
UI matches user expectations for this kind of tool. Record findings the same way.
|
||||
|
||||
Write both finding lists into the report's `findings.visual` / `findings.ux`,
|
||||
and add short per-view `notes`. Re-inject the manifest.
|
||||
|
||||
### 7. If `--fix`
|
||||
Only safe, self-contained frontend fixes (spacing, alignment, tokens, missing
|
||||
dark-mode colors, labels, aria, obvious copy). For each: edit the component/CSS,
|
||||
mark the finding `fixed:true` with what changed, then **re-run the spec** to
|
||||
re-capture the affected shots and regenerate the report. Run `task frontend:check`.
|
||||
Leave anything risky or ambiguous as a finding, not a change.
|
||||
|
||||
### 8. Deliver
|
||||
Tell the user the report path and give a tight chat summary: N views ×
|
||||
themes captured, top findings by severity, and (if `--fix`) what changed.
|
||||
Optionally `SendUserFile` the `walkthrough.html`.
|
||||
|
||||
## Gotchas
|
||||
- Stale `:5173` server serves old bundles - kill it before capturing (see step 2).
|
||||
- Missing `material-symbols-icons.json` → blank app → every shot times out. Run
|
||||
`generate-icons.js` first.
|
||||
- `await settle(page)` before shots or portals/transitions tear mid-capture.
|
||||
- Don't commit the generated `screenshots/` or the throwaway spec unless asked.
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Build a self-contained EXAMPLE.html from report-template.html with mock
|
||||
light/dark screenshots, so the viewer + global theme slider can be demoed
|
||||
without a real capture run. Run: python make_example.py"""
|
||||
import base64
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
|
||||
HERE = pathlib.Path(__file__).parent
|
||||
|
||||
|
||||
def svg(bg, fg, panel, accent, muted, label, kind):
|
||||
"""A simple fake 'screen' SVG: title bar, sidebar, content varies by kind."""
|
||||
parts = [
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="1600" height="900" viewBox="0 0 1600 900">',
|
||||
f'<rect width="1600" height="900" fill="{bg}"/>',
|
||||
# top bar
|
||||
f'<rect width="1600" height="64" fill="{panel}"/>',
|
||||
f'<circle cx="40" cy="32" r="12" fill="{accent}"/>',
|
||||
f'<rect x="64" y="24" width="160" height="16" rx="6" fill="{muted}"/>',
|
||||
f'<rect x="1430" y="20" width="130" height="24" rx="12" fill="{accent}"/>',
|
||||
# left sidebar
|
||||
f'<rect x="0" y="64" width="220" height="836" fill="{panel}"/>',
|
||||
]
|
||||
for i in range(6):
|
||||
y = 100 + i * 56
|
||||
parts.append(f'<rect x="24" y="{y}" width="172" height="32" rx="8" fill="{bg}"/>')
|
||||
if kind == "empty":
|
||||
parts += [
|
||||
f'<rect x="700" y="360" width="200" height="120" rx="16" fill="none" stroke="{muted}" stroke-width="3" stroke-dasharray="10 8"/>',
|
||||
f'<rect x="690" y="510" width="220" height="44" rx="10" fill="{accent}"/>',
|
||||
f'<text x="800" y="600" fill="{muted}" font-family="sans-serif" font-size="26" text-anchor="middle">{label}</text>',
|
||||
]
|
||||
elif kind == "form":
|
||||
for i in range(4):
|
||||
y = 140 + i * 90
|
||||
parts.append(f'<rect x="280" y="{y}" width="160" height="16" rx="6" fill="{muted}"/>')
|
||||
parts.append(f'<rect x="280" y="{y+26}" width="900" height="44" rx="8" fill="{panel}" stroke="{muted}" stroke-width="1"/>')
|
||||
parts.append(f'<rect x="280" y="560" width="200" height="50" rx="10" fill="{accent}"/>')
|
||||
parts.append(f'<text x="800" y="850" fill="{muted}" font-family="sans-serif" font-size="24" text-anchor="middle">{label}</text>')
|
||||
else: # dialog
|
||||
parts += [
|
||||
f'<rect width="1600" height="900" fill="{fg}" opacity="0.45"/>',
|
||||
f'<rect x="520" y="280" width="560" height="360" rx="18" fill="{panel}"/>',
|
||||
f'<rect x="556" y="320" width="280" height="22" rx="8" fill="{fg}"/>',
|
||||
f'<rect x="556" y="372" width="488" height="14" rx="6" fill="{muted}"/>',
|
||||
f'<rect x="556" y="398" width="420" height="14" rx="6" fill="{muted}"/>',
|
||||
f'<rect x="820" y="560" width="110" height="44" rx="9" fill="{bg}" stroke="{muted}"/>',
|
||||
f'<rect x="946" y="560" width="98" height="44" rx="9" fill="{accent}"/>',
|
||||
f'<text x="800" y="700" fill="#fff" font-family="sans-serif" font-size="24" text-anchor="middle">{label}</text>',
|
||||
]
|
||||
parts.append("</svg>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def data_uri(s):
|
||||
return "data:image/svg+xml;base64," + base64.b64encode(s.encode()).decode()
|
||||
|
||||
|
||||
LIGHT = dict(bg="#ffffff", fg="#111418", panel="#f1f3f6", accent="#2f6fed", muted="#c2c8d0")
|
||||
DARK = dict(bg="#16181c", fg="#000000", panel="#1f232a", accent="#5b8cff", muted="#3a414b")
|
||||
|
||||
|
||||
def pair(kind, label):
|
||||
return (
|
||||
data_uri(svg(LIGHT["bg"], LIGHT["fg"], LIGHT["panel"], LIGHT["accent"], LIGHT["muted"], label, kind)),
|
||||
data_uri(svg(DARK["bg"], DARK["fg"], DARK["panel"], DARK["accent"], DARK["muted"], label, kind)),
|
||||
)
|
||||
|
||||
|
||||
views = []
|
||||
for idx, (kind, title, label) in enumerate([
|
||||
("empty", "Empty state", "Drop a PDF to start"),
|
||||
("form", "Tool options panel", "Compress options"),
|
||||
("dialog", "Confirm dialog", "Replace original file?"),
|
||||
], start=1):
|
||||
light, dark = pair(kind, label)
|
||||
views.append({
|
||||
"id": f"{idx:02d}_{kind}",
|
||||
"title": title,
|
||||
"light": light,
|
||||
"dark": dark,
|
||||
"viewport": "1600x900",
|
||||
"notes": ["This is mock data to demo the viewer."],
|
||||
})
|
||||
|
||||
data = {
|
||||
"feature": "EXAMPLE - Compress PDF (mock data)",
|
||||
"branch": "demo",
|
||||
"generated": "example",
|
||||
"views": views,
|
||||
"findings": {
|
||||
"visual": [
|
||||
{"severity": "high", "view": "03_dialog", "title": "Dialog buttons too close",
|
||||
"detail": "Cancel/Confirm have only 8px gap; easy to misclick.",
|
||||
"fix": "Increase gap to var(--mantine-spacing-md)."},
|
||||
{"severity": "low", "view": "02_form", "title": "Field labels low contrast in dark mode",
|
||||
"detail": "Muted token fails WCAG AA on the dark panel.",
|
||||
"fix": "Use --mantine-color-dimmed instead of a hard-coded grey."},
|
||||
],
|
||||
"ux": [
|
||||
{"severity": "med", "view": "01_empty", "title": "Primary CTA below the dropzone",
|
||||
"detail": "Users expect the action button adjacent to the dropzone.",
|
||||
"fix": "Move the button directly under the dashed zone."},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
tpl = (HERE / "report-template.html").read_text(encoding="utf-8")
|
||||
out = re.sub(
|
||||
r"/\*__DATA__\*/.*?/\*__END__\*/",
|
||||
lambda _m: "/*__DATA__*/" + json.dumps(data) + "/*__END__*/",
|
||||
tpl, count=1, flags=re.S,
|
||||
)
|
||||
(HERE / "EXAMPLE.html").write_text(out, encoding="utf-8")
|
||||
print("wrote", (HERE / "EXAMPLE.html"))
|
||||
@@ -0,0 +1,298 @@
|
||||
<!doctype html>
|
||||
<!--
|
||||
UI Walkthrough report template (self-contained, works from file://).
|
||||
The ui-walkthrough skill replaces the JSON in the window.__WALKTHROUGH__ data
|
||||
block below with the captured manifest. Do not add external CDN deps - it must open offline.
|
||||
|
||||
Data shape:
|
||||
{
|
||||
"feature": "Compress PDF tool",
|
||||
"branch": "claude/...",
|
||||
"generated": "2026-06-21",
|
||||
"views": [
|
||||
{ "id": "01_empty", "title": "Empty state",
|
||||
"light": "screenshots/compress/01_empty_light.png",
|
||||
"dark": "screenshots/compress/01_empty_dark.png",
|
||||
"viewport": "1600x900",
|
||||
"notes": ["Heading is centered", "Primary CTA below the fold on mobile"] }
|
||||
],
|
||||
"findings": {
|
||||
"visual": [ { "severity":"high", "view":"01_empty", "title":"...", "detail":"...", "fix":"..." } ],
|
||||
"ux": [ { "severity":"med", "view":"03_dialog", "title":"...", "detail":"...", "fix":"..." } ]
|
||||
}
|
||||
}
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>UI Walkthrough</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #f6f7f9; --panel: #ffffff; --panel-2: #f0f2f5; --text: #1a1b1e;
|
||||
--muted: #6b7280; --border: #e2e5ea; --accent: #2f6fed; --accent-weak: #e8f0fe;
|
||||
--shadow: 0 1px 3px rgba(0,0,0,.08), 0 8px 24px rgba(0,0,0,.06);
|
||||
--hi: #d92d20; --med: #d98e00; --low: #2f6fed; --stage: #0b0c0e;
|
||||
}
|
||||
html[data-theme="dark"] {
|
||||
--bg: #0d0e10; --panel: #16181c; --panel-2: #1d2024; --text: #e6e8eb;
|
||||
--muted: #9aa3ad; --border: #2a2e35; --accent: #5b8cff; --accent-weak: #1a2336;
|
||||
--shadow: 0 1px 3px rgba(0,0,0,.5), 0 8px 24px rgba(0,0,0,.4); --stage: #000;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font: 14px/1.5 -apple-system, "Segoe UI", Roboto, system-ui, sans-serif;
|
||||
background: var(--bg); color: var(--text); }
|
||||
header { display: flex; align-items: center; gap: 16px; padding: 12px 20px;
|
||||
background: var(--panel); border-bottom: 1px solid var(--border); position: sticky; top: 0; z-index: 5; }
|
||||
header h1 { font-size: 15px; margin: 0; font-weight: 650; }
|
||||
header .sub { color: var(--muted); font-size: 12px; }
|
||||
.spacer { flex: 1; }
|
||||
.counter { color: var(--muted); font-variant-numeric: tabular-nums; font-size: 13px; }
|
||||
.tabs { display: flex; gap: 4px; }
|
||||
.tab { border: 1px solid var(--border); background: var(--panel-2); color: var(--text);
|
||||
padding: 6px 12px; border-radius: 8px; cursor: pointer; font-size: 13px; }
|
||||
.tab.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
|
||||
/* Light/Dark slider */
|
||||
.theme-toggle { display: flex; align-items: center; gap: 9px; user-select: none; }
|
||||
.theme-toggle .lbl { font-size: 12px; color: var(--muted); }
|
||||
.theme-toggle .lbl.on { color: var(--text); font-weight: 600; }
|
||||
.switch { position: relative; width: 52px; height: 28px; }
|
||||
.switch input { opacity: 0; width: 0; height: 0; }
|
||||
.slider { position: absolute; inset: 0; cursor: pointer; background: var(--panel-2);
|
||||
border: 1px solid var(--border); border-radius: 999px; transition: .2s; }
|
||||
.slider:before { content: ""; position: absolute; height: 20px; width: 20px; left: 3px; top: 3px;
|
||||
background: #fbbf24; border-radius: 50%; transition: .2s; box-shadow: 0 1px 2px rgba(0,0,0,.3); }
|
||||
.switch input:checked + .slider { background: var(--accent); }
|
||||
.switch input:checked + .slider:before { transform: translateX(24px); background: #c7d2fe; }
|
||||
|
||||
main { display: grid; grid-template-columns: 240px 1fr; height: calc(100vh - 53px); }
|
||||
.rail { border-right: 1px solid var(--border); overflow-y: auto; background: var(--panel); padding: 8px; }
|
||||
.rail .group-label { font-size: 11px; text-transform: uppercase; letter-spacing: .05em;
|
||||
color: var(--muted); padding: 10px 8px 4px; }
|
||||
.thumb { display: flex; gap: 9px; align-items: center; padding: 7px; border-radius: 8px;
|
||||
cursor: pointer; border: 1px solid transparent; }
|
||||
.thumb:hover { background: var(--panel-2); }
|
||||
.thumb.active { background: var(--accent-weak); border-color: var(--accent); }
|
||||
.thumb img { width: 64px; height: 40px; object-fit: cover; border-radius: 4px; border: 1px solid var(--border); background: var(--stage); }
|
||||
.thumb .t { font-size: 12.5px; line-height: 1.3; }
|
||||
.thumb .badge { font-size: 10px; color: var(--muted); }
|
||||
.thumb .dot { width: 7px; height: 7px; border-radius: 50%; margin-left: auto; flex: none; }
|
||||
|
||||
.stagewrap { display: flex; flex-direction: column; min-width: 0; }
|
||||
.stage { flex: 1; display: flex; align-items: center; justify-content: center; padding: 22px;
|
||||
background: var(--stage); position: relative; min-height: 0; }
|
||||
.stage img { max-width: 100%; max-height: 100%; object-fit: contain; border-radius: 8px;
|
||||
box-shadow: 0 4px 30px rgba(0,0,0,.4); background: #fff; }
|
||||
html[data-theme="dark"] .stage img { background: #16181c; }
|
||||
.nav-btn { position: absolute; top: 50%; transform: translateY(-50%); width: 42px; height: 42px;
|
||||
border-radius: 50%; border: 1px solid var(--border); background: var(--panel);
|
||||
color: var(--text); cursor: pointer; font-size: 18px; opacity: .85; }
|
||||
.nav-btn:hover { opacity: 1; } .nav-btn.prev { left: 16px; } .nav-btn.next { right: 16px; }
|
||||
.nav-btn:disabled { opacity: .25; cursor: default; }
|
||||
.missing { color: var(--muted); font-size: 13px; text-align: center; }
|
||||
|
||||
.detail { border-top: 1px solid var(--border); background: var(--panel); padding: 14px 20px;
|
||||
max-height: 38vh; overflow-y: auto; }
|
||||
.detail h2 { margin: 0 0 4px; font-size: 15px; }
|
||||
.detail .meta { color: var(--muted); font-size: 12px; margin-bottom: 10px; }
|
||||
.notes { list-style: none; padding: 0; margin: 0; display: grid; gap: 6px; }
|
||||
.notes li { display: flex; gap: 8px; align-items: flex-start; }
|
||||
.sev { font-size: 10px; font-weight: 700; text-transform: uppercase; padding: 2px 7px; border-radius: 999px;
|
||||
color: #fff; flex: none; margin-top: 1px; }
|
||||
.sev.high { background: var(--hi); } .sev.med { background: var(--med); } .sev.low { background: var(--low); }
|
||||
.finding .fix { color: var(--muted); font-size: 12.5px; }
|
||||
.finding .fix b { color: var(--text); font-weight: 600; }
|
||||
|
||||
/* Summary tab */
|
||||
.summary { padding: 20px 28px; overflow-y: auto; }
|
||||
.summary h2 { font-size: 16px; margin: 22px 0 8px; }
|
||||
.summary .empty { color: var(--muted); }
|
||||
.card { background: var(--panel); border: 1px solid var(--border); border-radius: 10px;
|
||||
padding: 12px 14px; margin-bottom: 8px; box-shadow: var(--shadow); }
|
||||
.card .head { display: flex; gap: 8px; align-items: center; }
|
||||
.card a { color: var(--accent); text-decoration: none; cursor: pointer; }
|
||||
.hide { display: none !important; }
|
||||
kbd { font: 11px ui-monospace, monospace; background: var(--panel-2); border: 1px solid var(--border);
|
||||
border-radius: 4px; padding: 1px 5px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<div>
|
||||
<h1 id="feature-title">UI Walkthrough</h1>
|
||||
<div class="sub" id="feature-sub"></div>
|
||||
</div>
|
||||
<div class="spacer"></div>
|
||||
<div class="tabs">
|
||||
<button class="tab active" data-tab="viewer">Walkthrough</button>
|
||||
<button class="tab" data-tab="summary">Findings</button>
|
||||
</div>
|
||||
<div class="counter" id="counter"></div>
|
||||
<label class="theme-toggle" title="Toggle light / dark for every screenshot">
|
||||
<span class="lbl" id="lbl-light">Light</span>
|
||||
<span class="switch"><input type="checkbox" id="theme-switch" /><span class="slider"></span></span>
|
||||
<span class="lbl" id="lbl-dark">Dark</span>
|
||||
</label>
|
||||
</header>
|
||||
|
||||
<main id="viewer-pane">
|
||||
<aside class="rail" id="rail"></aside>
|
||||
<section class="stagewrap">
|
||||
<div class="stage">
|
||||
<button class="nav-btn prev" id="prev" aria-label="Previous">‹</button>
|
||||
<img id="stage-img" alt="" />
|
||||
<div class="missing hide" id="missing"></div>
|
||||
<button class="nav-btn next" id="next" aria-label="Next">›</button>
|
||||
</div>
|
||||
<div class="detail">
|
||||
<h2 id="view-title"></h2>
|
||||
<div class="meta" id="view-meta"></div>
|
||||
<ul class="notes" id="view-notes"></ul>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<section class="summary hide" id="summary-pane"></section>
|
||||
|
||||
<script id="data">
|
||||
window.__WALKTHROUGH__ = /*__DATA__*/{"feature":"No data","branch":"","generated":"","views":[],"findings":{"visual":[],"ux":[]}}/*__END__*/;
|
||||
</script>
|
||||
<script>
|
||||
(function () {
|
||||
var D = window.__WALKTHROUGH__ || { views: [], findings: { visual: [], ux: [] } };
|
||||
var views = D.views || [];
|
||||
var state = { i: 0, theme: localStorage.getItem("ui-wt-theme") || "light", tab: "viewer" };
|
||||
|
||||
var $ = function (id) { return document.getElementById(id); };
|
||||
function sevClass(s) { return s === "high" ? "high" : s === "med" || s === "medium" ? "med" : "low"; }
|
||||
|
||||
function applyChrome() {
|
||||
document.documentElement.setAttribute("data-theme", state.theme);
|
||||
$("theme-switch").checked = state.theme === "dark";
|
||||
$("lbl-light").classList.toggle("on", state.theme === "light");
|
||||
$("lbl-dark").classList.toggle("on", state.theme === "dark");
|
||||
}
|
||||
|
||||
function srcFor(v) { return state.theme === "dark" ? (v.dark || v.light) : (v.light || v.dark); }
|
||||
|
||||
function findingsForView(id) {
|
||||
var all = (D.findings && D.findings.visual || []).concat(D.findings && D.findings.ux || []);
|
||||
return all.filter(function (f) { return f.view === id; });
|
||||
}
|
||||
|
||||
function renderRail() {
|
||||
var rail = $("rail");
|
||||
rail.innerHTML = "";
|
||||
if (!views.length) { rail.innerHTML = '<div class="group-label">No views captured</div>'; return; }
|
||||
views.forEach(function (v, idx) {
|
||||
var fs = findingsForView(v.id);
|
||||
var worst = fs.some(function (f){return sevClass(f.severity)==="high";}) ? "var(--hi)"
|
||||
: fs.some(function (f){return sevClass(f.severity)==="med";}) ? "var(--med)"
|
||||
: fs.length ? "var(--low)" : "transparent";
|
||||
var el = document.createElement("div");
|
||||
el.className = "thumb" + (idx === state.i ? " active" : "");
|
||||
el.innerHTML = '<img src="' + srcFor(v) + '" alt="" />' +
|
||||
'<div><div class="t">' + (v.title || v.id) + '</div>' +
|
||||
'<div class="badge">' + (v.viewport || "") + '</div></div>' +
|
||||
'<span class="dot" style="background:' + worst + '"></span>';
|
||||
el.onclick = function () { state.i = idx; render(); };
|
||||
rail.appendChild(el);
|
||||
});
|
||||
}
|
||||
|
||||
function render() {
|
||||
applyChrome();
|
||||
if (!views.length) {
|
||||
$("missing").classList.remove("hide"); $("stage-img").classList.add("hide");
|
||||
$("missing").textContent = "No screenshots in this report yet.";
|
||||
$("counter").textContent = ""; return;
|
||||
}
|
||||
var v = views[state.i];
|
||||
var src = srcFor(v);
|
||||
var img = $("stage-img");
|
||||
if (src) {
|
||||
img.classList.remove("hide"); $("missing").classList.add("hide");
|
||||
img.src = src; img.alt = v.title || v.id;
|
||||
} else {
|
||||
img.classList.add("hide"); $("missing").classList.remove("hide");
|
||||
$("missing").textContent = "No " + state.theme + " screenshot for this view.";
|
||||
}
|
||||
$("counter").textContent = (state.i + 1) + " / " + views.length;
|
||||
$("view-title").textContent = v.title || v.id;
|
||||
$("view-meta").textContent = [v.viewport, state.theme + " mode"].filter(Boolean).join(" · ");
|
||||
var notes = $("view-notes"); notes.innerHTML = "";
|
||||
var fs = findingsForView(v.id);
|
||||
(v.notes || []).forEach(function (n) {
|
||||
var li = document.createElement("li"); li.textContent = "· " + n; notes.appendChild(li);
|
||||
});
|
||||
fs.forEach(function (f) {
|
||||
var li = document.createElement("li"); li.className = "finding";
|
||||
li.innerHTML = '<span class="sev ' + sevClass(f.severity) + '">' + (f.severity || "note") + '</span>' +
|
||||
'<span><b>' + (f.title || "") + '</b> — ' + (f.detail || "") +
|
||||
(f.fix ? ' <span class="fix"><b>Fix:</b> ' + f.fix + '</span>' : '') + '</span>';
|
||||
notes.appendChild(li);
|
||||
});
|
||||
$("prev").disabled = state.i === 0;
|
||||
$("next").disabled = state.i === views.length - 1;
|
||||
renderRail();
|
||||
}
|
||||
|
||||
function renderSummary() {
|
||||
var pane = $("summary-pane");
|
||||
function block(title, arr) {
|
||||
var h = '<h2>' + title + ' (' + arr.length + ')</h2>';
|
||||
if (!arr.length) return h + '<div class="empty">None found.</div>';
|
||||
return h + arr.map(function (f) {
|
||||
return '<div class="card"><div class="head">' +
|
||||
'<span class="sev ' + sevClass(f.severity) + '">' + (f.severity || "note") + '</span>' +
|
||||
'<b>' + (f.title || "") + '</b>' +
|
||||
(f.view ? ' <a data-jump="' + f.view + '">' + f.view + '</a>' : '') + '</div>' +
|
||||
'<div style="margin-top:6px">' + (f.detail || "") + '</div>' +
|
||||
(f.fix ? '<div class="finding" style="margin-top:6px"><span class="fix"><b>Fix:</b> ' + f.fix + '</span></div>' : '') +
|
||||
'</div>';
|
||||
}).join("");
|
||||
}
|
||||
pane.innerHTML = block("Visual & consistency", (D.findings && D.findings.visual) || []) +
|
||||
block("UX & ease of use", (D.findings && D.findings.ux) || []);
|
||||
pane.querySelectorAll("[data-jump]").forEach(function (a) {
|
||||
a.onclick = function () {
|
||||
var id = a.getAttribute("data-jump");
|
||||
var idx = views.findIndex(function (v) { return v.id === id; });
|
||||
if (idx >= 0) { state.i = idx; setTab("viewer"); }
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function setTab(t) {
|
||||
state.tab = t;
|
||||
document.querySelectorAll(".tab").forEach(function (b) { b.classList.toggle("active", b.dataset.tab === t); });
|
||||
$("viewer-pane").classList.toggle("hide", t !== "viewer");
|
||||
$("summary-pane").classList.toggle("hide", t !== "summary");
|
||||
if (t === "viewer") $("viewer-pane").style.display = "grid";
|
||||
if (t === "summary") renderSummary();
|
||||
}
|
||||
|
||||
// wiring
|
||||
$("feature-title").textContent = D.feature || "UI Walkthrough";
|
||||
$("feature-sub").textContent = [D.branch, D.generated].filter(Boolean).join(" · ");
|
||||
$("theme-switch").onchange = function () {
|
||||
state.theme = this.checked ? "dark" : "light";
|
||||
localStorage.setItem("ui-wt-theme", state.theme);
|
||||
render();
|
||||
};
|
||||
$("prev").onclick = function () { if (state.i > 0) { state.i--; render(); } };
|
||||
$("next").onclick = function () { if (state.i < views.length - 1) { state.i++; render(); } };
|
||||
document.addEventListener("keydown", function (e) {
|
||||
if (state.tab !== "viewer") return;
|
||||
if (e.key === "ArrowLeft") $("prev").click();
|
||||
if (e.key === "ArrowRight") $("next").click();
|
||||
if (e.key.toLowerCase() === "t") $("theme-switch").click();
|
||||
});
|
||||
document.querySelectorAll(".tab").forEach(function (b) { b.onclick = function () { setTab(b.dataset.tab); }; });
|
||||
|
||||
render();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -27,7 +27,6 @@ node_modules/
|
||||
**/node_modules/
|
||||
frontend/node_modules/
|
||||
frontend/editor/dist/
|
||||
frontend/dist-portal/
|
||||
frontend/editor/playwright-report/
|
||||
.npm/
|
||||
.yarn/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-desktop
|
||||
pkgver=2.14.0
|
||||
pkgver=2.14.1
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-server-bin
|
||||
pkgver=2.14.0
|
||||
pkgver=2.14.1
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
|
||||
arch=('any')
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
build: &build
|
||||
- build.gradle
|
||||
- app/(common|core|proprietary)/build.gradle
|
||||
- app/(common|core|proprietary|saas)/build.gradle
|
||||
- Taskfile.yml
|
||||
- .taskfiles/backend.yml
|
||||
|
||||
openapi: &openapi
|
||||
- *build
|
||||
- app/(common|core|proprietary)/src/main/java/**
|
||||
- app/(common|core|proprietary|saas)/src/main/java/**
|
||||
|
||||
docker-base: &docker-base
|
||||
- docker/base/Dockerfile
|
||||
@@ -23,9 +23,9 @@ docker: &docker
|
||||
- *docker-base
|
||||
|
||||
project: &project
|
||||
- app/(common|core|proprietary)/src/(main|test)/java/**
|
||||
- app/(common|core|proprietary|saas)/src/(main|test)/java/**
|
||||
- *build
|
||||
- "app/(common|core|proprietary)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*"
|
||||
- "app/(common|core|proprietary|saas)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*"
|
||||
- exampleYmlFiles/**
|
||||
- gradle/**
|
||||
- libs/**
|
||||
@@ -82,11 +82,26 @@ tauri: &tauri
|
||||
# tool surfaces it generates models from.
|
||||
engine: &engine
|
||||
- engine/**
|
||||
- app/(common|core|proprietary)/src/main/java/**
|
||||
- app/(common|core|proprietary|saas)/src/main/java/**
|
||||
- .github/workflows/ai-engine.yml
|
||||
- Taskfile.yml
|
||||
- .taskfiles/engine.yml
|
||||
|
||||
# Files that can make the committed generated API models (frontend tool API
|
||||
# types + engine tool models) go stale: the Java tool surfaces they derive from,
|
||||
# the generators, the generated files themselves (to catch a hand-edit), and the
|
||||
# tasks that drive generation. Deliberately excludes the broad frontend/docker/
|
||||
# testing globs, so a CSS-only PR does not boot the backend to rebuild the spec.
|
||||
generated-models: &generated-models
|
||||
- *openapi
|
||||
- frontend/editor/scripts/generate-tool-api-types.mts
|
||||
- frontend/editor/src/core/types/toolApiTypes.ts
|
||||
- engine/scripts/generate_tool_models.py
|
||||
- engine/src/stirling/models/tool_models.py
|
||||
- .taskfiles/frontend.yml
|
||||
- .taskfiles/engine.yml
|
||||
- .github/workflows/check-generated-models.yml
|
||||
|
||||
licenses-frontend: &licenses-frontend
|
||||
- ".github/workflows/frontend-backend-licenses-update.yml"
|
||||
- "frontend/package.json"
|
||||
|
||||
@@ -63,6 +63,7 @@ labels:
|
||||
files:
|
||||
- 'app/core/src/main/resources/static/.*'
|
||||
- 'app/proprietary/src/main/resources/static/.*'
|
||||
- 'app/saas/src/main/resources/static/.*'
|
||||
- 'frontend/**'
|
||||
- 'frontend/.*'
|
||||
- 'frontend/**/.*'
|
||||
@@ -83,6 +84,7 @@ labels:
|
||||
- 'app/common/src/main/java/.*.java'
|
||||
- 'app/proprietary/src/main/java/.*.java'
|
||||
- 'app/core/src/main/java/.*.java'
|
||||
- 'app/saas/src/main/java/.*.java'
|
||||
|
||||
- label: 'Back End'
|
||||
files:
|
||||
@@ -90,6 +92,9 @@ labels:
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/controller/.*'
|
||||
- 'app/core/src/main/resources/settings.yml.template'
|
||||
- 'app/core/src/main/resources/application.properties'
|
||||
- 'app/proprietary/src/main/resources/application-proprietary.properties'
|
||||
- 'app/saas/src/main/resources/application-dev.properties'
|
||||
- 'app/saas/src/main/resources/application-saas.properties'
|
||||
- 'app/core/src/main/resources/banner.txt'
|
||||
- 'app/core/src/main/resources/static/python/png_to_webp.py'
|
||||
- 'app/core/src/main/resources/static/python/split_photos.py'
|
||||
@@ -153,6 +158,7 @@ labels:
|
||||
- 'app/common/src/test/.*'
|
||||
- 'app/proprietary/src/test/.*'
|
||||
- 'app/core/src/test/.*'
|
||||
- 'app/saas/src/test/.*'
|
||||
- 'testing/.*'
|
||||
- '.github/workflows/scorecards.yml'
|
||||
- 'exampleYmlFiles/test_cicd.yml'
|
||||
@@ -171,3 +177,4 @@ labels:
|
||||
- 'app/common/build.gradle'
|
||||
- 'app/proprietary/build.gradle'
|
||||
- 'app/core/build.gradle'
|
||||
- 'app/saas/build.gradle'
|
||||
|
||||
@@ -116,6 +116,9 @@ jobs:
|
||||
env:
|
||||
USE_DEPOT: ${{ needs.pick.outputs.is_fork != 'true' }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
# Single source of truth for whether this preview embeds the admin portal:
|
||||
# drives the image build-arg and the deployment comment.
|
||||
BUILD_PORTAL: "true"
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
@@ -246,12 +249,14 @@ jobs:
|
||||
file: ./docker/embedded/Dockerfile
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-${{ steps.commit-hash.outputs.app_short }}
|
||||
build-args: VERSION_TAG=v2-alpha
|
||||
build-args: |
|
||||
VERSION_TAG=v2-alpha
|
||||
BUILD_PORTAL=${{ env.BUILD_PORTAL }}
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push V2 image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-image.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
@@ -259,7 +264,9 @@ jobs:
|
||||
cache-from: type=gha,scope=stirling-pdf-latest
|
||||
cache-to: type=gha,mode=max,scope=stirling-pdf-latest
|
||||
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-${{ steps.commit-hash.outputs.app_short }}
|
||||
build-args: VERSION_TAG=v2-alpha
|
||||
build-args: |
|
||||
VERSION_TAG=v2-alpha
|
||||
BUILD_PORTAL=${{ env.BUILD_PORTAL }}
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Set up SSH
|
||||
@@ -290,6 +297,8 @@ jobs:
|
||||
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/storage:/storage:rw
|
||||
environment:
|
||||
DISABLE_ADDITIONAL_FEATURES: "false"
|
||||
POLICIES_ENABLED: "true"
|
||||
STIRLING_BILLING_ACCOUNT_LINK_ENABLED: "true"
|
||||
SECURITY_ENABLELOGIN: "true"
|
||||
SECURITY_INITIALLOGIN_USERNAME: "${{ secrets.TEST_LOGIN_USERNAME }}"
|
||||
SECURITY_INITIALLOGIN_PASSWORD: "${{ secrets.TEST_LOGIN_PASSWORD }}"
|
||||
@@ -333,9 +342,70 @@ jobs:
|
||||
# Set port for output
|
||||
echo "v2_port=${V2_PORT}" >> $GITHUB_OUTPUT
|
||||
|
||||
# ---- Storybook preview (only when this PR touches stories/.storybook) ----
|
||||
# Runs inside the same approved-contributor-gated deploy job, so it deploys
|
||||
# under the exact same access rules as the app preview.
|
||||
- name: Detect Storybook changes
|
||||
id: sb-changes
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
storybook:
|
||||
- 'frontend/**/*.stories.@(ts|tsx|mdx)'
|
||||
- 'frontend/**/*.mdx'
|
||||
- 'frontend/.storybook/**'
|
||||
|
||||
- name: Set up Node.js for Storybook
|
||||
if: steps.sb-changes.outputs.storybook == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install Task for Storybook
|
||||
if: steps.sb-changes.outputs.storybook == 'true'
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
- name: Build and deploy Storybook
|
||||
id: storybook
|
||||
if: steps.sb-changes.outputs.storybook == 'true'
|
||||
env:
|
||||
VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
|
||||
VPS_USER: ${{ secrets.NEW_VPS_USERNAME }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# `prepare` generates the icon set stories import (not committed).
|
||||
task frontend:prepare
|
||||
task frontend:storybook:build
|
||||
PR=${{ needs.check-pr.outputs.pr_number }}
|
||||
# Served at the ROOT of its own port so Storybook's global MSW worker
|
||||
# (/mockServiceWorker.js) resolves. Port = PR + 20000 (bijective, offset
|
||||
# from the app preview's bare-PR-number port).
|
||||
SB_PORT=$((PR + 20000))
|
||||
DIR=/stirling/SB-PR-$PR
|
||||
tar czf storybook.tgz -C frontend/storybook-static .
|
||||
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
storybook.tgz "$VPS_USER@$VPS_HOST:/tmp/storybook-$PR.tgz"
|
||||
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T \
|
||||
"$VPS_USER@$VPS_HOST" << ENDSSH
|
||||
set -e
|
||||
rm -rf "$DIR" && mkdir -p "$DIR"
|
||||
tar xzf /tmp/storybook-$PR.tgz -C "$DIR"
|
||||
rm -f /tmp/storybook-$PR.tgz
|
||||
docker rm -f storybook-pr-$PR 2>/dev/null || true
|
||||
docker run -d --name storybook-pr-$PR --restart unless-stopped \
|
||||
-p $SB_PORT:80 -v "$DIR":/usr/share/nginx/html:ro nginx:alpine
|
||||
ENDSSH
|
||||
echo "url=http://$VPS_HOST:$SB_PORT/" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Post V2 deployment URL to PR
|
||||
if: success()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SB_URL: ${{ steps.storybook.outputs.url }}
|
||||
SB_FILES: ${{ steps.sb-changes.outputs.storybook_files }}
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
@@ -359,12 +429,40 @@ jobs:
|
||||
}
|
||||
|
||||
const deploymentUrl = `http://${{ secrets.NEW_VPS_HOST }}:${v2Port}`;
|
||||
const httpsUrl = `https://${v2Port}.ssl.stirlingpdf.cloud`;
|
||||
|
||||
// Only mention the portal when this image actually embeds it.
|
||||
// Use the direct IP URL - the SSL hostname isn't supported yet.
|
||||
const withPortal = "${{ env.BUILD_PORTAL }}" === "true";
|
||||
const portalNote = withPortal
|
||||
? `🧩 **Admin portal** included - try it at [${deploymentUrl}/portal](${deploymentUrl}/portal).\n\n`
|
||||
: ``;
|
||||
|
||||
// Storybook preview: only present when this PR changed stories/config.
|
||||
const sbUrl = process.env.SB_URL;
|
||||
let storybookNote = "";
|
||||
if (sbUrl) {
|
||||
const files = JSON.parse(process.env.SB_FILES || "[]");
|
||||
const stories = files.filter((f) => /\.stories\.(ts|tsx|mdx)$/.test(f));
|
||||
const config = files.filter((f) => f.startsWith("frontend/.storybook/"));
|
||||
const shorten = (f) =>
|
||||
f.replace(/^frontend\/editor\/src\//, "").replace(/^frontend\//, "");
|
||||
const storyList = stories.map((f) => `- \`${shorten(f)}\``).join("\n");
|
||||
const configList = config.map((f) => `- \`${shorten(f)}\``).join("\n");
|
||||
const summary =
|
||||
`${stories.length} stor${stories.length === 1 ? "y" : "ies"} changed` +
|
||||
(config.length ? ` (+${config.length} config file${config.length === 1 ? "" : "s"})` : "");
|
||||
storybookNote =
|
||||
`📚 **Storybook:** [${sbUrl}](${sbUrl})\n\n` +
|
||||
`<details>\n<summary>${summary}</summary>\n\n` +
|
||||
(storyList ? `**Stories**\n${storyList}\n\n` : "") +
|
||||
(configList ? `**Config**\n${configList}\n` : "") +
|
||||
`</details>\n\n`;
|
||||
}
|
||||
|
||||
const commentBody = `## 🚀 V2 Auto-Deployment Complete!\n\n` +
|
||||
`Your V2 PR with embedded architecture has been deployed!\n\n` +
|
||||
`🔗 **Direct Test URL (non-SSL)** [${deploymentUrl}](${deploymentUrl})\n\n` +
|
||||
`🔐 **Secure HTTPS URL**: [${httpsUrl}](${httpsUrl})\n\n` +
|
||||
portalNote +
|
||||
storybookNote +
|
||||
`_This deployment will be automatically cleaned up when the PR is closed._\n\n` +
|
||||
`🔄 **Auto-deployed** for approved V2 contributors.`;
|
||||
|
||||
@@ -460,7 +558,11 @@ jobs:
|
||||
else
|
||||
echo "V2 PR directory not found, nothing to clean up"
|
||||
fi
|
||||
|
||||
|
||||
# Remove this PR's Storybook preview (container + files), if any.
|
||||
docker rm -f storybook-pr-${{ github.event.pull_request.number }} 2>/dev/null || true
|
||||
rm -rf /stirling/SB-PR-${{ github.event.pull_request.number }}
|
||||
|
||||
# Clean up old unused images (older than 2 weeks) but keep recent ones for reuse
|
||||
docker image prune -af --filter "until=336h" --filter "label!=keep=true" || true
|
||||
|
||||
|
||||
@@ -270,7 +270,7 @@ jobs:
|
||||
|
||||
- name: Build and push PR-specific image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
@@ -296,7 +296,7 @@ jobs:
|
||||
|
||||
- name: Build and push engine image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && needs.check-comment.outputs.enable_prototypes == 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: ./engine
|
||||
file: ./engine/Dockerfile
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
name: AI Engine CI
|
||||
|
||||
# Validates the Python AI engine: regenerates tool models and runs the
|
||||
# engine quality gate (lint, type-check, format-check, tests). Called from
|
||||
# build.yml on PRs and merge_group; also runs directly on push to main as
|
||||
# a post-merge safety net.
|
||||
# Runs the engine quality gate (lint, type-check, format-check, tests). Called
|
||||
# from build.yml on PRs and merge_group; also runs directly on push to main as
|
||||
# a post-merge safety net. Freshness of the generated tool_models.py is checked
|
||||
# by the shared check-generated-models workflow.
|
||||
on:
|
||||
workflow_call:
|
||||
push:
|
||||
@@ -34,104 +34,9 @@ jobs:
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- name: Set up JDK 25
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
with:
|
||||
java-version: "25"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
- name: Regenerate tool models
|
||||
run: task engine:tool-models
|
||||
|
||||
- name: Verify tool models are up to date
|
||||
id: tool-models-check
|
||||
continue-on-error: true
|
||||
run: git diff --exit-code engine/src/stirling/models/tool_models.py
|
||||
|
||||
- name: Comment on tool models check failure
|
||||
# Only post a comment on PRs. github-script's PR helpers need an
|
||||
# issue/PR number, which doesn't exist on merge_group runs.
|
||||
if: steps.tool-models-check.outcome == 'failure' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- tool-models-check -->';
|
||||
const body = [
|
||||
marker,
|
||||
'### Tool Models Check Failed',
|
||||
'',
|
||||
'The generated `engine/src/stirling/models/tool_models.py` is out of date with the Java OpenAPI spec and will need to be regenerated before it can be merged in.',
|
||||
'',
|
||||
'Run `task engine:tool-models` to regenerate, then commit the updated file.',
|
||||
].join('\n');
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Fail if tool models check failed
|
||||
if: steps.tool-models-check.outcome == 'failure'
|
||||
run: |
|
||||
echo "============================================"
|
||||
echo " Tool Models Check Failed"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "The generated engine/src/stirling/models/tool_models.py"
|
||||
echo "is out of date with the Java OpenAPI spec and will"
|
||||
echo "need to be regenerated before it can be merged in."
|
||||
echo ""
|
||||
echo "Run 'task engine:tool-models' to regenerate, then"
|
||||
echo "commit the updated file."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
|
||||
- name: Remove tool models check comment on success
|
||||
if: steps.tool-models-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- tool-models-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Quality-check engine
|
||||
id: engine-check
|
||||
run: task engine:check
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -2,7 +2,7 @@ name: Enterprise E2E (Playwright)
|
||||
|
||||
# Enterprise Playwright suite — exercises premium-key gated features (audit,
|
||||
# teams, analytics) plus full OAuth + SAML logins via the Keycloak compose
|
||||
# stacks under testing/compose. Slow and secret-gated, so it runs in three
|
||||
# stacks under testing/compose. Slow and secret-gated, so it runs in four
|
||||
# situations:
|
||||
#
|
||||
# - PRs that touch proprietary / premium / SSO compose / enterprise tests
|
||||
@@ -12,8 +12,6 @@ name: Enterprise E2E (Playwright)
|
||||
# - on a nightly cron schedule (catches Keycloak image drift, license
|
||||
# expiry, upstream proprietary changes),
|
||||
# - manual workflow_dispatch.
|
||||
#
|
||||
# Auto-skipped when secrets.PREMIUM_KEY_ENTERPRISE is missing (forks, dependabot).
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
@@ -52,6 +50,10 @@ jobs:
|
||||
|
||||
playwright-e2e-enterprise:
|
||||
needs: pick
|
||||
# Skip on fork PRs / untrusted authors: they have no PREMIUM_KEY_ENTERPRISE
|
||||
# (nor DEPOT_TOKEN), so the suite can't boot premium and would fail. See the
|
||||
# header comment. GitHub reports the skipped reusable workflow as success.
|
||||
if: needs.pick.outputs.is_fork != 'true'
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
@@ -165,6 +167,8 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise OAuth Playwright tests
|
||||
id: oauth-tests
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-oauth.json
|
||||
run: task e2e:enterprise -- --grep "OAuth"
|
||||
- name: Stop backend + tear down OAuth Keycloak
|
||||
if: always()
|
||||
@@ -238,6 +242,8 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise SAML Playwright tests
|
||||
id: saml-tests
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-saml.json
|
||||
run: task e2e:enterprise -- --grep "SAML"
|
||||
- name: Stop backend + tear down SAML Keycloak
|
||||
if: always()
|
||||
@@ -268,6 +274,8 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise feature Playwright tests
|
||||
id: feature-tests
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-feature.json
|
||||
run: task e2e:enterprise -- --grep "Enterprise license"
|
||||
- name: Print backend log on failure
|
||||
if: failure()
|
||||
@@ -280,10 +288,23 @@ jobs:
|
||||
run: |
|
||||
source /tmp/helpers.sh
|
||||
stop_backend
|
||||
- name: Flag flaky tests
|
||||
# Runs regardless of the test outcomes: a flaky test (passed on retry)
|
||||
# leaves its step green, so this is the only place it surfaces. Merges
|
||||
# all three phase reports (some may be absent if an earlier phase hard-
|
||||
# failed and skipped the rest). Emits ::warning:: annotations + a job
|
||||
# summary; never fails the job.
|
||||
if: always()
|
||||
working-directory: frontend
|
||||
run: >
|
||||
npx tsx editor/scripts/report-flaky-tests.mts
|
||||
"${{ github.workspace }}/frontend/playwright-report/results-oauth.json"
|
||||
"${{ github.workspace }}/frontend/playwright-report/results-saml.json"
|
||||
"${{ github.workspace }}/frontend/playwright-report/results-feature.json"
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-enterprise-${{ github.run_id }}
|
||||
path: frontend/editor/playwright-report/
|
||||
path: frontend/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
@@ -43,6 +43,7 @@ jobs:
|
||||
docker-base: ${{ steps.changes.outputs.docker-base }}
|
||||
tauri: ${{ steps.changes.outputs.tauri }}
|
||||
engine: ${{ steps.changes.outputs.engine }}
|
||||
generated-models: ${{ steps.changes.outputs.generated-models }}
|
||||
proprietary: ${{ steps.changes.outputs.proprietary }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
@@ -171,6 +172,20 @@ jobs:
|
||||
uses: ./.github/workflows/ai-engine.yml
|
||||
secrets: inherit
|
||||
|
||||
# The generated frontend types and engine tool models are both derived from
|
||||
# the Java OpenAPI spec. This job regenerates and diffs them; it boots the
|
||||
# backend, so it is gated on the narrow generated-models filter (spec source,
|
||||
# generators, generated files, generation tasks) rather than the broad
|
||||
# frontend filter, so a CSS-only PR does not pay for a backend build.
|
||||
generated-models:
|
||||
if: needs.files-changed.outputs.generated-models == 'true'
|
||||
needs: [files-changed]
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
uses: ./.github/workflows/check-generated-models.yml
|
||||
secrets: inherit
|
||||
|
||||
pre-commit:
|
||||
needs: [files-changed]
|
||||
permissions:
|
||||
@@ -202,6 +217,9 @@ jobs:
|
||||
contents: read
|
||||
uses: ./.github/workflows/coverage-aggregate.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
frontend-validation-result: ${{ needs.frontend-validation.result }}
|
||||
playwright-e2e-live-result: ${{ needs.playwright-e2e-live.result }}
|
||||
|
||||
# Single status check that branch protection should mark as required.
|
||||
# Succeeds when every upstream job is either `success` or `skipped` (path-
|
||||
@@ -225,6 +243,7 @@ jobs:
|
||||
- test-build-docker-images
|
||||
- tauri-build
|
||||
- ai-engine
|
||||
- generated-models
|
||||
- pre-commit
|
||||
- dependency-review
|
||||
runs-on: ubuntu-latest
|
||||
@@ -250,6 +269,7 @@ jobs:
|
||||
test-build-docker-images=${{ needs.test-build-docker-images.result }}
|
||||
tauri-build=${{ needs.tauri-build.result }}
|
||||
ai-engine=${{ needs.ai-engine.result }}
|
||||
generated-models=${{ needs.generated-models.result }}
|
||||
pre-commit=${{ needs.pre-commit.result }}
|
||||
dependency-review=${{ needs.dependency-review.result }}
|
||||
run: |
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
name: Check generated models
|
||||
|
||||
# Verifies the committed generated API models are still in sync with the Java
|
||||
# OpenAPI spec: the frontend tool API types
|
||||
# (frontend/editor/src/core/types/toolApiTypes.ts) and the engine tool
|
||||
# models (engine/src/stirling/models/tool_models.py). Regenerates both with the
|
||||
# single top-level `task tool-models` and fails if either committed file is
|
||||
# out of date. Called from build.yml when the backend Java, frontend, or engine
|
||||
# changes; also runs on push to main as a post-merge safety net.
|
||||
on:
|
||||
workflow_call:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
generated-models:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- name: Set up JDK 25
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
with:
|
||||
java-version: "25"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
# Rebuilds the OpenAPI spec from the current Java and regenerates both the
|
||||
# frontend types and the engine tool models from it.
|
||||
- name: Regenerate generated models
|
||||
run: task tool-models
|
||||
|
||||
- name: Verify generated models are up to date
|
||||
id: models-check
|
||||
continue-on-error: true
|
||||
run: |
|
||||
git diff --exit-code \
|
||||
frontend/editor/src/core/types/toolApiTypes.ts \
|
||||
engine/src/stirling/models/tool_models.py
|
||||
|
||||
- name: Comment on generated models check failure
|
||||
# Only post a comment on PRs. github-script's PR helpers need an
|
||||
# issue/PR number, which doesn't exist on merge_group runs.
|
||||
if: steps.models-check.outcome == 'failure' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- generated-models-check -->';
|
||||
const body = [
|
||||
marker,
|
||||
'### Generated Models Check Failed',
|
||||
'',
|
||||
'The generated `frontend/editor/src/core/types/toolApiTypes.ts` and/or `engine/src/stirling/models/tool_models.py` are out of date with the Java OpenAPI spec and will need to be regenerated before they can be merged in.',
|
||||
'',
|
||||
'Run `task tool-models` to regenerate both, then commit the updated files.',
|
||||
].join('\n');
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Fail if generated models check failed
|
||||
if: steps.models-check.outcome == 'failure'
|
||||
run: |
|
||||
echo "============================================"
|
||||
echo " Generated Models Check Failed"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "The generated frontend API types and/or engine tool"
|
||||
echo "models are out of date with the Java OpenAPI spec and"
|
||||
echo "will need to be regenerated before they can be merged in."
|
||||
echo ""
|
||||
echo "Run 'task tool-models' to regenerate both, then"
|
||||
echo "commit the updated files."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
|
||||
- name: Remove generated models check comment on success
|
||||
if: steps.models-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- generated-models-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -13,6 +13,17 @@ name: Aggregate backend coverage
|
||||
# producers themselves
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
frontend-validation-result:
|
||||
description: Result of the frontend-validation producer job
|
||||
required: false
|
||||
type: string
|
||||
default: skipped
|
||||
playwright-e2e-live-result:
|
||||
description: Result of the playwright-e2e-live producer job
|
||||
required: false
|
||||
type: string
|
||||
default: skipped
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -40,7 +51,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
@@ -196,9 +207,9 @@ jobs:
|
||||
# --------------------------------------------------------------
|
||||
- name: Download vitest coverage artifact
|
||||
# frontend-validation uploads as `frontend-coverage`. Tolerate
|
||||
# absence so a backend-only PR still produces the matrix with
|
||||
# just backend rows populated.
|
||||
if: always()
|
||||
# absence on backend-only runs by skipping the download entirely
|
||||
# when the producer job was not part of this workflow run.
|
||||
if: inputs.frontend-validation-result == 'success'
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
|
||||
with:
|
||||
name: frontend-coverage
|
||||
@@ -206,12 +217,12 @@ jobs:
|
||||
continue-on-error: true
|
||||
|
||||
- name: Download Playwright frontend coverage artifact
|
||||
# e2e-live uploads as `playwright-frontend-coverage-<run_id>`.
|
||||
# Same tolerance as vitest - matrix script handles missing inputs.
|
||||
if: always()
|
||||
# e2e-live uploads the artifact with a stable name. Skip the
|
||||
# download entirely when the producer job did not run.
|
||||
if: inputs.playwright-e2e-live-result == 'success'
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
|
||||
with:
|
||||
name: playwright-frontend-coverage-${{ github.run_id }}
|
||||
name: playwright-frontend-coverage
|
||||
path: matrix-inputs/playwright/
|
||||
continue-on-error: true
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
distribution: temurin
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Build and push frontend image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-frontend.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/frontend/Dockerfile
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
|
||||
- name: Build and push backend image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-backend.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/backend/Dockerfile
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -62,7 +62,17 @@ jobs:
|
||||
# .test-state/playwright/coverage-pw/ for the post-process step
|
||||
# to aggregate. Chromium-only - other engines silently skip.
|
||||
PW_COVERAGE: "1"
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
|
||||
run: task e2e:live
|
||||
- name: Flag flaky tests
|
||||
# Runs regardless of the test outcome: a flaky test (passed on retry)
|
||||
# leaves the step green, so this is the only place it surfaces. Emits
|
||||
# ::warning:: annotations + a job summary; never fails the job.
|
||||
if: always()
|
||||
working-directory: frontend
|
||||
run: npx tsx editor/scripts/report-flaky-tests.mts "$PLAYWRIGHT_JSON_OUTPUT_FILE"
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
|
||||
- name: Generate JaCoCo report from e2e:live .exec
|
||||
if: always()
|
||||
id: live-coverage
|
||||
@@ -169,7 +179,7 @@ jobs:
|
||||
if: always() && steps.pw-frontend-coverage.outputs.summary == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-frontend-coverage-${{ github.run_id }}
|
||||
name: playwright-frontend-coverage
|
||||
path: |
|
||||
.test-state/playwright/coverage-pw-summary/
|
||||
.test-state/playwright/coverage-pw/
|
||||
|
||||
@@ -44,11 +44,22 @@ jobs:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
- name: Run stubbed E2E tests (chromium)
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
|
||||
run: task e2e:stubbed -- --workers=3
|
||||
- name: Flag flaky tests
|
||||
# Runs regardless of the test outcome: a flaky test (passed on retry)
|
||||
# leaves the step green, so this is the only place it surfaces. Emits
|
||||
# ::warning:: annotations + a job summary; never fails the job.
|
||||
if: always()
|
||||
working-directory: frontend
|
||||
run: npx tsx editor/scripts/report-flaky-tests.mts "$PLAYWRIGHT_JSON_OUTPUT_FILE"
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-stubbed-${{ github.run_id }}
|
||||
path: frontend/editor/playwright-report/
|
||||
path: frontend/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
@@ -98,6 +98,13 @@ jobs:
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
- name: Generate frontend license report (Push only)
|
||||
if: github.event_name == 'push'
|
||||
env:
|
||||
PR_IS_FORK: "false"
|
||||
run: task frontend:licenses:generate
|
||||
|
||||
- name: Generate frontend license report (internal PR)
|
||||
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
|
||||
env:
|
||||
@@ -353,6 +360,7 @@ jobs:
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
- name: Check licenses and generate report
|
||||
id: license-check
|
||||
run: task backend:licenses:generate || echo "LICENSE_CHECK_FAILED=true" >> $GITHUB_ENV
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependencies
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
|
||||
@@ -53,8 +53,8 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-nightly-${{ github.run_id }}
|
||||
path: frontend/editor/playwright-report/
|
||||
name: playwright-report-nightly-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
retention-days: 14
|
||||
|
||||
# Builds all desktop platforms on a schedule so the Rust dependency cache is
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
|
||||
- name: Build and push base image
|
||||
id: build-push-base
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: docker/base
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependencies
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
@@ -145,7 +145,7 @@ jobs:
|
||||
id: build-push-latest
|
||||
# Empty-tag guard: build-push-action errors when asked to push with no tags.
|
||||
if: env.RUN_MAIN_APP == 'true' && steps.meta.outputs.tags != ''
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
|
||||
- name: Build and push Unified Dockerfile (fat variant)
|
||||
id: build-push-fat
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-fat.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
@@ -236,7 +236,7 @@ jobs:
|
||||
|
||||
- name: Build and push Unified Dockerfile (ultra-lite variant)
|
||||
id: build-push-lite
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-lite.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
@@ -365,7 +365,7 @@ jobs:
|
||||
- name: Build and push unoserver image
|
||||
id: build-push-unoserver
|
||||
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode != 'skip'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
@@ -186,7 +186,7 @@ jobs:
|
||||
# (multi-platform + gha cache) against the published base image.
|
||||
- name: Build ${{ matrix.docker-rev }} (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && inputs.docker-base-changed != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
@@ -259,7 +259,7 @@ jobs:
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Build and push test image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
|
||||
@@ -15,10 +15,10 @@ testing/compose/validate-mcp-test.sh:curl-auth-header:92
|
||||
testing/compose/validate-mcp-test.sh:curl-auth-header:116
|
||||
|
||||
# Storybook example showing curl with a fake Bearer token placeholder (sk_live_a3f8...).
|
||||
frontend/shared/components/CodeBlock.stories.tsx:curl-auth-header:4
|
||||
frontend/editor/src/proprietary/ui/CodeBlock.stories.tsx:curl-auth-header:5
|
||||
|
||||
# Truncated placeholder API key in portal docs example (sk_live_8f2c...e10) - not a real secret.
|
||||
frontend/portal/src/components/docs/GettingStartedSection.tsx:generic-api-key:31
|
||||
frontend/editor/src/portal/components/docs/GettingStartedSection.tsx:generic-api-key:30
|
||||
|
||||
# False positive: generic-api-key matches the Java type name "X509Certificate"
|
||||
# in a method signature (CreateSignatureBase.resolveSignatureAlgorithm) - not a secret.
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"ignoredFiles": [
|
||||
"frontend/editor/src-tauri/icons/icon.png"
|
||||
]
|
||||
}
|
||||
@@ -80,6 +80,12 @@ tasks:
|
||||
OPEN: '{{.OPEN | default ""}}'
|
||||
env:
|
||||
BACKEND_URL: '{{.BACKEND_URL}}'
|
||||
# Dev-only browser-tab label so concurrent worktrees are distinguishable.
|
||||
# Only the worktree folder basename (e.g. "wt1") is exposed — never the
|
||||
# full path, hostname, or user. Consumed at dev-serve time by vite.config
|
||||
# and dropped from production builds.
|
||||
STIRLING_DEV_LABEL:
|
||||
sh: basename "$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cmds:
|
||||
- npx vite editor --mode {{.MODE}} --port {{.PORT}}{{if .OPEN}} --open{{end}}
|
||||
|
||||
@@ -128,37 +134,6 @@ tasks:
|
||||
- task: dev:_run
|
||||
vars: { MODE: prototypes, PORT: '{{.PORT}}', BACKEND_URL: '{{.BACKEND_URL}}', OPEN: '{{.OPEN}}' }
|
||||
|
||||
dev:portal:
|
||||
desc: "Start developer portal dev server"
|
||||
ignore_error: true
|
||||
deps: [install]
|
||||
vars:
|
||||
PORT: '{{.PORT | default "5173"}}'
|
||||
BACKEND_URL: '{{.BACKEND_URL | default "http://localhost:8080"}}'
|
||||
EDITOR_URL: '{{.EDITOR_URL | default ""}}'
|
||||
OPEN: '{{.OPEN | default ""}}'
|
||||
SUBPATH: '{{.SUBPATH | default ""}}'
|
||||
MOCKS: '{{.MOCKS | default ""}}'
|
||||
env:
|
||||
BACKEND_URL: '{{.BACKEND_URL}}'
|
||||
cmds:
|
||||
- '{{if .SUBPATH}}RUN_SUBPATH={{.SUBPATH}} {{end}}{{if .MOCKS}}VITE_PORTAL_MOCKS={{.MOCKS}} {{end}}{{if .EDITOR_URL}}VITE_EDITOR_URL={{.EDITOR_URL}} {{end}}npx vite portal --port {{.PORT}}{{if .OPEN}} --open{{end}}'
|
||||
|
||||
dev:portal:proxy:serve:
|
||||
internal: true
|
||||
vars:
|
||||
PORT: '{{.PORT | default "3000"}}'
|
||||
BACKEND_URL: '{{.BACKEND_URL | default "http://localhost:8080"}}'
|
||||
EDITOR_DEV_URL: '{{.EDITOR_DEV_URL | default ""}}'
|
||||
PORTAL_DEV_URL: '{{.PORTAL_DEV_URL | default ""}}'
|
||||
env:
|
||||
PORT: '{{.PORT}}'
|
||||
BACKEND_URL: '{{.BACKEND_URL}}'
|
||||
EDITOR_DEV_URL: '{{.EDITOR_DEV_URL}}'
|
||||
PORTAL_DEV_URL: '{{.PORTAL_DEV_URL}}'
|
||||
cmds:
|
||||
- npx tsx scripts/dev-origin-proxy.ts
|
||||
|
||||
# ============================================================
|
||||
# Build
|
||||
# ============================================================
|
||||
@@ -205,29 +180,6 @@ tasks:
|
||||
cmds:
|
||||
- npx vite build editor --mode prototypes
|
||||
|
||||
build:portal:
|
||||
desc: "Build developer portal"
|
||||
deps: [install]
|
||||
vars:
|
||||
SUBPATH: '{{.SUBPATH | default ""}}'
|
||||
cmds:
|
||||
- '{{if .SUBPATH}}RUN_SUBPATH={{.SUBPATH}} {{end}}npx vite build portal'
|
||||
|
||||
preview:portal:proxy:
|
||||
desc: "Build + serve editor + portal behind one origin (prod-like auth testing)"
|
||||
deps: [prepare]
|
||||
vars:
|
||||
PORT: '{{.PORT | default "3000"}}'
|
||||
BACKEND_URL: '{{.BACKEND_URL | default "http://localhost:8080"}}'
|
||||
env:
|
||||
PORT: '{{.PORT}}'
|
||||
BACKEND_URL: '{{.BACKEND_URL}}'
|
||||
cmds:
|
||||
- task: build:proprietary
|
||||
vars: { PREVIEW: '1' }
|
||||
- task: build:portal
|
||||
vars: { SUBPATH: portal }
|
||||
- npx tsx scripts/dev-origin-proxy.ts
|
||||
|
||||
storybook:
|
||||
desc: "Start Storybook dev server"
|
||||
@@ -263,8 +215,8 @@ tasks:
|
||||
deps: [install]
|
||||
cmds:
|
||||
# Globs so dpdm walks the whole tree. dpdm expands the braces itself, so this is
|
||||
# shell-agnostic. Covers editor, portal, and the shared design system.
|
||||
- npx dpdm "editor/src/**/*.{ts,tsx}" "portal/src/**/*.{ts,tsx}" "shared/**/*.{ts,tsx}" --circular --no-warning --no-tree --exit-code circular:1
|
||||
# shell-agnostic. Covers the whole editor tree, including the portal layer.
|
||||
- npx dpdm "editor/src/**/*.{ts,tsx}" --circular --no-warning --no-tree --exit-code circular:1
|
||||
|
||||
lint:fix:
|
||||
desc: "Auto-fix lint issues"
|
||||
@@ -345,8 +297,6 @@ tasks:
|
||||
desc: "Typecheck scripts"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: scripts/tsconfig.json }
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/scripts/tsconfig.json }
|
||||
|
||||
@@ -362,14 +312,7 @@ tasks:
|
||||
deps: [install]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: portal/tsconfig.json }
|
||||
|
||||
typecheck:shared:
|
||||
desc: "Typecheck the shared design system"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: shared/tsconfig.json }
|
||||
vars: { PROJECT: editor/src/portal/tsconfig.json }
|
||||
|
||||
typecheck:all:
|
||||
desc: "Typecheck all build variants"
|
||||
@@ -382,7 +325,6 @@ tasks:
|
||||
- task: typecheck:scripts
|
||||
- task: typecheck:prototypes
|
||||
- task: typecheck:portal
|
||||
- task: typecheck:shared
|
||||
|
||||
# ============================================================
|
||||
# Quality Gate
|
||||
@@ -411,7 +353,6 @@ tasks:
|
||||
- task: lint
|
||||
- task: format:check
|
||||
- task: build
|
||||
- task: build:portal
|
||||
- task: test
|
||||
- task: storybook:build
|
||||
|
||||
@@ -423,7 +364,6 @@ tasks:
|
||||
desc: "Run tests"
|
||||
cmds:
|
||||
- task: test:editor
|
||||
- task: test:portal
|
||||
|
||||
test:editor:
|
||||
desc: "Run editor tests"
|
||||
@@ -431,12 +371,6 @@ tasks:
|
||||
cmds:
|
||||
- npx vitest run --root editor
|
||||
|
||||
test:portal:
|
||||
desc: "Run portal tests"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx vitest run --root portal
|
||||
|
||||
test:watch:
|
||||
desc: "Run tests in watch mode"
|
||||
deps: [prepare]
|
||||
@@ -468,6 +402,23 @@ tasks:
|
||||
# Code Generation
|
||||
# ============================================================
|
||||
|
||||
tool-models:
|
||||
desc: "Generate tool API types from the Java OpenAPI spec"
|
||||
deps: [install, ":backend:swagger"]
|
||||
cmds:
|
||||
- npx tsx editor/scripts/generate-tool-api-types.mts --spec ../SwaggerDoc.json --output editor/src/core/types/toolApiTypes.ts
|
||||
sources:
|
||||
- editor/scripts/generate-tool-api-types.mts
|
||||
- ../SwaggerDoc.json
|
||||
generates:
|
||||
- editor/src/core/types/toolApiTypes.ts
|
||||
|
||||
tool-models:check:
|
||||
desc: "Fail if committed tool API types are out of date"
|
||||
deps: [install, ":backend:swagger"]
|
||||
cmds:
|
||||
- npx tsx editor/scripts/generate-tool-api-types.mts --spec ../SwaggerDoc.json --output editor/src/core/types/toolApiTypes.ts --check
|
||||
|
||||
licenses:generate:
|
||||
desc: "Generate frontend license report"
|
||||
deps: [install]
|
||||
@@ -481,7 +432,7 @@ tasks:
|
||||
clean:
|
||||
desc: "Clean build artifacts and caches"
|
||||
cmds:
|
||||
- cmd: powershell rm -Recurse -Force -ErrorAction SilentlyContinue node_modules/.vite, editor/dist, dist, dist-portal
|
||||
- cmd: powershell rm -Recurse -Force -ErrorAction SilentlyContinue node_modules/.vite, editor/dist, dist
|
||||
platforms: [windows]
|
||||
- cmd: rm -rf node_modules/.vite editor/dist dist dist-portal
|
||||
- cmd: rm -rf node_modules/.vite editor/dist dist
|
||||
platforms: [linux, darwin]
|
||||
|
||||
@@ -139,7 +139,8 @@ The project structure is defined in `engine/pyproject.toml`. Any new dependencie
|
||||
|
||||
#### Environment Variables
|
||||
- All `VITE_*` variables must be declared in the appropriate committed env file:
|
||||
- `frontend/editor/.env` — core, proprietary, and shared vars
|
||||
- `frontend/editor/.env` — core and shared vars (base, loaded in every mode)
|
||||
- `frontend/editor/.env.proprietary` — proprietary-only vars, e.g. the admin portal's SaaS/account-link keys (layered on top of `.env` in proprietary mode)
|
||||
- `frontend/editor/.env.saas` — SaaS-only vars (layered on top of `.env` in SaaS mode)
|
||||
- `frontend/editor/.env.desktop` — desktop (Tauri)-only vars (layered on top of `.env` in desktop mode)
|
||||
- These files are committed to Git and must not contain private keys
|
||||
@@ -452,6 +453,7 @@ The frontend is organized with a clear separation of concerns:
|
||||
|
||||
- **CRITICAL**: Always update translations in `en-US` only - all other languages (including `en-GB`) are handled separately
|
||||
- Translation files are located in `frontend/editor/public/locales/`
|
||||
- After changing any translation file, run `task pre-commit:fix`
|
||||
|
||||
## Important Notes
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ Visit the [Lombok website](https://projectlombok.org/setup/) for installation in
|
||||
|
||||
5. Add environment variable
|
||||
For local testing, you should generally be testing the full 'Security' version of Stirling PDF. To do this, you must add the environment flag DISABLE_ADDITIONAL_FEATURES=false to your system and/or IDE build/run step.
|
||||
5. **Frontend Setup (Required for Stirling 2.0)**
|
||||
6. **Frontend Setup (Required for Stirling 2.0)**
|
||||
Navigate to the frontend directory and install dependencies using npm.
|
||||
|
||||
### Verify Setup
|
||||
@@ -275,7 +275,7 @@ Stirling-PDF uses different Docker images for various configurations. The build
|
||||
1. Set the security environment variable:
|
||||
|
||||
```bash
|
||||
export DISABLE_ADDITIONAL_FEATURES=true # or false for to enable login and security features for builds
|
||||
export DISABLE_ADDITIONAL_FEATURES=true # or false to enable login and security features for builds
|
||||
```
|
||||
|
||||
2. Build the project:
|
||||
@@ -305,7 +305,7 @@ Stirling-PDF uses different Docker images for various configurations. The build
|
||||
docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest-fat -f ./Dockerfile.fat .
|
||||
```
|
||||
|
||||
Note: The `--no-cache` and `--pull` flags ensure that the build process uses the latest base images and doesn't use cached layers, which is useful for testing and ensuring reproducible builds. however to improve build times these can often be removed depending on your usecase
|
||||
Note: The `--no-cache` and `--pull` flags ensure that the build process uses the latest base images and doesn't use cached layers, which is useful for testing and ensuring reproducible builds. However, to improve build times these can often be removed depending on your use case
|
||||
|
||||
## 7. Testing
|
||||
|
||||
|
||||
@@ -20,8 +20,8 @@ if that directory exists, is licensed under the license defined in "frontend/edi
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/cloud/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/prototypes/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/prototypes/LICENSE".
|
||||
* All content that resides under the "frontend/portal/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/portal/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/portal/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/portal/LICENSE".
|
||||
* Content outside of the above mentioned directories or restrictions above is
|
||||
available under the MIT License as defined below.
|
||||
|
||||
|
||||
@@ -53,8 +53,8 @@ For full installation options (including desktop and Kubernetes), see our [Docum
|
||||
|
||||
## Support
|
||||
|
||||
- **Community** [Discord](https://discord.gg/HYmhKj45pU)
|
||||
- **Bug Reports**: [Github issues](https://github.com/Stirling-Tools/Stirling-PDF/issues)
|
||||
- **Community**: [Discord](https://discord.gg/HYmhKj45pU)
|
||||
- **Bug Reports**: [GitHub Issues](https://github.com/Stirling-Tools/Stirling-PDF/issues)
|
||||
|
||||
## Contributing
|
||||
|
||||
|
||||
@@ -79,86 +79,12 @@ tasks:
|
||||
OPEN: "true"
|
||||
|
||||
dev:portal:
|
||||
desc: "Start backend + developer portal concurrently on free ports"
|
||||
desc: "Start backend + editor; the portal is an admin route at /portal"
|
||||
vars:
|
||||
PORTS:
|
||||
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 5173{{else}}{{.FIND_FREE_PORT_SH}} 8080 5173{{end}}'
|
||||
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
|
||||
PORTAL_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
deps:
|
||||
- task: backend:dev
|
||||
vars:
|
||||
PORT: '{{.BACKEND_PORT}}'
|
||||
SECURITY_ENABLELOGIN: "true"
|
||||
POLICIES_ENABLED: "true"
|
||||
- task: frontend:dev:portal
|
||||
vars:
|
||||
PORT: '{{.PORTAL_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
MOCKS: 'false'
|
||||
OPEN: "true"
|
||||
|
||||
dev:portal:all:
|
||||
desc: "Start backend + developer portal + editor concurrently on free ports"
|
||||
vars:
|
||||
PORTS:
|
||||
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 5173 5174{{else}}{{.FIND_FREE_PORT_SH}} 8080 5173 5174{{end}}'
|
||||
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
|
||||
PORTAL_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
EDITOR_PORT: '{{index (splitList "\n" .PORTS) 2}}'
|
||||
deps:
|
||||
- task: backend:dev
|
||||
vars:
|
||||
PORT: '{{.BACKEND_PORT}}'
|
||||
SECURITY_ENABLELOGIN: "true"
|
||||
POLICIES_ENABLED: "true"
|
||||
- task: frontend:dev:portal
|
||||
vars:
|
||||
PORT: '{{.PORTAL_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
# Point the portal's "Editor" app switcher at the editor we spawn here.
|
||||
EDITOR_URL: 'http://localhost:{{.EDITOR_PORT}}/'
|
||||
MOCKS: 'false'
|
||||
OPEN: "true"
|
||||
- task: frontend:dev
|
||||
vars:
|
||||
PORT: '{{.EDITOR_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
|
||||
dev:portal:all:saas:
|
||||
desc: "Start SaaS backend + developer portal + editor concurrently on free ports"
|
||||
vars:
|
||||
PORTS:
|
||||
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 5173 5174{{else}}{{.FIND_FREE_PORT_SH}} 8080 5173 5174{{end}}'
|
||||
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
|
||||
PORTAL_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
EDITOR_PORT: '{{index (splitList "\n" .PORTS) 2}}'
|
||||
deps:
|
||||
- task: backend:dev:saas
|
||||
vars:
|
||||
PORT: '{{.BACKEND_PORT}}'
|
||||
POLICIES_ENABLED: "true"
|
||||
- task: frontend:dev:portal
|
||||
vars:
|
||||
PORT: '{{.PORTAL_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
EDITOR_URL: 'http://localhost:{{.EDITOR_PORT}}/'
|
||||
MOCKS: 'false'
|
||||
OPEN: "true"
|
||||
- task: frontend:dev
|
||||
vars:
|
||||
PORT: '{{.EDITOR_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
|
||||
dev:portal:proxy:
|
||||
desc: "Editor + portal on ONE origin + backend via live dev servers (shared-token login)"
|
||||
vars:
|
||||
PORTS:
|
||||
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 3000 5173 5174{{else}}{{.FIND_FREE_PORT_SH}} 8080 3000 5173 5174{{end}}'
|
||||
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
|
||||
PROXY_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
EDITOR_PORT: '{{index (splitList "\n" .PORTS) 2}}'
|
||||
PORTAL_PORT: '{{index (splitList "\n" .PORTS) 3}}'
|
||||
EDITOR_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
deps:
|
||||
- task: backend:dev
|
||||
vars:
|
||||
@@ -169,18 +95,7 @@ tasks:
|
||||
vars:
|
||||
PORT: '{{.EDITOR_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
- task: frontend:dev:portal
|
||||
vars:
|
||||
PORT: '{{.PORTAL_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
SUBPATH: portal
|
||||
MOCKS: 'false'
|
||||
- task: frontend:dev:portal:proxy:serve
|
||||
vars:
|
||||
PORT: '{{.PROXY_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
EDITOR_DEV_URL: 'http://localhost:{{.EDITOR_PORT}}'
|
||||
PORTAL_DEV_URL: 'http://localhost:{{.PORTAL_PORT}}'
|
||||
OPEN: "true"
|
||||
|
||||
dev:saas:
|
||||
desc: "Start SaaS backend + frontend concurrently on free ports"
|
||||
@@ -228,24 +143,6 @@ tasks:
|
||||
- task: backend:build
|
||||
- task: frontend:build
|
||||
|
||||
preview:portal:proxy:
|
||||
desc: "Build + serve editor + portal on ONE origin + backend (prod-like auth test)"
|
||||
vars:
|
||||
PORTS:
|
||||
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 3000{{else}}{{.FIND_FREE_PORT_SH}} 8080 3000{{end}}'
|
||||
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
|
||||
PROXY_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
deps:
|
||||
- task: backend:dev
|
||||
vars:
|
||||
PORT: '{{.BACKEND_PORT}}'
|
||||
SECURITY_ENABLELOGIN: "true"
|
||||
POLICIES_ENABLED: "true"
|
||||
- task: frontend:preview:portal:proxy
|
||||
vars:
|
||||
PORT: '{{.PROXY_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
|
||||
# ============================================================
|
||||
# Test
|
||||
# ============================================================
|
||||
@@ -288,6 +185,16 @@ tasks:
|
||||
- task: frontend:format:check
|
||||
- task: engine:format:check
|
||||
|
||||
# ============================================================
|
||||
# Code generation
|
||||
# ============================================================
|
||||
|
||||
tool-models:
|
||||
desc: "Generate all API models from the Java OpenAPI spec"
|
||||
cmds:
|
||||
- task: frontend:tool-models
|
||||
- task: engine:tool-models
|
||||
|
||||
# ============================================================
|
||||
# Quality Gate
|
||||
# ============================================================
|
||||
|
||||
@@ -80,10 +80,18 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Apache License Version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Apache License version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Apache License, Version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Apache License, version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "The Apache License, Version 2.0"
|
||||
@@ -108,6 +116,10 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Mozilla Public License 2.0 (MPL-2.0)"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Mozilla Public License Version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "CDDL+GPL License"
|
||||
@@ -172,6 +184,14 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Eclipse Public License, Version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "EPL-2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "LGPL-2.1-only"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Ubuntu Font Licence 1.0"
|
||||
|
||||
@@ -132,7 +132,7 @@ public class AppConfig {
|
||||
return true;
|
||||
}
|
||||
Path mountInfo = Path.of("/proc/1/mountinfo");
|
||||
// this should always exist, if not some unknown usecase
|
||||
// this should always exist, if not some unknown use case
|
||||
if (!Files.exists(mountInfo)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -246,6 +246,14 @@ public class ApplicationProperties {
|
||||
* and paused runs are kept regardless of age.
|
||||
*/
|
||||
private int runExpiryMinutes = 30;
|
||||
|
||||
/**
|
||||
* Whether a policy S3 source's custom endpoint may resolve to a loopback, link-local, or
|
||||
* private address. Off by default so a user-supplied endpoint cannot be pointed at internal
|
||||
* services (e.g. the cloud metadata address); enable for a self-hosted MinIO or other
|
||||
* in-network object store.
|
||||
*/
|
||||
private boolean allowPrivateS3Endpoints = false;
|
||||
}
|
||||
|
||||
@Data
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
/**
|
||||
* Thread-scoped correlation id for one automation run — a single pipeline, policy, or AI-workflow
|
||||
* execution over its input file(s).
|
||||
*
|
||||
* <p>Automations dispatch each tool step as a separate internal loopback POST via {@link
|
||||
* InternalApiClient}. The orchestrator opens a run scope around its dispatch loop; {@code
|
||||
* InternalApiClient} reads {@link #current()} and stamps it on every sub-step request as {@link
|
||||
* #RUN_ID_HEADER}. The SaaS PAYG interceptor uses that header so all sub-steps of ONE run group
|
||||
* into a single charge, while two <em>separate</em> runs that happen to touch identical bytes stay
|
||||
* distinct charges (the old content+time-window grouping merged them).
|
||||
*
|
||||
* <p>Sub-steps dispatch synchronously on the orchestrator's own thread (loopback {@code
|
||||
* RestTemplate}), so this ThreadLocal is visible to {@code InternalApiClient}. The id then crosses
|
||||
* to the receiving request thread via the HTTP header — never via this ThreadLocal.
|
||||
*
|
||||
* <p>No-op when the id is absent (a standalone tool call): the interceptor treats a missing run id
|
||||
* as "its own charge", which is exactly what a one-off call should be.
|
||||
*/
|
||||
public final class AutomationRunContext {
|
||||
|
||||
/** Header carrying the run id on internal sub-step dispatches. */
|
||||
public static final String RUN_ID_HEADER = "X-Stirling-Run-Id";
|
||||
|
||||
private static final ThreadLocal<String> CURRENT = new ThreadLocal<>();
|
||||
|
||||
private AutomationRunContext() {}
|
||||
|
||||
/**
|
||||
* Opens a run scope on the current thread. Returns an {@link AutoCloseable} that restores the
|
||||
* previously-active id (nesting-safe) — use in try-with-resources around the dispatch loop.
|
||||
*/
|
||||
public static Scope open(String runId) {
|
||||
String previous = CURRENT.get();
|
||||
CURRENT.set(runId);
|
||||
return () -> {
|
||||
if (previous == null) {
|
||||
CURRENT.remove();
|
||||
} else {
|
||||
CURRENT.set(previous);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** The run id active on this thread, or {@code null} when not inside a run scope. */
|
||||
public static String current() {
|
||||
return CURRENT.get();
|
||||
}
|
||||
|
||||
/** AutoCloseable whose {@link #close()} declares no checked exception. */
|
||||
public interface Scope extends AutoCloseable {
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.slf4j.MDC;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
@@ -60,6 +61,17 @@ public class InternalApiClient {
|
||||
*/
|
||||
public static final String AUTOMATION_HEADER = "X-Stirling-Automation";
|
||||
|
||||
/**
|
||||
* Header carrying the parent policy's name onto each sub-step dispatch, read from MDC key
|
||||
* {@link #POLICY_NAME_MDC_KEY} (set by the policy runner on the worker thread). Lets the audit
|
||||
* layer attribute a tool step to the policy that ran it, instead of showing it as a bare direct
|
||||
* call.
|
||||
*/
|
||||
public static final String POLICY_NAME_HEADER = "X-Stirling-Policy-Name";
|
||||
|
||||
/** MDC key the policy runner stamps with the running policy's name; forwarded as a header. */
|
||||
public static final String POLICY_NAME_MDC_KEY = "auditPolicyName";
|
||||
|
||||
private final ServletContext servletContext;
|
||||
private final UserServiceInterface userService;
|
||||
private final TempFileManager tempFileManager;
|
||||
@@ -111,6 +123,27 @@ public class InternalApiClient {
|
||||
// step inside a policy run must bill as AUTOMATION, not AI). Set unconditionally because
|
||||
// every caller of this dispatcher is an automation surface by design.
|
||||
headers.add(AUTOMATION_HEADER, "true");
|
||||
// Propagate the current automation run id (set by the orchestrator around its dispatch
|
||||
// loop) so the PAYG interceptor groups every sub-step of this one run into a single charge,
|
||||
// and never merges two separate runs that happen to touch identical bytes. Absent → the
|
||||
// receiving call is treated as standalone. See AutomationRunContext.
|
||||
String runId = AutomationRunContext.current();
|
||||
if (runId != null && !runId.isEmpty()) {
|
||||
headers.add(AutomationRunContext.RUN_ID_HEADER, runId);
|
||||
}
|
||||
|
||||
// Forward the parent policy name (set in MDC by the policy runner) so the audited sub-step
|
||||
// ties back to its policy. Single-line, length-capped: it becomes an HTTP header value.
|
||||
String policyName = MDC.get(POLICY_NAME_MDC_KEY);
|
||||
if (policyName != null && !policyName.isBlank()) {
|
||||
String safe = policyName.replaceAll("[\\r\\n]", " ").trim();
|
||||
if (safe.length() > 200) {
|
||||
safe = safe.substring(0, 200);
|
||||
}
|
||||
if (!safe.isEmpty()) {
|
||||
headers.add(POLICY_NAME_HEADER, safe);
|
||||
}
|
||||
}
|
||||
|
||||
// A no-file ai/tools call (e.g. create-pdf-from-html-agent) sends only string params, so
|
||||
// without this RestTemplate would use urlencoded instead of the multipart the controller
|
||||
|
||||
@@ -7,6 +7,7 @@ import java.time.format.DateTimeFormatter;
|
||||
import java.util.Calendar;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -17,6 +18,9 @@ import stirling.software.common.model.PdfMetadata;
|
||||
@Service
|
||||
public class PdfMetadataService {
|
||||
|
||||
/** ({@code {labels}}). Written by the classify-and-label tool. */
|
||||
public static final String CLASSIFICATION_KEY = "StirlingPDFClassification";
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final String stirlingPDFLabel;
|
||||
private final UserServiceInterface userService;
|
||||
@@ -177,4 +181,14 @@ public class PdfMetadataService {
|
||||
}
|
||||
pdf.getDocumentInformation().setAuthor(author);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the document classifier's JSON result into the custom Info-dictionary field {@link
|
||||
* #CLASSIFICATION_KEY}, leaving all other metadata untouched.
|
||||
*/
|
||||
public void setClassificationMetadata(PDDocument pdf, String classificationJson) {
|
||||
PDDocumentInformation info = pdf.getDocumentInformation();
|
||||
info.setCustomMetadataValue(CLASSIFICATION_KEY, classificationJson);
|
||||
pdf.setDocumentInformation(info);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,8 +144,10 @@ public class TempFileCleanupService {
|
||||
int directoriesDeletedCount = 0;
|
||||
for (Path directory : registry.getTempDirectories()) {
|
||||
try {
|
||||
if (Files.exists(directory)) {
|
||||
if (Files.exists(directory)
|
||||
&& shouldDeleteRegisteredDirectory(directory, maxAgeMillis)) {
|
||||
GeneralUtils.deleteDirectory(directory);
|
||||
registry.unregisterDirectory(directory);
|
||||
directoriesDeletedCount++;
|
||||
log.debug("Cleaned up temporary directory: {}", directory);
|
||||
}
|
||||
@@ -275,6 +277,21 @@ public class TempFileCleanupService {
|
||||
return totalDeletedCount.get();
|
||||
}
|
||||
|
||||
private boolean shouldDeleteRegisteredDirectory(Path directory, long maxAgeMillis) {
|
||||
if (maxAgeMillis <= 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
long currentTime = System.currentTimeMillis();
|
||||
long lastModified = Files.getLastModifiedTime(directory).toMillis();
|
||||
return (currentTime - lastModified) > maxAgeMillis;
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not check directory age, skipping cleanup: {}", directory, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Get the system temp directory path based on configuration or system property. */
|
||||
private Path getSystemTempPath() {
|
||||
String systemTempDir =
|
||||
|
||||
@@ -57,6 +57,16 @@ public class RequestUriUtils {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Admin portal SPA shell (mounted at /processor — must match the frontend
|
||||
// PORTAL_BASENAME). Served publicly like the editor root so a direct nav /
|
||||
// refresh to /processor loads the app (the JWT lives in localStorage, not a
|
||||
// cookie, so the server can't authenticate the navigation itself). The
|
||||
// portal gates access via its own auth gate + RequirePortalAccess, and its
|
||||
// data APIs stay protected, so serving the shell pre-auth is safe.
|
||||
if (normalizedUri.equals("/processor") || normalizedUri.startsWith("/processor/")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Treat common static file extensions as static resources
|
||||
return normalizedUri.endsWith(".svg")
|
||||
|| normalizedUri.endsWith(".png")
|
||||
|
||||
@@ -155,6 +155,7 @@ public class TempFileManager {
|
||||
if (directory != null && Files.isDirectory(directory)) {
|
||||
try {
|
||||
GeneralUtils.deleteDirectory(directory);
|
||||
registry.unregisterDirectory(directory);
|
||||
log.debug("Deleted temp directory: {}", directory.toString());
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to delete temp directory: {}", directory.toString(), e);
|
||||
|
||||
@@ -85,6 +85,18 @@ public class TempFileRegistry {
|
||||
return directory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a temporary directory from the registry.
|
||||
*
|
||||
* @param directory The directory to unregister
|
||||
*/
|
||||
public void unregisterDirectory(Path directory) {
|
||||
if (directory != null) {
|
||||
tempDirectories.remove(directory);
|
||||
log.debug("Unregistered temp directory: {}", directory.toString());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a third-party temporary file that requires special handling.
|
||||
*
|
||||
|
||||
@@ -176,11 +176,13 @@ class TempFileCleanupServiceMoreTest {
|
||||
class ScheduledCleanup {
|
||||
|
||||
@Test
|
||||
@DisplayName("deletes registered temp directories and reports counts")
|
||||
@DisplayName("deletes stale registered temp directories and reports counts")
|
||||
void deletesRegisteredDirectories() throws IOException {
|
||||
when(tempFileManager.cleanupOldTempFiles(anyLong())).thenReturn(2);
|
||||
Path regDir = Files.createDirectories(tempDir.resolve("registeredDir"));
|
||||
Files.createFile(regDir.resolve("inside.txt"));
|
||||
Files.setLastModifiedTime(
|
||||
regDir, FileTime.fromMillis(System.currentTimeMillis() - 2L * 60 * 60 * 1000));
|
||||
Set<Path> dirs = new HashSet<>();
|
||||
dirs.add(regDir);
|
||||
when(registry.getTempDirectories()).thenReturn(dirs);
|
||||
@@ -193,6 +195,22 @@ class TempFileCleanupServiceMoreTest {
|
||||
verify(tempFileManager).cleanupOldTempFiles(anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("keeps a fresh registered temp directory")
|
||||
void keepsFreshRegisteredDirectory() throws IOException {
|
||||
when(tempFileManager.cleanupOldTempFiles(anyLong())).thenReturn(0);
|
||||
Path regDir = Files.createDirectories(tempDir.resolve("freshRegisteredDir"));
|
||||
Files.createFile(regDir.resolve("inside.txt"));
|
||||
Set<Path> dirs = new HashSet<>();
|
||||
dirs.add(regDir);
|
||||
when(registry.getTempDirectories()).thenReturn(dirs);
|
||||
lenient().when(registry.contains(any(File.class))).thenReturn(false);
|
||||
|
||||
withIsolatedUserHome(cleanupService::scheduledCleanup);
|
||||
|
||||
assertThat(Files.exists(regDir)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("skips a registered directory that no longer exists")
|
||||
void skipsMissingRegisteredDirectory() {
|
||||
|
||||
@@ -73,6 +73,14 @@ class RequestUriUtilsTest {
|
||||
assertTrue(RequestUriUtils.isStaticResource("/mobile-scanner"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIsStaticResource_portalShell() {
|
||||
// The admin portal SPA shell (/processor) is served pre-auth so it's directly navigable.
|
||||
assertTrue(RequestUriUtils.isStaticResource("/processor"));
|
||||
assertTrue(RequestUriUtils.isStaticResource("/processor/users"));
|
||||
assertTrue(RequestUriUtils.isStaticResource("/app", "/app/processor"));
|
||||
}
|
||||
|
||||
// --- isFrontendRoute tests ---
|
||||
|
||||
@Test
|
||||
|
||||
@@ -175,6 +175,14 @@ springBoot {
|
||||
// Frontend build tasks - only enabled with -PbuildWithFrontend=true
|
||||
def buildWithFrontend = project.hasProperty('buildWithFrontend') && project.property('buildWithFrontend') == 'true'
|
||||
def buildPrototypes = project.hasProperty('prototypesMode') && project.property('prototypesMode') == 'true'
|
||||
// The admin portal ships as a lazy route inside the editor bundle (see
|
||||
// proprietary/routes/adminRouteExtensions). -PbuildWithPortal=true includes that
|
||||
// chunk via VITE_INCLUDE_PORTAL on the editor build; the deploy GHA sets it when
|
||||
// the portal or AI layers change. Building the portal implies building the editor.
|
||||
def buildWithPortal = project.hasProperty('buildWithPortal') && project.property('buildWithPortal') == 'true'
|
||||
if (buildWithPortal) {
|
||||
buildWithFrontend = true
|
||||
}
|
||||
// Workspace root holds package.json and node_modules (shared across editor /
|
||||
// future portal). Editor-specific paths (src, public, dist, tauri) live one
|
||||
// level deeper under frontend/editor/.
|
||||
@@ -297,9 +305,11 @@ tasks.register('npmBuild', Exec) {
|
||||
// Override VITE_API_BASE_URL to use relative paths for production builds
|
||||
// This ensures JARs work regardless of how they're deployed (direct, proxied, etc.)
|
||||
environment 'VITE_API_BASE_URL', '/'
|
||||
// Include the admin portal's lazy route/chunk in the editor build when requested.
|
||||
environment 'VITE_INCLUDE_PORTAL', (buildWithPortal ? 'true' : 'false')
|
||||
|
||||
doFirst {
|
||||
println "Building editor frontend application for production (mode=${frontendMode}, VITE_API_BASE_URL=/)"
|
||||
println "Building editor frontend application for production (mode=${frontendMode}, VITE_API_BASE_URL=/, portal=${buildWithPortal})"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
@@ -27,6 +28,7 @@ import stirling.software.SPDF.model.PipelineConfig;
|
||||
import stirling.software.SPDF.model.PipelineOperation;
|
||||
import stirling.software.SPDF.model.PipelineResult;
|
||||
import stirling.software.SPDF.service.ApiDocService;
|
||||
import stirling.software.common.service.AutomationRunContext;
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.ZipExtractionUtils;
|
||||
@@ -71,6 +73,17 @@ public class PipelineProcessor {
|
||||
|
||||
PipelineResult runPipelineAgainstFiles(List<Resource> outputFiles, PipelineConfig config)
|
||||
throws Exception {
|
||||
// One pipeline execution = one automation run. Scope a run id so every tool sub-step
|
||||
// dispatched via InternalApiClient groups into a single charge on the SaaS billing side
|
||||
// (see AutomationRunContext); pipeline steps run synchronously on this thread.
|
||||
try (AutomationRunContext.Scope ignored =
|
||||
AutomationRunContext.open(UUID.randomUUID().toString())) {
|
||||
return runPipelineAgainstFilesInternal(outputFiles, config);
|
||||
}
|
||||
}
|
||||
|
||||
private PipelineResult runPipelineAgainstFilesInternal(
|
||||
List<Resource> outputFiles, PipelineConfig config) throws Exception {
|
||||
PipelineResult result = new PipelineResult();
|
||||
|
||||
ByteArrayOutputStream logStream = new ByteArrayOutputStream();
|
||||
|
||||
@@ -305,6 +305,23 @@ public class GetInfoOnPDF {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Info-dictionary keys exposed above via typed getters; any other key in the dictionary is
|
||||
* surfaced as custom metadata (e.g. the classification policy's StirlingPDFClassification
|
||||
* entry).
|
||||
*/
|
||||
private static final java.util.Set<String> STANDARD_INFO_KEYS =
|
||||
java.util.Set.of(
|
||||
"Title",
|
||||
"Author",
|
||||
"Subject",
|
||||
"Keywords",
|
||||
"Producer",
|
||||
"Creator",
|
||||
"CreationDate",
|
||||
"ModDate",
|
||||
"Trapped");
|
||||
|
||||
private static ObjectNode extractMetadata(PDDocument document) {
|
||||
ObjectNode metadata = objectMapper.createObjectNode();
|
||||
|
||||
@@ -335,6 +352,18 @@ public class GetInfoOnPDF {
|
||||
if (modificationDate != null) {
|
||||
metadata.put("ModificationDate", modificationDate);
|
||||
}
|
||||
|
||||
// Surface custom Info-dictionary entries (anything beyond the
|
||||
// standard fields above) — e.g. StirlingPDFClassification
|
||||
for (String key : info.getMetadataKeys()) {
|
||||
if (STANDARD_INFO_KEYS.contains(key)) {
|
||||
continue;
|
||||
}
|
||||
String value = info.getCustomMetadataValue(key);
|
||||
if (value != null && !value.isBlank()) {
|
||||
metadata.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error extracting metadata: {}", e.getMessage());
|
||||
|
||||
@@ -5,8 +5,12 @@ import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
@@ -22,6 +26,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
@@ -38,15 +43,18 @@ class ManualRedactionService {
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Area and page redaction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactAreas(List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
AreaRedactionResult redactAreas(
|
||||
List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Set<String> capturedStrings = new LinkedHashSet<>();
|
||||
Map<Integer, List<PDRectangle>> rectsByPage = new HashMap<>();
|
||||
Set<Integer> forceRasterPages = new LinkedHashSet<>();
|
||||
|
||||
if (redactionAreas == null || redactionAreas.isEmpty()) {
|
||||
return;
|
||||
return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings);
|
||||
}
|
||||
|
||||
Map<Integer, List<RedactionArea>> redactionsByPage = new HashMap<>();
|
||||
@@ -74,57 +82,57 @@ class ManualRedactionService {
|
||||
continue;
|
||||
}
|
||||
|
||||
PDPage page = allPages.get(pageNumber - 1);
|
||||
int pageIndex = pageNumber - 1;
|
||||
PDPage page = allPages.get(pageIndex);
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
for (RedactionArea redactionArea : areasForPage) {
|
||||
Color redactColor = decodeOrDefault(redactionArea.getColor());
|
||||
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
float x = redactionArea.getX().floatValue();
|
||||
float y = redactionArea.getY().floatValue();
|
||||
float width = redactionArea.getWidth().floatValue();
|
||||
float height = redactionArea.getHeight().floatValue();
|
||||
|
||||
float pdfY = page.getBBox().getHeight() - y - height;
|
||||
|
||||
contentStream.addRect(x, pdfY, width, height);
|
||||
contentStream.fill();
|
||||
}
|
||||
contentStream.restoreGraphicsState();
|
||||
// Group rects by their decoded colour so each area keeps its own overlay tint.
|
||||
Map<Color, List<PDRectangle>> byColor = new LinkedHashMap<>();
|
||||
List<PDRectangle> allRects = new ArrayList<>();
|
||||
for (RedactionArea area : areasForPage) {
|
||||
float x = area.getX().floatValue();
|
||||
float y = area.getY().floatValue();
|
||||
float width = area.getWidth().floatValue();
|
||||
float height = area.getHeight().floatValue();
|
||||
// Request coords are top-left origin; convert to PDF user space (bottom-left).
|
||||
float pdfY = pageHeight - y - height;
|
||||
PDRectangle rect = new PDRectangle(x, pdfY, width, height);
|
||||
allRects.add(rect);
|
||||
byColor.computeIfAbsent(decodeOrDefault(area.getColor()), k -> new ArrayList<>())
|
||||
.add(rect);
|
||||
}
|
||||
|
||||
for (Map.Entry<Color, List<PDRectangle>> colorEntry : byColor.entrySet()) {
|
||||
Map<Integer, List<PDRectangle>> singlePage = new HashMap<>();
|
||||
singlePage.put(pageIndex, colorEntry.getValue());
|
||||
RedactionPipeline.RedactionResult result =
|
||||
RedactionPipeline.redactAreas(document, singlePage, colorEntry.getKey());
|
||||
capturedStrings.addAll(result.getCapturedStrings());
|
||||
forceRasterPages.addAll(result.getForceRasterPages());
|
||||
}
|
||||
rectsByPage.put(pageIndex, allRects);
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Manual area redaction captured {} text run(s) across {} page(s)",
|
||||
capturedStrings.size(),
|
||||
rectsByPage.size());
|
||||
return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings);
|
||||
}
|
||||
|
||||
void redactPages(ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
List<Integer> redactPages(
|
||||
ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Color redactColor = decodeOrDefault(request.getPageRedactionColor());
|
||||
List<Integer> pageNumbers = getPageNumbers(request, allPages.getCount());
|
||||
List<Integer> pageIndexes = getPageNumbers(request, allPages.getCount());
|
||||
|
||||
for (Integer pageNumber : pageNumbers) {
|
||||
PDPage page = allPages.get(pageNumber);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
PDRectangle box = page.getBBox();
|
||||
contentStream.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
contentStream.fill();
|
||||
}
|
||||
}
|
||||
// Whole-page wipe: drop the content stream, resources and annotations, then fill.
|
||||
RedactionPipeline.redactWholePages(document, pageIndexes, redactColor);
|
||||
return new ArrayList<>(pageIndexes);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Overlay drawing
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactFoundText(
|
||||
PDDocument document,
|
||||
@@ -158,7 +166,10 @@ class ManualRedactionService {
|
||||
|
||||
try {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
PDRectangle pageBox = page.getBBox();
|
||||
// TextPosition coords are relative to the CropBox origin.
|
||||
PDRectangle crop = page.getCropBox();
|
||||
float cropX = crop.getLowerLeftX();
|
||||
float cropY = crop.getLowerLeftY();
|
||||
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
@@ -179,8 +190,8 @@ class ManualRedactionService {
|
||||
}
|
||||
|
||||
contentStream.addRect(
|
||||
boxX,
|
||||
pageBox.getHeight() - block.getY2() - padding,
|
||||
cropX + boxX,
|
||||
cropY + crop.getHeight() - block.getY2() - padding,
|
||||
boxWidth,
|
||||
block.getY2() - block.getY1() + 2 * padding);
|
||||
}
|
||||
@@ -192,8 +203,8 @@ class ManualRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent
|
||||
// users from hovering over redacted URLs and seeing the underlying destination.
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent users
|
||||
// from hovering over redacted URLs and seeing the underlying destination.
|
||||
try {
|
||||
float pageH = page.getBBox().getHeight();
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
@@ -247,14 +258,19 @@ class ManualRedactionService {
|
||||
continue;
|
||||
}
|
||||
PDPage page = pages.get(pageIdx);
|
||||
PDRectangle crop = page.getCropBox();
|
||||
float cropX = crop.getLowerLeftX();
|
||||
float cropY = crop.getLowerLeftY();
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(color);
|
||||
for (float[] box : entry.getValue()) {
|
||||
// Box coords are CropBox-relative; offset by the CropBox origin
|
||||
// (cropbox-overlay).
|
||||
float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4];
|
||||
cs.addRect(x1, y1, x2 - x1, y2 - y1);
|
||||
cs.addRect(cropX + x1, cropY + y1, x2 - x1, y2 - y1);
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
@@ -262,9 +278,7 @@ class ManualRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Page element extraction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns bounding boxes for every text line and image on {@code page} in PDF user-space
|
||||
@@ -288,9 +302,7 @@ class ManualRedactionService {
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Finalization
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
@@ -298,7 +310,37 @@ class ManualRedactionService {
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode)
|
||||
boolean isTextRemovalMode,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns)
|
||||
throws IOException {
|
||||
return finalizeRedaction(
|
||||
document,
|
||||
allFoundTextsByPage,
|
||||
colorString,
|
||||
customPadding,
|
||||
convertToImage,
|
||||
isTextRemovalMode,
|
||||
literalTargets,
|
||||
patterns,
|
||||
Collections.emptySet());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param geometricRasterPages 0-based pages carrying a range / image-box redaction, whose
|
||||
* covered content (text under an overlay, or an image) is not text-removable and so must be
|
||||
* rasterised to guarantee removal.
|
||||
*/
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> allFoundTextsByPage,
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> geometricRasterPages)
|
||||
throws IOException {
|
||||
|
||||
List<PDFText> allFoundTexts = new ArrayList<>();
|
||||
@@ -309,74 +351,84 @@ class ManualRedactionService {
|
||||
if (!allFoundTexts.isEmpty()) {
|
||||
Color redactColor = decodeOrDefault(colorString);
|
||||
redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode);
|
||||
cleanDocumentMetadata(document);
|
||||
}
|
||||
|
||||
byte[] outputBytes;
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
cleanDocumentMetadata(convertedPdf);
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
convertedPdf.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized (image mode): {} pages ➜ {} KB",
|
||||
convertedPdf.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
// Convert-to-image physically removes all text, so verification is a plain save.
|
||||
outputBytes =
|
||||
RedactionPipeline.finalize(
|
||||
convertedPdf, Collections.emptySet(), Collections.emptyList());
|
||||
}
|
||||
} else {
|
||||
// Strip matched glyphs, then scrub/verify/rasterise via finalize.
|
||||
RedactionPipeline.redactLiteralTerms(document, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(document, literalTargets, patterns);
|
||||
// Geometric (range/image-box) redactions can't be text-removed; rasterise their pages.
|
||||
outputBytes =
|
||||
RedactionPipeline.rasteriseSpecificPages(
|
||||
outputBytes, geometricRasterPages, literalTargets, patterns);
|
||||
}
|
||||
|
||||
return writeBytes(outputBytes, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
/** Finalize a manual redaction; forced + verified-leaking pages are rasterised. */
|
||||
TempFile finalizeManual(
|
||||
PDDocument document, AreaRedactionResult areaResult, Boolean convertToImage)
|
||||
throws IOException {
|
||||
|
||||
byte[] outputBytes;
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
outputBytes =
|
||||
RedactionPipeline.finalize(
|
||||
convertedPdf, Collections.emptySet(), Collections.emptyList());
|
||||
}
|
||||
} else {
|
||||
outputBytes =
|
||||
RedactionPipeline.finalizeAreas(
|
||||
document,
|
||||
areaResult.rectsByPage,
|
||||
areaResult.forceRasterPages,
|
||||
areaResult.capturedStrings);
|
||||
}
|
||||
|
||||
return writeBytes(outputBytes, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
private TempFile writeBytes(byte[] outputBytes, int pageCount) throws IOException {
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
document.save(tempOut.getFile());
|
||||
java.nio.file.Files.write(tempOut.getFile().toPath(), outputBytes);
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized: {} pages ➜ {} KB",
|
||||
document.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
log.info("Redaction finalized: {} pages -> {} KB", pageCount, outputBytes.length / 1024);
|
||||
return tempOut;
|
||||
}
|
||||
|
||||
private void cleanDocumentMetadata(PDDocument document) {
|
||||
try {
|
||||
var documentInfo = document.getDocumentInformation();
|
||||
if (documentInfo != null) {
|
||||
documentInfo.setAuthor(null);
|
||||
documentInfo.setSubject(null);
|
||||
documentInfo.setKeywords(null);
|
||||
documentInfo.setModificationDate(java.util.Calendar.getInstance());
|
||||
log.debug("Cleaned document metadata for security");
|
||||
}
|
||||
// Utilities
|
||||
|
||||
if (document.getDocumentCatalog() != null) {
|
||||
try {
|
||||
document.getDocumentCatalog().setMetadata(null);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not clear XMP metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
/** Redaction rectangles, pages to force-rasterise, and the text captured under the rects. */
|
||||
static final class AreaRedactionResult {
|
||||
final Map<Integer, List<PDRectangle>> rectsByPage;
|
||||
final Set<Integer> forceRasterPages;
|
||||
final Set<String> capturedStrings;
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to clean document metadata: {}", e.getMessage());
|
||||
AreaRedactionResult(
|
||||
Map<Integer, List<PDRectangle>> rectsByPage,
|
||||
Set<Integer> forceRasterPages,
|
||||
Set<String> capturedStrings) {
|
||||
this.rectsByPage = rectsByPage;
|
||||
this.forceRasterPages = forceRasterPages;
|
||||
this.capturedStrings = capturedStrings;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Utilities
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
static Color decodeOrDefault(String hex) {
|
||||
if (hex == null) {
|
||||
return Color.BLACK;
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
@@ -29,13 +35,13 @@ import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.SecurityApi;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
@@ -93,33 +99,24 @@ public class RedactController {
|
||||
throws IOException {
|
||||
|
||||
MultipartFile file = request.getFileInput();
|
||||
String filename =
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(file.getOriginalFilename())))
|
||||
+ "_redacted.pdf";
|
||||
|
||||
try (PDDocument document = pdfDocumentFactory.load(file)) {
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
|
||||
// Whole-page wipes drop content; areas drop glyphs + overlay, verified later.
|
||||
manualRedactionService.redactPages(request, document, allPages);
|
||||
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
|
||||
ManualRedactionService.AreaRedactionResult areaResult =
|
||||
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
|
||||
|
||||
if (Boolean.TRUE.equals(request.getConvertPDFToImage())) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
convertedPdf,
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(
|
||||
file.getOriginalFilename())))
|
||||
+ "_redacted.pdf",
|
||||
tempFileManager);
|
||||
}
|
||||
}
|
||||
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
document,
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(file.getOriginalFilename())))
|
||||
+ "_redacted.pdf",
|
||||
tempFileManager);
|
||||
TempFile out =
|
||||
manualRedactionService.finalizeManual(
|
||||
document, areaResult, request.getConvertPDFToImage());
|
||||
return WebResponseUtils.pdfFileToWebResponse(out, filename);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,11 +133,17 @@ public class RedactController {
|
||||
+ "Users can provide text patterns to redact, with options for regex and whole word matching. "
|
||||
+ "Input:PDF Output:PDF Type:SISO")
|
||||
public ResponseEntity<Resource> redactPdf(@ModelAttribute RedactPdfRequest request) {
|
||||
String[] listOfText = request.getListOfText().split("\n");
|
||||
String rawListOfText = request.getListOfText();
|
||||
boolean useRegex = Boolean.TRUE.equals(request.getUseRegex());
|
||||
boolean wholeWordSearchBool = Boolean.TRUE.equals(request.getWholeWordSearch());
|
||||
|
||||
if (listOfText.length == 0 || (listOfText.length == 1 && listOfText[0].trim().isEmpty())) {
|
||||
if (rawListOfText == null || rawListOfText.trim().isEmpty()) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.redaction.no.patterns", "No text patterns provided for redaction");
|
||||
}
|
||||
|
||||
String[] listOfText = rawListOfText.split("\n");
|
||||
if (listOfText.length == 1 && listOfText[0].trim().isEmpty()) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.redaction.no.patterns", "No text patterns provided for redaction");
|
||||
}
|
||||
@@ -182,9 +185,32 @@ public class RedactController {
|
||||
request.getFileInput().getOriginalFilename())))
|
||||
+ "_redacted.pdf";
|
||||
|
||||
Set<String> literalTargets =
|
||||
Arrays.stream(listOfText)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
List<Pattern> compiledPatterns =
|
||||
RedactionPipeline.buildPatterns(listOfText, useRegex, wholeWordSearchBool);
|
||||
// Bare literals match substrings; regex/whole-word rely on compiled patterns.
|
||||
Set<String> verificationTargets =
|
||||
(useRegex || wholeWordSearchBool) ? Collections.emptySet() : literalTargets;
|
||||
|
||||
if (allFoundTextsByPage.isEmpty()) {
|
||||
log.info("No text found matching redaction patterns");
|
||||
return WebResponseUtils.pdfDocToWebResponse(document, filename, tempFileManager);
|
||||
// No page hit, but the target may live in a bookmark/annotation/form/JS carrier, so
|
||||
// still run the scrub + verify path rather than just wiping metadata.
|
||||
log.info("No page text matched; scrubbing catalog carriers and verifying");
|
||||
TempFile finalized =
|
||||
manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
Collections.emptyMap(),
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
true,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
}
|
||||
|
||||
boolean fallbackToBoxOnlyMode;
|
||||
@@ -205,7 +231,8 @@ public class RedactController {
|
||||
|
||||
if (fallbackToBoxOnlyMode) {
|
||||
log.warn(
|
||||
"Font compatibility issues detected. Using box-only redaction mode for better reliability.");
|
||||
"Font compatibility issue in placeholder pass; the true-removal pass and "
|
||||
+ "verification still guarantee the target is gone.");
|
||||
|
||||
fallbackDocument = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
@@ -220,7 +247,9 @@ public class RedactController {
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
false);
|
||||
false,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
}
|
||||
@@ -232,7 +261,9 @@ public class RedactController {
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
true);
|
||||
true,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
|
||||
@@ -241,11 +272,10 @@ public class RedactController {
|
||||
throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e);
|
||||
|
||||
} finally {
|
||||
// Both are distinct PDDocument handles.
|
||||
if (document != null) {
|
||||
try {
|
||||
if (fallbackDocument == null) {
|
||||
document.close();
|
||||
}
|
||||
document.close();
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to close main document: {}", e.getMessage());
|
||||
}
|
||||
|
||||
@@ -7,8 +7,11 @@ import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
@@ -30,6 +33,7 @@ import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyl
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
@@ -90,8 +94,8 @@ class RedactExecuteService {
|
||||
try {
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF
|
||||
// stripper only once across the entire execute() call rather than once per target.
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF stripper only
|
||||
// once across the entire execute() call rather than once per target.
|
||||
Map<Integer, List<PDFText>> foundTexts =
|
||||
hasTextOps ? collectTextMatches(document, request) : new HashMap<>();
|
||||
|
||||
@@ -101,13 +105,13 @@ class RedactExecuteService {
|
||||
totalMatches,
|
||||
foundTexts.size());
|
||||
|
||||
// Text removal (content-stream rewriting) — skipped in overlay-only mode.
|
||||
// Text removal (content-stream rewriting) - skipped in overlay-only mode.
|
||||
boolean needsOverlayOnly = overlayOnly;
|
||||
if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) {
|
||||
needsOverlayOnly = applyTextRemoval(document, request);
|
||||
} else if (overlayOnly) {
|
||||
log.info(
|
||||
"[redact/execute] overlay-only mode requested — skipping content-stream rewriting");
|
||||
"[redact/execute] overlay-only mode requested - skipping content-stream rewriting");
|
||||
}
|
||||
|
||||
// Reload fresh document on fallback so we overlay onto clean content.
|
||||
@@ -128,25 +132,46 @@ class RedactExecuteService {
|
||||
applyPageWipe(document, wipePages, style);
|
||||
}
|
||||
|
||||
// Range + image-box redactions are geometric overlays.
|
||||
Set<Integer> geometricRasterPages = new HashSet<>();
|
||||
for (TextRange range : ranges) {
|
||||
applyRangeRedaction(document, range, style, layoutCache);
|
||||
geometricRasterPages.addAll(
|
||||
applyRangeRedaction(document, range, style, layoutCache));
|
||||
}
|
||||
|
||||
for (ImageBox box : imageBoxes) {
|
||||
applyImageBoxRedaction(document, box, style);
|
||||
geometricRasterPages.addAll(applyImageBoxRedaction(document, box, style));
|
||||
}
|
||||
|
||||
if (request.getRedactImagePages() != null) {
|
||||
applyAllImagesRedaction(document, request.getRedactImagePages(), style);
|
||||
}
|
||||
|
||||
// Overlay-only skips removal/verify; font-fallback overlays still pass targets.
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
for (String value : textValues) {
|
||||
String trimmed = value == null ? "" : value.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
literalTargets.add(trimmed);
|
||||
}
|
||||
}
|
||||
List<Pattern> verificationPatterns =
|
||||
RedactionPipeline.buildPatterns(
|
||||
regexPatterns.toArray(new String[0]), true, false);
|
||||
Set<String> finalizeTargets = overlayOnly ? Collections.emptySet() : literalTargets;
|
||||
List<Pattern> finalizePatterns =
|
||||
overlayOnly ? Collections.emptyList() : verificationPatterns;
|
||||
|
||||
return manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
foundTexts,
|
||||
style.getColor(),
|
||||
style.getPadding(),
|
||||
convertToImage,
|
||||
!needsOverlayOnly);
|
||||
!needsOverlayOnly,
|
||||
finalizeTargets,
|
||||
finalizePatterns,
|
||||
geometricRasterPages);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Execute redaction failed: {}", e.getMessage(), e);
|
||||
@@ -162,9 +187,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Single-pass text scan (one stripper pass per execute() call)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit
|
||||
@@ -197,9 +220,7 @@ class RedactExecuteService {
|
||||
return found;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Text removal (content-stream rewriting)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the
|
||||
@@ -233,7 +254,7 @@ class RedactExecuteService {
|
||||
|
||||
if (fallback) {
|
||||
log.warn(
|
||||
"[redact/execute] font compatibility issue — falling back to overlay-only");
|
||||
"[redact/execute] font compatibility issue - falling back to overlay-only");
|
||||
} else {
|
||||
log.info("[redact/execute] content-stream text removal applied successfully");
|
||||
}
|
||||
@@ -246,9 +267,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Per-operation dispatch methods
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void applyPageWipe(PDDocument document, List<Integer> pageNumbers, RedactStyle style)
|
||||
throws IOException {
|
||||
@@ -304,7 +323,10 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
private void applyRangeRedaction(
|
||||
/**
|
||||
* @return 0-based pages covered by the range overlay; those pages must be rasterised.
|
||||
*/
|
||||
private Set<Integer> applyRangeRedaction(
|
||||
PDDocument document,
|
||||
TextRange range,
|
||||
RedactStyle style,
|
||||
@@ -322,6 +344,11 @@ class RedactExecuteService {
|
||||
style.getPadding(),
|
||||
ManualRedactionService.decodeOrDefault(style.getColor()),
|
||||
false);
|
||||
Set<Integer> pages = new HashSet<>();
|
||||
for (PDFText block : blocks) {
|
||||
pages.add(block.getPageIndex());
|
||||
}
|
||||
return pages;
|
||||
} else {
|
||||
log.warn(
|
||||
"[redact/execute] range not found: start='{}' end='{}'",
|
||||
@@ -331,10 +358,14 @@ class RedactExecuteService {
|
||||
} catch (Exception e) {
|
||||
log.warn("[redact/execute] range redaction failed: {}", e.getMessage());
|
||||
}
|
||||
return Collections.emptySet();
|
||||
}
|
||||
|
||||
private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style)
|
||||
throws IOException {
|
||||
/**
|
||||
* @return 0-based page covered by the image-box overlay; that page must be rasterised.
|
||||
*/
|
||||
private Set<Integer> applyImageBoxRedaction(
|
||||
PDDocument document, ImageBox box, RedactStyle style) throws IOException {
|
||||
List<float[]> boxes =
|
||||
List.of(
|
||||
new float[] {
|
||||
@@ -343,6 +374,7 @@ class RedactExecuteService {
|
||||
log.info("[redact/execute] image box overlay on page {}", box.pageIndex());
|
||||
Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
manualRedactionService.redactImageBoxes(document, boxes, boxColor);
|
||||
return Set.of(box.pageIndex());
|
||||
}
|
||||
|
||||
private void applyAllImagesRedaction(
|
||||
@@ -386,9 +418,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range collection helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text
|
||||
@@ -458,7 +488,7 @@ class RedactExecuteService {
|
||||
}
|
||||
if (end == null) {
|
||||
log.warn(
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping",
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) - skipping",
|
||||
start.page + 1,
|
||||
start.col,
|
||||
start.y);
|
||||
@@ -479,7 +509,7 @@ class RedactExecuteService {
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range '{}'→'{}': {} total blocks",
|
||||
"[redact/execute] range '{}'->'{}': {} total blocks",
|
||||
startStr,
|
||||
openEnded ? "<end of document>" : endStr,
|
||||
blocks.size());
|
||||
@@ -511,7 +541,6 @@ class RedactExecuteService {
|
||||
float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2();
|
||||
|
||||
// Line-box cache: populated lazily per page, reused across range iterations.
|
||||
// Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException.
|
||||
Map<Integer, List<float[]>> lineBoxCache = new HashMap<>();
|
||||
|
||||
for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) {
|
||||
@@ -677,7 +706,7 @@ class RedactExecuteService {
|
||||
if (i > 0) sb.append(", ");
|
||||
sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y));
|
||||
}
|
||||
if (anchors.size() > max) sb.append(", …");
|
||||
if (anchors.size() > max) sb.append(", ...");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@@ -703,9 +732,8 @@ class RedactExecuteService {
|
||||
candidates.add(new Candidate(tolerant, true));
|
||||
}
|
||||
|
||||
// If the anchor spans multiple lines (model provided entire paragraph instead of a short
|
||||
// phrase), try just the first non-empty line — it's usually sufficient to locate the
|
||||
// position and avoids mismatches from mid-paragraph text extraction artifacts.
|
||||
// Long multi-line anchors mismatch on extraction artifacts, so try just the first non-empty
|
||||
// line.
|
||||
if (trimmed.contains("\n")) {
|
||||
String firstLine =
|
||||
Arrays.stream(trimmed.split("\n"))
|
||||
@@ -733,7 +761,7 @@ class RedactExecuteService {
|
||||
if (!m.isEmpty()) {
|
||||
if (!c.pattern.equals(trimmed)) {
|
||||
log.info(
|
||||
"[redact/execute] range boundary matched via fallback: '{}' → '{}'",
|
||||
"[redact/execute] range boundary matched via fallback: '{}' -> '{}'",
|
||||
trimmed,
|
||||
c.pattern);
|
||||
}
|
||||
@@ -745,9 +773,7 @@ class RedactExecuteService {
|
||||
|
||||
private record Candidate(String pattern, boolean useRegex) {}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Static helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation
|
||||
|
||||
@@ -46,12 +46,11 @@ class TextRedactionService {
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
private static final float PRECISION_THRESHOLD = 1e-3f;
|
||||
private static final int FONT_SCALE_FACTOR = 1000;
|
||||
private static final Set<String> TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\"");
|
||||
private static final Set<String> TEXT_SHOWING_OPERATORS =
|
||||
stirling.software.SPDF.pdf.redaction.RedactionPipeline.TEXT_SHOWING_OPERATORS;
|
||||
private static final COSString EMPTY_COS_STRING = new COSString("");
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Public API
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
Map<Integer, List<PDFText>> findTextToRedact(
|
||||
PDDocument document, String[] listOfText, boolean useRegex, boolean wholeWordSearch) {
|
||||
@@ -135,9 +134,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Content stream manipulation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
List<Object> createTokensWithoutTargetText(
|
||||
PDDocument document,
|
||||
@@ -271,9 +268,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Placeholder creation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
String createPlaceholderWithFont(String originalWord, PDFont font) {
|
||||
if (originalWord == null || originalWord.isEmpty()) {
|
||||
@@ -410,9 +405,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Width calculation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private float safeGetStringWidth(PDFont font, String text) {
|
||||
if (font == null || text == null || text.isEmpty()) {
|
||||
@@ -601,9 +594,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Token and segment operations
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void processPageXObjects(
|
||||
PDDocument document,
|
||||
@@ -706,16 +697,12 @@ class TextRedactionService {
|
||||
private void writeRedactedContentToXObject(
|
||||
PDDocument document, PDFormXObject formXObject, List<Object> redactedTokens)
|
||||
throws IOException {
|
||||
|
||||
PDStream newStream = new PDStream(document);
|
||||
|
||||
try (var out = newStream.createOutputStream()) {
|
||||
// A form XObject's content IS its own stream body; overwrite it in place.
|
||||
PDStream formStream = new PDStream(formXObject.getCOSObject());
|
||||
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
ContentStreamWriter writer = new ContentStreamWriter(out);
|
||||
writer.writeTokens(redactedTokens);
|
||||
}
|
||||
|
||||
formXObject.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
formXObject.getCOSObject().setItem(COSName.CONTENTS, newStream.getCOSObject());
|
||||
}
|
||||
|
||||
private List<TextSegment> extractTextSegments(PDPage page, List<Object> tokens) {
|
||||
@@ -1168,9 +1155,7 @@ class TextRedactionService {
|
||||
};
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Inner data classes
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@Data
|
||||
private static class GraphicsState {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package stirling.software.SPDF.model.api.general;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
@@ -17,20 +19,8 @@ public class PosterPdfRequest extends PDFFile {
|
||||
allowableValues = {"A4", "Letter", "A3", "A5", "Legal", "Tabloid"})
|
||||
private String pageSize = "A4";
|
||||
|
||||
@Schema(
|
||||
description = "Horizontal decimation factor (how many columns to split into)",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
defaultValue = "2",
|
||||
minimum = "1",
|
||||
maximum = "10")
|
||||
private int xFactor = 2;
|
||||
|
||||
@Schema(
|
||||
description = "Vertical decimation factor (how many rows to split into)",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
defaultValue = "2",
|
||||
minimum = "1",
|
||||
maximum = "10")
|
||||
private int yFactor = 2;
|
||||
|
||||
@Schema(
|
||||
@@ -38,4 +28,36 @@ public class PosterPdfRequest extends PDFFile {
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
defaultValue = "false")
|
||||
private boolean rightToLeft = false;
|
||||
|
||||
@JsonProperty("xFactor")
|
||||
@Schema(
|
||||
description = "Horizontal decimation factor (how many columns to split into)",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
defaultValue = "2",
|
||||
minimum = "1",
|
||||
maximum = "10")
|
||||
public int getXFactor() {
|
||||
return xFactor;
|
||||
}
|
||||
|
||||
@JsonProperty("xFactor")
|
||||
public void setXFactor(int xFactor) {
|
||||
this.xFactor = xFactor;
|
||||
}
|
||||
|
||||
@JsonProperty("yFactor")
|
||||
@Schema(
|
||||
description = "Vertical decimation factor (how many rows to split into)",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
defaultValue = "2",
|
||||
minimum = "1",
|
||||
maximum = "10")
|
||||
public int getYFactor() {
|
||||
return yFactor;
|
||||
}
|
||||
|
||||
@JsonProperty("yFactor")
|
||||
public void setYFactor(int yFactor) {
|
||||
this.yFactor = yFactor;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,8 @@ public class AddPasswordRequest extends PDFFile {
|
||||
description = "The length of the encryption key",
|
||||
type = "integer",
|
||||
allowableValues = {"40", "128", "256"},
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
defaultValue = "256")
|
||||
private int keyLength = 256;
|
||||
|
||||
@Schema(description = "Whether document assembly is prevented", defaultValue = "false")
|
||||
|
||||
@@ -0,0 +1,755 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSObject;
|
||||
import org.apache.pdfbox.cos.COSStream;
|
||||
import org.apache.pdfbox.cos.COSString;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentNameDictionary;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDMetadata;
|
||||
import org.apache.pdfbox.pdmodel.common.PDNameTreeNode;
|
||||
import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureTreeRoot;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocumentOutline;
|
||||
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/** Removes/rewrites every catalog carrier that could leak redacted text. */
|
||||
@Slf4j
|
||||
public final class CatalogScrubber {
|
||||
|
||||
// A literal shorter than this won't delete a whole carrier (JS/XFA/action/embedded file).
|
||||
private static final int MIN_CARRIER_DROP_LITERAL = 3;
|
||||
|
||||
private CatalogScrubber() {}
|
||||
|
||||
/** Scrub all catalog-level carriers of the given literal/regex targets. */
|
||||
public static void scrub(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (document == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
PDDocumentCatalog catalog = document.getDocumentCatalog();
|
||||
if (catalog == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
scrubOutline(catalog.getDocumentOutline(), literalTargets, patterns);
|
||||
scrubAcroForm(catalog.getAcroForm(), literalTargets, patterns);
|
||||
scrubAnnotations(document, literalTargets, patterns);
|
||||
scrubStructTree(catalog.getStructureTreeRoot(), literalTargets, patterns);
|
||||
scrubNames(catalog.getNames(), literalTargets, patterns);
|
||||
scrubCatalogActions(catalog, literalTargets, patterns);
|
||||
scrubDocumentInfo(document, literalTargets, patterns);
|
||||
}
|
||||
|
||||
// Catalog actions: OpenAction, AA, and any JavaScript / URI payloads
|
||||
|
||||
private static void scrubCatalogActions(
|
||||
PDDocumentCatalog catalog, Set<String> targets, List<Pattern> patterns) {
|
||||
COSDictionary root = catalog.getCOSObject();
|
||||
if (root == null) {
|
||||
return;
|
||||
}
|
||||
// OpenAction may be either an action dict (with /URI or /JS) or an explicit
|
||||
scrubActionIfMatching(root, COSName.getPDFName("OpenAction"), targets, patterns);
|
||||
scrubActionIfMatching(root, COSName.getPDFName("AA"), targets, patterns);
|
||||
}
|
||||
|
||||
/** Drop the action at key if any target appears in its URI/JS payload. */
|
||||
private static void scrubActionIfMatching(
|
||||
COSDictionary parent, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
if (parent == null || key == null) {
|
||||
return;
|
||||
}
|
||||
COSBase value = parent.getDictionaryObject(key);
|
||||
if (value == null) {
|
||||
return;
|
||||
}
|
||||
if (containsTarget(value, carrierDropLiterals(targets), patterns, new HashSet<>())) {
|
||||
log.debug("Removing catalog {} due to target match", key.getName());
|
||||
parent.removeItem(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Literals specific enough to justify DELETING an entire carrier (JS / XFA / action / embedded
|
||||
* file). Sub-threshold literals (a single digit, a 2-char run) would nuke unrelated carriers,
|
||||
* so they are excluded from whole-carrier drops - they are still removed in-string by
|
||||
* stripMatches where the carrier is a plain string.
|
||||
*/
|
||||
private static Set<String> carrierDropLiterals(Set<String> targets) {
|
||||
if (targets == null) {
|
||||
return java.util.Collections.emptySet();
|
||||
}
|
||||
Set<String> specific = new LinkedHashSet<>();
|
||||
for (String t : targets) {
|
||||
if (t != null && t.trim().length() >= MIN_CARRIER_DROP_LITERAL) {
|
||||
specific.add(t);
|
||||
}
|
||||
}
|
||||
return specific;
|
||||
}
|
||||
|
||||
private static boolean containsTarget(
|
||||
COSBase base, Set<String> targets, List<Pattern> patterns, Set<COSBase> seen) {
|
||||
if (base == null) {
|
||||
return false;
|
||||
}
|
||||
COSBase resolved = base instanceof COSObject obj ? obj.getObject() : base;
|
||||
if (resolved == null || !seen.add(resolved)) {
|
||||
return false;
|
||||
}
|
||||
if (resolved instanceof COSString cs) {
|
||||
return matches(cs.getString(), targets, patterns);
|
||||
}
|
||||
if (resolved instanceof COSStream stream) {
|
||||
// Streams in XFA / OpenAction contexts are text (XML, JavaScript).
|
||||
try (java.io.InputStream is = stream.createInputStream()) {
|
||||
byte[] buf = new byte[2 * 1024 * 1024];
|
||||
int total = 0;
|
||||
int n;
|
||||
while ((n = is.read(buf, total, buf.length - total)) > 0) {
|
||||
total += n;
|
||||
if (total >= buf.length) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
String text = new String(buf, 0, total, java.nio.charset.StandardCharsets.UTF_8);
|
||||
if (matches(text, targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
// Fail closed: content past the 2 MiB cap is unproven, so treat as a match.
|
||||
return total >= buf.length && is.read() >= 0;
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scan stream for targets: {}", e.getMessage());
|
||||
// Fail closed: if we cannot read it we cannot prove it is clean, so treat
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (resolved instanceof COSDictionary dict) {
|
||||
for (COSName k : new HashSet<>(dict.keySet())) {
|
||||
if (containsTarget(dict.getItem(k), targets, patterns, seen)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (resolved instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
if (containsTarget(array.getObject(i), targets, patterns, seen)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the redaction target from /Info string entries and drop the XMP packet only if it
|
||||
* carries a target. Non-matching document properties (Title, Author, dates) are left intact -
|
||||
* redaction removes the target, it does not blanket-wipe metadata.
|
||||
*/
|
||||
private static void scrubDocumentInfo(
|
||||
PDDocument document, Set<String> targets, List<Pattern> patterns) {
|
||||
PDDocumentInformation info = document.getDocumentInformation();
|
||||
if (info != null && info.getCOSObject() != null) {
|
||||
COSDictionary infoDict = info.getCOSObject();
|
||||
for (COSName key : new HashSet<>(infoDict.keySet())) {
|
||||
if (infoDict.getDictionaryObject(key) instanceof COSString cs) {
|
||||
String stripped = stripMatches(cs.getString(), targets, patterns);
|
||||
if (!stripped.equals(cs.getString())) {
|
||||
if (stripped.isEmpty()) {
|
||||
infoDict.removeItem(key);
|
||||
} else {
|
||||
infoDict.setString(key, stripped);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PDDocumentCatalog catalog = document.getDocumentCatalog();
|
||||
if (catalog != null && catalog.getMetadata() != null) {
|
||||
scrubXmp(document, catalog, targets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* XMP is RDF/XML that usually mirrors /Info. Strip only the target occurrences so non-matching
|
||||
* properties (dates, rights, custom schema) survive; fall back to dropping the whole packet if
|
||||
* the edit can't be proven to have removed the target (e.g. entity-encoded) or anything throws.
|
||||
*/
|
||||
private static void scrubXmp(
|
||||
PDDocument document,
|
||||
PDDocumentCatalog catalog,
|
||||
Set<String> targets,
|
||||
List<Pattern> patterns) {
|
||||
String xmp;
|
||||
try {
|
||||
xmp = new String(catalog.getMetadata().toByteArray(), StandardCharsets.UTF_8);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not read XMP metadata: {}", e.getMessage());
|
||||
return;
|
||||
}
|
||||
if (!matches(xmp, targets, patterns)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String stripped = stripMatches(xmp, targets, patterns);
|
||||
// Only keep the edited packet if the target is provably gone from it.
|
||||
if (!matches(stripped, targets, patterns)) {
|
||||
catalog.setMetadata(
|
||||
new PDMetadata(
|
||||
document,
|
||||
new ByteArrayInputStream(
|
||||
stripped.getBytes(StandardCharsets.UTF_8))));
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Surgical XMP scrub failed; dropping packet: {}", e.getMessage());
|
||||
}
|
||||
catalog.setMetadata(null);
|
||||
}
|
||||
|
||||
// Outline
|
||||
|
||||
private static void scrubOutline(
|
||||
PDDocumentOutline outline, Set<String> targets, List<Pattern> patterns) {
|
||||
if (outline == null) {
|
||||
return;
|
||||
}
|
||||
scrubOutlineItems(outline.children(), targets, patterns);
|
||||
}
|
||||
|
||||
private static void scrubOutlineItems(
|
||||
Iterable<PDOutlineItem> items, Set<String> targets, List<Pattern> patterns) {
|
||||
if (items == null) {
|
||||
return;
|
||||
}
|
||||
for (PDOutlineItem item : items) {
|
||||
try {
|
||||
String title = item.getTitle();
|
||||
if (title != null) {
|
||||
String stripped = stripMatches(title, targets, patterns);
|
||||
if (!stripped.equals(title)) {
|
||||
item.setTitle(stripped);
|
||||
}
|
||||
}
|
||||
// Bookmark actions: /A is an action dict which may carry a /URI or /JS
|
||||
COSDictionary itemDict = item.getCOSObject();
|
||||
if (itemDict != null) {
|
||||
scrubActionIfMatching(itemDict, COSName.A, targets, patterns);
|
||||
scrubActionIfMatching(itemDict, COSName.getPDFName("AA"), targets, patterns);
|
||||
}
|
||||
scrubOutlineItems(item.children(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub outline item: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AcroForm
|
||||
|
||||
private static void scrubAcroForm(
|
||||
PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
|
||||
if (form == null) {
|
||||
return;
|
||||
}
|
||||
// XFA forms: scrubbed separately because the XFA XML packet carries
|
||||
scrubXfa(form, targets, patterns);
|
||||
|
||||
try {
|
||||
for (PDField field : form.getFieldTree()) {
|
||||
scrubField(field, targets, patterns);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to walk AcroForm field tree: {}", e.getMessage());
|
||||
}
|
||||
|
||||
// Force viewers to regenerate appearance streams from the (scrubbed) /V
|
||||
try {
|
||||
form.setNeedAppearances(true);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to set /NeedAppearances on AcroForm: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubXfa(PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
COSBase xfaBase = form.getCOSObject().getDictionaryObject(COSName.XFA);
|
||||
if (xfaBase == null) {
|
||||
return;
|
||||
}
|
||||
boolean hit =
|
||||
containsTarget(
|
||||
xfaBase, carrierDropLiterals(targets), patterns, new HashSet<>());
|
||||
if (hit) {
|
||||
// Simplest safe move: strip the XFA entry entirely.
|
||||
log.warn(
|
||||
"Removing XFA form packet from AcroForm - XFA XML contained a redaction "
|
||||
+ "target and has been dropped so viewers render AcroForm widgets "
|
||||
+ "instead.");
|
||||
form.getCOSObject().removeItem(COSName.XFA);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub XFA: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubField(PDField field, Set<String> targets, List<Pattern> patterns) {
|
||||
if (field == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
COSDictionary dict = field.getCOSObject();
|
||||
// Only touch fields whose own values actually contain a target: clearing /AP
|
||||
// document-wide blanks unrelated fields in viewers that ignore /NeedAppearances.
|
||||
boolean matched =
|
||||
dictValueMatches(dict, COSName.V, targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.DV, targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.getPDFName("RV"), targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.getPDFName("TU"), targets, patterns)
|
||||
|| buttonCaptionMatches(dict, targets, patterns)
|
||||
|| fieldValueMatches(field, targets, patterns);
|
||||
if (!matched) {
|
||||
return;
|
||||
}
|
||||
scrubDictStrings(dict, COSName.V, targets, patterns);
|
||||
scrubDictStrings(dict, COSName.DV, targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("RV"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("TU"), targets, patterns);
|
||||
// Keep field appearance streams in sync with value where possible.
|
||||
try {
|
||||
if (field.getValueAsString() != null) {
|
||||
String stripped = stripMatches(field.getValueAsString(), targets, patterns);
|
||||
if (!stripped.equals(field.getValueAsString())) {
|
||||
field.setValue(stripped);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to rewrite field value via setValue: {}", e.getMessage());
|
||||
}
|
||||
if (!isButtonField(dict)) {
|
||||
clearWidgetAppearances(dict);
|
||||
} else if (scrubButtonCaption(dict, targets, patterns)) {
|
||||
// A button caption (/MK /CA,/RC,/AC) that carried the target renders via /AP that
|
||||
// /NeedAppearances can't rebuild, so drop the stale /AP too.
|
||||
clearWidgetAppearances(dict);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub field: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static final String[] MK_CAPTION_KEYS = {"CA", "RC", "AC"};
|
||||
|
||||
/** True if a button widget's /MK caption (down/rollover/alternate) contains a target. */
|
||||
private static boolean buttonCaptionMatches(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
for (COSDictionary mk : mkDicts(dict)) {
|
||||
for (String k : MK_CAPTION_KEYS) {
|
||||
if (dictValueMatches(mk, COSName.getPDFName(k), targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Strip target substrings from a button widget's /MK captions; true if any matched. */
|
||||
private static boolean scrubButtonCaption(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
boolean hit = false;
|
||||
for (COSDictionary mk : mkDicts(dict)) {
|
||||
for (String k : MK_CAPTION_KEYS) {
|
||||
COSName key = COSName.getPDFName(k);
|
||||
if (dictValueMatches(mk, key, targets, patterns)) {
|
||||
hit = true;
|
||||
scrubDictStrings(mk, key, targets, patterns);
|
||||
}
|
||||
}
|
||||
}
|
||||
return hit;
|
||||
}
|
||||
|
||||
/** The /MK appearance-characteristics dicts on a field and any widget kids. */
|
||||
private static List<COSDictionary> mkDicts(COSDictionary dict) {
|
||||
List<COSDictionary> out = new java.util.ArrayList<>();
|
||||
if (dict.getDictionaryObject(COSName.getPDFName("MK")) instanceof COSDictionary mk) {
|
||||
out.add(mk);
|
||||
}
|
||||
if (dict.getDictionaryObject(COSName.KIDS) instanceof COSArray kids) {
|
||||
for (int i = 0; i < kids.size(); i++) {
|
||||
if (kids.getObject(i) instanceof COSDictionary kid
|
||||
&& kid.getDictionaryObject(COSName.getPDFName("MK"))
|
||||
instanceof COSDictionary mk) {
|
||||
out.add(mk);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static boolean isButtonField(COSDictionary dict) {
|
||||
COSBase ftBase = dict.getDictionaryObject(COSName.FT);
|
||||
COSName ft = ftBase instanceof COSName n ? n : null;
|
||||
if (ft == null) {
|
||||
COSBase parent = dict.getDictionaryObject(COSName.PARENT);
|
||||
if (parent instanceof COSDictionary p
|
||||
&& p.getDictionaryObject(COSName.FT) instanceof COSName pn) {
|
||||
ft = pn;
|
||||
}
|
||||
}
|
||||
return COSName.getPDFName("Btn").equals(ft);
|
||||
}
|
||||
|
||||
private static boolean dictValueMatches(
|
||||
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
COSBase value = dict.getDictionaryObject(key);
|
||||
if (value instanceof COSString cs) {
|
||||
return matches(cs.getString(), targets, patterns);
|
||||
}
|
||||
if (value instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
if (array.getObject(i) instanceof COSString element
|
||||
&& matches(element.getString(), targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static boolean fieldValueMatches(
|
||||
PDField field, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
String value = field.getValueAsString();
|
||||
return value != null && matches(value, targets, patterns);
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static void clearWidgetAppearances(COSDictionary fieldDict) {
|
||||
if (fieldDict == null) {
|
||||
return;
|
||||
}
|
||||
// The field itself may be a widget (single-widget field) and/or have Kids.
|
||||
fieldDict.removeItem(COSName.AP);
|
||||
COSBase kids = fieldDict.getDictionaryObject(COSName.KIDS);
|
||||
if (kids instanceof COSArray arr) {
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase kidBase = arr.getObject(i);
|
||||
if (kidBase instanceof COSDictionary kidDict) {
|
||||
kidDict.removeItem(COSName.AP);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Annotations
|
||||
|
||||
private static void scrubAnnotations(
|
||||
PDDocument document, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
for (PDPage page : document.getPages()) {
|
||||
// Page-level additional actions (/AA - open/close JS) can carry the target too.
|
||||
scrubActionIfMatching(
|
||||
page.getCOSObject(), COSName.getPDFName("AA"), targets, patterns);
|
||||
List<PDAnnotation> annotations;
|
||||
try {
|
||||
annotations = page.getAnnotations();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to load annotations for page: {}", e.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (annotations == null) {
|
||||
continue;
|
||||
}
|
||||
for (PDAnnotation annotation : annotations) {
|
||||
scrubAnnotation(annotation, targets, patterns);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Annotation scrub walk failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubAnnotation(
|
||||
PDAnnotation annotation, Set<String> targets, List<Pattern> patterns) {
|
||||
if (annotation == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String contents = annotation.getContents();
|
||||
if (contents != null) {
|
||||
String stripped = stripMatches(contents, targets, patterns);
|
||||
if (!stripped.equals(contents)) {
|
||||
annotation.setContents(stripped);
|
||||
}
|
||||
}
|
||||
COSDictionary dict = annotation.getCOSObject();
|
||||
scrubDictStrings(dict, COSName.getPDFName("RC"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Subj"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("NM"), targets, patterns);
|
||||
// Link/widget annotation actions can encode the target in a /URI or JS body.
|
||||
scrubActionIfMatching(dict, COSName.A, targets, patterns);
|
||||
scrubActionIfMatching(dict, COSName.getPDFName("AA"), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub annotation: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Structure tree
|
||||
|
||||
private static void scrubStructTree(
|
||||
PDStructureTreeRoot root, Set<String> targets, List<Pattern> patterns) {
|
||||
if (root == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
scrubStructDict(root.getCOSObject(), targets, patterns, new HashSet<>());
|
||||
} catch (Exception e) {
|
||||
log.debug("Structure tree scrub failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubStructDict(
|
||||
COSBase base, Set<String> targets, List<Pattern> patterns, Set<COSBase> seen) {
|
||||
if (base == null) {
|
||||
return;
|
||||
}
|
||||
COSBase resolved = base instanceof COSObject obj ? obj.getObject() : base;
|
||||
if (resolved == null || !seen.add(resolved)) {
|
||||
return;
|
||||
}
|
||||
if (resolved instanceof COSDictionary dict) {
|
||||
// Do not walk into content streams - those are handled by content-stream
|
||||
if (resolved instanceof COSStream) {
|
||||
return;
|
||||
}
|
||||
scrubDictStrings(dict, COSName.getPDFName("ActualText"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Alt"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("E"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Lang"), targets, patterns);
|
||||
for (COSName key : new HashSet<>(dict.keySet())) {
|
||||
COSBase value = dict.getItem(key);
|
||||
if (value instanceof COSDictionary
|
||||
|| value instanceof COSArray
|
||||
|| value instanceof COSObject) {
|
||||
scrubStructDict(value, targets, patterns, seen);
|
||||
}
|
||||
}
|
||||
} else if (resolved instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
scrubStructDict(array.getObject(i), targets, patterns, seen);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Names tree (JavaScript + embedded files)
|
||||
|
||||
private static void scrubNames(
|
||||
PDDocumentNameDictionary names, Set<String> targets, List<Pattern> patterns) {
|
||||
if (names == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
dropMatchingNames(names.getJavaScript(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub JavaScript names: {}", e.getMessage());
|
||||
}
|
||||
try {
|
||||
dropMatchingNames(names.getEmbeddedFiles(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub embedded-file names: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void dropMatchingNames(
|
||||
PDNameTreeNode<?> node, Set<String> targets, List<Pattern> patterns) {
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
COSDictionary dict = node.getCOSObject();
|
||||
if (dict == null) {
|
||||
return;
|
||||
}
|
||||
scrubNameTreeDict(dict, targets, patterns);
|
||||
}
|
||||
|
||||
private static void scrubNameTreeDict(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
if (dict == null) {
|
||||
return;
|
||||
}
|
||||
// Guarded casts: a malformed /Names or /Kids must skip that carrier, not throw a swallowed
|
||||
// ClassCastException.
|
||||
COSArray namesArray =
|
||||
dict.getDictionaryObject(COSName.NAMES) instanceof COSArray a ? a : null;
|
||||
if (namesArray != null) {
|
||||
Set<String> dropLiterals = carrierDropLiterals(targets);
|
||||
for (int i = namesArray.size() - 2; i >= 0; i -= 2) {
|
||||
COSBase keyBase = namesArray.getObject(i);
|
||||
String key = keyBase instanceof COSString s ? s.getString() : null;
|
||||
// Drop the pair when the KEY or the VALUE (JS /JS stream, embedded-file bytes)
|
||||
// contains a target - not just the key.
|
||||
boolean keyHit = key != null && matches(key, dropLiterals, patterns);
|
||||
boolean valueHit =
|
||||
i + 1 < namesArray.size()
|
||||
&& containsTarget(
|
||||
namesArray.getObject(i + 1),
|
||||
dropLiterals,
|
||||
patterns,
|
||||
new HashSet<>());
|
||||
if (keyHit || valueHit) {
|
||||
namesArray.remove(i + 1);
|
||||
namesArray.remove(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
COSArray kids = dict.getDictionaryObject(COSName.KIDS) instanceof COSArray a ? a : null;
|
||||
if (kids != null) {
|
||||
for (int i = 0; i < kids.size(); i++) {
|
||||
COSBase kid = kids.getObject(i);
|
||||
if (kid instanceof COSDictionary kidDict) {
|
||||
scrubNameTreeDict(kidDict, targets, patterns);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helpers
|
||||
|
||||
private static void scrubDictStrings(
|
||||
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
if (dict == null || key == null) {
|
||||
return;
|
||||
}
|
||||
COSBase value = dict.getDictionaryObject(key);
|
||||
if (value instanceof COSString cosString) {
|
||||
String stripped = stripMatches(cosString.getString(), targets, patterns);
|
||||
if (!stripped.equals(cosString.getString())) {
|
||||
dict.setString(key, stripped);
|
||||
}
|
||||
} else if (value instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
COSBase element = array.getObject(i);
|
||||
if (element instanceof COSString elementString) {
|
||||
String stripped = stripMatches(elementString.getString(), targets, patterns);
|
||||
if (!stripped.equals(elementString.getString())) {
|
||||
array.set(i, new COSString(stripped));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static String stripMatches(String source, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (source == null || source.isEmpty()) {
|
||||
return source;
|
||||
}
|
||||
String result = source;
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// Case-insensitive literal removal. Verification is case-insensitive
|
||||
result = caseInsensitiveReplaceAll(result, target);
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
// Force case-insensitive matching for catalog carriers regardless
|
||||
Pattern ci = withCaseInsensitive(pattern);
|
||||
result = ci.matcher(DeadlineCharSequence.of(result)).replaceAll("");
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
// Fail closed: a throwing regex means we cannot prove the carrier clean, so
|
||||
// drop the whole string rather than leaving it intact.
|
||||
log.warn("Pattern replace failed for {}; dropping carrier text", pattern);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static boolean matches(String source, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (source == null || source.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
String lower = source.toLowerCase(Locale.ROOT);
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target != null
|
||||
&& !target.isEmpty()
|
||||
&& lower.contains(target.toLowerCase(Locale.ROOT))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
if (withCaseInsensitive(pattern)
|
||||
.matcher(DeadlineCharSequence.of(source))
|
||||
.find()) {
|
||||
return true;
|
||||
}
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
// Fail closed: a throwing regex counts as a match so the carrier is scrubbed.
|
||||
log.warn("Pattern match failed for {}; treating carrier as a match", pattern);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String caseInsensitiveReplaceAll(String source, String target) {
|
||||
if (target.isEmpty()) {
|
||||
return source;
|
||||
}
|
||||
Pattern literal =
|
||||
Pattern.compile(
|
||||
Pattern.quote(target), Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
|
||||
return literal.matcher(source).replaceAll("");
|
||||
}
|
||||
|
||||
private static Pattern withCaseInsensitive(Pattern pattern) {
|
||||
if ((pattern.flags() & Pattern.CASE_INSENSITIVE) != 0) {
|
||||
return pattern;
|
||||
}
|
||||
try {
|
||||
return Pattern.compile(
|
||||
pattern.pattern(),
|
||||
pattern.flags() | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
|
||||
} catch (Exception e) {
|
||||
return pattern;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
/**
|
||||
* Wraps a match subject so a runaway (catastrophic-backtracking) regex aborts instead of hanging
|
||||
* the request thread: every {@link #charAt(int)} the matcher performs checks a wall-clock deadline
|
||||
* and throws once it is exceeded. The thrown {@link RegexTimeoutException} is a {@link
|
||||
* RuntimeException}, so the redaction match sites' existing fail-closed catches handle it.
|
||||
*/
|
||||
final class DeadlineCharSequence implements CharSequence {
|
||||
|
||||
/** Per-match wall-clock budget. A legitimate whole-document match finishes far inside this. */
|
||||
static final long DEFAULT_BUDGET_MILLIS = 2_000L;
|
||||
|
||||
static final class RegexTimeoutException extends RuntimeException {
|
||||
RegexTimeoutException(long budgetMillis) {
|
||||
super("Regex evaluation exceeded " + budgetMillis + " ms (possible ReDoS)");
|
||||
}
|
||||
}
|
||||
|
||||
private final CharSequence inner;
|
||||
private final long deadlineNanos;
|
||||
private final long budgetMillis;
|
||||
|
||||
private DeadlineCharSequence(CharSequence inner, long budgetMillis) {
|
||||
this.inner = inner;
|
||||
this.budgetMillis = budgetMillis;
|
||||
this.deadlineNanos = System.nanoTime() + budgetMillis * 1_000_000L;
|
||||
}
|
||||
|
||||
/** Wrap {@code text} with the default budget; null becomes an empty sequence. */
|
||||
static DeadlineCharSequence of(String text) {
|
||||
return new DeadlineCharSequence(text == null ? "" : text, DEFAULT_BUDGET_MILLIS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public char charAt(int index) {
|
||||
if (System.nanoTime() > deadlineNanos) {
|
||||
throw new RegexTimeoutException(budgetMillis);
|
||||
}
|
||||
return inner.charAt(index);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int length() {
|
||||
return inner.length();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharSequence subSequence(int start, int end) {
|
||||
return inner.subSequence(start, end);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return inner.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,748 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSFloat;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSString;
|
||||
import org.apache.pdfbox.pdfparser.PDFStreamParser;
|
||||
import org.apache.pdfbox.pdfwriter.ContentStreamWriter;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.common.PDStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.text.PDFTextStripperByArea;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
|
||||
/**
|
||||
* Content-stream editing: physically removes glyphs from page and form-XObject streams, both by
|
||||
* literal/regex match (auto-word) and by rect intersection (manual area), plus the area-capture and
|
||||
* overlay/annotation helpers those paths rely on.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RedactionContentEditor {
|
||||
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
|
||||
private RedactionContentEditor() {}
|
||||
|
||||
/** True only for upright pages whose CropBox origin is (0,0), where the rect flip is exact. */
|
||||
static boolean isSurgicallySafe(PDPage page) {
|
||||
if (page.getRotation() != 0) {
|
||||
return false;
|
||||
}
|
||||
PDRectangle crop = page.getCropBox();
|
||||
return crop.getLowerLeftX() == 0f && crop.getLowerLeftY() == 0f;
|
||||
}
|
||||
|
||||
/** Drop annotations whose rectangle overlaps any redaction rect. */
|
||||
static void removeOverlappingAnnotations(PDPage page, List<PDRectangle> rects) {
|
||||
try {
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
for (var ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
boolean overlaps = false;
|
||||
if (ar != null) {
|
||||
Rectangle2D.Float a =
|
||||
new Rectangle2D.Float(
|
||||
ar.getLowerLeftX(),
|
||||
ar.getLowerLeftY(),
|
||||
ar.getWidth(),
|
||||
ar.getHeight());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (a.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
overlaps = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!overlaps) {
|
||||
kept.add(ann);
|
||||
}
|
||||
}
|
||||
page.setAnnotations(kept);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not strip overlapping annotations: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
static List<Pattern> effectivePatterns(Set<String> literalTargets, List<Pattern> patterns) {
|
||||
List<Pattern> result = new ArrayList<>();
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// Case-insensitive to match TextFinderUtils; removal must not miss finder hits.
|
||||
result.add(
|
||||
Pattern.compile(
|
||||
Pattern.quote(target),
|
||||
Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE));
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
result.addAll(patterns);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Per-page content-stream rewrite (literal/regex based)
|
||||
|
||||
static void rewritePageContent(PDDocument document, PDPage page, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
PDResources resources = page.getResources();
|
||||
if (resources == null) {
|
||||
return;
|
||||
}
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(page));
|
||||
boolean modified = rewriteTokens(tokens, resources, patterns);
|
||||
if (modified) {
|
||||
writePageTokens(document, page, tokens);
|
||||
}
|
||||
// Recurse into form XObjects referenced by this page (shared visited set + depth cap).
|
||||
rewriteFormXObjects(resources, resources, patterns, new HashSet<>(), 0);
|
||||
}
|
||||
|
||||
private static void rewriteFormXObjects(
|
||||
PDResources resources,
|
||||
PDResources parentResources,
|
||||
List<Pattern> patterns,
|
||||
Set<COSBase> visited,
|
||||
int depth)
|
||||
throws IOException {
|
||||
if (depth > MAX_XOBJECT_DEPTH) {
|
||||
log.warn("XObject nesting exceeded {}; stopping recursion", MAX_XOBJECT_DEPTH);
|
||||
return;
|
||||
}
|
||||
for (COSName name : resources.getXObjectNames()) {
|
||||
try {
|
||||
var xobj = resources.getXObject(name);
|
||||
if (!(xobj instanceof PDFormXObject form)) {
|
||||
continue;
|
||||
}
|
||||
if (!visited.add(form.getCOSObject())) {
|
||||
continue;
|
||||
}
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(form));
|
||||
// Forms may inherit fonts from the parent when they carry no own /Resources.
|
||||
PDResources formResources =
|
||||
form.getResources() != null ? form.getResources() : parentResources;
|
||||
boolean modified = rewriteTokens(tokens, formResources, patterns);
|
||||
if (modified) {
|
||||
// A form XObject's content IS its own stream body; overwrite it in place.
|
||||
PDStream formStream = new PDStream(form.getCOSObject());
|
||||
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
new ContentStreamWriter(out).writeTokens(tokens);
|
||||
}
|
||||
}
|
||||
rewriteFormXObjects(formResources, formResources, patterns, visited, depth + 1);
|
||||
} catch (IOException e) {
|
||||
log.debug("Failed to rewrite XObject {}: {}", name.getName(), e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Object> parseTokens(PDFStreamParser parser) throws IOException {
|
||||
List<Object> tokens = new ArrayList<>();
|
||||
Object t;
|
||||
while ((t = parser.parseNextToken()) != null) {
|
||||
tokens.add(t);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/** Walk tokens keeping a tiny text state (current font). */
|
||||
private static boolean rewriteTokens(
|
||||
List<Object> tokens, PDResources resources, List<Pattern> patterns) {
|
||||
boolean modified = false;
|
||||
PDFont currentFont = null;
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
Object tok = tokens.get(i);
|
||||
if (!(tok instanceof Operator op)) {
|
||||
continue;
|
||||
}
|
||||
String name = op.getName();
|
||||
if ("Tf".equals(name) && i >= 2) {
|
||||
Object fontNameTok = tokens.get(i - 2);
|
||||
if (fontNameTok instanceof COSName fontName) {
|
||||
try {
|
||||
currentFont = resources.getFont(fontName);
|
||||
} catch (IOException ex) {
|
||||
log.debug(
|
||||
"Could not resolve font {}: {}",
|
||||
fontName.getName(),
|
||||
ex.getMessage());
|
||||
currentFont = null;
|
||||
}
|
||||
}
|
||||
} else if (RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name) && i >= 1) {
|
||||
int operandIdx = i - 1;
|
||||
Object operand = tokens.get(operandIdx);
|
||||
if (operand instanceof COSString cosString) {
|
||||
COSString replacement = rewriteCosString(cosString, currentFont, patterns);
|
||||
if (replacement != null) {
|
||||
tokens.set(operandIdx, replacement);
|
||||
modified = true;
|
||||
}
|
||||
} else if (operand instanceof COSArray arr) {
|
||||
COSArray newArr = rewriteCosArray(arr, currentFont, patterns);
|
||||
if (newArr != null) {
|
||||
tokens.set(operandIdx, newArr);
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return modified;
|
||||
}
|
||||
|
||||
private static COSString rewriteCosString(
|
||||
COSString cosString, PDFont font, List<Pattern> patterns) {
|
||||
if (font == null) {
|
||||
// Without a font we cannot decode safely.
|
||||
return rewriteRawLatin(cosString, patterns);
|
||||
}
|
||||
DecodeResult decoded = decodeCosString(cosString, font);
|
||||
if (decoded == null) {
|
||||
return rewriteRawLatin(cosString, patterns);
|
||||
}
|
||||
boolean[] drop = findDroppedCharsMask(decoded.text, patterns);
|
||||
if (drop == null) {
|
||||
return null;
|
||||
}
|
||||
return buildFilteredCosString(decoded, drop, font);
|
||||
}
|
||||
|
||||
private static COSArray rewriteCosArray(COSArray arr, PDFont font, List<Pattern> patterns) {
|
||||
// Build a concatenated decode across all COSString elements so that matches
|
||||
List<DecodeResult> parts = new ArrayList<>();
|
||||
StringBuilder concat = new StringBuilder();
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase elem = arr.get(i);
|
||||
if (elem instanceof COSString cs) {
|
||||
DecodeResult decoded = font != null ? decodeCosString(cs, font) : null;
|
||||
if (decoded == null) {
|
||||
decoded = decodeAsLatin(cs);
|
||||
}
|
||||
parts.add(decoded);
|
||||
concat.append(decoded.text);
|
||||
} else {
|
||||
parts.add(null);
|
||||
}
|
||||
}
|
||||
boolean[] fullDrop = findDroppedCharsMask(concat.toString(), patterns);
|
||||
if (fullDrop == null) {
|
||||
return null;
|
||||
}
|
||||
COSArray out = new COSArray();
|
||||
int cursor = 0;
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase elem = arr.get(i);
|
||||
if (elem instanceof COSString cs) {
|
||||
DecodeResult decoded = parts.get(i);
|
||||
int partLen = decoded.text.length();
|
||||
boolean[] partDrop = new boolean[partLen];
|
||||
System.arraycopy(fullDrop, cursor, partDrop, 0, partLen);
|
||||
cursor += partLen;
|
||||
COSString rebuilt =
|
||||
buildFilteredCosStringRaw(decoded, partDrop, font, cs.getBytes());
|
||||
// A zero-length COSString is valid; PDFBox emits it as () producing no
|
||||
out.add(rebuilt);
|
||||
} else {
|
||||
out.add(elem);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** For cases where we have no font - treat the bytes as latin-1 characters. */
|
||||
private static COSString rewriteRawLatin(COSString cosString, List<Pattern> patterns) {
|
||||
DecodeResult decoded = decodeAsLatin(cosString);
|
||||
boolean[] drop = findDroppedCharsMask(decoded.text, patterns);
|
||||
if (drop == null) {
|
||||
return null;
|
||||
}
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
for (int i = 0; i < decoded.text.length(); i++) {
|
||||
if (drop[i]) continue;
|
||||
out.write(decoded.text.charAt(i) & 0xFF);
|
||||
}
|
||||
return new COSString(out.toByteArray());
|
||||
}
|
||||
|
||||
private static DecodeResult decodeAsLatin(COSString cosString) {
|
||||
byte[] bytes = cosString.getBytes();
|
||||
StringBuilder sb = new StringBuilder(bytes.length);
|
||||
int[] codeStart = new int[bytes.length];
|
||||
int[] codeLen = new int[bytes.length];
|
||||
for (int i = 0; i < bytes.length; i++) {
|
||||
sb.append((char) (bytes[i] & 0xFF));
|
||||
codeStart[i] = i;
|
||||
codeLen[i] = 1;
|
||||
}
|
||||
return new DecodeResult(sb.toString(), bytes, codeStart, codeLen);
|
||||
}
|
||||
|
||||
/** Decode a COSString to Unicode via the font; null if decoding fails. */
|
||||
private static DecodeResult decodeCosString(COSString cosString, PDFont font) {
|
||||
byte[] bytes = cosString.getBytes();
|
||||
StringBuilder text = new StringBuilder();
|
||||
List<Integer> starts = new ArrayList<>();
|
||||
List<Integer> lens = new ArrayList<>();
|
||||
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes)) {
|
||||
int pos = 0;
|
||||
while (in.available() > 0) {
|
||||
int before = in.available();
|
||||
int code;
|
||||
try {
|
||||
code = font.readCode(in);
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
log.debug(
|
||||
"Font {} failed to decode byte sequence: {}",
|
||||
font.getName(),
|
||||
ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
int consumed = before - in.available();
|
||||
String unicode;
|
||||
try {
|
||||
unicode = font.toUnicode(code);
|
||||
} catch (Exception ex) {
|
||||
unicode = null;
|
||||
}
|
||||
if (unicode == null) {
|
||||
// If the font has no ToUnicode mapping we cannot match reliably - return
|
||||
return null;
|
||||
}
|
||||
text.append(unicode);
|
||||
// Associate every Unicode character produced with the same code byte range
|
||||
for (int c = 0; c < unicode.length(); c++) {
|
||||
starts.add(pos);
|
||||
lens.add(consumed);
|
||||
}
|
||||
if (consumed == 0) {
|
||||
// Defensive: avoid infinite loop on malformed fonts.
|
||||
break;
|
||||
}
|
||||
pos += consumed;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
}
|
||||
int[] startArr = starts.stream().mapToInt(Integer::intValue).toArray();
|
||||
int[] lenArr = lens.stream().mapToInt(Integer::intValue).toArray();
|
||||
return new DecodeResult(text.toString(), bytes, startArr, lenArr);
|
||||
}
|
||||
|
||||
/** Null if no pattern matches; else a per-char drop mask over the text. */
|
||||
private static boolean[] findDroppedCharsMask(String text, List<Pattern> patterns) {
|
||||
boolean any = false;
|
||||
boolean[] mask = new boolean[text.length()];
|
||||
for (Pattern pattern : patterns) {
|
||||
Matcher m;
|
||||
try {
|
||||
m = pattern.matcher(DeadlineCharSequence.of(text));
|
||||
} catch (Exception ex) {
|
||||
continue;
|
||||
}
|
||||
while (m.find()) {
|
||||
int s = m.start();
|
||||
int e = m.end();
|
||||
if (e <= s) continue;
|
||||
for (int i = s; i < e; i++) {
|
||||
mask[i] = true;
|
||||
}
|
||||
any = true;
|
||||
}
|
||||
}
|
||||
return any ? mask : null;
|
||||
}
|
||||
|
||||
private static COSString buildFilteredCosString(
|
||||
DecodeResult decoded, boolean[] drop, PDFont font) {
|
||||
return buildFilteredCosStringRaw(decoded, drop, font, decoded.bytes);
|
||||
}
|
||||
|
||||
private static COSString buildFilteredCosStringRaw(
|
||||
DecodeResult decoded, boolean[] drop, PDFont font, byte[] originalBytes) {
|
||||
// Collect code byte-ranges to drop.
|
||||
Set<Integer> dropStarts = new HashSet<>();
|
||||
for (int i = 0; i < drop.length; i++) {
|
||||
if (drop[i]) {
|
||||
dropStarts.add(decoded.codeStarts[i]);
|
||||
}
|
||||
}
|
||||
// Map byte-start -> code length once (O(n)); findCodeLenAt was an O(n^2) linear scan.
|
||||
Map<Integer, Integer> lenByStart = new HashMap<>();
|
||||
for (int j = 0; j < decoded.codeStarts.length; j++) {
|
||||
lenByStart.putIfAbsent(decoded.codeStarts[j], decoded.codeLens[j]);
|
||||
}
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
int i = 0;
|
||||
while (i < originalBytes.length) {
|
||||
int start = i;
|
||||
int len = lenByStart.getOrDefault(start, -1);
|
||||
if (len <= 0) {
|
||||
// Unknown - keep the byte verbatim.
|
||||
out.write(originalBytes[i] & 0xFF);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (dropStarts.contains(start)) {
|
||||
// Replace dropped code with encoded space if possible.
|
||||
byte[] spaceBytes = tryEncodeSpace(font);
|
||||
if (spaceBytes != null) {
|
||||
out.write(spaceBytes, 0, spaceBytes.length);
|
||||
}
|
||||
} else {
|
||||
out.write(originalBytes, start, len);
|
||||
}
|
||||
i += len;
|
||||
}
|
||||
return new COSString(out.toByteArray());
|
||||
}
|
||||
|
||||
private static byte[] tryEncodeSpace(PDFont font) {
|
||||
if (font == null) {
|
||||
return new byte[] {0x20};
|
||||
}
|
||||
try {
|
||||
return font.encode(" ");
|
||||
} catch (Exception e) {
|
||||
// Some fonts cannot encode a space at all (e.g. Type3 throws
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class DecodeResult {
|
||||
final String text;
|
||||
final byte[] bytes;
|
||||
final int[] codeStarts;
|
||||
final int[] codeLens;
|
||||
|
||||
DecodeResult(String text, byte[] bytes, int[] codeStarts, int[] codeLens) {
|
||||
this.text = text;
|
||||
this.bytes = bytes;
|
||||
this.codeStarts = codeStarts;
|
||||
this.codeLens = codeLens;
|
||||
}
|
||||
}
|
||||
|
||||
// Area capture
|
||||
|
||||
static List<String> captureTextInRects(PDPage page, List<PDRectangle> rects)
|
||||
throws IOException {
|
||||
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
|
||||
stripper.setSortByPosition(true);
|
||||
for (int i = 0; i < rects.size(); i++) {
|
||||
PDRectangle rect = rects.get(i);
|
||||
// PDFTextStripperByArea uses a Java2D rectangle in the same coordinate
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
Rectangle2D.Float region =
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight());
|
||||
stripper.addRegion("r" + i, region);
|
||||
}
|
||||
try {
|
||||
stripper.extractRegions(page);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to extract text in rects: {}", e.getMessage());
|
||||
return Collections.emptyList();
|
||||
}
|
||||
Set<String> captured = new LinkedHashSet<>();
|
||||
for (int i = 0; i < rects.size(); i++) {
|
||||
String text = stripper.getTextForRegion("r" + i);
|
||||
if (text == null) {
|
||||
continue;
|
||||
}
|
||||
for (String token : text.split("\\s+")) {
|
||||
String trimmed = token.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
captured.add(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ArrayList<>(captured);
|
||||
}
|
||||
|
||||
// Content-stream rewriting (rect-driven glyph removal)
|
||||
|
||||
/**
|
||||
* Blanks show-text operands whose glyphs fall in a rect. Returns false (surgical removal is
|
||||
* unreliable, caller must rasterise) when text-bearing form XObjects skew the operator ordinals
|
||||
* or an image sits under a rect - leaving the page stream untouched so the raster is correct.
|
||||
*/
|
||||
static boolean removeTokensIntersectingRects(
|
||||
PDDocument document, PDPage page, List<PDRectangle> rects) throws IOException {
|
||||
|
||||
List<Rectangle2D.Float> areaRects = new ArrayList<>();
|
||||
for (PDRectangle rect : rects) {
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
areaRects.add(
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight()));
|
||||
}
|
||||
|
||||
int pageIndex = document.getPages().indexOf(page);
|
||||
TokenIndexCollector collector = new TokenIndexCollector(areaRects);
|
||||
collector.setStartPage(pageIndex + 1);
|
||||
collector.setEndPage(pageIndex + 1);
|
||||
collector.getText(document);
|
||||
int collectorTextOps = collector.totalTextOps();
|
||||
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(page));
|
||||
int pageTextOps = 0;
|
||||
for (Object t : tokens) {
|
||||
if (t instanceof Operator op
|
||||
&& RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(op.getName())) {
|
||||
pageTextOps++;
|
||||
}
|
||||
}
|
||||
// Collector ordinals span form XObjects; images under a rect can't be surgically removed.
|
||||
if (collectorTextOps != pageTextOps || imageIntersectsAnyRect(page, pageIndex, rects)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Drop ONLY the in-rect glyphs of each operand (not the whole run), tracking the active
|
||||
// font so codes decode. Bail to rasterise if the font can't decode or glyph!=code counts.
|
||||
PDResources resources = page.getResources();
|
||||
PDFont currentFont = null;
|
||||
int opCounter = -1;
|
||||
boolean modified = false;
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
Object tok = tokens.get(i);
|
||||
if (!(tok instanceof Operator op)) {
|
||||
continue;
|
||||
}
|
||||
String name = op.getName();
|
||||
if ("Tf".equals(name) && i >= 2 && tokens.get(i - 2) instanceof COSName fn) {
|
||||
try {
|
||||
currentFont = resources != null ? resources.getFont(fn) : null;
|
||||
} catch (IOException e) {
|
||||
currentFont = null;
|
||||
}
|
||||
} else if (RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name)) {
|
||||
opCounter++;
|
||||
Set<Integer> dropGlyphs = collector.dropGlyphsByOp.get(opCounter);
|
||||
if (dropGlyphs == null || dropGlyphs.isEmpty() || i < 1) {
|
||||
continue;
|
||||
}
|
||||
// ' and " also move to the next line; converting them to TJ would lose that -
|
||||
// raster.
|
||||
if ("'".equals(name) || "\"".equals(name)) {
|
||||
return false;
|
||||
}
|
||||
int expected = collector.glyphCountByOp.getOrDefault(opCounter, -1);
|
||||
COSArray rebuilt =
|
||||
rebuildAdvancePreserving(
|
||||
tokens.get(i - 1), currentFont, dropGlyphs, expected);
|
||||
if (rebuilt == null) {
|
||||
return false; // undecodable / glyph-count mismatch - rasterise instead
|
||||
}
|
||||
// Emit as TJ so the removed glyphs' advance is preserved (no reflow).
|
||||
tokens.set(i - 1, rebuilt);
|
||||
tokens.set(i, Operator.getOperator("TJ"));
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
writePageTokens(document, page, tokens);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Distinct code byte-starts in order (one per glyph/code) from a per-char DecodeResult. */
|
||||
private static int[] distinctCodeStarts(DecodeResult d) {
|
||||
List<Integer> starts = new ArrayList<>();
|
||||
int last = -1;
|
||||
for (int s : d.codeStarts) {
|
||||
if (s != last) {
|
||||
starts.add(s);
|
||||
last = s;
|
||||
}
|
||||
}
|
||||
return starts.stream().mapToInt(Integer::intValue).toArray();
|
||||
}
|
||||
|
||||
private static int codeAt(PDFont font, byte[] bytes, int start, int len) throws IOException {
|
||||
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes, start, len)) {
|
||||
return font.readCode(in);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild a Tj/TJ operand as a TJ array that keeps every glyph EXCEPT the given 0-based
|
||||
* indexes, replacing each dropped run with a numeric adjustment equal to its advance so the
|
||||
* surviving text does not reflow into (or out of) the box. Returns null (caller rasterises) if
|
||||
* anything can't be decoded or the glyph count doesn't line up with the position pass.
|
||||
*/
|
||||
private static COSArray rebuildAdvancePreserving(
|
||||
Object operand, PDFont font, Set<Integer> dropGlyphs, int expectedGlyphs) {
|
||||
if (font == null) {
|
||||
return null;
|
||||
}
|
||||
List<COSBase> elements = new ArrayList<>();
|
||||
if (operand instanceof COSString cs) {
|
||||
elements.add(cs);
|
||||
} else if (operand instanceof COSArray arr) {
|
||||
for (COSBase b : arr) {
|
||||
elements.add(b);
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
COSArray out = new COSArray();
|
||||
ByteArrayOutputStream seg = new ByteArrayOutputStream();
|
||||
float pendingAdvance = 0f; // accumulated width of dropped codes, in 1/1000 text units
|
||||
int globalCode = 0;
|
||||
for (COSBase el : elements) {
|
||||
if (!(el instanceof COSString s)) {
|
||||
// Existing numeric adjustment: flush whichever run is pending, then preserve it.
|
||||
if (seg.size() > 0) {
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
seg.reset();
|
||||
} else if (pendingAdvance != 0f) {
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
pendingAdvance = 0f;
|
||||
}
|
||||
out.add(el);
|
||||
continue;
|
||||
}
|
||||
DecodeResult d = decodeCosString(s, font);
|
||||
if (d == null) {
|
||||
return null;
|
||||
}
|
||||
byte[] b = s.getBytes();
|
||||
int[] starts = distinctCodeStarts(d);
|
||||
Map<Integer, Integer> lenByStart = new HashMap<>();
|
||||
for (int j = 0; j < d.codeStarts.length; j++) {
|
||||
lenByStart.putIfAbsent(d.codeStarts[j], d.codeLens[j]);
|
||||
}
|
||||
for (int k = 0; k < starts.length; k++) {
|
||||
int bs = starts[k];
|
||||
int len = lenByStart.getOrDefault(bs, -1);
|
||||
if (len <= 0) {
|
||||
return null;
|
||||
}
|
||||
if (dropGlyphs.contains(globalCode)) {
|
||||
if (seg.size() > 0) { // close the kept run before accumulating advance
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
seg.reset();
|
||||
}
|
||||
try {
|
||||
pendingAdvance += font.getWidth(codeAt(font, b, bs, len));
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
if (pendingAdvance != 0f) { // emit the removed run's advance, then keep glyphs
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
pendingAdvance = 0f;
|
||||
}
|
||||
seg.write(b, bs, len);
|
||||
}
|
||||
globalCode++;
|
||||
}
|
||||
}
|
||||
if (pendingAdvance != 0f) {
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
}
|
||||
if (seg.size() > 0) {
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
}
|
||||
if (expectedGlyphs >= 0 && globalCode != expectedGlyphs) {
|
||||
return null;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any image on the page overlaps a redaction rect (would survive under the overlay).
|
||||
*/
|
||||
private static boolean imageIntersectsAnyRect(
|
||||
PDPage page, int pageIndex, List<PDRectangle> rects) {
|
||||
try {
|
||||
PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
locator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) {
|
||||
Rectangle2D.Float img =
|
||||
new Rectangle2D.Float(
|
||||
ib.x1(), ib.y1(), ib.x2() - ib.x1(), ib.y2() - ib.y1());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (img.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
// Cannot prove no image sits under a box - treat as unsafe (force raster).
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static void writePageTokens(PDDocument document, PDPage page, List<Object> tokens)
|
||||
throws IOException {
|
||||
PDStream stream = new PDStream(document);
|
||||
try (var out = stream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
ContentStreamWriter writer = new ContentStreamWriter(out);
|
||||
writer.writeTokens(tokens);
|
||||
}
|
||||
page.setContents(stream);
|
||||
}
|
||||
|
||||
static void drawOverlay(
|
||||
PDDocument document, PDPage page, List<PDRectangle> rects, Color overlayColor)
|
||||
throws IOException {
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(overlayColor);
|
||||
for (PDRectangle rect : rects) {
|
||||
cs.addRect(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight());
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.utils.text.TextFinderUtils;
|
||||
import stirling.software.common.util.RegexPatternUtils;
|
||||
|
||||
/**
|
||||
* Public facade for the manual-area, whole-page and auto-word redaction paths. Orchestrates the
|
||||
* collaborators that do the work: {@link RedactionContentEditor} (content-stream glyph removal),
|
||||
* {@link RedactionVerifier} (fail-closed independent verification) and {@link RedactionRasteriser}
|
||||
* (last-resort rasterisation), with {@link CatalogScrubber} for out-of-page carriers.
|
||||
*/
|
||||
@Slf4j
|
||||
public final class RedactionPipeline {
|
||||
|
||||
public static final Set<String> TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\"");
|
||||
|
||||
private RedactionPipeline() {}
|
||||
|
||||
/** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */
|
||||
static void setJpdfiumAvailableForTest(boolean available) {
|
||||
RedactionVerifier.setJpdfiumAvailableForTest(available);
|
||||
}
|
||||
|
||||
/** Redact page-local rects (PDF user-space), dropping intersecting glyphs + overlay. */
|
||||
public static RedactionResult redactAreas(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDRectangle>> rectsByPageIndex,
|
||||
Color overlayColor)
|
||||
throws IOException {
|
||||
|
||||
Set<String> capturedStrings = new LinkedHashSet<>();
|
||||
Set<Integer> forceRasterPages = new LinkedHashSet<>();
|
||||
|
||||
for (Map.Entry<Integer, List<PDRectangle>> entry : rectsByPageIndex.entrySet()) {
|
||||
int pageIndex = entry.getKey();
|
||||
List<PDRectangle> rects = entry.getValue();
|
||||
if (pageIndex < 0 || pageIndex >= document.getNumberOfPages() || rects.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
capturedStrings.addAll(RedactionContentEditor.captureTextInRects(page, rects));
|
||||
|
||||
// Rotation / non-zero CropBox origin break the coordinate flip, and text-bearing forms
|
||||
// or in-rect images defeat page-stream ordinal removal.
|
||||
boolean surgical =
|
||||
RedactionContentEditor.isSurgicallySafe(page)
|
||||
&& RedactionContentEditor.removeTokensIntersectingRects(
|
||||
document, page, rects);
|
||||
if (!surgical) {
|
||||
forceRasterPages.add(pageIndex);
|
||||
}
|
||||
RedactionContentEditor.removeOverlappingAnnotations(page, rects);
|
||||
RedactionContentEditor.drawOverlay(document, page, rects, overlayColor);
|
||||
}
|
||||
|
||||
return new RedactionResult(capturedStrings, forceRasterPages);
|
||||
}
|
||||
|
||||
/** Wipe whole pages: drop all content/resources, fill with a rectangle. */
|
||||
public static void redactWholePages(
|
||||
PDDocument document, List<Integer> pageIndexes, Color overlayColor) throws IOException {
|
||||
for (Integer pageIndex : pageIndexes) {
|
||||
if (pageIndex == null || pageIndex < 0 || pageIndex >= document.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PDRectangle media = page.getMediaBox();
|
||||
|
||||
// Drop existing content streams and page resources outright.
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
page.getCOSObject().removeItem(COSName.ANNOTS);
|
||||
// /Thumb is a rendered image of the original page - drop it too.
|
||||
page.getCOSObject().removeItem(COSName.getPDFName("Thumb"));
|
||||
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.OVERWRITE, true, true)) {
|
||||
cs.setNonStrokingColor(overlayColor);
|
||||
cs.addRect(
|
||||
media.getLowerLeftX(),
|
||||
media.getLowerLeftY(),
|
||||
media.getWidth(),
|
||||
media.getHeight());
|
||||
cs.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Physically remove every literal/regex match from all page content streams. */
|
||||
public static void redactLiteralTerms(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
List<Pattern> effectivePatterns =
|
||||
RedactionContentEditor.effectivePatterns(literalTargets, patterns);
|
||||
if (effectivePatterns.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int pageIndex = 0;
|
||||
for (PDPage page : document.getPages()) {
|
||||
try {
|
||||
RedactionContentEditor.rewritePageContent(document, page, effectivePatterns);
|
||||
} catch (IOException | RuntimeException e) {
|
||||
// Never let one page's font quirk (e.g. Type3 encode, damaged program)
|
||||
log.warn(
|
||||
"Content-stream rewrite failed on page {} ({}); leaving it for the "
|
||||
+ "verification/rasterisation pass.",
|
||||
pageIndex + 1,
|
||||
e.toString());
|
||||
}
|
||||
pageIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalize with document-wide verification: scrub, wipe metadata, save, verify. */
|
||||
public static byte[] finalize(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
return finalize(document, literalTargets, patterns, null);
|
||||
}
|
||||
|
||||
/** Finalize with verification scoped to a specific set of page indexes. */
|
||||
public static byte[] finalize(
|
||||
PDDocument document,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> affectedPages)
|
||||
throws IOException {
|
||||
|
||||
CatalogScrubber.scrub(document, literalTargets, patterns);
|
||||
RedactionVerifier.warnAboutEmbeddedFontGlyphs(document);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
try {
|
||||
RedactionVerifier.verify(bytes, literalTargets, patterns, affectedPages);
|
||||
return bytes;
|
||||
} catch (RedactionVerificationFailedException primaryFailure) {
|
||||
// Rewriter could not guarantee removal; rasterise as a last resort.
|
||||
log.warn(
|
||||
"Primary redaction verification failed ({}); falling back to page-scoped "
|
||||
+ "rasterisation to guarantee removal.",
|
||||
primaryFailure.getMessage());
|
||||
// With no page scope (auto-word), rasterise only the pages that still leak.
|
||||
Set<Integer> pagesToRaster =
|
||||
(affectedPages == null || affectedPages.isEmpty())
|
||||
? RedactionVerifier.findLeakingPages(bytes, literalTargets, patterns)
|
||||
: new HashSet<>(affectedPages);
|
||||
try (PDDocument rasterised = RedactionRasteriser.rasterisePages(bytes, pagesToRaster)) {
|
||||
CatalogScrubber.scrub(rasterised, literalTargets, patterns);
|
||||
ByteArrayOutputStream rasterOut = new ByteArrayOutputStream();
|
||||
rasterised.save(rasterOut);
|
||||
byte[] rasterBytes = rasterOut.toByteArray();
|
||||
RedactionVerifier.verify(rasterBytes, literalTargets, patterns, affectedPages);
|
||||
return rasterBytes;
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Rasterisation fallback failed after primary redaction leak", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalize a manual area redaction; rasterises forced + verified-leaking pages. */
|
||||
public static byte[] finalizeAreas(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDRectangle>> rectsByPage,
|
||||
Set<Integer> forceRasterPages,
|
||||
Set<String> capturedTargets)
|
||||
throws IOException {
|
||||
|
||||
// Text captured under the rects may also live in a bookmark / form value / annotation / JS
|
||||
// carrier.
|
||||
Set<String> carrierTargets =
|
||||
capturedTargets == null ? Collections.emptySet() : capturedTargets;
|
||||
CatalogScrubber.scrub(document, carrierTargets, Collections.emptyList());
|
||||
RedactionVerifier.warnAboutEmbeddedFontGlyphs(document);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
Set<Integer> toRaster =
|
||||
new HashSet<>(forceRasterPages == null ? Set.of() : forceRasterPages);
|
||||
toRaster.addAll(RedactionVerifier.findLeakingRectPages(bytes, rectsByPage));
|
||||
if (toRaster.isEmpty()) {
|
||||
return bytes;
|
||||
}
|
||||
log.warn("Manual redaction rasterising page(s) {} to guarantee removal", toRaster);
|
||||
try (PDDocument rasterised = RedactionRasteriser.rasterisePages(bytes, toRaster)) {
|
||||
ByteArrayOutputStream rasterOut = new ByteArrayOutputStream();
|
||||
rasterised.save(rasterOut);
|
||||
return rasterOut.toByteArray();
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Rasterisation fallback failed after manual redaction leak", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rasterise the given pages of already-finalized bytes to guarantee removal of geometric (range
|
||||
* / image-box) redactions whose covered content - text under an overlay, or an image - is not
|
||||
* removed by content-stream text redaction. Scrubs carriers and re-saves.
|
||||
*/
|
||||
public static byte[] rasteriseSpecificPages(
|
||||
byte[] bytes, Set<Integer> pages, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
if (pages == null || pages.isEmpty()) {
|
||||
return bytes;
|
||||
}
|
||||
log.warn("Rasterising page(s) {} to guarantee geometric redaction removal", pages);
|
||||
try (PDDocument rasterised =
|
||||
RedactionRasteriser.rasterisePages(bytes, new HashSet<>(pages))) {
|
||||
CatalogScrubber.scrub(rasterised, literalTargets, patterns);
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
rasterised.save(out);
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
/** Build case-insensitive regex patterns from user input. */
|
||||
public static List<Pattern> buildPatterns(
|
||||
String[] rawEntries, boolean useRegex, boolean wholeWordSearch) {
|
||||
List<Pattern> patterns = new ArrayList<>();
|
||||
if (rawEntries == null) {
|
||||
return patterns;
|
||||
}
|
||||
for (String raw : rawEntries) {
|
||||
if (raw == null) {
|
||||
continue;
|
||||
}
|
||||
String trimmed = raw.trim();
|
||||
if (trimmed.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String core = useRegex ? trimmed : Pattern.quote(trimmed);
|
||||
if (wholeWordSearch) {
|
||||
// Shared with the finder so removal + verification use identical boundaries.
|
||||
core = TextFinderUtils.applyWordBoundaries(trimmed, core);
|
||||
}
|
||||
// Compile via the shared cache (same path as the finder) for consistent
|
||||
// handling of user-supplied regex; ReDoS is bounded by DeadlineCharSequence.
|
||||
patterns.add(RegexPatternUtils.getInstance().createSearchPattern(core, true));
|
||||
} catch (PatternSyntaxException e) {
|
||||
log.debug("Skipping invalid regex '{}': {}", trimmed, e.getMessage());
|
||||
}
|
||||
}
|
||||
return patterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any font is not provably reliable for PDFBox glyph extraction (delegates to {@link
|
||||
* RedactionVerifier}); exposed here for the pipeline's callers and tests.
|
||||
*/
|
||||
static boolean documentHasUnreliableFont(PDDocument document) {
|
||||
return RedactionVerifier.documentHasUnreliableFont(document);
|
||||
}
|
||||
|
||||
/** Captured strings plus pages that must be rasterised (surgical removal was unreliable). */
|
||||
public static final class RedactionResult {
|
||||
private final Set<String> capturedStrings;
|
||||
private final Set<Integer> forceRasterPages;
|
||||
|
||||
public RedactionResult(Set<String> capturedStrings, Set<Integer> forceRasterPages) {
|
||||
this.capturedStrings = capturedStrings == null ? Set.of() : capturedStrings;
|
||||
this.forceRasterPages = forceRasterPages == null ? Set.of() : forceRasterPages;
|
||||
}
|
||||
|
||||
public Set<String> getCapturedStrings() {
|
||||
return capturedStrings;
|
||||
}
|
||||
|
||||
public Set<Integer> getForceRasterPages() {
|
||||
return forceRasterPages;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.rendering.ImageType;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/** Rasterisation fallback: renders whole pages to an image so no text/vector content survives. */
|
||||
@Slf4j
|
||||
final class RedactionRasteriser {
|
||||
|
||||
private RedactionRasteriser() {}
|
||||
|
||||
/** Rasterise the listed pages (all when null) at 150 DPI, replacing their content. */
|
||||
static PDDocument rasterisePages(byte[] sourceBytes, Set<Integer> pagesToRaster)
|
||||
throws IOException {
|
||||
// Load the document directly and mutate in place: rewriting
|
||||
PDDocument source = Loader.loadPDF(sourceBytes);
|
||||
try {
|
||||
PDFRenderer renderer = new PDFRenderer(source);
|
||||
int pageCount = source.getNumberOfPages();
|
||||
for (int i = 0; i < pageCount; i++) {
|
||||
if (pagesToRaster != null && !pagesToRaster.contains(i)) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = source.getPage(i);
|
||||
// PDFRenderer renders the CropBox region, so draw the raster over the CropBox (not
|
||||
// the MediaBox) or a CropBox != MediaBox page is stretched/offset.
|
||||
PDRectangle crop = page.getCropBox();
|
||||
|
||||
BufferedImage img = renderer.renderImageWithDPI(i, 150, ImageType.RGB);
|
||||
ByteArrayOutputStream imgOut = new ByteArrayOutputStream();
|
||||
ImageIO.write(img, "png", imgOut);
|
||||
PDImageXObject imageXObject =
|
||||
PDImageXObject.createFromByteArray(
|
||||
source, imgOut.toByteArray(), "redacted-page-" + i);
|
||||
|
||||
// Drop all prior content / resources / annotations / thumbnail; the raster is the
|
||||
// page.
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
page.getCOSObject().removeItem(COSName.ANNOTS);
|
||||
page.getCOSObject().removeItem(COSName.getPDFName("Thumb"));
|
||||
// Rotation is already baked into the rendered image, so reset it to zero.
|
||||
page.setRotation(0);
|
||||
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
source,
|
||||
page,
|
||||
PDPageContentStream.AppendMode.OVERWRITE,
|
||||
false,
|
||||
true)) {
|
||||
// The rendered image already has the rotation baked in visually
|
||||
cs.drawImage(
|
||||
imageXObject,
|
||||
crop.getLowerLeftX(),
|
||||
crop.getLowerLeftY(),
|
||||
crop.getWidth(),
|
||||
crop.getHeight());
|
||||
}
|
||||
}
|
||||
return source;
|
||||
} catch (IOException | RuntimeException e) {
|
||||
source.close();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
/** Thrown when post-redaction verification still finds a target in the saved PDF. */
|
||||
public class RedactionVerificationFailedException extends RuntimeException {
|
||||
|
||||
public RedactionVerificationFailedException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public RedactionVerificationFailedException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,495 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.text.PdfTextExtractor;
|
||||
|
||||
/**
|
||||
* Independent, fail-closed verification that redacted text is truly gone: an /ActualText-blind
|
||||
* PDFBox pass plus an additive native (PDFium) pass for fonts PDFBox cannot reliably extract, with
|
||||
* per-page leak localisation for the rasterisation fallback.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RedactionVerifier {
|
||||
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
|
||||
// Latched false when the native PDFium binding can't load, so the host falls back to the PDFBox
|
||||
// pass.
|
||||
private static volatile boolean jpdfiumAvailable = true;
|
||||
|
||||
// Skip the additive native pass above this size to bound off-heap copy + native runtime on
|
||||
// adversarial inputs; the PDFBox glyph-blind pass still verifies.
|
||||
private static final long MAX_JPDFIUM_VERIFY_BYTES = 100L * 1024 * 1024;
|
||||
|
||||
private RedactionVerifier() {}
|
||||
|
||||
/** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */
|
||||
static void setJpdfiumAvailableForTest(boolean available) {
|
||||
jpdfiumAvailable = available;
|
||||
}
|
||||
|
||||
/** Warns when the document still carries embedded Type0/TrueType font */
|
||||
static void warnAboutEmbeddedFontGlyphs(PDDocument document) {
|
||||
boolean anyEmbedded = false;
|
||||
Set<PDFont> visited = new HashSet<>();
|
||||
for (PDPage page : document.getPages()) {
|
||||
PDResources resources = page.getResources();
|
||||
if (resources == null) {
|
||||
continue;
|
||||
}
|
||||
for (COSName name : resources.getFontNames()) {
|
||||
PDFont font;
|
||||
try {
|
||||
font = resources.getFont(name);
|
||||
} catch (IOException ioe) {
|
||||
continue;
|
||||
}
|
||||
if (font == null || !visited.add(font)) {
|
||||
continue;
|
||||
}
|
||||
if (font instanceof PDType0Font || font instanceof PDTrueTypeFont) {
|
||||
anyEmbedded = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (anyEmbedded) {
|
||||
log.warn(
|
||||
"Redacted document contains embedded Type0/TrueType fonts; glyph outlines for "
|
||||
+ "redacted characters may remain in the font program. Text is not "
|
||||
+ "extractable via content-stream reading, but raw font inspection can "
|
||||
+ "still recover glyph shapes. Use the convert-to-image fallback for "
|
||||
+ "maximum assurance.");
|
||||
}
|
||||
}
|
||||
|
||||
static void verify(
|
||||
byte[] bytes,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> affectedPages) {
|
||||
if ((literalTargets == null || literalTargets.isEmpty())
|
||||
&& (patterns == null || patterns.isEmpty())) {
|
||||
return;
|
||||
}
|
||||
Set<Integer> pageSet =
|
||||
(affectedPages == null || affectedPages.isEmpty())
|
||||
? null
|
||||
: new TreeSet<>(affectedPages);
|
||||
// PDFBox pass, blind to /ActualText so a benign override can't mask real glyphs.
|
||||
boolean needNativePass;
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
assertNoTarget(extractText(reopened, pageSet), literalTargets, patterns);
|
||||
needNativePass = documentHasUnreliableFont(reopened);
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Failed to reopen redacted PDF for verification", e);
|
||||
}
|
||||
// Additive producer-independent pass: native PDFium sees glyphs PDFBox may miss (fonts with
|
||||
// no ToUnicode).
|
||||
if (needNativePass) {
|
||||
String nativeText = extractTextJPDFium(bytes, pageSet);
|
||||
if (nativeText == null) {
|
||||
// Required independent pass could not run (native unavailable or doc over the size
|
||||
// guard); fail closed.
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Independent native verification could not run for a document whose fonts "
|
||||
+ "PDFBox cannot reliably extract; cannot confirm removal");
|
||||
}
|
||||
assertNoTarget(nativeText, literalTargets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any font (on any page or nested form XObject) is not provably reliable for PDFBox
|
||||
* glyph extraction - i.e. it is neither a built-in Standard-14 font nor carries a /ToUnicode
|
||||
* map. These are exactly the fonts (CID/Type3/symbolic without ToUnicode) where the PDFBox pass
|
||||
* can go blind, so the independent native pass earns its cost. Biased to {@code true} on any
|
||||
* inspection failure so the native pass runs whenever reliability is uncertain.
|
||||
*/
|
||||
static boolean documentHasUnreliableFont(PDDocument document) {
|
||||
try {
|
||||
Set<COSBase> visited = new HashSet<>();
|
||||
for (PDPage page : document.getPages()) {
|
||||
if (resourcesHaveUnreliableFont(page.getResources(), visited, 0)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} catch (RuntimeException e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean resourcesHaveUnreliableFont(
|
||||
PDResources res, Set<COSBase> visited, int depth) {
|
||||
if (res == null) {
|
||||
return false;
|
||||
}
|
||||
if (depth > MAX_XOBJECT_DEPTH) {
|
||||
return true; // too deeply nested to fully verify - run the native pass to be safe
|
||||
}
|
||||
if (!visited.add(res.getCOSObject())) {
|
||||
return false; // already inspected this resource dictionary
|
||||
}
|
||||
for (COSName name : res.getFontNames()) {
|
||||
PDFont font;
|
||||
try {
|
||||
font = res.getFont(name);
|
||||
} catch (Exception e) {
|
||||
return true; // font won't load for inspection - assume unreliable
|
||||
}
|
||||
// Reliable only for built-in Standard-14 fonts or a /ToUnicode map we can trust.
|
||||
if (font != null
|
||||
&& !font.isStandard14()
|
||||
&& (!font.getCOSObject().containsKey(COSName.TO_UNICODE)
|
||||
|| isSubsetFont(font))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for (COSName name : res.getXObjectNames()) {
|
||||
try {
|
||||
if (res.getXObject(name) instanceof PDFormXObject form
|
||||
&& resourcesHaveUnreliableFont(form.getResources(), visited, depth + 1)) {
|
||||
return true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return true; // can't inspect the XObject - assume unreliable
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* A subset-embedded font carries a 6-uppercase-letter '+' BaseFont tag (e.g. {@code ABCDEF+}).
|
||||
* Its /ToUnicode is custom-built and cannot be trusted for the reliability gate: a partial map
|
||||
* silently loses text, and a crafted map can defeat text-based redaction entirely (only
|
||||
* convert-to-image fully mitigates that adversarial case).
|
||||
*/
|
||||
private static boolean isSubsetFont(PDFont font) {
|
||||
String name = font.getName();
|
||||
if (name == null || name.length() < 8 || name.charAt(6) != '+') {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < 6; i++) {
|
||||
char c = name.charAt(i);
|
||||
if (c < 'A' || c > 'Z') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Fail-closed match check with whitespace-normalised literals and X2 regex semantics. */
|
||||
private static void assertNoTarget(
|
||||
String extracted, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (extracted == null) {
|
||||
return;
|
||||
}
|
||||
String normalised = extracted.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
// De-hyphenated view catches a target split by a soft hyphen (U+00AD) or a line-break
|
||||
// hyphen ("-" + space).
|
||||
String dehyphenated = normalised.replace("\u00ad", "").replaceAll("-\\s+", "");
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
String needle = target.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
if (normalised.contains(needle) || dehyphenated.contains(needle)) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Redacted text still extractable: '" + target + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
if (pattern.matcher(DeadlineCharSequence.of(extracted)).find()) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Redacted pattern still extractable: " + pattern.pattern());
|
||||
}
|
||||
} catch (RedactionVerificationFailedException rvf) {
|
||||
throw rvf;
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Verification regex failed ("
|
||||
+ pattern.pattern()
|
||||
+ "): "
|
||||
+ e.getMessage(),
|
||||
e instanceof Exception ? (Exception) e : new Exception(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Extract text from pageIndexes (0-based) using an /ActualText-blind stripper. */
|
||||
private static String extractText(PDDocument document, Set<Integer> pageIndexes)
|
||||
throws IOException {
|
||||
PDFTextStripper stripper = new GlyphOnlyTextStripper();
|
||||
if (pageIndexes == null) {
|
||||
return stripper.getText(document);
|
||||
}
|
||||
StringBuilder out = new StringBuilder();
|
||||
for (Integer pageIdx : pageIndexes) {
|
||||
if (pageIdx == null || pageIdx < 0 || pageIdx >= document.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
stripper.setStartPage(pageIdx + 1);
|
||||
stripper.setEndPage(pageIdx + 1);
|
||||
String pageText = stripper.getText(document);
|
||||
if (pageText != null) {
|
||||
out.append(pageText);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Independent native (PDFium) extraction; null if the binding is unavailable (additive only).
|
||||
*/
|
||||
private static String extractTextJPDFium(byte[] bytes, Set<Integer> pageIndexes) {
|
||||
if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) {
|
||||
return null;
|
||||
}
|
||||
try (PdfDocument doc = PdfDocument.open(bytes)) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int n = doc.pageCount();
|
||||
if (pageIndexes == null) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
sb.append(jpdfiumPlainText(doc, i)).append('\n');
|
||||
}
|
||||
} else {
|
||||
for (Integer p : pageIndexes) {
|
||||
if (p != null && p >= 0 && p < n) {
|
||||
sb.append(jpdfiumPlainText(doc, p)).append('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (RuntimeException | Error e) {
|
||||
onJpdfiumFailure(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String jpdfiumPlainText(PdfDocument doc, int i) {
|
||||
try {
|
||||
return PdfTextExtractor.extractPage(doc, i).plainText();
|
||||
} catch (RuntimeException | Error e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** One native open, all pages' plain text; null if the binding is unavailable. */
|
||||
private static List<String> extractPagesJPDFium(byte[] bytes) {
|
||||
if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) {
|
||||
return null;
|
||||
}
|
||||
try (PdfDocument doc = PdfDocument.open(bytes)) {
|
||||
List<String> pages = new ArrayList<>();
|
||||
int n = doc.pageCount();
|
||||
for (int i = 0; i < n; i++) {
|
||||
pages.add(jpdfiumPlainText(doc, i));
|
||||
}
|
||||
return pages;
|
||||
} catch (RuntimeException | Error e) {
|
||||
onJpdfiumFailure(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A native-binding load error latches the pass off process-wide (warn once) so a host without
|
||||
* the native stops retrying; a per-document error only skips this one document (debug).
|
||||
*/
|
||||
private static void onJpdfiumFailure(Throwable e) {
|
||||
boolean nativeUnavailable =
|
||||
e instanceof UnsatisfiedLinkError
|
||||
|| e instanceof NoClassDefFoundError
|
||||
|| e instanceof ExceptionInInitializerError
|
||||
|| e.getClass().getSimpleName().contains("NativeLoad");
|
||||
if (nativeUnavailable) {
|
||||
jpdfiumAvailable = false;
|
||||
log.warn(
|
||||
"JPDFium native unavailable; redaction verification will use the PDFBox pass "
|
||||
+ "only: {}",
|
||||
e.toString());
|
||||
} else {
|
||||
log.debug("JPDFium verification skipped for this document: {}", e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
/** Pages whose surviving text (PDFBox glyph-blind OR native PDFium) still matches a target. */
|
||||
static Set<Integer> findLeakingPages(
|
||||
byte[] bytes, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
List<String> jpdfiumPages = extractPagesJPDFium(bytes);
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
// Native pass required but unavailable: PDFBox can't localise the leak, so rasterise
|
||||
// every page.
|
||||
if (jpdfiumPages == null && documentHasUnreliableFont(reopened)) {
|
||||
log.warn(
|
||||
"Independent native pass unavailable on an unreliable-font document; "
|
||||
+ "rasterising all pages to guarantee removal.");
|
||||
return null;
|
||||
}
|
||||
Set<Integer> leaking = new TreeSet<>();
|
||||
GlyphOnlyTextStripper stripper = new GlyphOnlyTextStripper();
|
||||
for (int i = 0; i < reopened.getNumberOfPages(); i++) {
|
||||
stripper.setStartPage(i + 1);
|
||||
stripper.setEndPage(i + 1);
|
||||
String pdfboxText = stripper.getText(reopened);
|
||||
String jpdfiumText =
|
||||
jpdfiumPages != null && i < jpdfiumPages.size() ? jpdfiumPages.get(i) : "";
|
||||
String combined = (pdfboxText == null ? "" : pdfboxText) + "\n" + jpdfiumText;
|
||||
if (pageLeaks(combined, literalTargets, patterns)) {
|
||||
leaking.add(i);
|
||||
}
|
||||
}
|
||||
if (leaking.isEmpty()) {
|
||||
log.warn(
|
||||
"Verification failed but no single page leaks in isolation; rasterising "
|
||||
+ "all pages to be safe.");
|
||||
return null;
|
||||
}
|
||||
log.info("Leak detection: rasterising only page(s) {}", leaking);
|
||||
return leaking;
|
||||
} catch (Exception e) {
|
||||
log.warn("Per-page leak detection failed ({}); rasterising all pages.", e.toString());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean pageLeaks(
|
||||
String pageText, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
try {
|
||||
assertNoTarget(pageText, literalTargets, patterns);
|
||||
return false;
|
||||
} catch (RedactionVerificationFailedException e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/** Redacted pages that still show text inside a rect or carry an overlapping annotation. */
|
||||
static Set<Integer> findLeakingRectPages(
|
||||
byte[] bytes, Map<Integer, List<PDRectangle>> rectsByPage) {
|
||||
Set<Integer> leaking = new HashSet<>();
|
||||
if (rectsByPage == null || rectsByPage.isEmpty()) {
|
||||
return leaking;
|
||||
}
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
for (Map.Entry<Integer, List<PDRectangle>> entry : rectsByPage.entrySet()) {
|
||||
int pageIndex = entry.getKey();
|
||||
if (pageIndex < 0 || pageIndex >= reopened.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = reopened.getPage(pageIndex);
|
||||
if (glyphStillInRects(reopened, page, pageIndex, entry.getValue())
|
||||
|| annotationOverlapsRect(page, entry.getValue())) {
|
||||
leaking.add(pageIndex);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Cannot verify - rasterise every redacted page to be safe.
|
||||
return new HashSet<>(rectsByPage.keySet());
|
||||
}
|
||||
return leaking;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any non-blank glyph is still painted inside a rect. Position-based (not ToUnicode),
|
||||
* so it catches residual CID/Type3/no-ToUnicode glyphs the text stripper would miss; fails
|
||||
* closed.
|
||||
*/
|
||||
private static boolean glyphStillInRects(
|
||||
PDDocument doc, PDPage page, int pageIndex, List<PDRectangle> rects) {
|
||||
try {
|
||||
List<Rectangle2D.Float> areaRects = new ArrayList<>();
|
||||
for (PDRectangle rect : rects) {
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
areaRects.add(
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight()));
|
||||
}
|
||||
TokenIndexCollector collector = new TokenIndexCollector(areaRects);
|
||||
collector.setStartPage(pageIndex + 1);
|
||||
collector.setEndPage(pageIndex + 1);
|
||||
collector.getText(doc);
|
||||
return collector.anyGlyphInRect();
|
||||
} catch (Exception e) {
|
||||
return true; // cannot prove the rect is clean - rasterise to be safe
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean annotationOverlapsRect(PDPage page, List<PDRectangle> rects) {
|
||||
try {
|
||||
for (var ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
if (ar == null) {
|
||||
continue;
|
||||
}
|
||||
Rectangle2D.Float a =
|
||||
new Rectangle2D.Float(
|
||||
ar.getLowerLeftX(),
|
||||
ar.getLowerLeftY(),
|
||||
ar.getWidth(),
|
||||
ar.getHeight());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (a.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** PDFTextStripper that strips /ActualText so verification sees the real glyph stream. */
|
||||
private static final class GlyphOnlyTextStripper extends PDFTextStripper {
|
||||
private static final COSName ACTUAL_TEXT = COSName.getPDFName("ActualText");
|
||||
|
||||
GlyphOnlyTextStripper() throws IOException {}
|
||||
|
||||
@Override
|
||||
public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {
|
||||
COSDictionary safe = properties;
|
||||
if (properties != null && properties.containsKey(ACTUAL_TEXT)) {
|
||||
safe = new COSDictionary(properties);
|
||||
safe.removeItem(ACTUAL_TEXT);
|
||||
}
|
||||
super.beginMarkedContentSequence(tag, safe);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* Position-based (ToUnicode-independent) glyph locator: records, per text-showing operator, the
|
||||
* 0-based indexes of the non-blank glyphs whose box intersects a rect, plus each operator's total
|
||||
* glyph count. Blank glyphs are ignored so a space left behind by removal is not a "leak".
|
||||
*/
|
||||
final class TokenIndexCollector extends PDFTextStripper {
|
||||
private final List<Rectangle2D.Float> rects;
|
||||
final Map<Integer, Set<Integer>> dropGlyphsByOp = new HashMap<>();
|
||||
final Map<Integer, Integer> glyphCountByOp = new HashMap<>();
|
||||
private int showTextOpCounter = -1;
|
||||
private int glyphInOp = 0;
|
||||
|
||||
TokenIndexCollector(List<Rectangle2D.Float> rects) throws IOException {
|
||||
this.rects = rects;
|
||||
setSortByPosition(false);
|
||||
}
|
||||
|
||||
int totalTextOps() {
|
||||
return showTextOpCounter + 1;
|
||||
}
|
||||
|
||||
boolean anyGlyphInRect() {
|
||||
return !dropGlyphsByOp.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processTextPosition(TextPosition text) {
|
||||
int op = showTextOpCounter;
|
||||
int idx = glyphInOp++;
|
||||
String u = text.getUnicode();
|
||||
if (u != null && !u.isBlank()) {
|
||||
// PDFBox reports coordinates with top-left origin here.
|
||||
float x = text.getX();
|
||||
float y = text.getY() - text.getHeight();
|
||||
Rectangle2D.Float glyph =
|
||||
new Rectangle2D.Float(x, y, text.getWidth(), text.getHeight());
|
||||
for (Rectangle2D.Float rect : rects) {
|
||||
if (rect.intersects(glyph)) {
|
||||
dropGlyphsByOp.computeIfAbsent(op, k -> new HashSet<>()).add(idx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
super.processTextPosition(text);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processOperator(Operator operator, List<COSBase> operands) throws IOException {
|
||||
String name = operator.getName();
|
||||
boolean textOp = RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name);
|
||||
if (textOp) {
|
||||
showTextOpCounter++;
|
||||
glyphInOp = 0;
|
||||
}
|
||||
super.processOperator(operator, operands);
|
||||
if (textOp) {
|
||||
glyphCountByOp.put(showTextOpCounter, glyphInOp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -86,14 +86,21 @@ public class TextFinderUtils {
|
||||
return patterns;
|
||||
}
|
||||
|
||||
private static String applyWordBoundaries(String originalTerm, String patternString) {
|
||||
if (originalTerm.length() == 1 && Character.isDigit(originalTerm.charAt(0))) {
|
||||
/** Shared whole-word wrapper so the finder and RedactionPipeline never diverge (see F2). */
|
||||
public static String applyWordBoundaries(String originalTerm, String patternString) {
|
||||
// Lookarounds for single/non-word-edge terms (\b no-ops on chars like '&'); \b otherwise.
|
||||
boolean nonWordEdge =
|
||||
originalTerm.length() == 1
|
||||
|| !isWordChar(originalTerm.charAt(0))
|
||||
|| !isWordChar(originalTerm.charAt(originalTerm.length() - 1));
|
||||
if (nonWordEdge) {
|
||||
return "(?<![\\w])" + patternString + "(?![\\w])";
|
||||
} else if (originalTerm.length() == 1) {
|
||||
return "(?<![\\w])" + patternString + "(?![\\w])";
|
||||
} else {
|
||||
return "\\b" + patternString + "\\b";
|
||||
}
|
||||
return "\\b" + patternString + "\\b";
|
||||
}
|
||||
|
||||
private static boolean isWordChar(char c) {
|
||||
return Character.isLetterOrDigit(c) || c == '_';
|
||||
}
|
||||
|
||||
public static boolean hasProblematicFonts(PDPage page) {
|
||||
|
||||
|
Before Width: | Height: | Size: 6.4 KiB After Width: | Height: | Size: 6.4 KiB |
@@ -1,18 +1 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 24 24">
|
||||
<symbol id="icon-redact-auto" viewBox="0 0 24 24"> <g id="Layer_2" data-name="Layer 2">
|
||||
<g id="Layer_1-2" data-name="Layer 1">
|
||||
<rect width="24" height="24" style="fill: none"/>
|
||||
<g>
|
||||
<path d="M17.541,15.64258a.91793.91793,0,0,1,.55469-.18555h1.1084a.91586.91586,0,0,1,.55469.18555,1.30889,1.30889,0,0,1,.40429.499,1.57206,1.57206,0,0,1,.15039.68457v5.47754H19.2041V20.21094H18.0957v2.09277H16.9873V16.82617a1.55843,1.55843,0,0,1,.15039-.68457A1.2979,1.2979,0,0,1,17.541,15.64258Zm1.66309,1.10547H18.0957v2.17187h1.1084Z" style="fill: currentColor"/>
|
||||
<path d="M5.68653,22.30351a2.00588,2.00588,0,0,1-2-2v-16A1.92585,1.92585,0,0,1,4.274,2.891a1.92585,1.92585,0,0,1,1.4125-.5875h8l6,6v5.66931h-2V9.30351h-5v-5h-7v16h9.74021v2Z" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="10.43189" width="4.33778" height="0.79501" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="12.16889" width="7.31192" height="1.21288" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="17.14555" width="7.31192" height="1.21288" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="14.32375" width="7.31192" height="0.57517" style="fill: currentColor"/>
|
||||
<rect x="7.69809" y="15.79848" width="5.25578" height="0.4475" style="fill: currentColor"/>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</symbol>
|
||||
</svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 24 24"><symbol id="icon-redact-auto" viewBox="0 0 24 24"><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><rect width="24" height="24" style="fill:none"/><g><path d="M17.541,15.64258a.91793.91793,0,0,1,.55469-.18555h1.1084a.91586.91586,0,0,1,.55469.18555,1.30889,1.30889,0,0,1,.40429.499,1.57206,1.57206,0,0,1,.15039.68457v5.47754H19.2041V20.21094H18.0957v2.09277H16.9873V16.82617a1.55843,1.55843,0,0,1,.15039-.68457A1.2979,1.2979,0,0,1,17.541,15.64258Zm1.66309,1.10547H18.0957v2.17187h1.1084Z" style="fill:currentColor"/><path d="M5.68653,22.30351a2.00588,2.00588,0,0,1-2-2v-16A1.92585,1.92585,0,0,1,4.274,2.891a1.92585,1.92585,0,0,1,1.4125-.5875h8l6,6v5.66931h-2V9.30351h-5v-5h-7v16h9.74021v2Z" style="fill:currentColor"/><rect width="4.338" height=".795" x="7.698" y="10.432" style="fill:currentColor"/><rect width="7.312" height="1.213" x="7.698" y="12.169" style="fill:currentColor"/><rect width="7.312" height="1.213" x="7.698" y="17.146" style="fill:currentColor"/><rect width="7.312" height=".575" x="7.698" y="14.324" style="fill:currentColor"/><rect width="5.256" height=".448" x="7.698" y="15.798" style="fill:currentColor"/></g></g></g></symbol></svg>
|
||||
|
Before Width: | Height: | Size: 1.5 KiB After Width: | Height: | Size: 1.3 KiB |
@@ -1,13 +1 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
|
||||
<svg width="800px" height="800px" viewBox="0 0 512 512" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<title>rename</title>
|
||||
<symbol id="icon-rename" viewBox="0 0 512 512">
|
||||
<g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
|
||||
<g id="Combined-Shape" fill="currentColor" transform="translate(42.666667, 64.000000)">
|
||||
<path d="M362.666667,1.42108547e-14 L362.666667,21.3333333 L320,21.333 L320,362.666 L362.666667,362.666667 L362.666667,384 L320,383.999 L320,384 L298.666667,384 L298.666,383.999 L256,384 L256,362.666667 L298.666,362.666 L298.666,21.333 L256,21.3333333 L256,1.42108547e-14 L362.666667,1.42108547e-14 Z M426.666667,64 L426.666667,320 L341.333333,320 L341.333333,277.333333 L384,277.333333 L384,106.666667 L341.333333,106.666667 L341.333333,64 L426.666667,64 Z M277.333333,64 L277.333333,320 L3.55271368e-14,320 L3.55271368e-14,64 L277.333333,64 Z M179.2,89.6 L149.333333,89.6 L149.333333,234.666667 C149.333333,248 148.5,256.333333 147.875,264.354167 L147.792993,265.422171 L147.792993,265.422171 L147.714003,266.48894 C147.417695,270.579012 147.2,274.696296 147.2,279.466667 L147.2,279.466667 L177.066667,279.466667 L177.066667,260.266667 C184.941497,273.926888 199.708077,282.130544 215.466667,281.6 C229.540046,281.805757 242.921593,275.508559 251.733333,264.533333 C263.162478,248.989677 269.832496,230.461848 270.933333,211.2 C270.933333,170.666667 249.6,142.933333 217.6,142.933333 C202.507405,142.999748 188.308689,150.099106 179.2,162.133333 L179.2,162.133333 L179.2,89.6 Z M119.466667,162.133333 C107.961824,149.843793 91.4322333,143.546807 74.6666667,145.066667 C57.6785115,144.485924 40.8138255,148.15216 25.6,155.733333 L25.6,155.733333 L34.1333333,177.066667 C45.3979052,171.147831 57.7246848,167.522308 70.4,166.4 C78.5613135,165.511423 86.6853595,168.371259 92.4903835,174.176283 C98.2954074,179.981307 101.155244,188.105353 100.266667,196.266667 L100.266667,196.266667 L100.266667,198.4 L78.9333333,198.4 C65.8181975,197.679203 52.705771,199.864608 40.5333333,204.8 C26.2806563,210.950309 17.6507691,225.621117 19.2,241.066667 C19.0625857,252.057651 23.6679763,262.574827 31.8381493,269.927982 C40.0083223,277.281138 50.9508304,280.757072 61.8666667,279.466667 C77.2795695,280.291768 92.2192911,274.001359 102.4,262.4 L102.4,262.4 L102.4,277.333333 L130.133333,277.333333 C128.292479,266.054406 127.577851,254.620365 128,243.2 L128,243.2 L128,204.8 C129.999138,190.023932 126.995128,175.003882 119.466667,162.133333 Z M98.1333333,213.333333 L98.1333333,238.933333 C92.082572,249.988391 80.836024,257.218314 68.2666667,258.133333 C63.0655139,258.520242 57.9538681,256.621996 54.2659359,252.934064 C50.5780036,249.246132 48.6797582,244.134486 49.0666667,238.933333 C49.0666667,224 59.7333333,215.466667 85.3333333,213.333333 L85.3333333,213.333333 L98.1333333,213.333333 Z M209.066667,166.4 C226.133333,166.4 238.933333,183.466667 238.933333,211.2 C238.933333,238.933333 228.266667,256 211.2,256 C197.298049,255.69869 184.825037,247.383349 179.2,234.666667 L179.2,234.666667 L179.2,187.733333 C185.154203,176.240507 196.263981,168.304951 209.066667,166.4 Z">
|
||||
</path>
|
||||
</g>
|
||||
</g>
|
||||
</symbol>
|
||||
</svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="800" height="800" version="1.1" viewBox="0 0 512 512"><title>rename</title><symbol id="icon-rename" viewBox="0 0 512 512"><g id="Page-1" fill="none" fill-rule="evenodd" stroke="none" stroke-width="1"><g id="Combined-Shape" fill="currentColor"><path d="M362.666667,1.42108547e-14 L362.666667,21.3333333 L320,21.333 L320,362.666 L362.666667,362.666667 L362.666667,384 L320,383.999 L320,384 L298.666667,384 L298.666,383.999 L256,384 L256,362.666667 L298.666,362.666 L298.666,21.333 L256,21.3333333 L256,1.42108547e-14 L362.666667,1.42108547e-14 Z M426.666667,64 L426.666667,320 L341.333333,320 L341.333333,277.333333 L384,277.333333 L384,106.666667 L341.333333,106.666667 L341.333333,64 L426.666667,64 Z M277.333333,64 L277.333333,320 L3.55271368e-14,320 L3.55271368e-14,64 L277.333333,64 Z M179.2,89.6 L149.333333,89.6 L149.333333,234.666667 C149.333333,248 148.5,256.333333 147.875,264.354167 L147.792993,265.422171 L147.792993,265.422171 L147.714003,266.48894 C147.417695,270.579012 147.2,274.696296 147.2,279.466667 L147.2,279.466667 L177.066667,279.466667 L177.066667,260.266667 C184.941497,273.926888 199.708077,282.130544 215.466667,281.6 C229.540046,281.805757 242.921593,275.508559 251.733333,264.533333 C263.162478,248.989677 269.832496,230.461848 270.933333,211.2 C270.933333,170.666667 249.6,142.933333 217.6,142.933333 C202.507405,142.999748 188.308689,150.099106 179.2,162.133333 L179.2,162.133333 L179.2,89.6 Z M119.466667,162.133333 C107.961824,149.843793 91.4322333,143.546807 74.6666667,145.066667 C57.6785115,144.485924 40.8138255,148.15216 25.6,155.733333 L25.6,155.733333 L34.1333333,177.066667 C45.3979052,171.147831 57.7246848,167.522308 70.4,166.4 C78.5613135,165.511423 86.6853595,168.371259 92.4903835,174.176283 C98.2954074,179.981307 101.155244,188.105353 100.266667,196.266667 L100.266667,196.266667 L100.266667,198.4 L78.9333333,198.4 C65.8181975,197.679203 52.705771,199.864608 40.5333333,204.8 C26.2806563,210.950309 17.6507691,225.621117 19.2,241.066667 C19.0625857,252.057651 23.6679763,262.574827 31.8381493,269.927982 C40.0083223,277.281138 50.9508304,280.757072 61.8666667,279.466667 C77.2795695,280.291768 92.2192911,274.001359 102.4,262.4 L102.4,262.4 L102.4,277.333333 L130.133333,277.333333 C128.292479,266.054406 127.577851,254.620365 128,243.2 L128,243.2 L128,204.8 C129.999138,190.023932 126.995128,175.003882 119.466667,162.133333 Z M98.1333333,213.333333 L98.1333333,238.933333 C92.082572,249.988391 80.836024,257.218314 68.2666667,258.133333 C63.0655139,258.520242 57.9538681,256.621996 54.2659359,252.934064 C50.5780036,249.246132 48.6797582,244.134486 49.0666667,238.933333 C49.0666667,224 59.7333333,215.466667 85.3333333,213.333333 L85.3333333,213.333333 L98.1333333,213.333333 Z M209.066667,166.4 C226.133333,166.4 238.933333,183.466667 238.933333,211.2 C238.933333,238.933333 228.266667,256 211.2,256 C197.298049,255.69869 184.825037,247.383349 179.2,234.666667 L179.2,234.666667 L179.2,187.733333 C185.154203,176.240507 196.263981,168.304951 209.066667,166.4 Z" transform="translate(42.666667, 64.000000)"/></g></g></symbol></svg>
|
||||
|
Before Width: | Height: | Size: 3.3 KiB After Width: | Height: | Size: 3.1 KiB |
|
Before Width: | Height: | Size: 6.1 KiB After Width: | Height: | Size: 5.3 KiB |
@@ -19,6 +19,7 @@ import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -568,7 +569,15 @@ class ManualRedactionServiceTest {
|
||||
byPage.put(0, new ArrayList<>(Arrays.asList(text(0, 72, 690, 300, 710))));
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 1.0f, false, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
1.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertNotNull(result.getFile());
|
||||
@@ -590,7 +599,15 @@ class ManualRedactionServiceTest {
|
||||
Map<Integer, List<PDFText>> byPage = new HashMap<>();
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 0.0f, false, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getFile().exists());
|
||||
@@ -610,7 +627,15 @@ class ManualRedactionServiceTest {
|
||||
Map<Integer, List<PDFText>> byPage = new HashMap<>();
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 0.0f, null, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
null,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getFile().exists());
|
||||
@@ -630,7 +655,15 @@ class ManualRedactionServiceTest {
|
||||
byPage.put(0, new ArrayList<>(Arrays.asList(text(0, 72, 690, 300, 710))));
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#FF0000", 2.0f, false, true);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#FF0000",
|
||||
2.0f,
|
||||
false,
|
||||
true,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
try (PDDocument reloaded = Loader.loadPDF(result.getFile())) {
|
||||
@@ -658,7 +691,14 @@ class ManualRedactionServiceTest {
|
||||
IOException.class,
|
||||
() ->
|
||||
service.finalizeRedaction(
|
||||
doc, byPage, "#000000", 0.0f, false, false));
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList()));
|
||||
// The failing temp file is closed on the error path.
|
||||
verify(failing).close();
|
||||
} finally {
|
||||
|
||||
@@ -299,6 +299,18 @@ class RedactControllerMoreTest {
|
||||
verify(pdfDocumentFactory, never()).load(any(MultipartFile.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("null listOfText throws an illegal-argument error before any load")
|
||||
void nullPatternsThrows() throws Exception {
|
||||
RedactPdfRequest request = new RedactPdfRequest();
|
||||
request.setFileInput(pdfFile(new byte[] {1, 2, 3}));
|
||||
request.setListOfText(null);
|
||||
|
||||
assertThatThrownBy(() -> controller.redactPdf(request))
|
||||
.isInstanceOf(RuntimeException.class);
|
||||
verify(pdfDocumentFactory, never()).load(any(MultipartFile.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("null file input is reported as a failure")
|
||||
void nullFileThrows() {
|
||||
|
||||
@@ -192,15 +192,27 @@ class RedactControllerTest {
|
||||
when(mockCOSStream.createOutputStream()).thenReturn(mockOutputStream);
|
||||
when(mockCOSStream.createOutputStream(any())).thenReturn(mockOutputStream);
|
||||
|
||||
// The pipeline saves then reloads the bytes to verify, so the mock must emit a real,
|
||||
// reloadable PDF (not a placeholder) or verification throws on reopen.
|
||||
byte[] realPdfBytes = createSimplePdfContent();
|
||||
lenient()
|
||||
.doAnswer(
|
||||
inv -> {
|
||||
File f = inv.getArgument(0);
|
||||
java.nio.file.Files.write(f.toPath(), "mock pdf".getBytes());
|
||||
java.nio.file.Files.write(f.toPath(), realPdfBytes);
|
||||
return null;
|
||||
})
|
||||
.when(mockDocument)
|
||||
.save(any(File.class));
|
||||
lenient()
|
||||
.doAnswer(
|
||||
inv -> {
|
||||
java.io.OutputStream os = inv.getArgument(0);
|
||||
os.write(realPdfBytes);
|
||||
return null;
|
||||
})
|
||||
.when(mockDocument)
|
||||
.save(any(java.io.OutputStream.class));
|
||||
doNothing().when(mockDocument).close();
|
||||
|
||||
// Build real service instances so tests exercise actual logic
|
||||
@@ -347,7 +359,7 @@ class RedactControllerTest {
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
|
||||
verify(mockDocument).save(any(File.class));
|
||||
verify(mockDocument).save(any(java.io.OutputStream.class));
|
||||
verify(mockDocument).close();
|
||||
}
|
||||
}
|
||||
@@ -753,7 +765,7 @@ class RedactControllerTest {
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response.getBody());
|
||||
assertTrue(drainBody(response).length > 0);
|
||||
verify(mockDocument, times(1)).save(any(File.class));
|
||||
verify(mockDocument, times(1)).save(any(java.io.OutputStream.class));
|
||||
verify(mockDocument, times(1)).close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
@@ -776,7 +788,7 @@ class RedactControllerTest {
|
||||
if (response != null) {
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
verify(mockDocument, times(1)).save(any(File.class));
|
||||
verify(mockDocument, times(1)).save(any(java.io.OutputStream.class));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.info("Manual redaction test completed with graceful handling: {}", e.getMessage());
|
||||
|
||||
@@ -0,0 +1,755 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSInteger;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSStream;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDFormContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.common.PDStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFontDescriptor;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.MacRomanEncoding;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.WinAnsiEncoding;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
/** Security matrix for redaction across PDF shapes: standard and embedded */
|
||||
@DisplayName("Redaction PDF-variety security matrix")
|
||||
class RedactionPdfVarietyTest {
|
||||
|
||||
private static final String LIBERATION =
|
||||
"/org/apache/pdfbox/resources/ttf/LiberationSans-Regular.ttf";
|
||||
private static final float FONT_SIZE = 12f;
|
||||
private static final float LEFT_X = 72f;
|
||||
private static final float TOP_Y = PDRectangle.LETTER.getHeight() - 80f;
|
||||
|
||||
private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private TempFileManager tempFileManager;
|
||||
private RedactController controller;
|
||||
|
||||
private final List<File> createdTempFiles = new ArrayList<>();
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws IOException {
|
||||
pdfDocumentFactory = mock(CustomPDFDocumentFactory.class);
|
||||
tempFileManager = mock(TempFileManager.class);
|
||||
|
||||
lenient()
|
||||
.when(tempFileManager.createManagedTempFile(anyString()))
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
File f =
|
||||
Files.createTempFile(
|
||||
"redact-variety", inv.<String>getArgument(0))
|
||||
.toFile();
|
||||
createdTempFiles.add(f);
|
||||
TempFile tf = mock(TempFile.class);
|
||||
lenient().when(tf.getFile()).thenReturn(f);
|
||||
lenient().when(tf.getPath()).thenReturn(f.toPath());
|
||||
return tf;
|
||||
});
|
||||
|
||||
controller =
|
||||
new RedactController(
|
||||
pdfDocumentFactory,
|
||||
tempFileManager,
|
||||
new ManualRedactionService(tempFileManager),
|
||||
new TextRedactionService(),
|
||||
mock(RedactExecuteService.class));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
for (File f : createdTempFiles) {
|
||||
if (f != null && f.exists()) {
|
||||
f.delete();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Matrix cases (all through the real /auto-redact controller path)
|
||||
|
||||
@Test
|
||||
@DisplayName("standard Helvetica: target gone, neighbours survive")
|
||||
void standard14Helvetica() throws IOException {
|
||||
byte[] out = autoRedact(helveticaPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("standard-14 Times and Courier: target gone, neighbours survive")
|
||||
void standard14TimesAndCourier() throws IOException {
|
||||
for (Standard14Fonts.FontName fn :
|
||||
new Standard14Fonts.FontName[] {
|
||||
Standard14Fonts.FontName.TIMES_ROMAN, Standard14Fonts.FontName.COURIER
|
||||
}) {
|
||||
byte[] pdf = std14Pdf(fn, "alpha SECRET omega");
|
||||
byte[] out = autoRedact(pdf, "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).as("%s keeps neighbours", fn).contains("alpha");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple TrueType with MacRoman encoding: target gone, neighbours survive")
|
||||
void simpleTrueTypeMacRoman() throws IOException {
|
||||
byte[] out = autoRedact(macRomanTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded Type0 with /ToUnicode stripped: target still gone, neighbours survive")
|
||||
void type0WithoutToUnicode() throws IOException {
|
||||
byte[] out = autoRedact(type0NoToUnicodePdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("symbolic TrueType (no encoding, no ToUnicode): target not extractable")
|
||||
void symbolicTrueType() throws IOException {
|
||||
// PDFBox extraction is glyph-blind here, so removal is guaranteed by the native pass +
|
||||
// rasterise fallback rather than surgical decode; this pins that the pipeline handles a
|
||||
// symbolic font without error and leaves no extractable target.
|
||||
byte[] out = autoRedact(symbolicTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("subset-tagged font (ABCDEF+): target gone, neighbours survive")
|
||||
void subsetTaggedFont() throws IOException {
|
||||
byte[] out = autoRedact(subsetTaggedPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("real Type3 font (glyphs as content streams, no ToUnicode): target gone")
|
||||
void type3GlyphProcs() throws IOException {
|
||||
// crop_test.pdf uses DejaVuSans embedded as a subset Type3 font with no
|
||||
byte[] input;
|
||||
try (InputStream in = getClass().getResourceAsStream("/redaction/type3_dejavu.pdf")) {
|
||||
input = in.readAllBytes();
|
||||
}
|
||||
byte[] out = autoRedact(input, "EXAMPLE");
|
||||
assertGone(out, "EXAMPLE");
|
||||
assertThat(pdfText(out)).contains("CROP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded subset Type0 (CID) font: target gone, neighbours survive")
|
||||
void embeddedSubsetType0() throws IOException {
|
||||
byte[] out = autoRedact(type0Pdf(true, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded full Type0 (CID) font: target gone, neighbours survive")
|
||||
void embeddedFullType0() throws IOException {
|
||||
byte[] out = autoRedact(type0Pdf(false, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded Type1 (PFB) font: target gone, neighbours survive")
|
||||
void embeddedType1() throws IOException {
|
||||
byte[] out = autoRedact(type1Pdf("PUBLICA SECRETWORD KEEP"), "SECRETWORD");
|
||||
assertGone(out, "SECRETWORD");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded CJK Type0 (CIDFontType2) font: CJK target gone, neighbours survive")
|
||||
void embeddedCjkType0() throws IOException {
|
||||
byte[] out = autoRedact(cjkPdf("PUBLICA \u79D8\u5BC6 KEEP"), "\u79D8\u5BC6");
|
||||
assertGone(out, "\u79D8\u5BC6");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded CIDFontType0 (CFF) font: target gone, neighbours survive")
|
||||
void embeddedCidFontType0() throws IOException {
|
||||
byte[] out = autoRedact(cidFontType0Pdf("PUBLICA SECRETWORD KEEP"), "SECRETWORD");
|
||||
assertGone(out, "SECRETWORD");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple embedded TrueType (WinAnsi) font: target gone, neighbours survive")
|
||||
void simpleTrueTypeWinAnsi() throws IOException {
|
||||
byte[] out = autoRedact(simpleTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rotated pages (90/180/270): target gone on every rotation")
|
||||
void rotatedPages() throws IOException {
|
||||
for (int rotation : new int[] {90, 180, 270}) {
|
||||
byte[] out = autoRedact(rotatedPdf(rotation, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).as("rotation %d keeps neighbours", rotation).contains("alpha");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target split across TJ array operands is removed")
|
||||
void tjArraySplit() throws IOException {
|
||||
byte[] out = autoRedact(tjSplitPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("public");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target split across separate Tj operators is removed, other pages keep text")
|
||||
void crossOperatorSplit() throws IOException {
|
||||
byte[] out = autoRedact(crossOperatorPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
// Page 2 was never touched; whatever path handled page 1, page 2 text must
|
||||
assertThat(pdfText(out)).contains("PUBLIC PAGE TWO");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target inside a Form XObject is removed")
|
||||
void formXObjectText() throws IOException {
|
||||
byte[] out = autoRedact(formXObjectPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("CropBox smaller than MediaBox: target gone, neighbours survive")
|
||||
void cropBoxOffset() throws IOException {
|
||||
byte[] out = autoRedact(cropBoxPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("multi-page: target on middle page only, other pages keep their text")
|
||||
void multiPageMiddleTarget() throws IOException {
|
||||
byte[] out =
|
||||
autoRedact(
|
||||
multiPagePdf("PUBLIC ONE", "middle SECRET line", "PUBLIC THREE"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("PUBLIC ONE").contains("PUBLIC THREE");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("case variant: doc contains 'Secret', target 'SECRET' removes it")
|
||||
void caseVariantRemoved() throws IOException {
|
||||
byte[] out = autoRedact(helveticaPdf("alpha Secret omega"), "SECRET");
|
||||
assertGone(out, "Secret");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("whole-word with case variant: 'Cat' removed, substrings survive")
|
||||
void wholeWordCaseVariant() throws IOException {
|
||||
byte[] bytes = helveticaPdf("Cat classification scatter");
|
||||
factoryReturns(bytes);
|
||||
RedactPdfRequest request = baseRequest(bytes, "cat");
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
byte[] out = drainBody(controller.redactPdf(request));
|
||||
String text = pdfText(out);
|
||||
assertThat(text).contains("classification").contains("scatter");
|
||||
// The standalone word must be gone in any case variant.
|
||||
assertThat(text).doesNotContainPattern("(?i)\\bcat\\b");
|
||||
}
|
||||
|
||||
// Page-scoped rasterisation (pipeline-level, deterministic)
|
||||
|
||||
@Test
|
||||
@DisplayName("verification leak rasterises only the leaking page, others keep text")
|
||||
void leakRasterisesOnlyLeakingPage() throws IOException {
|
||||
// Cross-operator split defeats the per-operand literal rewriter
|
||||
byte[] input = crossOperatorPdf();
|
||||
byte[] out;
|
||||
try (PDDocument doc = Loader.loadPDF(input)) {
|
||||
Set<String> targets = Set.of("SECRET");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"SECRET"}, false, false);
|
||||
RedactionPipeline.redactLiteralTerms(doc, targets, patterns);
|
||||
out = RedactionPipeline.finalize(doc, targets, patterns);
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(out)) {
|
||||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
stripper.setStartPage(1);
|
||||
stripper.setEndPage(1);
|
||||
String page1 = stripper.getText(reopened);
|
||||
stripper.setStartPage(2);
|
||||
stripper.setEndPage(2);
|
||||
String page2 = stripper.getText(reopened);
|
||||
|
||||
assertThat(page1.toLowerCase()).doesNotContain("secret");
|
||||
assertThat(page2).contains("PUBLIC PAGE TWO");
|
||||
}
|
||||
}
|
||||
|
||||
// Drivers and assertions
|
||||
|
||||
private byte[] autoRedact(byte[] pdfBytes, String target) throws IOException {
|
||||
factoryReturns(pdfBytes);
|
||||
RedactPdfRequest request = baseRequest(pdfBytes, target);
|
||||
ResponseEntity<Resource> response = controller.redactPdf(request);
|
||||
assertThat(response.getStatusCode().value()).isEqualTo(200);
|
||||
return drainBody(response);
|
||||
}
|
||||
|
||||
private RedactPdfRequest baseRequest(byte[] pdfBytes, String target) {
|
||||
RedactPdfRequest request = new RedactPdfRequest();
|
||||
request.setFileInput(pdfFile(pdfBytes));
|
||||
request.setListOfText(target);
|
||||
request.setUseRegex(false);
|
||||
request.setWholeWordSearch(false);
|
||||
request.setRedactColor("#000000");
|
||||
request.setConvertPDFToImage(false);
|
||||
return request;
|
||||
}
|
||||
|
||||
private void assertGone(byte[] out, String target) throws IOException {
|
||||
assertThat(pdfText(out).toLowerCase()).doesNotContain(target.toLowerCase());
|
||||
}
|
||||
|
||||
private void factoryReturns(byte[] pdfBytes) throws IOException {
|
||||
lenient()
|
||||
.when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(pdfBytes));
|
||||
}
|
||||
|
||||
private MockMultipartFile pdfFile(byte[] bytes) {
|
||||
return new MockMultipartFile("fileInput", "doc.pdf", "application/pdf", bytes);
|
||||
}
|
||||
|
||||
private byte[] drainBody(ResponseEntity<Resource> response) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try (InputStream in = response.getBody().getInputStream()) {
|
||||
in.transferTo(baos);
|
||||
}
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
private String pdfText(byte[] pdfBytes) throws IOException {
|
||||
try (PDDocument doc = Loader.loadPDF(pdfBytes)) {
|
||||
return new PDFTextStripper().getText(doc);
|
||||
}
|
||||
}
|
||||
|
||||
// PDF builders
|
||||
|
||||
private static PDFont helvetica() {
|
||||
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
|
||||
}
|
||||
|
||||
private byte[] helveticaPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] std14Pdf(Standard14Fonts.FontName fontName, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, new PDType1Font(fontName), line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] macRomanTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, MacRomanEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
// GID order emitted by generators/gen_cff_cidfont.py: 0=.notdef 1=space then the letters.
|
||||
private static final String CFF_GLYPHS = "\0 ABCDEIKLOPRSTUW";
|
||||
|
||||
private static int cffGid(char c) {
|
||||
int i = CFF_GLYPHS.indexOf(c);
|
||||
return i < 0 ? 0 : i;
|
||||
}
|
||||
|
||||
/**
|
||||
* PDFBox 3.0.7 has no CFF embedder, so hand-assemble a Type0 -> CIDFontType0 (CFF in
|
||||
* /FontFile3, Identity-H, CID==GID, explicit /ToUnicode) from the generated CFF program.
|
||||
*/
|
||||
private byte[] cidFontType0Pdf(String line) throws IOException {
|
||||
byte[] cff;
|
||||
try (InputStream in = getClass().getResourceAsStream("/redaction/cidfont0_cff.bin")) {
|
||||
cff = in.readAllBytes();
|
||||
}
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
|
||||
PDStream fontFile = new PDStream(doc);
|
||||
try (var os = fontFile.createOutputStream()) {
|
||||
os.write(cff);
|
||||
}
|
||||
COSStream ff = fontFile.getCOSObject();
|
||||
ff.setItem(COSName.SUBTYPE, COSName.getPDFName("CIDFontType0C"));
|
||||
|
||||
COSDictionary fdesc = new COSDictionary();
|
||||
fdesc.setItem(COSName.TYPE, COSName.FONT_DESC);
|
||||
fdesc.setName(COSName.FONT_NAME, "MinimalCFFCID");
|
||||
fdesc.setInt(COSName.FLAGS, 4);
|
||||
COSArray bbox = new COSArray();
|
||||
for (int v : new int[] {0, -200, 600, 800}) {
|
||||
bbox.add(COSInteger.get(v));
|
||||
}
|
||||
fdesc.setItem(COSName.FONT_BBOX, bbox);
|
||||
fdesc.setInt(COSName.ITALIC_ANGLE, 0);
|
||||
fdesc.setInt(COSName.ASCENT, 800);
|
||||
fdesc.setInt(COSName.DESCENT, -200);
|
||||
fdesc.setInt(COSName.CAP_HEIGHT, 700);
|
||||
fdesc.setInt(COSName.STEM_V, 80);
|
||||
fdesc.setItem(COSName.getPDFName("FontFile3"), ff);
|
||||
|
||||
COSDictionary cidSystemInfo = new COSDictionary();
|
||||
cidSystemInfo.setString(COSName.REGISTRY, "Adobe");
|
||||
cidSystemInfo.setString(COSName.ORDERING, "Identity");
|
||||
cidSystemInfo.setInt(COSName.SUPPLEMENT, 0);
|
||||
|
||||
COSDictionary cidFont = new COSDictionary();
|
||||
cidFont.setItem(COSName.TYPE, COSName.FONT);
|
||||
cidFont.setItem(COSName.SUBTYPE, COSName.CID_FONT_TYPE0);
|
||||
cidFont.setName(COSName.BASE_FONT, "MinimalCFFCID");
|
||||
cidFont.setItem(COSName.CIDSYSTEMINFO, cidSystemInfo);
|
||||
cidFont.setItem(COSName.FONT_DESC, fdesc);
|
||||
cidFont.setInt(COSName.DW, 600);
|
||||
|
||||
COSStream toUni = new PDStream(doc).getCOSObject();
|
||||
try (var os = toUni.createOutputStream()) {
|
||||
os.write(cffToUnicode().getBytes(java.nio.charset.StandardCharsets.US_ASCII));
|
||||
}
|
||||
|
||||
COSDictionary type0 = new COSDictionary();
|
||||
type0.setItem(COSName.TYPE, COSName.FONT);
|
||||
type0.setItem(COSName.SUBTYPE, COSName.TYPE0);
|
||||
type0.setName(COSName.BASE_FONT, "MinimalCFFCID");
|
||||
type0.setItem(COSName.ENCODING, COSName.IDENTITY_H);
|
||||
COSArray descendants = new COSArray();
|
||||
descendants.add(cidFont);
|
||||
type0.setItem(COSName.DESCENDANT_FONTS, descendants);
|
||||
type0.setItem(COSName.TO_UNICODE, toUni);
|
||||
|
||||
PDFont font = org.apache.pdfbox.pdmodel.font.PDFontFactory.createFont(type0);
|
||||
PDResources res = new PDResources();
|
||||
COSName fName = res.add(font);
|
||||
page.setResources(res);
|
||||
|
||||
StringBuilder csb = new StringBuilder();
|
||||
csb.append("BT\n/").append(fName.getName()).append(" 24 Tf\n50 700 Td\n<");
|
||||
for (int i = 0; i < line.length(); i++) {
|
||||
csb.append(String.format("%04X", cffGid(line.charAt(i))));
|
||||
}
|
||||
csb.append("> Tj\nET\n");
|
||||
PDStream content = new PDStream(doc);
|
||||
try (var os = content.createOutputStream()) {
|
||||
os.write(csb.toString().getBytes(java.nio.charset.StandardCharsets.US_ASCII));
|
||||
}
|
||||
page.setContents(content);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private static String cffToUnicode() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n");
|
||||
sb.append("/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n");
|
||||
sb.append("/CMapName /Adobe-Identity-UCS def\n/CMapType 2 def\n");
|
||||
sb.append("1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n");
|
||||
StringBuilder body = new StringBuilder();
|
||||
int count = 0;
|
||||
for (int gid = 1; gid < CFF_GLYPHS.length(); gid++) {
|
||||
body.append(String.format("<%04X> <%04X>\n", gid, (int) CFF_GLYPHS.charAt(gid)));
|
||||
count++;
|
||||
}
|
||||
sb.append(count).append(" beginbfchar\n").append(body).append("endbfchar\n");
|
||||
sb.append("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private byte[] type1Pdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream pfb = getClass().getResourceAsStream("/redaction/type1_minimal.pfb")) {
|
||||
font = new PDType1Font(doc, pfb, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] cjkPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = getClass().getResourceAsStream("/redaction/cjk_minimal.ttf")) {
|
||||
font = PDType0Font.load(doc, ttf, true);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] symbolicTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDTrueTypeFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
// Strip /Encoding and flag the descriptor Symbolic so extraction must use the cmap.
|
||||
font.getCOSObject().removeItem(org.apache.pdfbox.cos.COSName.ENCODING);
|
||||
PDFontDescriptor fd = font.getFontDescriptor();
|
||||
fd.setFlags((fd.getFlags() | 4) & ~32);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] type0NoToUnicodePdf(String line) throws IOException {
|
||||
byte[] pdf;
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
// Full embed, not subset: the subsetter regenerates /ToUnicode at save, undoing the
|
||||
// strip.
|
||||
font = PDType0Font.load(doc, ttf, false);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
font.getCOSObject().removeItem(org.apache.pdfbox.cos.COSName.TO_UNICODE);
|
||||
pdf = save(doc);
|
||||
}
|
||||
// Guard: the strip must survive save so this fixture genuinely exercises the no-mapping
|
||||
// path.
|
||||
try (PDDocument reopened = Loader.loadPDF(pdf)) {
|
||||
PDResources res = reopened.getPage(0).getResources();
|
||||
for (org.apache.pdfbox.cos.COSName n : res.getFontNames()) {
|
||||
assertThat(
|
||||
res.getFont(n)
|
||||
.getCOSObject()
|
||||
.containsKey(org.apache.pdfbox.cos.COSName.TO_UNICODE))
|
||||
.as("no-ToUnicode fixture must not ship /ToUnicode")
|
||||
.isFalse();
|
||||
}
|
||||
}
|
||||
return pdf;
|
||||
}
|
||||
|
||||
private byte[] subsetTaggedPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDType0Font.load(doc, ttf, true);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
String tagged = "ABCDEF+" + font.getName();
|
||||
font.getCOSObject().setName(org.apache.pdfbox.cos.COSName.BASE_FONT, tagged);
|
||||
if (font.getFontDescriptor() != null) {
|
||||
font.getFontDescriptor()
|
||||
.getCOSObject()
|
||||
.setName(org.apache.pdfbox.cos.COSName.FONT_NAME, tagged);
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] type0Pdf(boolean subset, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDType0Font.load(doc, ttf, subset);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] simpleTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] rotatedPdf(int rotation, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, rotation, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] cropBoxPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, 0, new PDRectangle(40, 40, 500, 700));
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] multiPagePdf(String... pageLines) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
for (String line : pageLines) {
|
||||
addTextPage(doc, helvetica(), line, 0, null);
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
/** One page whose text is emitted as a TJ array: ["public ", "SEC", -20 */
|
||||
private byte[] tjSplitPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(helvetica(), FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
cs.showTextWithPositioning(
|
||||
new Object[] {"public ", "SEC", Float.valueOf(-20f), "RET", " end"});
|
||||
cs.endText();
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
/** Page 1 shows "SEC" and "RET" as separate adjacent Tj operators; page 2 */
|
||||
private byte[] crossOperatorPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
PDFont font = helvetica();
|
||||
float secWidth = font.getStringWidth("SEC") / 1000f * FONT_SIZE;
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
cs.showText("SEC");
|
||||
cs.endText();
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X + secWidth, TOP_Y);
|
||||
cs.showText("RET");
|
||||
cs.endText();
|
||||
}
|
||||
addTextPage(doc, helvetica(), "PUBLIC PAGE TWO", 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] formXObjectPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
|
||||
PDFormXObject form = new PDFormXObject(doc);
|
||||
form.setBBox(new PDRectangle(0, 0, 400, 60));
|
||||
form.setResources(new PDResources());
|
||||
try (PDFormContentStream fcs = new PDFormContentStream(form)) {
|
||||
fcs.beginText();
|
||||
fcs.setFont(helvetica(), FONT_SIZE);
|
||||
fcs.newLineAtOffset(10, 20);
|
||||
fcs.showText("xobj SECRET payload");
|
||||
fcs.endText();
|
||||
}
|
||||
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.transform(Matrix.getTranslateInstance(LEFT_X, TOP_Y - 60));
|
||||
cs.drawForm(form);
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private void addTextPage(
|
||||
PDDocument doc, PDFont font, String line, int rotation, PDRectangle cropBox)
|
||||
throws IOException {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
if (rotation != 0) {
|
||||
page.setRotation(rotation);
|
||||
}
|
||||
if (cropBox != null) {
|
||||
page.setCropBox(cropBox);
|
||||
}
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
if (rotation != 0) {
|
||||
// Real generators compensate the text matrix so text reads upright
|
||||
cs.setTextMatrix(
|
||||
Matrix.getRotateInstance(
|
||||
Math.toRadians(rotation),
|
||||
PDRectangle.LETTER.getWidth() / 2,
|
||||
PDRectangle.LETTER.getHeight() / 2));
|
||||
} else {
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
}
|
||||
cs.showText(line);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] save(PDDocument doc) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
doc.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
@@ -287,8 +287,12 @@ class TextRedactionServiceExtraTest {
|
||||
|
||||
service.createTokensWithoutTargetText(doc, page, Set.of("SECRET"), false, false);
|
||||
|
||||
// The deep traversal must have rewritten the inner form's content stream.
|
||||
assertThat(inner.getCOSObject().containsKey(COSName.CONTENTS)).isTrue();
|
||||
// The deep traversal must have rewritten the inner form's own stream in place (F3).
|
||||
String innerContent;
|
||||
try (var in = inner.getContents()) {
|
||||
innerContent = new String(in.readAllBytes(), StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
assertThat(innerContent).doesNotContain("SECRET");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -226,19 +226,23 @@ class TextRedactionServiceMoreTest {
|
||||
}
|
||||
page.setContents(pageStream);
|
||||
|
||||
// Processing the page walks into the XObject graph; when a match is found inside
|
||||
// the
|
||||
// form, writeRedactedContentToXObject runs and sets a /Contents item on the form's
|
||||
// COS dictionary. Asserting that item appears proves the XObject redaction path
|
||||
// executed end-to-end without throwing.
|
||||
// Processing the page walks into the XObject graph and rewrites the form's own
|
||||
// stream body in place (F3), so the target is physically gone from the form.
|
||||
List<Object> tokens =
|
||||
service.createTokensWithoutTargetText(
|
||||
doc, page, Set.of("SECRET"), false, false);
|
||||
|
||||
assertThat(tokens).isNotNull();
|
||||
assertThat(form.getCOSObject().containsKey(COSName.CONTENTS))
|
||||
.as("form XObject redaction path should have written a new content item")
|
||||
.isTrue();
|
||||
String rewrittenForm;
|
||||
try (var in = form.getContents()) {
|
||||
rewrittenForm =
|
||||
new String(
|
||||
in.readAllBytes(),
|
||||
java.nio.charset.StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
assertThat(rewrittenForm)
|
||||
.as("form XObject content should have the target removed in place")
|
||||
.doesNotContain("SECRET");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Integration tests that exercise the real redaction pipeline against PDFs */
|
||||
class RedactionPipelineIntegrationTest {
|
||||
|
||||
private static byte[] loadFixture() throws Exception {
|
||||
try (InputStream in =
|
||||
RedactionPipelineIntegrationTest.class.getResourceAsStream(
|
||||
"/redaction/test_pdf_1.pdf")) {
|
||||
assertNotNull(in, "fixture resource must exist on classpath");
|
||||
return in.readAllBytes();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"finalize on rotated ReportLab PDF without a rewrite pass falls back to rasterisation so target disappears")
|
||||
void finaliseFallsBackToRasterisationWhenTargetWouldSurvive() throws Exception {
|
||||
byte[] fixtureBytes = loadFixture();
|
||||
byte[] outputBytes;
|
||||
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Test");
|
||||
// No content-stream rewriting performed.
|
||||
outputBytes = RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
|
||||
}
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("test"),
|
||||
"Rasterisation fallback must have removed target. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"finalize with zero rewrite + synthetic upright PDF triggers RedactionVerificationFailedException only when fallback disabled")
|
||||
void verificationExceptionWiringSanityCheck() throws Exception {
|
||||
// This test proves the verification hook itself still throws -
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
cs.newLineAtOffset(100, 700);
|
||||
cs.showText("Top Smith Classified");
|
||||
cs.endText();
|
||||
}
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Smith");
|
||||
// Even with the rasterisation fallback active, the output must not contain
|
||||
byte[] outBytes =
|
||||
RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
|
||||
try (PDDocument reopened = Loader.loadPDF(outBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("smith"),
|
||||
"Neither primary pass nor rasterisation removed target. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"auto-word redact on rotated PDF with target packed in single Tj removes the word from text stream")
|
||||
void autoWordRedactRemovesTargetOnRotatedSingleTjPdf() throws Exception {
|
||||
byte[] fixtureBytes = loadFixture();
|
||||
byte[] outputBytes;
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Test");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"Test"}, false, false);
|
||||
|
||||
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
|
||||
RedactionPipeline.redactLiteralTerms(doc, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(doc, literalTargets, patterns);
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
String extracted = stripper.getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("test"),
|
||||
"Target term must not be extractable after redact. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple upright Helvetica fixture still gets word redacted via the pipeline")
|
||||
void autoWordRedactRemovesTargetOnSyntheticPdf() throws Exception {
|
||||
byte[] outputBytes;
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Secret");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"Secret"}, false, false);
|
||||
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
cs.newLineAtOffset(100, 700);
|
||||
cs.showText("Top Secret Classified");
|
||||
cs.endText();
|
||||
}
|
||||
ByteArrayOutputStream tmp = new ByteArrayOutputStream();
|
||||
doc.save(tmp);
|
||||
// reload from bytes so we are processing an already-saved PDF similar
|
||||
try (PDDocument reloaded = Loader.loadPDF(tmp.toByteArray())) {
|
||||
RedactionPipeline.redactLiteralTerms(reloaded, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(reloaded, literalTargets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("secret"),
|
||||
"Target term must not be extractable after redact on synthetic PDF. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a minimal OpenType-CFF (.otf) font for a CIDFontType0 redaction fixture.
|
||||
|
||||
Why this exists
|
||||
---------------
|
||||
Real-world Adobe/InDesign PDFs embed CFF-based CID fonts (Type0 -> CIDFontType0,
|
||||
program in /FontFile3 /Subtype /CIDFontType0C). PDFBox 3.0.7 can *read* those but
|
||||
ships NO CFF embedder: every PDType0Font.load(...) overload routes CFF outlines
|
||||
through the TrueType path (which either rejects "TTF using CFF outlines" or, via the
|
||||
OTFParser+TrueTypeFont overload, emits a CIDFontType2 and then fails to subset CFF
|
||||
because there is no glyf table). So to build a genuine CIDFontType0 fixture you must:
|
||||
1. generate the CFF .otf with fontTools (this script), and
|
||||
2. assemble the Type0/CIDFontType0 dictionary by hand, attaching the raw CFF table
|
||||
as /FontFile3 /Subtype /CIDFontType0C (see the companion Java harness).
|
||||
|
||||
This script emits BOTH minimal_cff.otf and cff_raw.bin (the raw CFF table bytes,
|
||||
ready to drop into /FontFile3).
|
||||
|
||||
Glyph coverage: only the ASCII letters in "PUBLICA SECRETWORD KEEP"
|
||||
(P U B L I C A S E R T W O D K) plus .notdef and space. Outlines are trivial filled
|
||||
boxes; only the cmap / encoding correctness matters so the drawn text extracts.
|
||||
"""
|
||||
import sys
|
||||
from fontTools.fontBuilder import FontBuilder
|
||||
from fontTools.pens.t2CharStringPen import T2CharStringPen
|
||||
from fontTools.ttLib import TTFont
|
||||
|
||||
UPM = 1000
|
||||
LETTERS = sorted(set("PUBLICASECRETWORDKEEP")) # A B C D E I K L O P R S T U W
|
||||
|
||||
|
||||
def box_charstring(width=600, height=700):
|
||||
"""A simple filled rectangle as a Type2 (CFF) charstring."""
|
||||
pen = T2CharStringPen(width, None)
|
||||
pen.moveTo((50, 0))
|
||||
pen.lineTo((width - 50, 0))
|
||||
pen.lineTo((width - 50, height))
|
||||
pen.lineTo((50, height))
|
||||
pen.closePath()
|
||||
return pen.getCharString()
|
||||
|
||||
|
||||
def build_otf(out_otf):
|
||||
# GID order: .notdef (GID 0), space (GID 1), then the letters. The companion
|
||||
# harness relies on this exact order for its Identity-H CID==GID mapping.
|
||||
glyph_order = [".notdef", "space"] + list(LETTERS)
|
||||
|
||||
fb = FontBuilder(unitsPerEm=UPM, isTTF=False) # isTTF=False -> CFF/OTF
|
||||
fb.setupGlyphOrder(glyph_order)
|
||||
|
||||
# Unicode cmap: codepoint -> glyph name. PDFBox reads this to build /ToUnicode,
|
||||
# and it also lets any consumer resolve the glyphs by character.
|
||||
cmap = {0x20: "space"}
|
||||
for c in LETTERS:
|
||||
cmap[ord(c)] = c
|
||||
fb.setupCharacterMap(cmap)
|
||||
|
||||
charstrings = {
|
||||
".notdef": box_charstring(),
|
||||
"space": box_charstring(width=250, height=0), # empty advance-only glyph
|
||||
}
|
||||
for c in LETTERS:
|
||||
charstrings[c] = box_charstring()
|
||||
|
||||
fb.setupCFF(
|
||||
psName="MinimalCFFCID",
|
||||
charStringsDict=charstrings,
|
||||
fontInfo={
|
||||
"FullName": "Minimal CFF CID",
|
||||
"FamilyName": "MinimalCFFCID",
|
||||
"Weight": "Regular",
|
||||
},
|
||||
privateDict={},
|
||||
)
|
||||
|
||||
metrics = {".notdef": (600, 50), "space": (250, 0)}
|
||||
for c in LETTERS:
|
||||
metrics[c] = (600, 50)
|
||||
fb.setupHorizontalMetrics(metrics)
|
||||
fb.setupHorizontalHeader(ascent=800, descent=-200)
|
||||
|
||||
fb.setupNameTable(
|
||||
{
|
||||
"familyName": "MinimalCFFCID",
|
||||
"styleName": "Regular",
|
||||
"psName": "MinimalCFFCID",
|
||||
"fullName": "Minimal CFF CID",
|
||||
"version": "Version 1.000",
|
||||
"uniqueFontIdentifier": "MinimalCFFCID;1.000",
|
||||
}
|
||||
)
|
||||
fb.setupOS2(
|
||||
sTypoAscender=800,
|
||||
sTypoDescender=-200,
|
||||
usWinAscent=800,
|
||||
usWinDescent=200,
|
||||
sCapHeight=700,
|
||||
)
|
||||
fb.setupPost(keepGlyphNames=True)
|
||||
|
||||
fb.save(out_otf)
|
||||
print("wrote", out_otf, "(", len(glyph_order), "glyphs )")
|
||||
|
||||
|
||||
def dump_cff(out_otf, out_cff):
|
||||
"""Extract the raw CFF table bytes for use as PDF /FontFile3."""
|
||||
f = TTFont(out_otf)
|
||||
cff = f["CFF "].compile(f)
|
||||
with open(out_cff, "wb") as fh:
|
||||
fh.write(cff)
|
||||
print("wrote", out_cff, "(", len(cff), "bytes CFF )")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
otf = sys.argv[1] if len(sys.argv) > 1 else "minimal_cff.otf"
|
||||
cffbin = sys.argv[2] if len(sys.argv) > 2 else "cff_raw.bin"
|
||||
build_otf(otf)
|
||||
dump_cff(otf, cffbin)
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate a minimal TrueType font (.ttf) that covers exactly the characters
|
||||
needed for a CJK Type0 (Identity-H) redaction test:
|
||||
|
||||
- ASCII letters in "PUBLICA SECRETWORD KEEP": P U B L I C A S E R T W O D K + space
|
||||
- Two CJK ideographs used as the redaction target:
|
||||
U+79D8 (secret1) and U+5BC6 (secret2) -> "秘密"
|
||||
- .notdef
|
||||
|
||||
Glyph outlines are trivial boxes; the point of the test is that the cmap /
|
||||
encoding is correct so PDFBox can build a working /ToUnicode and the text is
|
||||
EXTRACTABLE. When PDFBox loads this via PDType0Font.load(doc, in, true) it
|
||||
becomes an Identity-H Type0 font with a CIDFontType2 descendant.
|
||||
|
||||
Usage: python gen_cjk_font.py <output.ttf>
|
||||
"""
|
||||
import sys
|
||||
from fontTools.fontBuilder import FontBuilder
|
||||
from fontTools.pens.ttGlyphPen import TTGlyphPen
|
||||
|
||||
UPM = 1000
|
||||
|
||||
|
||||
def box_glyph(width_units):
|
||||
"""A trivial filled box outline so the glyph has real contours."""
|
||||
pen = TTGlyphPen(None)
|
||||
x0, y0, x1, y1 = 80, 0, max(120, width_units - 80), 700
|
||||
pen.moveTo((x0, y0))
|
||||
pen.lineTo((x0, y1))
|
||||
pen.lineTo((x1, y1))
|
||||
pen.lineTo((x1, y0))
|
||||
pen.closePath()
|
||||
return pen.glyph()
|
||||
|
||||
|
||||
def empty_glyph():
|
||||
return TTGlyphPen(None).glyph()
|
||||
|
||||
|
||||
def main(out_path):
|
||||
# Characters we must cover. Order defines glyph order (after .notdef).
|
||||
ascii_chars = list("PUBLICA SECRETWORD KEEP")
|
||||
ascii_unique = []
|
||||
for ch in ascii_chars:
|
||||
if ch not in ascii_unique:
|
||||
ascii_unique.append(ch)
|
||||
|
||||
cjk = ["秘", "密"] # 秘 密
|
||||
|
||||
# glyph name -> codepoint mapping
|
||||
def gname(cp):
|
||||
if cp == 0x20:
|
||||
return "space"
|
||||
return "uni%04X" % cp
|
||||
|
||||
codepoints = []
|
||||
for ch in ascii_unique:
|
||||
codepoints.append(ord(ch))
|
||||
for ch in cjk:
|
||||
codepoints.append(ord(ch))
|
||||
|
||||
glyph_order = [".notdef"] + [gname(cp) for cp in codepoints]
|
||||
|
||||
fb = FontBuilder(UPM, isTTF=True)
|
||||
fb.setupGlyphOrder(glyph_order)
|
||||
|
||||
# cmap: map every codepoint to its glyph. FontBuilder builds format-4
|
||||
# (BMP) + format-12 as needed. Both CJK codepoints are BMP so format-4
|
||||
# covers them.
|
||||
cmap = {cp: gname(cp) for cp in codepoints}
|
||||
fb.setupCharacterMap(cmap)
|
||||
|
||||
# Build glyf table with box outlines; advance widths.
|
||||
glyphs = {".notdef": empty_glyph()}
|
||||
metrics = {".notdef": (600, 0)}
|
||||
for cp in codepoints:
|
||||
name = gname(cp)
|
||||
if cp == 0x20:
|
||||
glyphs[name] = empty_glyph()
|
||||
metrics[name] = (300, 0) # space advance, no outline
|
||||
elif cp > 0x2000: # CJK -> full-width
|
||||
glyphs[name] = box_glyph(900)
|
||||
metrics[name] = (1000, 80)
|
||||
else:
|
||||
glyphs[name] = box_glyph(560)
|
||||
metrics[name] = (600, 80)
|
||||
|
||||
fb.setupGlyf(glyphs)
|
||||
fb.setupHorizontalMetrics(metrics)
|
||||
fb.setupHorizontalHeader(ascent=800, descent=-200)
|
||||
|
||||
# Names + the rest of the required tables.
|
||||
fb.setupNameTable({
|
||||
"familyName": "CJKRedactTest",
|
||||
"styleName": "Regular",
|
||||
"psName": "CJKRedactTest-Regular",
|
||||
"fullName": "CJKRedactTest Regular",
|
||||
})
|
||||
fb.setupOS2(sTypoAscender=800, sTypoDescender=-200, usWinAscent=800,
|
||||
usWinDescent=200)
|
||||
fb.setupPost()
|
||||
|
||||
fb.save(out_path)
|
||||
print("Wrote", out_path)
|
||||
print("Glyph order:", glyph_order)
|
||||
print("cmap entries:", {("U+%04X" % k): v for k, v in cmap.items()})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: gen_cjk_font.py <output.ttf>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
main(sys.argv[1])
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a minimal real Type1 font as a classic PFB (0x80 0x01 segment format).
|
||||
|
||||
Pure-python, uses fontTools.t1Lib.T1Font.createData() + writePFB. The font contains
|
||||
.notdef plus the ASCII letters needed for "PUBLICA SECRETWORD KEEP", with a proper
|
||||
Encoding array + standard PostScript glyph names. Because the glyph names are the
|
||||
standard ones and the PDF font uses WinAnsiEncoding, PDFBox extracts text via the
|
||||
base encoding (code -> glyphname -> Unicode); no /ToUnicode CMap is required.
|
||||
|
||||
Usage: python gen_type1.py <out.pfb>
|
||||
"""
|
||||
import sys
|
||||
from fontTools import t1Lib
|
||||
from fontTools.misc import psCharStrings
|
||||
|
||||
|
||||
# Glyph name for each character we must render. PostScript standard glyph names.
|
||||
CHARS = {
|
||||
"P": "P", "U": "U", "B": "B", "L": "L", "I": "I", "C": "C", "A": "A",
|
||||
"S": "S", "E": "E", "R": "R", "T": "T", "W": "W", "O": "O", "D": "D",
|
||||
"K": "K", " ": "space",
|
||||
}
|
||||
|
||||
EM = 1000
|
||||
WIDTH = 600
|
||||
BOX_L, BOX_B, BOX_R, BOX_T = 50, 0, 550, 700
|
||||
|
||||
|
||||
def box_charstring(width):
|
||||
"""A trivial filled box outline. hsbw sets sidebearing+width; then a closed rect."""
|
||||
prog = [
|
||||
0, width, "hsbw", # sb=0, width
|
||||
BOX_L, BOX_B, "rmoveto", # move to bottom-left (relative from 0,0)
|
||||
(BOX_R - BOX_L), "hlineto", # right
|
||||
(BOX_T - BOX_B), "vlineto", # up
|
||||
-(BOX_R - BOX_L), "hlineto", # left
|
||||
"closepath",
|
||||
"endchar",
|
||||
]
|
||||
cs = psCharStrings.T1CharString(program=prog)
|
||||
return cs
|
||||
|
||||
|
||||
def space_charstring(width):
|
||||
prog = [0, width, "hsbw", "endchar"]
|
||||
return psCharStrings.T1CharString(program=prog)
|
||||
|
||||
|
||||
def build_font(font_name="MinType1Test"):
|
||||
# 256-slot encoding, default .notdef, filled at each char's codepoint.
|
||||
encoding = [".notdef"] * 256
|
||||
for ch, gname in CHARS.items():
|
||||
encoding[ord(ch)] = gname
|
||||
|
||||
charstrings = {".notdef": box_charstring(WIDTH)}
|
||||
for ch, gname in CHARS.items():
|
||||
if gname == "space":
|
||||
charstrings[gname] = space_charstring(WIDTH)
|
||||
else:
|
||||
charstrings[gname] = box_charstring(WIDTH)
|
||||
|
||||
# Subrs: the 5 standard Type1 subrs (division/hint machinery). Provide as
|
||||
# T1CharString objects so createData can compile+encrypt them.
|
||||
subrs = [psCharStrings.T1CharString(bytecode=s) for s in t1Lib.std_subrs]
|
||||
|
||||
# Order matters. fontTools emits a stray "put" after the Subrs array; fontbox's
|
||||
# Type1Parser breaks its Private loop on the first non-LITERAL after Subrs, so
|
||||
# every key AFTER Subrs is dropped. Keep Subrs LAST so nothing gets skipped.
|
||||
private = {
|
||||
"RD": t1Lib.RD_value, # ('string','currentfile','exch','readstring','pop')
|
||||
"ND": t1Lib.ND_values[0], # ('def',) codespell:ignore
|
||||
"NP": t1Lib.PD_values[0], # ('put',)
|
||||
"MinFeature": [16, 16],
|
||||
# PDFBox reads BlueValues[2] for CapHeight, so >=3 entries required.
|
||||
# Pairs: baseline (0 0), then cap-height zone (700 712).
|
||||
"BlueValues": [0, 0, BOX_T, BOX_T + 12],
|
||||
"StdHW": [60],
|
||||
"StdVW": [80],
|
||||
"lenIV": 4,
|
||||
"password": 5839,
|
||||
"Subrs": subrs,
|
||||
}
|
||||
|
||||
font_info = {
|
||||
"version": "001.001",
|
||||
"FullName": font_name,
|
||||
"FamilyName": font_name,
|
||||
"Weight": "Regular",
|
||||
"ItalicAngle": 0,
|
||||
"isFixedPitch": False,
|
||||
"UnderlinePosition": -100,
|
||||
"UnderlineThickness": 50,
|
||||
}
|
||||
|
||||
font = {
|
||||
"FontName": font_name,
|
||||
"FontInfo": font_info,
|
||||
"FontType": 1,
|
||||
"PaintType": 0,
|
||||
"FontMatrix": [0.001, 0, 0, 0.001, 0, 0],
|
||||
"FontBBox": [0, 0, WIDTH, BOX_T],
|
||||
"Encoding": encoding,
|
||||
"Private": private,
|
||||
"CharStrings": charstrings,
|
||||
}
|
||||
return font
|
||||
|
||||
|
||||
def main():
|
||||
out = sys.argv[1] if len(sys.argv) > 1 else "min.pfb"
|
||||
# Construct a T1Font without reading a file, then hand-build .font.
|
||||
t1 = t1Lib.T1Font.__new__(t1Lib.T1Font)
|
||||
t1.encoding = "ascii"
|
||||
t1.font = build_font()
|
||||
data = t1.createData() # full ASCII+eexec Type1 program (binary eexec inline)
|
||||
t1.data = data
|
||||
t1.saveAs(out, "PFB") # split into 0x80 0x01 / 0x80 0x02 / 0x80 0x03 segments
|
||||
# Report first bytes to confirm PFB marker.
|
||||
with open(out, "rb") as f:
|
||||
head = f.read(6)
|
||||
print("WROTE", out)
|
||||
print("PFB head bytes:", " ".join(f"{b:02x}" for b in head))
|
||||
assert head[0] == 0x80 and head[1] == 0x01, "not a classic PFB 0x80 0x01 header"
|
||||
print("OK: classic PFB 0x80 0x01 ASCII segment marker present")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,7 +4,9 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import stirling.software.proprietary.access.service.DefaultPrincipalResolver;
|
||||
import stirling.software.proprietary.access.service.DefaultTeamLeadLookup;
|
||||
import stirling.software.proprietary.access.service.PrincipalResolver;
|
||||
import stirling.software.proprietary.access.service.TeamLeadLookup;
|
||||
|
||||
/** Access-layer bean wiring. */
|
||||
@@ -17,4 +19,11 @@ public class AccessConfig {
|
||||
TeamLeadLookup defaultTeamLeadLookup() {
|
||||
return new DefaultTeamLeadLookup();
|
||||
}
|
||||
|
||||
/** USER/TEAM projection unless another bean is defined (e.g. the saas resolver). */
|
||||
@Bean
|
||||
@ConditionalOnMissingBean(PrincipalResolver.class)
|
||||
PrincipalResolver defaultPrincipalResolver() {
|
||||
return new DefaultPrincipalResolver();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,9 @@ import stirling.software.proprietary.access.model.PrincipalType;
|
||||
import stirling.software.proprietary.access.model.ResourceGrant;
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
import stirling.software.proprietary.access.service.ResourceAccessService;
|
||||
import stirling.software.proprietary.security.database.repository.UserRepository;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
|
||||
/** Admin endpoints to grant/revoke access to gated resources (the portal, integration configs). */
|
||||
@RestController
|
||||
@@ -36,6 +38,8 @@ import stirling.software.proprietary.security.model.User;
|
||||
public class ResourceGrantController {
|
||||
|
||||
private final ResourceAccessService accessService;
|
||||
private final UserRepository userRepository;
|
||||
private final TeamRepository teamRepository;
|
||||
|
||||
@GetMapping("/grants")
|
||||
public ResponseEntity<?> list(
|
||||
@@ -45,6 +49,14 @@ public class ResourceGrantController {
|
||||
return ResponseEntity.ok(grants.stream().map(this::toDto).toList());
|
||||
}
|
||||
|
||||
@GetMapping("/grants/by-principal")
|
||||
public ResponseEntity<?> listByPrincipal(
|
||||
@RequestParam PrincipalType principalType, @RequestParam Long principalId) {
|
||||
List<ResourceGrant> grants =
|
||||
accessService.listGrantsForPrincipal(principalType, principalId);
|
||||
return ResponseEntity.ok(grants.stream().map(this::toDto).toList());
|
||||
}
|
||||
|
||||
@PostMapping("/grants")
|
||||
public ResponseEntity<?> create(
|
||||
@RequestBody GrantRequest request, @AuthenticationPrincipal User admin) {
|
||||
@@ -57,17 +69,26 @@ public class ResourceGrantController {
|
||||
"error",
|
||||
"resourceType, principalType and principalId are required"));
|
||||
}
|
||||
// PORTAL is a singleton (empty resourceId); every other type must name a resource.
|
||||
boolean portal = request.resourceType() == ResourceType.PORTAL;
|
||||
if (!portal && (request.resourceId() == null || request.resourceId().isBlank())) {
|
||||
return ResponseEntity.badRequest()
|
||||
.body(Map.of("error", "resourceId is required for " + request.resourceType()));
|
||||
}
|
||||
Long principalId = request.principalId();
|
||||
String principalError = validatePrincipalExists(request.principalType(), principalId);
|
||||
if (principalError != null) {
|
||||
return ResponseEntity.badRequest().body(Map.of("error", principalError));
|
||||
}
|
||||
AccessPermission permission =
|
||||
request.permission() == null ? AccessPermission.USE : request.permission();
|
||||
// PORTAL is a singleton resource; its grants always target the whole type.
|
||||
String resourceId =
|
||||
request.resourceType() == ResourceType.PORTAL ? "" : request.resourceId();
|
||||
String resourceId = portal ? "" : request.resourceId();
|
||||
ResourceGrant grant =
|
||||
accessService.grant(
|
||||
request.resourceType(),
|
||||
resourceId,
|
||||
request.principalType(),
|
||||
request.principalId(),
|
||||
principalId,
|
||||
permission,
|
||||
admin);
|
||||
return ResponseEntity.ok(toDto(grant));
|
||||
@@ -79,6 +100,14 @@ public class ResourceGrantController {
|
||||
return ResponseEntity.ok(Map.of("message", "Grant revoked"));
|
||||
}
|
||||
|
||||
// Rejects grants to nonexistent principals (dead rows otherwise).
|
||||
private String validatePrincipalExists(PrincipalType type, Long id) {
|
||||
return switch (type) {
|
||||
case USER -> userRepository.existsById(id) ? null : "User " + id + " does not exist";
|
||||
case TEAM -> teamRepository.existsById(id) ? null : "Team " + id + " does not exist";
|
||||
};
|
||||
}
|
||||
|
||||
private Map<String, Object> toDto(ResourceGrant g) {
|
||||
Map<String, Object> m = new HashMap<>();
|
||||
m.put("id", g.getId());
|
||||
|
||||
@@ -54,4 +54,15 @@ public abstract class OwnedResource {
|
||||
public Long getOwnerTeamId() {
|
||||
return ownerTeam != null ? ownerTeam.getId() : null;
|
||||
}
|
||||
|
||||
/** Owner as a principal ref; null when server-owned (admin-only ownership). */
|
||||
public PrincipalRef getOwnerRef() {
|
||||
if (getOwnerUserId() != null) {
|
||||
return PrincipalRef.user(getOwnerUserId());
|
||||
}
|
||||
if (getOwnerTeamId() != null) {
|
||||
return PrincipalRef.team(getOwnerTeamId());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package stirling.software.proprietary.access.model;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/** A (type, id) principal pair; the atom grants and ownership are expressed in. */
|
||||
public record PrincipalRef(PrincipalType type, Long id) {
|
||||
|
||||
public static PrincipalRef user(Long id) {
|
||||
return new PrincipalRef(PrincipalType.USER, id);
|
||||
}
|
||||
|
||||
public static PrincipalRef team(Long id) {
|
||||
return new PrincipalRef(PrincipalType.TEAM, id);
|
||||
}
|
||||
|
||||
/** Canonical engine wire form, e.g. "user:12". */
|
||||
public String token() {
|
||||
return type.name().toLowerCase(Locale.ROOT) + ":" + id;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
package stirling.software.proprietary.access.model;
|
||||
|
||||
/** Who a {@link ResourceGrant} is granted to. Org-wide access is expressed via default policy. */
|
||||
/** Who a {@link ResourceGrant} is granted to. */
|
||||
public enum PrincipalType {
|
||||
USER,
|
||||
TEAM
|
||||
|
||||