haeaeae/Stirling-PDF
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden true-removal redaction and gate independent verification

Browse Source
This commit is contained in:
Anthony Stirling
2026-07-02 22:59:58 +01:00
parent 18cc6dcdc8
commit b88cbe6bfc
16 changed files with 994 additions and 627 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java
+31 -26
View File
@@ -5,6 +5,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -52,9 +53,10 @@ class ManualRedactionService {
Set<String> capturedStrings = new LinkedHashSet<>();
Map<Integer, List<PDRectangle>> rectsByPage = new HashMap<>();
Set<Integer> forceRasterPages = new LinkedHashSet<>();
if (redactionAreas == null || redactionAreas.isEmpty()) {
return new AreaRedactionResult(rectsByPage);
return new AreaRedactionResult(rectsByPage, forceRasterPages);
}
Map<Integer, List<RedactionArea>> redactionsByPage = new HashMap<>();
@@ -86,8 +88,9 @@ class ManualRedactionService {
PDPage page = allPages.get(pageIndex);
float pageHeight = page.getBBox().getHeight();
List<PDRectangle> rects = new ArrayList<>();
Color overlayColor = Color.BLACK;
// Group rects by their decoded colour so each area keeps its own overlay tint (F9a).
Map<Color, List<PDRectangle>> byColor = new LinkedHashMap<>();
List<PDRectangle> allRects = new ArrayList<>();
for (RedactionArea area : areasForPage) {
float x = area.getX().floatValue();
float y = area.getY().floatValue();
@@ -95,24 +98,28 @@ class ManualRedactionService {
float height = area.getHeight().floatValue();
// Request coords are top-left origin; convert to PDF user space (bottom-left).
float pdfY = pageHeight - y - height;
rects.add(new PDRectangle(x, pdfY, width, height));
overlayColor = decodeOrDefault(area.getColor());
PDRectangle rect = new PDRectangle(x, pdfY, width, height);
allRects.add(rect);
byColor.computeIfAbsent(decodeOrDefault(area.getColor()), k -> new ArrayList<>())
.add(rect);
}
// Physically drop intersecting glyphs and draw the overlay rectangle over the area.
Map<Integer, List<PDRectangle>> singlePage = new HashMap<>();
singlePage.put(pageIndex, rects);
RedactionPipeline.RedactionResult result =
RedactionPipeline.redactAreas(document, singlePage, overlayColor);
capturedStrings.addAll(result.getCapturedStrings());
rectsByPage.put(pageIndex, rects);
for (Map.Entry<Color, List<PDRectangle>> colorEntry : byColor.entrySet()) {
Map<Integer, List<PDRectangle>> singlePage = new HashMap<>();
singlePage.put(pageIndex, colorEntry.getValue());
RedactionPipeline.RedactionResult result =
RedactionPipeline.redactAreas(document, singlePage, colorEntry.getKey());
capturedStrings.addAll(result.getCapturedStrings());
forceRasterPages.addAll(result.getForceRasterPages());
}
rectsByPage.put(pageIndex, allRects);
}
log.debug(
"Manual area redaction captured {} text run(s) across {} page(s)",
capturedStrings.size(),
rectsByPage.size());
return new AreaRedactionResult(rectsByPage);
return new AreaRedactionResult(rectsByPage, forceRasterPages);
}
List<Integer> redactPages(
@@ -327,8 +334,7 @@ class ManualRedactionService {
convertedPdf, Collections.emptySet(), Collections.emptyList());
}
} else {
// True-removal pass: physically strip matched glyph bytes from every content stream,
// then scrub catalog carriers, verify, and rasterise affected pages on any leak.
// Strip matched glyphs, then scrub/verify/rasterise via finalize.
RedactionPipeline.redactLiteralTerms(document, literalTargets, patterns);
outputBytes = RedactionPipeline.finalize(document, literalTargets, patterns);
}
@@ -336,15 +342,9 @@ class ManualRedactionService {
return writeBytes(outputBytes, document.getNumberOfPages());
}
/**
* Finalize a manual area/page redaction. The overlay rectangles are already drawn and the
* intersecting glyphs already dropped. Verification is region-based: each redaction rectangle
* is re-scanned and must be empty (whole-page wipes carry no rects and are guaranteed clean).
*/
/** Finalize a manual redaction; forced + verified-leaking pages are rasterised. */
TempFile finalizeManual(
PDDocument document,
Map<Integer, List<PDRectangle>> rectsByPage,
Boolean convertToImage)
PDDocument document, AreaRedactionResult areaResult, Boolean convertToImage)
throws IOException {
byte[] outputBytes;
@@ -355,7 +355,9 @@ class ManualRedactionService {
convertedPdf, Collections.emptySet(), Collections.emptyList());
}
} else {
outputBytes = RedactionPipeline.finalizeAreas(document, rectsByPage);
outputBytes =
RedactionPipeline.finalizeAreas(
document, areaResult.rectsByPage, areaResult.forceRasterPages);
}
return writeBytes(outputBytes, document.getNumberOfPages());
@@ -378,12 +380,15 @@ class ManualRedactionService {
// Utilities
// -----------------------------------------------------------------------
/** Redaction rectangles (per 0-based page index) applied by a manual area pass. */
/** Redaction rectangles + pages to force-rasterise, per 0-based page index. */
static final class AreaRedactionResult {
final Map<Integer, List<PDRectangle>> rectsByPage;
final Set<Integer> forceRasterPages;
AreaRedactionResult(Map<Integer, List<PDRectangle>> rectsByPage) {
AreaRedactionResult(
Map<Integer, List<PDRectangle>> rectsByPage, Set<Integer> forceRasterPages) {
this.rectsByPage = rectsByPage;
this.forceRasterPages = forceRasterPages;
}
}
app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java
+20 -16
View File
@@ -108,15 +108,14 @@ public class RedactController {
try (PDDocument document = pdfDocumentFactory.load(file)) {
PDPageTree allPages = document.getDocumentCatalog().getPages();
// Whole-page wipes drop content outright (guaranteed clean); area redactions drop
// intersecting glyphs, draw an overlay, and are verified per-rectangle at finalize.
// Whole-page wipes drop content; areas drop glyphs + overlay, verified later.
manualRedactionService.redactPages(request, document, allPages);
ManualRedactionService.AreaRedactionResult areaResult =
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
TempFile out =
manualRedactionService.finalizeManual(
document, areaResult.rectsByPage, request.getConvertPDFToImage());
document, areaResult, request.getConvertPDFToImage());
return WebResponseUtils.pdfFileToWebResponse(out, filename);
}
}
@@ -180,16 +179,6 @@ public class RedactController {
request.getFileInput().getOriginalFilename())))
+ "_redacted.pdf";
if (allFoundTextsByPage.isEmpty()) {
log.info("No text found matching redaction patterns");
// Still finalize so metadata is scrubbed and the document is rewritten
// consistently.
TempFile finalized =
manualRedactionService.finalizeManual(
document, Collections.emptyMap(), request.getConvertPDFToImage());
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
}
Set<String> literalTargets =
Arrays.stream(listOfText)
.map(String::trim)
@@ -197,12 +186,27 @@ public class RedactController {
.collect(Collectors.toCollection(LinkedHashSet::new));
List<Pattern> compiledPatterns =
RedactionPipeline.buildPatterns(listOfText, useRegex, wholeWordSearchBool);
// Bare literal targets match substrings, so they are only safe when the search is a
// plain literal. In regex or whole-word mode the boundary/regex semantics live entirely
// in the compiled patterns, so pass no literal targets to avoid stripping substrings.
// Bare literals match substrings; regex/whole-word rely on compiled patterns.
Set<String> verificationTargets =
(useRegex || wholeWordSearchBool) ? Collections.emptySet() : literalTargets;
if (allFoundTextsByPage.isEmpty()) {
// No page hit, but the target may live in a bookmark/annotation/form/JS carrier, so
// still run the scrub + verify path rather than just wiping metadata (F5).
log.info("No page text matched; scrubbing catalog carriers and verifying");
TempFile finalized =
manualRedactionService.finalizeRedaction(
document,
Collections.emptyMap(),
request.getRedactColor(),
request.getCustomPadding(),
request.getConvertPDFToImage(),
true,
verificationTargets,
compiledPatterns);
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
}
boolean fallbackToBoxOnlyMode;
try {
fallbackToBoxOnlyMode =
app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java
+1 -3
View File
@@ -143,9 +143,7 @@ class RedactExecuteService {
applyAllImagesRedaction(document, request.getRedactImagePages(), style);
}
// Explicit overlay-only requests must not rewrite content or verify. When overlay-only
// was forced by a font fallback (not user choice) we still pass the targets so the
// pipeline's true-removal + verification + page-scoped raster fallback run.
// Overlay-only skips removal/verify; font-fallback overlays still pass targets.
Set<String> literalTargets = new LinkedHashSet<>();
for (String value : textValues) {
String trimmed = value == null ? "" : value.trim();
app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java
+4 -7
View File
@@ -706,16 +706,13 @@ class TextRedactionService {
private void writeRedactedContentToXObject(
PDDocument document, PDFormXObject formXObject, List<Object> redactedTokens)
throws IOException {
PDStream newStream = new PDStream(document);
try (var out = newStream.createOutputStream()) {
// A form XObject's content IS its own stream body; overwrite it in place. Attaching a
// /Contents key would be a no-op (its content is not read from /Contents).
PDStream formStream = new PDStream(formXObject.getCOSObject());
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
ContentStreamWriter writer = new ContentStreamWriter(out);
writer.writeTokens(redactedTokens);
}
formXObject.getCOSObject().removeItem(COSName.CONTENTS);
formXObject.getCOSObject().setItem(COSName.CONTENTS, newStream.getCOSObject());
}
private List<TextSegment> extractTextSegments(PDPage page, List<Object> tokens) {
app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java
+96 -88
View File
@@ -29,35 +29,13 @@ import org.apache.pdfbox.pdmodel.interactive.form.PDField;
import lombok.extern.slf4j.Slf4j;
/**
* Walks a {@link PDDocument} and physically removes or rewrites every carrier that a PDF can use to
* leak text which the user asked to redact.
*
* <p>Covers:
*
* <ul>
* <li>{@link PDDocumentInformation} (Info dict) + XMP metadata stream
* <li>{@link PDDocumentOutline} bookmark titles
* <li>{@link PDAcroForm} field values (V, DV) and rich text (RV)
* <li>Every {@link PDAnnotation} Contents and RC
* <li>Structure tree ActualText, Alt, T, E, Lang entries
* <li>Names tree: JavaScript entries and embedded files (dropped entirely when matching)
* </ul>
*
* <p>When applied after the content-stream rewrite it closes the secondary leak paths flagged in
* the redaction security audit.
*/
/** Removes/rewrites every catalog carrier that could leak redacted text. */
@Slf4j
public final class CatalogScrubber {
private CatalogScrubber() {}
/**
* Remove occurrences of every {@code target} string (and any regex/whole-word pattern form
* produced by {@link RedactionPipeline#buildPatterns}) from all catalog-level carriers of the
* document. When {@code wipeAllMetadata} is {@code true} the document Info dict entries and XMP
* metadata stream are wiped wholesale; this is the safe default after a redaction operation.
*/
/** Scrub all catalog-level carriers of the given literal/regex targets. */
public static void scrub(
PDDocument document, Set<String> literalTargets, List<Pattern> patterns) {
if (document == null) {
@@ -77,9 +55,7 @@ public final class CatalogScrubber {
scrubCatalogActions(catalog, literalTargets, patterns);
}
// ---------------------------------------------------------------------
// Catalog actions: OpenAction, AA, and any JavaScript / URI payloads on the catalog
// ---------------------------------------------------------------------
// Catalog actions: OpenAction, AA, and any JavaScript / URI payloads
private static void scrubCatalogActions(
PDDocumentCatalog catalog, Set<String> targets, List<Pattern> patterns) {
@@ -87,16 +63,12 @@ public final class CatalogScrubber {
if (root == null) {
return;
}
// OpenAction may be either an action dict (with /URI or /JS) or an explicit destination
// (array). We scrub strings in both cases; if the OpenAction matches a target we clear it.
// OpenAction may be either an action dict (with /URI or /JS) or an explicit
scrubActionIfMatching(root, COSName.getPDFName("OpenAction"), targets, patterns);
scrubActionIfMatching(root, COSName.getPDFName("AA"), targets, patterns);
}
/**
* If the action dictionary at {@code key} contains any target literal in a URI or JS payload,
* wipe the key entirely. Otherwise recursively scrub string fields inside it.
*/
/** Drop the action at key if any target appears in its URI/JS payload. */
private static void scrubActionIfMatching(
COSDictionary parent, COSName key, Set<String> targets, List<Pattern> patterns) {
if (parent == null || key == null) {
@@ -125,9 +97,7 @@ public final class CatalogScrubber {
return matches(cs.getString(), targets, patterns);
}
if (resolved instanceof COSStream stream) {
// Streams in XFA / OpenAction contexts are text (XML, JavaScript). Read the bytes as
// UTF-8 and test for target literals. We cap read length to avoid pathological memory
// use; 2 MiB is plenty for XFA packets and far beyond any realistic JS action.
// Streams in XFA / OpenAction contexts are text (XML, JavaScript).
try (java.io.InputStream is = stream.createInputStream()) {
byte[] buf = new byte[2 * 1024 * 1024];
int total = 0;
@@ -139,11 +109,14 @@ public final class CatalogScrubber {
}
}
String text = new String(buf, 0, total, java.nio.charset.StandardCharsets.UTF_8);
return matches(text, targets, patterns);
if (matches(text, targets, patterns)) {
return true;
}
// Fail closed: content past the 2 MiB cap is unproven, so treat as a match (F7c).
return total >= buf.length && is.read() >= 0;
} catch (Exception e) {
log.debug("Failed to scan stream for targets: {}", e.getMessage());
// Fail closed: if we cannot read it we cannot prove it is clean, so treat as a
// match so the caller drops the stream. This is conservative by design.
// Fail closed: if we cannot read it we cannot prove it is clean, so treat
return true;
}
}
@@ -166,10 +139,7 @@ public final class CatalogScrubber {
return false;
}
/**
* Clean potentially sensitive metadata carriers. Called after {@link #scrub} so that surviving
* references to author/subject/keywords/XMP descriptors do not leak redacted values.
*/
/** Wipe Info-dict entries and the XMP metadata stream. */
public static void wipeMetadata(PDDocument document) {
if (document == null) {
return;
@@ -194,9 +164,7 @@ public final class CatalogScrubber {
}
}
// ---------------------------------------------------------------------
// Outline
// ---------------------------------------------------------------------
private static void scrubOutline(
PDDocumentOutline outline, Set<String> targets, List<Pattern> patterns) {
@@ -220,9 +188,7 @@ public final class CatalogScrubber {
item.setTitle(stripped);
}
}
// Bookmark actions: /A is an action dict which may carry a /URI or /JS payload.
// If any target literal appears anywhere inside the action subtree, drop the
// action entirely so the URI / script cannot leak the target.
// Bookmark actions: /A is an action dict which may carry a /URI or /JS
COSDictionary itemDict = item.getCOSObject();
if (itemDict != null) {
scrubActionIfMatching(itemDict, COSName.A, targets, patterns);
@@ -235,18 +201,14 @@ public final class CatalogScrubber {
}
}
// ---------------------------------------------------------------------
// AcroForm
// ---------------------------------------------------------------------
private static void scrubAcroForm(
PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
if (form == null) {
return;
}
// XFA forms: scrubbed separately because the XFA XML packet carries the "real" field
// values for XFA-enabled PDFs. Handle XFA before walking the field tree so we fail closed
// if XFA scrubbing throws.
// XFA forms: scrubbed separately because the XFA XML packet carries
scrubXfa(form, targets, patterns);
try {
@@ -257,10 +219,7 @@ public final class CatalogScrubber {
log.debug("Failed to walk AcroForm field tree: {}", e.getMessage());
}
// Force viewers to regenerate appearance streams from the (scrubbed) /V values rather
// than reusing any cached /AP /N that still contains the target text. Belt-and-braces:
// scrubField has also cleared per-widget /AP dicts, but /NeedAppearances ensures any
// future change still triggers regeneration.
// Force viewers to regenerate appearance streams from the (scrubbed) /V
try {
form.setNeedAppearances(true);
} catch (Exception e) {
@@ -276,10 +235,7 @@ public final class CatalogScrubber {
}
boolean hit = containsTarget(xfaBase, targets, patterns, new HashSet<>());
if (hit) {
// Simplest safe move: strip the XFA entry entirely. Viewers fall back to the
// AcroForm widgets which we have already scrubbed. Leaving a "partially scrubbed"
// XFA packet risks regex failures on partial XML and re-encoded entities leaking
// the target.
// Simplest safe move: strip the XFA entry entirely.
log.warn(
"Removing XFA form packet from AcroForm - XFA XML contained a redaction "
+ "target and has been dropped so viewers render AcroForm widgets "
@@ -297,9 +253,21 @@ public final class CatalogScrubber {
}
try {
COSDictionary dict = field.getCOSObject();
// Only touch fields whose own values actually contain a target (F7a): clearing /AP
// document-wide blanks unrelated fields in viewers that ignore /NeedAppearances.
boolean matched =
dictValueMatches(dict, COSName.V, targets, patterns)
|| dictValueMatches(dict, COSName.DV, targets, patterns)
|| dictValueMatches(dict, COSName.getPDFName("RV"), targets, patterns)
|| dictValueMatches(dict, COSName.getPDFName("TU"), targets, patterns)
|| fieldValueMatches(field, targets, patterns);
if (!matched) {
return;
}
scrubDictStrings(dict, COSName.V, targets, patterns);
scrubDictStrings(dict, COSName.DV, targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("RV"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("TU"), targets, patterns);
// Keep field appearance streams in sync with value where possible.
try {
if (field.getValueAsString() != null) {
@@ -311,16 +279,54 @@ public final class CatalogScrubber {
} catch (Exception e) {
log.debug("Failed to rewrite field value via setValue: {}", e.getMessage());
}
// Drop per-widget appearance streams (/AP dict) for every widget kid of this field.
// The cached appearance stream contains the pre-redaction value baked in as glyph
// data; simply rewriting /V leaves it visually unchanged in many viewers. Removing /AP
// plus /NeedAppearances at the form level forces regeneration.
clearWidgetAppearances(dict);
// Button on-states (/Btn) cannot be rebuilt from /V by /NeedAppearances, so leave their
// /AP intact and rely on value scrubbing; drop /AP only for matched text fields.
if (!isButtonField(dict)) {
clearWidgetAppearances(dict);
}
} catch (Exception e) {
log.debug("Failed to scrub field: {}", e.getMessage());
}
}
private static boolean isButtonField(COSDictionary dict) {
COSName ft = (COSName) dict.getDictionaryObject(COSName.FT);
if (ft == null) {
COSBase parent = dict.getDictionaryObject(COSName.PARENT);
if (parent instanceof COSDictionary p) {
ft = (COSName) p.getDictionaryObject(COSName.FT);
}
}
return COSName.getPDFName("Btn").equals(ft);
}
private static boolean dictValueMatches(
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
COSBase value = dict.getDictionaryObject(key);
if (value instanceof COSString cs) {
return matches(cs.getString(), targets, patterns);
}
if (value instanceof COSArray array) {
for (int i = 0; i < array.size(); i++) {
if (array.getObject(i) instanceof COSString element
&& matches(element.getString(), targets, patterns)) {
return true;
}
}
}
return false;
}
private static boolean fieldValueMatches(
PDField field, Set<String> targets, List<Pattern> patterns) {
try {
String value = field.getValueAsString();
return value != null && matches(value, targets, patterns);
} catch (Exception e) {
return false;
}
}
private static void clearWidgetAppearances(COSDictionary fieldDict) {
if (fieldDict == null) {
return;
@@ -338,9 +344,7 @@ public final class CatalogScrubber {
}
}
// ---------------------------------------------------------------------
// Annotations
// ---------------------------------------------------------------------
private static void scrubAnnotations(
PDDocument document, Set<String> targets, List<Pattern> patterns) {
@@ -388,9 +392,7 @@ public final class CatalogScrubber {
}
}
// ---------------------------------------------------------------------
// Structure tree
// ---------------------------------------------------------------------
private static void scrubStructTree(
PDStructureTreeRoot root, Set<String> targets, List<Pattern> patterns) {
@@ -414,7 +416,7 @@ public final class CatalogScrubber {
return;
}
if (resolved instanceof COSDictionary dict) {
// Do not walk into content streams - those are handled by content-stream rewrite.
// Do not walk into content streams - those are handled by content-stream
if (resolved instanceof COSStream) {
return;
}
@@ -438,9 +440,7 @@ public final class CatalogScrubber {
}
}
// ---------------------------------------------------------------------
// Names tree (JavaScript + embedded files)
// ---------------------------------------------------------------------
private static void scrubNames(
PDDocumentNameDictionary names, Set<String> targets, List<Pattern> patterns) {
@@ -481,7 +481,17 @@ public final class CatalogScrubber {
for (int i = namesArray.size() - 2; i >= 0; i -= 2) {
COSBase keyBase = namesArray.getObject(i);
String key = keyBase instanceof COSString s ? s.getString() : null;
if (key != null && matches(key, targets, patterns)) {
// Drop the pair when the KEY or the VALUE (JS /JS stream, embedded-file bytes)
// contains a target - not just the key (F7b).
boolean keyHit = key != null && matches(key, targets, patterns);
boolean valueHit =
i + 1 < namesArray.size()
&& containsTarget(
namesArray.getObject(i + 1),
targets,
patterns,
new HashSet<>());
if (keyHit || valueHit) {
namesArray.remove(i + 1);
namesArray.remove(i);
}
@@ -498,9 +508,7 @@ public final class CatalogScrubber {
}
}
// ---------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------
private static void scrubDictStrings(
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
@@ -536,24 +544,21 @@ public final class CatalogScrubber {
if (target == null || target.isEmpty()) {
continue;
}
// Case-insensitive literal removal. Verification is case-insensitive, so scrubbing
// MUST be too or mixed-case ("SMITH" in a catalog string vs "Smith" in the target
// list) will fail verification and trip the rasterisation fallback - or worse, on
// carriers that are not verified, leak the string untouched.
// Case-insensitive literal removal. Verification is case-insensitive
result = caseInsensitiveReplaceAll(result, target);
}
}
if (patterns != null) {
for (Pattern pattern : patterns) {
try {
// Force case-insensitive matching for catalog carriers regardless of the flags
// the pattern was compiled with. User-supplied redaction targets should not
// silently miss because the author typed the name in different case.
// Force case-insensitive matching for catalog carriers regardless
Pattern ci = withCaseInsensitive(pattern);
result = ci.matcher(result).replaceAll("");
} catch (Exception e) {
log.debug(
"Pattern replace failed for {}: {}", pattern.pattern(), e.getMessage());
} catch (RuntimeException | StackOverflowError e) {
// Fail closed (X2): a throwing regex means we cannot prove the carrier clean,
// so drop the whole string rather than leaving it intact.
log.warn("Pattern replace failed for {}; dropping carrier text", pattern);
return "";
}
}
}
@@ -580,8 +585,11 @@ public final class CatalogScrubber {
if (withCaseInsensitive(pattern).matcher(source).find()) {
return true;
}
} catch (Exception e) {
log.debug("Pattern match failed for {}: {}", pattern.pattern(), e.getMessage());
} catch (RuntimeException | StackOverflowError e) {
// Fail closed (X2): a throwing regex counts as a match so the carrier is
// scrubbed.
log.warn("Pattern match failed for {}; treating carrier as a match", pattern);
return true;
}
}
}
app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java
+489 -386
View File
File diff suppressed because it is too large Load Diff
app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionVerificationFailedException.java
+1 -5
View File
@@ -1,10 +1,6 @@
package stirling.software.SPDF.pdf.redaction;
/**
* Thrown when a post-redaction verification pass still finds any of the target strings in the
* re-parsed PDF text. Indicates that the redaction pipeline did not fully remove the targeted
* content and the output must not be treated as safe to release.
*/
/** Thrown when post-redaction verification still finds a target in the saved PDF. */
public class RedactionVerificationFailedException extends RuntimeException {
public RedactionVerificationFailedException(String message) {
app/core/src/main/java/stirling/software/SPDF/utils/text/TextFinderUtils.java
+13 -6
View File
@@ -86,14 +86,21 @@ public class TextFinderUtils {
return patterns;
}
private static String applyWordBoundaries(String originalTerm, String patternString) {
if (originalTerm.length() == 1 && Character.isDigit(originalTerm.charAt(0))) {
/** Shared whole-word wrapper so the finder and RedactionPipeline never diverge (see F2). */
public static String applyWordBoundaries(String originalTerm, String patternString) {
// Lookarounds for single/non-word-edge terms (\b no-ops on chars like '&'); \b otherwise.
boolean nonWordEdge =
originalTerm.length() == 1
|| !isWordChar(originalTerm.charAt(0))
|| !isWordChar(originalTerm.charAt(originalTerm.length() - 1));
if (nonWordEdge) {
return "(?<![\\w])" + patternString + "(?![\\w])";
} else if (originalTerm.length() == 1) {
return "(?<![\\w])" + patternString + "(?![\\w])";
} else {
return "\\b" + patternString + "\\b";
}
return "\\b" + patternString + "\\b";
}
private static boolean isWordChar(char c) {
return Character.isLetterOrDigit(c) || c == '_';
}
public static boolean hasProblematicFonts(PDPage page) {
app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerTest.java
+5 -4
View File
@@ -192,22 +192,23 @@ class RedactControllerTest {
when(mockCOSStream.createOutputStream()).thenReturn(mockOutputStream);
when(mockCOSStream.createOutputStream(any())).thenReturn(mockOutputStream);
// The pipeline saves then reloads the bytes to verify, so the mock must emit a real,
// reloadable PDF (not a placeholder) or verification throws on reopen.
byte[] realPdfBytes = createSimplePdfContent();
lenient()
.doAnswer(
inv -> {
File f = inv.getArgument(0);
java.nio.file.Files.write(f.toPath(), "mock pdf".getBytes());
java.nio.file.Files.write(f.toPath(), realPdfBytes);
return null;
})
.when(mockDocument)
.save(any(File.class));
// RedactionPipeline serialises via OutputStream before handing the bytes to TempFile, so
// the mock must emit a non-empty payload on that code path too.
lenient()
.doAnswer(
inv -> {
java.io.OutputStream os = inv.getArgument(0);
os.write("mock pdf".getBytes());
os.write(realPdfBytes);
return null;
})
.when(mockDocument)
app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactionPdfVarietyTest.java
+7 -25
View File
@@ -48,13 +48,7 @@ import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
/**
* Security matrix for redaction across PDF shapes: standard and embedded (subset and full) fonts,
* rotated pages, TJ splits, cross-operator splits, Form XObjects, CropBox offsets, multi-page and
* case variants. Every case asserts the target is unrecoverable from the output text layer, and
* (where removal should be surgical) that neighbouring text survives. Uses the LiberationSans TTF
* bundled inside the PDFBox jar so the embedded-font cases run on any CI platform.
*/
/** Security matrix for redaction across PDF shapes: standard and embedded */
@DisplayName("Redaction PDF-variety security matrix")
class RedactionPdfVarietyTest {
@@ -108,9 +102,7 @@ class RedactionPdfVarietyTest {
}
}
// -----------------------------------------------------------------------
// Matrix cases (all through the real /auto-redact controller path)
// -----------------------------------------------------------------------
@Test
@DisplayName("standard Helvetica: target gone, neighbours survive")
@@ -161,9 +153,7 @@ class RedactionPdfVarietyTest {
@Test
@DisplayName("real Type3 font (glyphs as content streams, no ToUnicode): target gone")
void type3GlyphProcs() throws IOException {
// crop_test.pdf uses DejaVuSans embedded as a subset Type3 font with no ToUnicode map.
// Redacting it must not throw (Type3 font.encode() throws UnsupportedOperationException)
// and must remove the target from the extractable text layer.
// crop_test.pdf uses DejaVuSans embedded as a subset Type3 font with no
byte[] input;
try (InputStream in = getClass().getResourceAsStream("/redaction/type3_dejavu.pdf")) {
input = in.readAllBytes();
@@ -220,7 +210,7 @@ class RedactionPdfVarietyTest {
void crossOperatorSplit() throws IOException {
byte[] out = autoRedact(crossOperatorPdf(), "SECRET");
assertGone(out, "SECRET");
// Page 2 was never touched; whatever path handled page 1, page 2 text must survive.
// Page 2 was never touched; whatever path handled page 1, page 2 text must
assertThat(pdfText(out)).contains("PUBLIC PAGE TWO");
}
@@ -272,15 +262,12 @@ class RedactionPdfVarietyTest {
assertThat(text).doesNotContainPattern("(?i)\\bcat\\b");
}
// -----------------------------------------------------------------------
// Page-scoped rasterisation (pipeline-level, deterministic)
// -----------------------------------------------------------------------
@Test
@DisplayName("verification leak rasterises only the leaking page, others keep text")
void leakRasterisesOnlyLeakingPage() throws IOException {
// Cross-operator split defeats the per-operand literal rewriter, so verification must
// catch the leak and rasterise page 1 only, leaving page 2 searchable.
// Cross-operator split defeats the per-operand literal rewriter
byte[] input = crossOperatorPdf();
byte[] out;
try (PDDocument doc = Loader.loadPDF(input)) {
@@ -305,9 +292,7 @@ class RedactionPdfVarietyTest {
}
}
// -----------------------------------------------------------------------
// Drivers and assertions
// -----------------------------------------------------------------------
private byte[] autoRedact(byte[] pdfBytes, String target) throws IOException {
factoryReturns(pdfBytes);
@@ -356,9 +341,7 @@ class RedactionPdfVarietyTest {
}
}
// -----------------------------------------------------------------------
// PDF builders
// -----------------------------------------------------------------------
private static PDFont helvetica() {
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
@@ -464,7 +447,7 @@ class RedactionPdfVarietyTest {
}
}
/** One page whose text is emitted as a TJ array: ["public ", "SEC", -20, "RET", " end"]. */
/** One page whose text is emitted as a TJ array: ["public ", "SEC", -20 */
private byte[] tjSplitPdf() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
@@ -481,7 +464,7 @@ class RedactionPdfVarietyTest {
}
}
/** Page 1 shows "SEC" and "RET" as separate adjacent Tj operators; page 2 is clean. */
/** Page 1 shows "SEC" and "RET" as separate adjacent Tj operators; page 2 */
private byte[] crossOperatorPdf() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
@@ -546,8 +529,7 @@ class RedactionPdfVarietyTest {
cs.beginText();
cs.setFont(font, FONT_SIZE);
if (rotation != 0) {
// Real generators compensate the text matrix so text reads upright on rotated
// pages; anchor at page centre so every rotation keeps the line on-page.
// Real generators compensate the text matrix so text reads upright
cs.setTextMatrix(
Matrix.getRotateInstance(
Math.toRadians(rotation),
app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceExtraTest.java
+6 -2
View File
@@ -287,8 +287,12 @@ class TextRedactionServiceExtraTest {
service.createTokensWithoutTargetText(doc, page, Set.of("SECRET"), false, false);
// The deep traversal must have rewritten the inner form's content stream.
assertThat(inner.getCOSObject().containsKey(COSName.CONTENTS)).isTrue();
// The deep traversal must have rewritten the inner form's own stream in place (F3).
String innerContent;
try (var in = inner.getContents()) {
innerContent = new String(in.readAllBytes(), StandardCharsets.ISO_8859_1);
}
assertThat(innerContent).doesNotContain("SECRET");
}
}
app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceMoreTest.java
+12 -8
View File
@@ -226,19 +226,23 @@ class TextRedactionServiceMoreTest {
}
page.setContents(pageStream);
// Processing the page walks into the XObject graph; when a match is found inside
// the
// form, writeRedactedContentToXObject runs and sets a /Contents item on the form's
// COS dictionary. Asserting that item appears proves the XObject redaction path
// executed end-to-end without throwing.
// Processing the page walks into the XObject graph and rewrites the form's own
// stream body in place (F3), so the target is physically gone from the form.
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
assertThat(tokens).isNotNull();
assertThat(form.getCOSObject().containsKey(COSName.CONTENTS))
.as("form XObject redaction path should have written a new content item")
.isTrue();
String rewrittenForm;
try (var in = form.getContents()) {
rewrittenForm =
new String(
in.readAllBytes(),
java.nio.charset.StandardCharsets.ISO_8859_1);
}
assertThat(rewrittenForm)
.as("form XObject content should have the target removed in place")
.doesNotContain("SECRET");
}
}
}
app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineIntegrationTest.java
+5 -20
View File
@@ -23,15 +23,7 @@ import org.apache.pdfbox.text.PDFTextStripper;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
/**
* Integration tests that exercise the real redaction pipeline against PDFs with characteristics
* that have tripped up the implementation on real user files: page rotation (/Rotate 90), entire
* phrases packed into one Tj operator, and standard Type1 fonts with WinAnsiEncoding.
*
* <p>The critical assertion for every test is that a fresh {@link PDFTextStripper} with default
* configuration run over the saved bytes does not contain the target search term
* (case-insensitive).
*/
/** Integration tests that exercise the real redaction pipeline against PDFs */
class RedactionPipelineIntegrationTest {
private static byte[] loadFixture() throws Exception {
@@ -52,10 +44,7 @@ class RedactionPipelineIntegrationTest {
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
Set<String> literalTargets = new LinkedHashSet<>();
literalTargets.add("Test");
// No content-stream rewriting performed. The primary verification pass inside
// finalize must see the surviving target, switch to the rasterisation fallback,
// and return bytes that no longer extract as text. This proves the safety net is
// wired - without it the output would still contain the target.
// No content-stream rewriting performed.
outputBytes = RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
}
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
@@ -73,9 +62,7 @@ class RedactionPipelineIntegrationTest {
@DisplayName(
"finalize with zero rewrite + synthetic upright PDF triggers RedactionVerificationFailedException only when fallback disabled")
void verificationExceptionWiringSanityCheck() throws Exception {
// This test proves the verification hook itself still throws - when finalize cannot
// rasterise (we pass an already-closed document by letting finalize re-save an empty
// fresh document that still has the target text baked in).
// This test proves the verification hook itself still throws -
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
@@ -88,8 +75,7 @@ class RedactionPipelineIntegrationTest {
}
Set<String> literalTargets = new LinkedHashSet<>();
literalTargets.add("Smith");
// Even with the rasterisation fallback active, the output must not contain the
// target. Either the primary pass or the fallback must succeed; both is a bug.
// Even with the rasterisation fallback active, the output must not contain
byte[] outBytes =
RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
try (PDDocument reopened = Loader.loadPDF(outBytes)) {
@@ -153,8 +139,7 @@ class RedactionPipelineIntegrationTest {
}
ByteArrayOutputStream tmp = new ByteArrayOutputStream();
doc.save(tmp);
// reload from bytes so we are processing an already-saved PDF similar to the
// real upload flow.
// reload from bytes so we are processing an already-saved PDF similar
try (PDDocument reloaded = Loader.loadPDF(tmp.toByteArray())) {
RedactionPipeline.redactLiteralTerms(reloaded, literalTargets, patterns);
outputBytes = RedactionPipeline.finalize(reloaded, literalTargets, patterns);
app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java
+302 -25
View File
@@ -2,11 +2,13 @@ package stirling.software.SPDF.pdf.redaction;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.awt.Color;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -17,15 +19,24 @@ import java.util.Set;
import java.util.regex.Pattern;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSStream;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.common.PDStream;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.apache.pdfbox.pdmodel.font.encoding.WinAnsiEncoding;
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationHighlight;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocumentOutline;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
@@ -54,7 +65,7 @@ class RedactionPipelineTest {
}
Map<Integer, List<PDRectangle>> rects = new HashMap<>();
// The text above sits around y=700 with height ~12. Cover it fully.
// The text above sits around y=700 with height ~12.
rects.put(0, List.of(new PDRectangle(90, 695, 260, 25)));
RedactionPipeline.redactAreas(doc, rects, Color.BLACK);
@@ -131,9 +142,7 @@ class RedactionPipelineTest {
item.setTitle("See page on Smith case");
outline.addLast(item);
// AcroForm field carrying the redacted string. We set the V entry directly on the
// COS dictionary to avoid triggering AppearanceGeneratorHelper which requires a full
// /DA + /DR resource graph - CatalogScrubber is supposed to rewrite V regardless.
// AcroForm field carrying the redacted string.
PDAcroForm form = new PDAcroForm(doc);
doc.getDocumentCatalog().setAcroForm(form);
PDTextField field = new PDTextField(form);
@@ -197,8 +206,7 @@ class RedactionPipelineTest {
Set<String> targets = new LinkedHashSet<>();
targets.add("Smith");
// No content-stream rewriting was done. The primary verification must trip and the
// rasterisation fallback must kick in so the final bytes still have no target.
// No content-stream rewriting was done.
bytes = RedactionPipeline.finalize(doc, targets, Collections.emptyList());
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
@@ -213,10 +221,7 @@ class RedactionPipelineTest {
@DisplayName(
"Manual rect redaction feeds captured text into scoped verification - raster fallback kicks in when text survives")
void manualRectVerificationUsesCapturedStrings() throws Exception {
// Simulate the failure mode: a rect is drawn over "LEAKED" but the content-stream rewrite
// is intentionally bypassed by only calling the CatalogScrubber path (finalize with
// targets=["LEAKED"] and affectedPages=[0]). Verification must see the surviving text and
// trigger rasterisation of page 0. Page 1 must remain text-searchable.
// Simulate the failure mode: a rect is drawn over "LEAKED"
byte[] bytes;
try (PDDocument doc = new PDDocument()) {
PDPage p0 = new PDPage(PDRectangle.A4);
@@ -316,17 +321,7 @@ class RedactionPipelineTest {
@DisplayName(
"Pathological verification regex triggers verification FAIL (not silent pass) and engages fallback")
void pathologicalVerificationRegexFailsClosed() throws Exception {
// A regex that throws on .matcher(...).find() - build a pattern that causes a runtime
// exception by constructing a matcher over a degenerate character sequence. We simulate
// the pathological case by supplying a pattern whose matcher throws StackOverflowError via
// deep alternation. Because JVM reliably triggering SOE is tricky, we instead construct
// a pattern where .find() throws an unchecked exception using a custom Pattern subclass
// is not possible (Pattern is final). The realistic pathological case is catastrophic
// backtracking, but we cannot rely on timeouts in a unit test. Instead we verify the
// IMPORTANT invariant indirectly: when finalize is called with no content rewrite and a
// surviving target, verification must FAIL and the raster fallback must engage - this
// proves the verify path is not silently swallowing exceptions (which would return the
// unrasterised bytes).
// A regex that throws on .matcher(...).find() - build a pattern that causes
byte[] bytes;
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
@@ -338,15 +333,13 @@ class RedactionPipelineTest {
cs.showText("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!");
cs.endText();
}
// Pattern that matches the content - if regex exceptions were swallowed, verification
// would return unrasterised bytes with the match still present.
// Pattern that matches the content - if regex exceptions were swallowed
List<Pattern> patterns = List.of(Pattern.compile("a{5,}"));
bytes = RedactionPipeline.finalize(doc, Collections.emptySet(), patterns);
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
String text = new PDFTextStripper().getText(reopened);
// Rasterisation should have eliminated text extractability entirely on the affected
// page (no affectedPages passed means whole-document rasterisation fallback).
// Rasterisation should have eliminated text extractability entirely
assertFalse(
Pattern.compile("a{5,}").matcher(text).find(),
"Regex match must not survive verification fallback");
@@ -526,4 +519,288 @@ class RedactionPipelineTest {
List<Pattern> regex = RedactionPipeline.buildPatterns(new String[] {"\\d{3}"}, true, false);
assertTrue(regex.get(0).matcher("ID 123 issued").find());
}
@Test
@DisplayName(
"whole-word single-char non-word target (&) matches; finder and pipeline agree (F2)")
void wholeWordSingleCharNonWordTarget() {
List<Pattern> pipeline = RedactionPipeline.buildPatterns(new String[] {"&"}, false, true);
assertEquals(1, pipeline.size());
assertTrue(
pipeline.get(0).matcher("a & b").find(),
"\\b&\\b no-ops on '&'; the shared lookaround wrapper must match");
List<Pattern> finder =
stirling.software.SPDF.utils.text.TextFinderUtils.createOptimizedSearchPatterns(
Set.of("&"), false, true);
assertTrue(finder.get(0).matcher("a & b").find(), "finder and pipeline must agree");
}
@Test
@DisplayName("form XObject text is physically removed, not left for rasterisation (F3)")
void formXObjectTextPhysicallyRemoved() throws Exception {
byte[] bytes;
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
PDFormXObject form = new PDFormXObject(doc);
form.setResources(new PDResources());
form.getResources()
.put(
COSName.getPDFName("F1"),
new PDType1Font(Standard14Fonts.FontName.HELVETICA));
form.setBBox(new PDRectangle(0, 0, 300, 50));
try (var out = form.getStream().createOutputStream()) {
out.write(
"BT /F1 12 Tf 0 10 Td (SECRET payload) Tj ET"
.getBytes(StandardCharsets.ISO_8859_1));
}
PDResources pageRes = new PDResources();
COSName fn = pageRes.add(form);
page.setResources(pageRes);
PDStream ps = new PDStream(doc);
try (var out = ps.createOutputStream()) {
out.write(
("q 1 0 0 1 100 700 cm /" + fn.getName() + " Do Q")
.getBytes(StandardCharsets.ISO_8859_1));
}
page.setContents(ps);
Set<String> targets = new LinkedHashSet<>(Set.of("SECRET"));
List<Pattern> pats =
RedactionPipeline.buildPatterns(new String[] {"SECRET"}, false, false);
RedactionPipeline.redactLiteralTerms(doc, targets, pats);
bytes = RedactionPipeline.finalize(doc, targets, pats);
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
assertFalse(
new PDFTextStripper().getText(reopened).contains("SECRET"),
"form XObject text must not be extractable");
assertFalse(
pageHasImage(reopened.getPage(0)),
"removal should be surgical (form rewritten), not full rasterisation");
}
}
@Test
@DisplayName(
"verify() sees glyphs masked by /ActualText and rasterises the survivor (discovery)")
void actualTextMaskedSurvivorIsCaught() throws Exception {
byte[] bytes;
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
PDResources res = new PDResources();
COSName f = res.add(new PDType1Font(Standard14Fonts.FontName.HELVETICA));
page.setResources(res);
// Real glyphs paint SECRET but /ActualText claims the run is SAFE.
String content =
"/Span <</ActualText (SAFE)>> BDC BT /"
+ f.getName()
+ " 12 Tf 100 700 Td (SECRET) Tj ET EMC";
PDStream ps = new PDStream(doc);
try (var out = ps.createOutputStream()) {
out.write(content.getBytes(StandardCharsets.ISO_8859_1));
}
page.setContents(ps);
// No removal pass: rely on finalize's verify to catch the masked survivor.
bytes =
RedactionPipeline.finalize(
doc, new LinkedHashSet<>(Set.of("SECRET")), List.of());
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
// A default stripper would read 'SAFE' and pass; the glyph-blind verify caught 'SECRET'
// and rasterised, so the page is now an image and nothing extractable survives.
assertTrue(
pageHasImage(reopened.getPage(0)),
"ActualText-masked survivor must be detected and rasterised");
assertFalse(new PDFTextStripper().getText(reopened).contains("SECRET"));
}
}
@Test
@DisplayName("scrub keeps /AP on form fields that do not contain the target (F7a)")
void untouchedFormFieldKeepsAppearance() throws Exception {
try (PDDocument doc = new PDDocument()) {
doc.addPage(new PDPage(PDRectangle.A4));
PDAcroForm form = new PDAcroForm(doc);
doc.getDocumentCatalog().setAcroForm(form);
PDTextField hit = new PDTextField(form);
hit.setPartialName("hit");
hit.getCOSObject().setString(COSName.V, "paid by Smith");
hit.getCOSObject().setItem(COSName.AP, new COSDictionary());
form.getFields().add(hit);
PDTextField clean = new PDTextField(form);
clean.setPartialName("clean");
clean.getCOSObject().setString(COSName.V, "nothing sensitive");
clean.getCOSObject().setItem(COSName.AP, new COSDictionary());
form.getFields().add(clean);
CatalogScrubber.scrub(
doc, new LinkedHashSet<>(Set.of("Smith")), Collections.emptyList());
assertNull(
hit.getCOSObject().getDictionaryObject(COSName.AP),
"matched field /AP should be dropped for regeneration");
assertNotNull(
clean.getCOSObject().getDictionaryObject(COSName.AP),
"untouched field must keep its /AP (F7a)");
}
}
private static final String LIBERATION =
"/org/apache/pdfbox/resources/ttf/LiberationSans-Regular.ttf";
@Test
@DisplayName("verify gate skips the native pass when all fonts are Standard-14")
void gateSkipsForStandard14() throws Exception {
byte[] bytes;
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.beginText();
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
cs.newLineAtOffset(72, 700);
cs.showText("plain Helvetica text");
cs.endText();
}
ByteArrayOutputStream b = new ByteArrayOutputStream();
doc.save(b);
bytes = b.toByteArray();
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
assertFalse(
RedactionPipeline.documentHasUnreliableFont(reopened),
"Standard-14 fonts extract reliably in PDFBox; native pass is redundant");
}
}
@Test
@DisplayName("verify gate runs the native pass for an embedded font without /ToUnicode")
void gateRunsForNonStandardFontWithoutToUnicode() throws Exception {
byte[] bytes;
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
PDFont font;
try (var ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
}
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.beginText();
cs.setFont(font, 12);
cs.newLineAtOffset(72, 700);
cs.showText("embedded TrueType no ToUnicode");
cs.endText();
}
ByteArrayOutputStream b = new ByteArrayOutputStream();
doc.save(b);
bytes = b.toByteArray();
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
assertFalse(
reopened.getPage(0)
.getResources()
.getFontNames()
.iterator()
.next()
.getName()
.isEmpty());
assertTrue(
RedactionPipeline.documentHasUnreliableFont(reopened),
"embedded non-Standard-14 font without /ToUnicode needs the native pass");
}
}
@Test
@DisplayName("verify gate skips the native pass when a Type0 font carries /ToUnicode")
void gateSkipsForType0WithToUnicode() throws Exception {
byte[] bytes;
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
PDFont font;
try (var ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
font = PDType0Font.load(doc, ttf, false);
}
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.beginText();
cs.setFont(font, 12);
cs.newLineAtOffset(72, 700);
cs.showText("full Type0 with ToUnicode");
cs.endText();
}
ByteArrayOutputStream b = new ByteArrayOutputStream();
doc.save(b);
bytes = b.toByteArray();
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
assertFalse(
RedactionPipeline.documentHasUnreliableFont(reopened),
"a /ToUnicode map makes PDFBox extraction authoritative; native pass redundant");
}
}
@Test
@DisplayName("verify gate recurses into form XObjects to find an unreliable font")
void gateDetectsUnreliableFontInsideXObject() throws Exception {
byte[] bytes;
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
// Page-level font is reliable (Standard-14); the risky font hides in a form XObject.
PDResources pageRes = new PDResources();
pageRes.put(
COSName.getPDFName("PF"),
new PDType1Font(Standard14Fonts.FontName.HELVETICA));
PDFormXObject form = new PDFormXObject(doc);
PDResources formRes = new PDResources();
PDFont embedded;
try (var ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
embedded = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
}
formRes.put(COSName.getPDFName("F1"), embedded);
form.setResources(formRes);
form.setBBox(new PDRectangle(0, 0, 200, 50));
try (var out = form.getStream().createOutputStream()) {
out.write(
"BT /F1 12 Tf 0 10 Td (hidden) Tj ET"
.getBytes(StandardCharsets.ISO_8859_1));
}
COSName formName = pageRes.add(form);
page.setResources(pageRes);
PDStream ps = new PDStream(doc);
try (var out = ps.createOutputStream()) {
out.write(
("q 1 0 0 1 50 700 cm /" + formName.getName() + " Do Q")
.getBytes(StandardCharsets.ISO_8859_1));
}
page.setContents(ps);
ByteArrayOutputStream b = new ByteArrayOutputStream();
doc.save(b);
bytes = b.toByteArray();
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
assertTrue(
RedactionPipeline.documentHasUnreliableFont(reopened),
"an unreliable font inside a form XObject must be detected by recursion");
}
}
private static boolean pageHasImage(PDPage page) throws Exception {
PDResources res = page.getResources();
if (res == null) {
return false;
}
for (COSName n : res.getXObjectNames()) {
if (res.getXObject(n) instanceof PDImageXObject) {
return true;
}
}
return false;
}
}
testing/cucumber/features/redact.feature
+1 -4
View File
@@ -1,9 +1,6 @@
@security @redact
Feature: PDF redaction physically removes text
Redaction must destroy the underlying text, not merely cover it with a box.
These scenarios push known text through the redaction endpoints and assert the
target is gone from the extracted text layer (and catalog carriers) while other
text survives.
Target text must be gone from the extracted text layer, not just covered by a box.
Scenario: Auto-redact physically removes the target word
Given I generate a PDF file as "fileInput"
testing/cucumber/features/steps/step_definitions.py
+1 -2
View File
@@ -17,8 +17,7 @@ from PIL import Image, ImageDraw
API_HEADERS = {"X-API-KEY": "123456789"}
# Base URL of the backend under test. Defaults to the CI value; override with
# STIRLING_BASE_URL to point at a backend on another port (e.g. a local sidecar run).
# Backend under test; override STIRLING_BASE_URL to target another port.
BASE_URL = os.environ.get("STIRLING_BASE_URL", "http://localhost:8080")
#########