From b88cbe6bfc500d6fcd99186d2ac7d4853cf8bdad Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:59:58 +0100 Subject: [PATCH] Harden true-removal redaction and gate independent verification --- .../api/security/ManualRedactionService.java | 57 +- .../api/security/RedactController.java | 36 +- .../api/security/RedactExecuteService.java | 4 +- .../api/security/TextRedactionService.java | 11 +- .../SPDF/pdf/redaction/CatalogScrubber.java | 184 ++-- .../SPDF/pdf/redaction/RedactionPipeline.java | 875 ++++++++++-------- .../RedactionVerificationFailedException.java | 6 +- .../SPDF/utils/text/TextFinderUtils.java | 19 +- .../api/security/RedactControllerTest.java | 9 +- .../api/security/RedactionPdfVarietyTest.java | 32 +- .../TextRedactionServiceExtraTest.java | 8 +- .../TextRedactionServiceMoreTest.java | 20 +- .../RedactionPipelineIntegrationTest.java | 25 +- .../pdf/redaction/RedactionPipelineTest.java | 327 ++++++- testing/cucumber/features/redact.feature | 5 +- .../features/steps/step_definitions.py | 3 +- 16 files changed, 994 insertions(+), 627 deletions(-) diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java index c8a4c5776d..1aaa05edc5 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -52,9 +53,10 @@ class ManualRedactionService { Set capturedStrings = new LinkedHashSet<>(); Map> rectsByPage = new HashMap<>(); + Set forceRasterPages = new LinkedHashSet<>(); if (redactionAreas == null || redactionAreas.isEmpty()) { - return new AreaRedactionResult(rectsByPage); + return new AreaRedactionResult(rectsByPage, forceRasterPages); } Map> redactionsByPage = new HashMap<>(); @@ -86,8 +88,9 @@ class ManualRedactionService { PDPage page = allPages.get(pageIndex); float pageHeight = page.getBBox().getHeight(); - List rects = new ArrayList<>(); - Color overlayColor = Color.BLACK; + // Group rects by their decoded colour so each area keeps its own overlay tint (F9a). + Map> byColor = new LinkedHashMap<>(); + List allRects = new ArrayList<>(); for (RedactionArea area : areasForPage) { float x = area.getX().floatValue(); float y = area.getY().floatValue(); @@ -95,24 +98,28 @@ class ManualRedactionService { float height = area.getHeight().floatValue(); // Request coords are top-left origin; convert to PDF user space (bottom-left). float pdfY = pageHeight - y - height; - rects.add(new PDRectangle(x, pdfY, width, height)); - overlayColor = decodeOrDefault(area.getColor()); + PDRectangle rect = new PDRectangle(x, pdfY, width, height); + allRects.add(rect); + byColor.computeIfAbsent(decodeOrDefault(area.getColor()), k -> new ArrayList<>()) + .add(rect); } - // Physically drop intersecting glyphs and draw the overlay rectangle over the area. - Map> singlePage = new HashMap<>(); - singlePage.put(pageIndex, rects); - RedactionPipeline.RedactionResult result = - RedactionPipeline.redactAreas(document, singlePage, overlayColor); - capturedStrings.addAll(result.getCapturedStrings()); - rectsByPage.put(pageIndex, rects); + for (Map.Entry> colorEntry : byColor.entrySet()) { + Map> singlePage = new HashMap<>(); + singlePage.put(pageIndex, colorEntry.getValue()); + RedactionPipeline.RedactionResult result = + RedactionPipeline.redactAreas(document, singlePage, colorEntry.getKey()); + capturedStrings.addAll(result.getCapturedStrings()); + forceRasterPages.addAll(result.getForceRasterPages()); + } + rectsByPage.put(pageIndex, allRects); } log.debug( "Manual area redaction captured {} text run(s) across {} page(s)", capturedStrings.size(), rectsByPage.size()); - return new AreaRedactionResult(rectsByPage); + return new AreaRedactionResult(rectsByPage, forceRasterPages); } List redactPages( @@ -327,8 +334,7 @@ class ManualRedactionService { convertedPdf, Collections.emptySet(), Collections.emptyList()); } } else { - // True-removal pass: physically strip matched glyph bytes from every content stream, - // then scrub catalog carriers, verify, and rasterise affected pages on any leak. + // Strip matched glyphs, then scrub/verify/rasterise via finalize. RedactionPipeline.redactLiteralTerms(document, literalTargets, patterns); outputBytes = RedactionPipeline.finalize(document, literalTargets, patterns); } @@ -336,15 +342,9 @@ class ManualRedactionService { return writeBytes(outputBytes, document.getNumberOfPages()); } - /** - * Finalize a manual area/page redaction. The overlay rectangles are already drawn and the - * intersecting glyphs already dropped. Verification is region-based: each redaction rectangle - * is re-scanned and must be empty (whole-page wipes carry no rects and are guaranteed clean). - */ + /** Finalize a manual redaction; forced + verified-leaking pages are rasterised. */ TempFile finalizeManual( - PDDocument document, - Map> rectsByPage, - Boolean convertToImage) + PDDocument document, AreaRedactionResult areaResult, Boolean convertToImage) throws IOException { byte[] outputBytes; @@ -355,7 +355,9 @@ class ManualRedactionService { convertedPdf, Collections.emptySet(), Collections.emptyList()); } } else { - outputBytes = RedactionPipeline.finalizeAreas(document, rectsByPage); + outputBytes = + RedactionPipeline.finalizeAreas( + document, areaResult.rectsByPage, areaResult.forceRasterPages); } return writeBytes(outputBytes, document.getNumberOfPages()); @@ -378,12 +380,15 @@ class ManualRedactionService { // Utilities // ----------------------------------------------------------------------- - /** Redaction rectangles (per 0-based page index) applied by a manual area pass. */ + /** Redaction rectangles + pages to force-rasterise, per 0-based page index. */ static final class AreaRedactionResult { final Map> rectsByPage; + final Set forceRasterPages; - AreaRedactionResult(Map> rectsByPage) { + AreaRedactionResult( + Map> rectsByPage, Set forceRasterPages) { this.rectsByPage = rectsByPage; + this.forceRasterPages = forceRasterPages; } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java index 98e2c73642..8fe26a841c 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java @@ -108,15 +108,14 @@ public class RedactController { try (PDDocument document = pdfDocumentFactory.load(file)) { PDPageTree allPages = document.getDocumentCatalog().getPages(); - // Whole-page wipes drop content outright (guaranteed clean); area redactions drop - // intersecting glyphs, draw an overlay, and are verified per-rectangle at finalize. + // Whole-page wipes drop content; areas drop glyphs + overlay, verified later. manualRedactionService.redactPages(request, document, allPages); ManualRedactionService.AreaRedactionResult areaResult = manualRedactionService.redactAreas(request.getRedactions(), document, allPages); TempFile out = manualRedactionService.finalizeManual( - document, areaResult.rectsByPage, request.getConvertPDFToImage()); + document, areaResult, request.getConvertPDFToImage()); return WebResponseUtils.pdfFileToWebResponse(out, filename); } } @@ -180,16 +179,6 @@ public class RedactController { request.getFileInput().getOriginalFilename()))) + "_redacted.pdf"; - if (allFoundTextsByPage.isEmpty()) { - log.info("No text found matching redaction patterns"); - // Still finalize so metadata is scrubbed and the document is rewritten - // consistently. - TempFile finalized = - manualRedactionService.finalizeManual( - document, Collections.emptyMap(), request.getConvertPDFToImage()); - return WebResponseUtils.pdfFileToWebResponse(finalized, filename); - } - Set literalTargets = Arrays.stream(listOfText) .map(String::trim) @@ -197,12 +186,27 @@ public class RedactController { .collect(Collectors.toCollection(LinkedHashSet::new)); List compiledPatterns = RedactionPipeline.buildPatterns(listOfText, useRegex, wholeWordSearchBool); - // Bare literal targets match substrings, so they are only safe when the search is a - // plain literal. In regex or whole-word mode the boundary/regex semantics live entirely - // in the compiled patterns, so pass no literal targets to avoid stripping substrings. + // Bare literals match substrings; regex/whole-word rely on compiled patterns. Set verificationTargets = (useRegex || wholeWordSearchBool) ? Collections.emptySet() : literalTargets; + if (allFoundTextsByPage.isEmpty()) { + // No page hit, but the target may live in a bookmark/annotation/form/JS carrier, so + // still run the scrub + verify path rather than just wiping metadata (F5). + log.info("No page text matched; scrubbing catalog carriers and verifying"); + TempFile finalized = + manualRedactionService.finalizeRedaction( + document, + Collections.emptyMap(), + request.getRedactColor(), + request.getCustomPadding(), + request.getConvertPDFToImage(), + true, + verificationTargets, + compiledPatterns); + return WebResponseUtils.pdfFileToWebResponse(finalized, filename); + } + boolean fallbackToBoxOnlyMode; try { fallbackToBoxOnlyMode = diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java index 6c637d4db1..6bfa97abcd 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java @@ -143,9 +143,7 @@ class RedactExecuteService { applyAllImagesRedaction(document, request.getRedactImagePages(), style); } - // Explicit overlay-only requests must not rewrite content or verify. When overlay-only - // was forced by a font fallback (not user choice) we still pass the targets so the - // pipeline's true-removal + verification + page-scoped raster fallback run. + // Overlay-only skips removal/verify; font-fallback overlays still pass targets. Set literalTargets = new LinkedHashSet<>(); for (String value : textValues) { String trimmed = value == null ? "" : value.trim(); diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java index a9633ffa39..5bb25a5ca7 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java @@ -706,16 +706,13 @@ class TextRedactionService { private void writeRedactedContentToXObject( PDDocument document, PDFormXObject formXObject, List redactedTokens) throws IOException { - - PDStream newStream = new PDStream(document); - - try (var out = newStream.createOutputStream()) { + // A form XObject's content IS its own stream body; overwrite it in place. Attaching a + // /Contents key would be a no-op (its content is not read from /Contents). + PDStream formStream = new PDStream(formXObject.getCOSObject()); + try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) { ContentStreamWriter writer = new ContentStreamWriter(out); writer.writeTokens(redactedTokens); } - - formXObject.getCOSObject().removeItem(COSName.CONTENTS); - formXObject.getCOSObject().setItem(COSName.CONTENTS, newStream.getCOSObject()); } private List extractTextSegments(PDPage page, List tokens) { diff --git a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java index 51db943808..05996d86e0 100644 --- a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java +++ b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java @@ -29,35 +29,13 @@ import org.apache.pdfbox.pdmodel.interactive.form.PDField; import lombok.extern.slf4j.Slf4j; -/** - * Walks a {@link PDDocument} and physically removes or rewrites every carrier that a PDF can use to - * leak text which the user asked to redact. - * - *

Covers: - * - *

    - *
  • {@link PDDocumentInformation} (Info dict) + XMP metadata stream - *
  • {@link PDDocumentOutline} bookmark titles - *
  • {@link PDAcroForm} field values (V, DV) and rich text (RV) - *
  • Every {@link PDAnnotation} Contents and RC - *
  • Structure tree ActualText, Alt, T, E, Lang entries - *
  • Names tree: JavaScript entries and embedded files (dropped entirely when matching) - *
- * - *

When applied after the content-stream rewrite it closes the secondary leak paths flagged in - * the redaction security audit. - */ +/** Removes/rewrites every catalog carrier that could leak redacted text. */ @Slf4j public final class CatalogScrubber { private CatalogScrubber() {} - /** - * Remove occurrences of every {@code target} string (and any regex/whole-word pattern form - * produced by {@link RedactionPipeline#buildPatterns}) from all catalog-level carriers of the - * document. When {@code wipeAllMetadata} is {@code true} the document Info dict entries and XMP - * metadata stream are wiped wholesale; this is the safe default after a redaction operation. - */ + /** Scrub all catalog-level carriers of the given literal/regex targets. */ public static void scrub( PDDocument document, Set literalTargets, List patterns) { if (document == null) { @@ -77,9 +55,7 @@ public final class CatalogScrubber { scrubCatalogActions(catalog, literalTargets, patterns); } - // --------------------------------------------------------------------- - // Catalog actions: OpenAction, AA, and any JavaScript / URI payloads on the catalog - // --------------------------------------------------------------------- + // Catalog actions: OpenAction, AA, and any JavaScript / URI payloads private static void scrubCatalogActions( PDDocumentCatalog catalog, Set targets, List patterns) { @@ -87,16 +63,12 @@ public final class CatalogScrubber { if (root == null) { return; } - // OpenAction may be either an action dict (with /URI or /JS) or an explicit destination - // (array). We scrub strings in both cases; if the OpenAction matches a target we clear it. + // OpenAction may be either an action dict (with /URI or /JS) or an explicit scrubActionIfMatching(root, COSName.getPDFName("OpenAction"), targets, patterns); scrubActionIfMatching(root, COSName.getPDFName("AA"), targets, patterns); } - /** - * If the action dictionary at {@code key} contains any target literal in a URI or JS payload, - * wipe the key entirely. Otherwise recursively scrub string fields inside it. - */ + /** Drop the action at key if any target appears in its URI/JS payload. */ private static void scrubActionIfMatching( COSDictionary parent, COSName key, Set targets, List patterns) { if (parent == null || key == null) { @@ -125,9 +97,7 @@ public final class CatalogScrubber { return matches(cs.getString(), targets, patterns); } if (resolved instanceof COSStream stream) { - // Streams in XFA / OpenAction contexts are text (XML, JavaScript). Read the bytes as - // UTF-8 and test for target literals. We cap read length to avoid pathological memory - // use; 2 MiB is plenty for XFA packets and far beyond any realistic JS action. + // Streams in XFA / OpenAction contexts are text (XML, JavaScript). try (java.io.InputStream is = stream.createInputStream()) { byte[] buf = new byte[2 * 1024 * 1024]; int total = 0; @@ -139,11 +109,14 @@ public final class CatalogScrubber { } } String text = new String(buf, 0, total, java.nio.charset.StandardCharsets.UTF_8); - return matches(text, targets, patterns); + if (matches(text, targets, patterns)) { + return true; + } + // Fail closed: content past the 2 MiB cap is unproven, so treat as a match (F7c). + return total >= buf.length && is.read() >= 0; } catch (Exception e) { log.debug("Failed to scan stream for targets: {}", e.getMessage()); - // Fail closed: if we cannot read it we cannot prove it is clean, so treat as a - // match so the caller drops the stream. This is conservative by design. + // Fail closed: if we cannot read it we cannot prove it is clean, so treat return true; } } @@ -166,10 +139,7 @@ public final class CatalogScrubber { return false; } - /** - * Clean potentially sensitive metadata carriers. Called after {@link #scrub} so that surviving - * references to author/subject/keywords/XMP descriptors do not leak redacted values. - */ + /** Wipe Info-dict entries and the XMP metadata stream. */ public static void wipeMetadata(PDDocument document) { if (document == null) { return; @@ -194,9 +164,7 @@ public final class CatalogScrubber { } } - // --------------------------------------------------------------------- // Outline - // --------------------------------------------------------------------- private static void scrubOutline( PDDocumentOutline outline, Set targets, List patterns) { @@ -220,9 +188,7 @@ public final class CatalogScrubber { item.setTitle(stripped); } } - // Bookmark actions: /A is an action dict which may carry a /URI or /JS payload. - // If any target literal appears anywhere inside the action subtree, drop the - // action entirely so the URI / script cannot leak the target. + // Bookmark actions: /A is an action dict which may carry a /URI or /JS COSDictionary itemDict = item.getCOSObject(); if (itemDict != null) { scrubActionIfMatching(itemDict, COSName.A, targets, patterns); @@ -235,18 +201,14 @@ public final class CatalogScrubber { } } - // --------------------------------------------------------------------- // AcroForm - // --------------------------------------------------------------------- private static void scrubAcroForm( PDAcroForm form, Set targets, List patterns) { if (form == null) { return; } - // XFA forms: scrubbed separately because the XFA XML packet carries the "real" field - // values for XFA-enabled PDFs. Handle XFA before walking the field tree so we fail closed - // if XFA scrubbing throws. + // XFA forms: scrubbed separately because the XFA XML packet carries scrubXfa(form, targets, patterns); try { @@ -257,10 +219,7 @@ public final class CatalogScrubber { log.debug("Failed to walk AcroForm field tree: {}", e.getMessage()); } - // Force viewers to regenerate appearance streams from the (scrubbed) /V values rather - // than reusing any cached /AP /N that still contains the target text. Belt-and-braces: - // scrubField has also cleared per-widget /AP dicts, but /NeedAppearances ensures any - // future change still triggers regeneration. + // Force viewers to regenerate appearance streams from the (scrubbed) /V try { form.setNeedAppearances(true); } catch (Exception e) { @@ -276,10 +235,7 @@ public final class CatalogScrubber { } boolean hit = containsTarget(xfaBase, targets, patterns, new HashSet<>()); if (hit) { - // Simplest safe move: strip the XFA entry entirely. Viewers fall back to the - // AcroForm widgets which we have already scrubbed. Leaving a "partially scrubbed" - // XFA packet risks regex failures on partial XML and re-encoded entities leaking - // the target. + // Simplest safe move: strip the XFA entry entirely. log.warn( "Removing XFA form packet from AcroForm - XFA XML contained a redaction " + "target and has been dropped so viewers render AcroForm widgets " @@ -297,9 +253,21 @@ public final class CatalogScrubber { } try { COSDictionary dict = field.getCOSObject(); + // Only touch fields whose own values actually contain a target (F7a): clearing /AP + // document-wide blanks unrelated fields in viewers that ignore /NeedAppearances. + boolean matched = + dictValueMatches(dict, COSName.V, targets, patterns) + || dictValueMatches(dict, COSName.DV, targets, patterns) + || dictValueMatches(dict, COSName.getPDFName("RV"), targets, patterns) + || dictValueMatches(dict, COSName.getPDFName("TU"), targets, patterns) + || fieldValueMatches(field, targets, patterns); + if (!matched) { + return; + } scrubDictStrings(dict, COSName.V, targets, patterns); scrubDictStrings(dict, COSName.DV, targets, patterns); scrubDictStrings(dict, COSName.getPDFName("RV"), targets, patterns); + scrubDictStrings(dict, COSName.getPDFName("TU"), targets, patterns); // Keep field appearance streams in sync with value where possible. try { if (field.getValueAsString() != null) { @@ -311,16 +279,54 @@ public final class CatalogScrubber { } catch (Exception e) { log.debug("Failed to rewrite field value via setValue: {}", e.getMessage()); } - // Drop per-widget appearance streams (/AP dict) for every widget kid of this field. - // The cached appearance stream contains the pre-redaction value baked in as glyph - // data; simply rewriting /V leaves it visually unchanged in many viewers. Removing /AP - // plus /NeedAppearances at the form level forces regeneration. - clearWidgetAppearances(dict); + // Button on-states (/Btn) cannot be rebuilt from /V by /NeedAppearances, so leave their + // /AP intact and rely on value scrubbing; drop /AP only for matched text fields. + if (!isButtonField(dict)) { + clearWidgetAppearances(dict); + } } catch (Exception e) { log.debug("Failed to scrub field: {}", e.getMessage()); } } + private static boolean isButtonField(COSDictionary dict) { + COSName ft = (COSName) dict.getDictionaryObject(COSName.FT); + if (ft == null) { + COSBase parent = dict.getDictionaryObject(COSName.PARENT); + if (parent instanceof COSDictionary p) { + ft = (COSName) p.getDictionaryObject(COSName.FT); + } + } + return COSName.getPDFName("Btn").equals(ft); + } + + private static boolean dictValueMatches( + COSDictionary dict, COSName key, Set targets, List patterns) { + COSBase value = dict.getDictionaryObject(key); + if (value instanceof COSString cs) { + return matches(cs.getString(), targets, patterns); + } + if (value instanceof COSArray array) { + for (int i = 0; i < array.size(); i++) { + if (array.getObject(i) instanceof COSString element + && matches(element.getString(), targets, patterns)) { + return true; + } + } + } + return false; + } + + private static boolean fieldValueMatches( + PDField field, Set targets, List patterns) { + try { + String value = field.getValueAsString(); + return value != null && matches(value, targets, patterns); + } catch (Exception e) { + return false; + } + } + private static void clearWidgetAppearances(COSDictionary fieldDict) { if (fieldDict == null) { return; @@ -338,9 +344,7 @@ public final class CatalogScrubber { } } - // --------------------------------------------------------------------- // Annotations - // --------------------------------------------------------------------- private static void scrubAnnotations( PDDocument document, Set targets, List patterns) { @@ -388,9 +392,7 @@ public final class CatalogScrubber { } } - // --------------------------------------------------------------------- // Structure tree - // --------------------------------------------------------------------- private static void scrubStructTree( PDStructureTreeRoot root, Set targets, List patterns) { @@ -414,7 +416,7 @@ public final class CatalogScrubber { return; } if (resolved instanceof COSDictionary dict) { - // Do not walk into content streams - those are handled by content-stream rewrite. + // Do not walk into content streams - those are handled by content-stream if (resolved instanceof COSStream) { return; } @@ -438,9 +440,7 @@ public final class CatalogScrubber { } } - // --------------------------------------------------------------------- // Names tree (JavaScript + embedded files) - // --------------------------------------------------------------------- private static void scrubNames( PDDocumentNameDictionary names, Set targets, List patterns) { @@ -481,7 +481,17 @@ public final class CatalogScrubber { for (int i = namesArray.size() - 2; i >= 0; i -= 2) { COSBase keyBase = namesArray.getObject(i); String key = keyBase instanceof COSString s ? s.getString() : null; - if (key != null && matches(key, targets, patterns)) { + // Drop the pair when the KEY or the VALUE (JS /JS stream, embedded-file bytes) + // contains a target - not just the key (F7b). + boolean keyHit = key != null && matches(key, targets, patterns); + boolean valueHit = + i + 1 < namesArray.size() + && containsTarget( + namesArray.getObject(i + 1), + targets, + patterns, + new HashSet<>()); + if (keyHit || valueHit) { namesArray.remove(i + 1); namesArray.remove(i); } @@ -498,9 +508,7 @@ public final class CatalogScrubber { } } - // --------------------------------------------------------------------- // Helpers - // --------------------------------------------------------------------- private static void scrubDictStrings( COSDictionary dict, COSName key, Set targets, List patterns) { @@ -536,24 +544,21 @@ public final class CatalogScrubber { if (target == null || target.isEmpty()) { continue; } - // Case-insensitive literal removal. Verification is case-insensitive, so scrubbing - // MUST be too or mixed-case ("SMITH" in a catalog string vs "Smith" in the target - // list) will fail verification and trip the rasterisation fallback - or worse, on - // carriers that are not verified, leak the string untouched. + // Case-insensitive literal removal. Verification is case-insensitive result = caseInsensitiveReplaceAll(result, target); } } if (patterns != null) { for (Pattern pattern : patterns) { try { - // Force case-insensitive matching for catalog carriers regardless of the flags - // the pattern was compiled with. User-supplied redaction targets should not - // silently miss because the author typed the name in different case. + // Force case-insensitive matching for catalog carriers regardless Pattern ci = withCaseInsensitive(pattern); result = ci.matcher(result).replaceAll(""); - } catch (Exception e) { - log.debug( - "Pattern replace failed for {}: {}", pattern.pattern(), e.getMessage()); + } catch (RuntimeException | StackOverflowError e) { + // Fail closed (X2): a throwing regex means we cannot prove the carrier clean, + // so drop the whole string rather than leaving it intact. + log.warn("Pattern replace failed for {}; dropping carrier text", pattern); + return ""; } } } @@ -580,8 +585,11 @@ public final class CatalogScrubber { if (withCaseInsensitive(pattern).matcher(source).find()) { return true; } - } catch (Exception e) { - log.debug("Pattern match failed for {}: {}", pattern.pattern(), e.getMessage()); + } catch (RuntimeException | StackOverflowError e) { + // Fail closed (X2): a throwing regex counts as a match so the carrier is + // scrubbed. + log.warn("Pattern match failed for {}; treating carrier as a match", pattern); + return true; } } } diff --git a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java index f178aed52f..33bb91ad77 100644 --- a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java +++ b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java @@ -8,6 +8,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -24,6 +25,7 @@ import javax.imageio.ImageIO; import org.apache.pdfbox.contentstream.operator.Operator; import org.apache.pdfbox.cos.COSArray; import org.apache.pdfbox.cos.COSBase; +import org.apache.pdfbox.cos.COSDictionary; import org.apache.pdfbox.cos.COSName; import org.apache.pdfbox.cos.COSString; import org.apache.pdfbox.pdfparser.PDFStreamParser; @@ -47,33 +49,27 @@ import org.apache.pdfbox.text.TextPosition; import lombok.extern.slf4j.Slf4j; -/** - * Shared plumbing for the three redaction paths (manual areas, whole pages, auto-word). - * - *

Every path funnels through this class so that the following guarantees hold uniformly: - * - *

    - *
  1. Text tokens that fall inside a target rect are physically dropped from the content stream. - *
  2. Catalog-level carriers (outline, AcroForm, annotations, struct tree, names tree) are - * scrubbed so redacted strings cannot survive outside the page content. - *
  3. Info dict + XMP metadata is wiped. - *
  4. The document is rewritten with a fresh xref (no incremental save) and then verified with a - * fresh {@link PDFTextStripper} pass; surviving target text throws {@link - * RedactionVerificationFailedException}. The literal/pattern path verifies the whole (or - * affected) page text; the manual-area path verifies each redaction rectangle is empty. - *
- */ +import stirling.software.SPDF.pdf.parser.PageImageLocator; +import stirling.software.SPDF.utils.text.TextFinderUtils; + +/** Shared plumbing for the manual-area, whole-page and auto-word redaction paths. */ @Slf4j public final class RedactionPipeline { private static final Set TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\""); + private static final int MAX_XOBJECT_DEPTH = 10; + + // Latched false if the native PDFium binding can't load, so a host without it (musl/exotic) + // logs once and falls back to the PDFBox pass instead of retrying the native open every redaction. + private static volatile boolean jpdfiumAvailable = true; + + // Skip the additive native pass above this size to bound off-heap copy + native runtime on + // adversarial inputs; the PDFBox glyph-blind pass still verifies. 100 MB. + private static final long MAX_JPDFIUM_VERIFY_BYTES = 100L * 1024 * 1024; private RedactionPipeline() {} - /** - * Apply a list of page-local redaction rects (coordinates in PDF user space) against the - * supplied document. - */ + /** Redact page-local rects (PDF user-space), dropping intersecting glyphs + overlay. */ public static RedactionResult redactAreas( PDDocument document, Map> rectsByPageIndex, @@ -81,6 +77,7 @@ public final class RedactionPipeline { throws IOException { Set capturedStrings = new LinkedHashSet<>(); + Set forceRasterPages = new LinkedHashSet<>(); for (Map.Entry> entry : rectsByPageIndex.entrySet()) { int pageIndex = entry.getKey(); @@ -89,21 +86,69 @@ public final class RedactionPipeline { continue; } PDPage page = document.getPage(pageIndex); - List captured = captureTextInRects(page, rects); - capturedStrings.addAll(captured); + capturedStrings.addAll(captureTextInRects(page, rects)); - removeTokensIntersectingRects(document, page, rects); + // Rotation / non-zero CropBox origin break the coordinate flip, and text-bearing forms + // or in-rect images defeat page-stream ordinal removal. Draw the overlay but force the + // coordinate-independent rasterisation fallback for such pages (F1/F4/F12a). + boolean surgical = + isSurgicallySafe(page) && removeTokensIntersectingRects(document, page, rects); + if (!surgical) { + forceRasterPages.add(pageIndex); + } + removeOverlappingAnnotations(page, rects); drawOverlay(document, page, rects, overlayColor); } - return new RedactionResult(capturedStrings); + return new RedactionResult(capturedStrings, forceRasterPages); } - /** - * Replace the entire contents of the listed pages with a single filled rectangle in the page - * media box. All underlying text and images are dropped from the content stream and from the - * page resources. - */ + /** True only for upright pages whose CropBox origin is (0,0), where the rect flip is exact. */ + private static boolean isSurgicallySafe(PDPage page) { + if (page.getRotation() != 0) { + return false; + } + PDRectangle crop = page.getCropBox(); + return crop.getLowerLeftX() == 0f && crop.getLowerLeftY() == 0f; + } + + /** Drop annotations whose rectangle overlaps any redaction rect (F6a). */ + private static void removeOverlappingAnnotations(PDPage page, List rects) { + try { + List kept = + new ArrayList<>(); + for (var ann : page.getAnnotations()) { + PDRectangle ar = ann.getRectangle(); + boolean overlaps = false; + if (ar != null) { + Rectangle2D.Float a = + new Rectangle2D.Float( + ar.getLowerLeftX(), + ar.getLowerLeftY(), + ar.getWidth(), + ar.getHeight()); + for (PDRectangle rect : rects) { + if (a.intersects( + rect.getLowerLeftX(), + rect.getLowerLeftY(), + rect.getWidth(), + rect.getHeight())) { + overlaps = true; + break; + } + } + } + if (!overlaps) { + kept.add(ann); + } + } + page.setAnnotations(kept); + } catch (Exception e) { + log.debug("Could not strip overlapping annotations: {}", e.getMessage()); + } + } + + /** Wipe whole pages: drop all content/resources, fill with a rectangle. */ public static void redactWholePages( PDDocument document, List pageIndexes, Color overlayColor) throws IOException { for (Integer pageIndex : pageIndexes) { @@ -132,24 +177,7 @@ public final class RedactionPipeline { } } - /** - * Rewrites every content stream on every page so that any occurrence of any literal target or - * regex pattern is physically removed from the glyph bytes. Handles: - * - *
    - *
  • simple Type1 fonts with WinAnsiEncoding (e.g. ReportLab {@code (Test PDF #1) Tj}), - *
  • Type0/CID fonts with multi-byte codes, - *
  • pages with {@code /Rotate 90} or other rotations, - *
  • text split across multiple operands inside a single {@code TJ} array. - *
- * - *

The approach decodes each {@link COSString} operand byte-by-byte using the font that is - * current at that point in the content stream, reconstructs the Unicode text, runs the target - * patterns against that text, and rebuilds the byte operand omitting the matched character - * codes. Width adjustment ({@code TJ} numeric kerning) is reinserted so downstream layout is - * preserved even when the operand shrinks. The page layout may look sparse but the glyphs for - * the target term are guaranteed to be gone. - */ + /** Physically remove every literal/regex match from all page content streams. */ public static void redactLiteralTerms( PDDocument document, Set literalTargets, List patterns) throws IOException { @@ -162,8 +190,7 @@ public final class RedactionPipeline { try { rewritePageContent(document, page, effectivePatterns); } catch (IOException | RuntimeException e) { - // Never let one page's font quirk (e.g. Type3 encode, damaged program) abort the - // whole document. Leave this page for the verify+rasterise safety net to catch. + // Never let one page's font quirk (e.g. Type3 encode, damaged program) log.warn( "Content-stream rewrite failed on page {} ({}); leaving it for the " + "verification/rasterisation pass.", @@ -174,32 +201,14 @@ public final class RedactionPipeline { } } - /** - * Finalize the document with document-wide verification: scrub catalog carriers, wipe metadata, - * subset embedded fonts, save with a fresh xref and verify that no literal target survives - * anywhere in the saved PDF. - * - *

Use this overload for auto-word redaction where every occurrence of a literal target is a - * deliberate removal target. For manual rect redaction use {@link #finalize(PDDocument, Set, - * List, Set)} so verification is scoped to the affected pages (since the same word can - * legitimately remain on non-targeted pages). - */ + /** Finalize with document-wide verification: scrub, wipe metadata, save, verify. */ public static byte[] finalize( PDDocument document, Set literalTargets, List patterns) throws IOException { return finalize(document, literalTargets, patterns, null); } - /** - * Finalize with verification scoped to a specific set of page indexes. When {@code - * affectedPages} is null the verification runs over the whole document. When non-null, only the - * listed pages are checked for surviving literal targets; the rasterisation fallback (if - * triggered) also rasterises only those pages, preserving every other page verbatim so - * unrelated content remains text-searchable. - * - *

Pass an empty set together with empty targets/patterns to skip verification entirely (for - * manual redactions that captured no text and drew no image boxes). - */ + /** Finalize with verification scoped to a specific set of page indexes. */ public static byte[] finalize( PDDocument document, Set literalTargets, @@ -219,18 +228,12 @@ public final class RedactionPipeline { verify(bytes, literalTargets, patterns, affectedPages); return bytes; } catch (RedactionVerificationFailedException primaryFailure) { - // Last-resort: if the content-stream rewriter could not guarantee removal (unusual - // fonts, encrypted streams, non-text glyph carriers), rasterise only the affected - // pages so the target is physically gone without destroying the text layer on pages - // that were never touched. This is logged loudly so operators know redaction fell - // back to the image path. + // Rewriter could not guarantee removal; rasterise as a last resort. log.warn( "Primary redaction verification failed ({}); falling back to page-scoped " + "rasterisation to guarantee removal.", primaryFailure.getMessage()); - // When the caller gave no page scope (auto-word mode), locate the leaking pages so - // only those are rasterised; clean pages keep their searchable text layer. If leak - // detection itself fails, fall back to rasterising everything (null). + // With no page scope (auto-word), rasterise only the pages that still leak. Set pagesToRaster = (affectedPages == null || affectedPages.isEmpty()) ? findLeakingPages(bytes, literalTargets, patterns) @@ -250,16 +253,12 @@ public final class RedactionPipeline { } } - /** - * Finalize a manual area redaction. Unlike the literal/pattern path, verification here is - * region-based: after saving, each redaction rectangle is re-scanned and must contain no text. - * This avoids the substring false positives that whole-page string verification hits when a box - * clips a word into a common fragment (for example a clipped "HEADER" becoming "HE", which - * would otherwise match "here" elsewhere on the page). On a real leak the affected pages are - * rasterised so untouched pages keep their text layer. - */ + /** Finalize a manual area redaction; rasterises forced + verified-leaking pages. */ public static byte[] finalizeAreas( - PDDocument document, Map> rectsByPage) throws IOException { + PDDocument document, + Map> rectsByPage, + Set forceRasterPages) + throws IOException { CatalogScrubber.wipeMetadata(document); warnAboutEmbeddedFontGlyphs(document); @@ -268,31 +267,30 @@ public final class RedactionPipeline { document.save(baos); byte[] bytes = baos.toByteArray(); - try { - verifyRectsEmpty(bytes, rectsByPage); + Set toRaster = + new HashSet<>(forceRasterPages == null ? Set.of() : forceRasterPages); + toRaster.addAll(findLeakingRectPages(bytes, rectsByPage)); + if (toRaster.isEmpty()) { return bytes; - } catch (RedactionVerificationFailedException primaryFailure) { - log.warn( - "Manual-area redaction verification failed ({}); rasterising affected pages to " - + "guarantee removal.", - primaryFailure.getMessage()); - Set pagesToRaster = new HashSet<>(rectsByPage.keySet()); - try (PDDocument rasterised = rasterisePages(bytes, pagesToRaster)) { - CatalogScrubber.wipeMetadata(rasterised); - ByteArrayOutputStream rasterOut = new ByteArrayOutputStream(); - rasterised.save(rasterOut); - return rasterOut.toByteArray(); - } catch (IOException e) { - throw new RedactionVerificationFailedException( - "Rasterisation fallback failed after manual redaction leak", e); - } + } + log.warn("Manual redaction rasterising page(s) {} to guarantee removal", toRaster); + try (PDDocument rasterised = rasterisePages(bytes, toRaster)) { + CatalogScrubber.wipeMetadata(rasterised); + ByteArrayOutputStream rasterOut = new ByteArrayOutputStream(); + rasterised.save(rasterOut); + return rasterOut.toByteArray(); + } catch (IOException e) { + throw new RedactionVerificationFailedException( + "Rasterisation fallback failed after manual redaction leak", e); } } - private static void verifyRectsEmpty( + /** Redacted pages that still show text inside a rect or carry an overlapping annotation. */ + private static Set findLeakingRectPages( byte[] bytes, Map> rectsByPage) { + Set leaking = new HashSet<>(); if (rectsByPage == null || rectsByPage.isEmpty()) { - return; + return leaking; } try (PDDocument reopened = org.apache.pdfbox.Loader.loadPDF(bytes)) { for (Map.Entry> entry : rectsByPage.entrySet()) { @@ -300,23 +298,48 @@ public final class RedactionPipeline { if (pageIndex < 0 || pageIndex >= reopened.getNumberOfPages()) { continue; } - List remaining = - captureTextInRects(reopened.getPage(pageIndex), entry.getValue()); - for (String fragment : remaining) { - if (fragment != null && !fragment.isBlank()) { - throw new RedactionVerificationFailedException( - "Text still present inside redaction area on page " - + (pageIndex + 1) - + ": '" - + fragment - + "'"); - } + PDPage page = reopened.getPage(pageIndex); + boolean textLeft = + captureTextInRects(page, entry.getValue()).stream() + .anyMatch(s -> s != null && !s.isBlank()); + if (textLeft || annotationOverlapsRect(page, entry.getValue())) { + leaking.add(pageIndex); } } } catch (IOException e) { - throw new RedactionVerificationFailedException( - "Failed to reopen redacted PDF for area verification", e); + // Cannot verify - rasterise every redacted page to be safe. + return new HashSet<>(rectsByPage.keySet()); } + return leaking; + } + + private static boolean annotationOverlapsRect(PDPage page, List rects) { + try { + for (var ann : page.getAnnotations()) { + PDRectangle ar = ann.getRectangle(); + if (ar == null) { + continue; + } + Rectangle2D.Float a = + new Rectangle2D.Float( + ar.getLowerLeftX(), + ar.getLowerLeftY(), + ar.getWidth(), + ar.getHeight()); + for (PDRectangle rect : rects) { + if (a.intersects( + rect.getLowerLeftX(), + rect.getLowerLeftY(), + rect.getWidth(), + rect.getHeight())) { + return true; + } + } + } + } catch (Exception e) { + return true; + } + return false; } private static List effectivePatterns( @@ -327,8 +350,7 @@ public final class RedactionPipeline { if (target == null || target.isEmpty()) { continue; } - // Case-insensitive to match TextFinderUtils' search semantics; removal must never - // be narrower than what the finder matched (or verification would raster pages). + // Case-insensitive to match TextFinderUtils; removal must not miss finder hits. result.add( Pattern.compile( Pattern.quote(target), @@ -341,9 +363,7 @@ public final class RedactionPipeline { return result; } - // --------------------------------------------------------------------- // Per-page content-stream rewrite (literal/regex based) - // --------------------------------------------------------------------- private static void rewritePageContent(PDDocument document, PDPage page, List patterns) throws IOException { @@ -356,16 +376,21 @@ public final class RedactionPipeline { if (modified) { writePageTokens(document, page, tokens); } - // Recurse into form XObjects referenced by this page. - rewriteFormXObjects(document, resources, patterns, new HashSet<>()); + // Recurse into form XObjects referenced by this page (shared visited set + depth cap). + rewriteFormXObjects(resources, resources, patterns, new HashSet<>(), 0); } private static void rewriteFormXObjects( - PDDocument document, PDResources resources, + PDResources parentResources, List patterns, - Set visited) + Set visited, + int depth) throws IOException { + if (depth > MAX_XOBJECT_DEPTH) { + log.warn("XObject nesting exceeded {}; stopping recursion", MAX_XOBJECT_DEPTH); + return; + } for (COSName name : resources.getXObjectNames()) { try { var xobj = resources.getXObject(name); @@ -376,20 +401,19 @@ public final class RedactionPipeline { continue; } List tokens = parseTokens(new PDFStreamParser(form)); - PDResources formResources = form.getResources(); - if (formResources == null) { - continue; - } + // Forms may inherit fonts from the parent when they carry no own /Resources. + PDResources formResources = + form.getResources() != null ? form.getResources() : parentResources; boolean modified = rewriteTokens(tokens, formResources, patterns); if (modified) { - PDStream stream = new PDStream(document); - try (var out = stream.createOutputStream(COSName.FLATE_DECODE)) { + // A form XObject's content IS its own stream body; overwrite it in place. + // Writing a /Contents key would be a no-op (F3). + PDStream formStream = new PDStream(form.getCOSObject()); + try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) { new ContentStreamWriter(out).writeTokens(tokens); } - form.getCOSObject().removeItem(COSName.CONTENTS); - form.getCOSObject().setItem(COSName.CONTENTS, stream.getCOSObject()); } - rewriteFormXObjects(document, formResources, patterns, visited); + rewriteFormXObjects(formResources, formResources, patterns, visited, depth + 1); } catch (IOException e) { log.debug("Failed to rewrite XObject {}: {}", name.getName(), e.getMessage()); } @@ -405,13 +429,7 @@ public final class RedactionPipeline { return tokens; } - /** - * Walk tokens keeping a tiny text state (current font). When a text-showing operator is - * encountered its string operand(s) are decoded via the current font, run against every - * pattern, and rewritten with matching character codes removed. - * - * @return true if any operand was modified. - */ + /** Walk tokens keeping a tiny text state (current font). */ private static boolean rewriteTokens( List tokens, PDResources resources, List patterns) { boolean modified = false; @@ -459,7 +477,7 @@ public final class RedactionPipeline { private static COSString rewriteCosString( COSString cosString, PDFont font, List patterns) { if (font == null) { - // Without a font we cannot decode safely. Try a best-effort latin-1 interpretation. + // Without a font we cannot decode safely. return rewriteRawLatin(cosString, patterns); } DecodeResult decoded = decodeCosString(cosString, font); @@ -474,9 +492,7 @@ public final class RedactionPipeline { } private static COSArray rewriteCosArray(COSArray arr, PDFont font, List patterns) { - // Build a concatenated decode across all COSString elements so that matches spanning - // multiple operands are found. Non-string elements are kerning adjustments; keep them - // in place and re-emit them between the rewritten strings. + // Build a concatenated decode across all COSString elements so that matches List parts = new ArrayList<>(); StringBuilder concat = new StringBuilder(); for (int i = 0; i < arr.size(); i++) { @@ -508,7 +524,7 @@ public final class RedactionPipeline { cursor += partLen; COSString rebuilt = buildFilteredCosStringRaw(decoded, partDrop, font, cs.getBytes()); - // A zero-length COSString is valid; PDFBox emits it as () producing no glyphs. + // A zero-length COSString is valid; PDFBox emits it as () producing no out.add(rebuilt); } else { out.add(elem); @@ -545,10 +561,7 @@ public final class RedactionPipeline { return new DecodeResult(sb.toString(), bytes, codeStart, codeLen); } - /** - * Decode a {@link COSString} into Unicode characters using the supplied font. Returns null if - * decoding fails at any point (e.g. malformed byte sequence). - */ + /** Decode a COSString to Unicode via the font; null if decoding fails. */ private static DecodeResult decodeCosString(COSString cosString, PDFont font) { byte[] bytes = cosString.getBytes(); StringBuilder text = new StringBuilder(); @@ -576,14 +589,11 @@ public final class RedactionPipeline { unicode = null; } if (unicode == null) { - // If the font has no ToUnicode mapping we cannot match reliably - return null - // and let the caller either fall back to latin-1 (rare) or rasterisation via - // the final verification pass. + // If the font has no ToUnicode mapping we cannot match reliably - return return null; } text.append(unicode); - // Associate every Unicode character produced with the same code byte range so - // that dropping any one of them drops the whole code. + // Associate every Unicode character produced with the same code byte range for (int c = 0; c < unicode.length(); c++) { starts.add(pos); lens.add(consumed); @@ -602,10 +612,7 @@ public final class RedactionPipeline { return new DecodeResult(text.toString(), bytes, startArr, lenArr); } - /** - * Returns null if none of the patterns match; otherwise a boolean mask the same length as - * {@code text} where {@code true} means "this character belongs to a matched range". - */ + /** Null if no pattern matches; else a per-char drop mask over the text. */ private static boolean[] findDroppedCharsMask(String text, List patterns) { boolean any = false; boolean[] mask = new boolean[text.length()]; @@ -636,19 +643,23 @@ public final class RedactionPipeline { private static COSString buildFilteredCosStringRaw( DecodeResult decoded, boolean[] drop, PDFont font, byte[] originalBytes) { - // Collect code byte-ranges to drop. Because one code can produce multiple chars, we drop - // the code if ANY of its chars is flagged. + // Collect code byte-ranges to drop. Set dropStarts = new HashSet<>(); for (int i = 0; i < drop.length; i++) { if (drop[i]) { dropStarts.add(decoded.codeStarts[i]); } } + // Map byte-start -> code length once (O(n)); findCodeLenAt was an O(n^2) linear scan (F10). + Map lenByStart = new HashMap<>(); + for (int j = 0; j < decoded.codeStarts.length; j++) { + lenByStart.putIfAbsent(decoded.codeStarts[j], decoded.codeLens[j]); + } ByteArrayOutputStream out = new ByteArrayOutputStream(); int i = 0; while (i < originalBytes.length) { int start = i; - int len = findCodeLenAt(decoded, start); + int len = lenByStart.getOrDefault(start, -1); if (len <= 0) { // Unknown - keep the byte verbatim. out.write(originalBytes[i] & 0xFF); @@ -656,8 +667,7 @@ public final class RedactionPipeline { continue; } if (dropStarts.contains(start)) { - // Replace dropped code with encoded space if possible. This preserves rough - // layout; if encoding a space fails we simply drop the bytes. + // Replace dropped code with encoded space if possible. byte[] spaceBytes = tryEncodeSpace(font); if (spaceBytes != null) { out.write(spaceBytes, 0, spaceBytes.length); @@ -670,15 +680,6 @@ public final class RedactionPipeline { return new COSString(out.toByteArray()); } - private static int findCodeLenAt(DecodeResult decoded, int byteStart) { - for (int j = 0; j < decoded.codeStarts.length; j++) { - if (decoded.codeStarts[j] == byteStart) { - return decoded.codeLens[j]; - } - } - return -1; - } - private static byte[] tryEncodeSpace(PDFont font) { if (font == null) { return new byte[] {0x20}; @@ -687,9 +688,6 @@ public final class RedactionPipeline { return font.encode(" "); } catch (Exception e) { // Some fonts cannot encode a space at all (e.g. Type3 throws - // UnsupportedOperationException - // from encode()). Returning null drops the code with no placeholder - the target glyph - // is still physically removed, which is what matters for redaction. return null; } } @@ -708,20 +706,12 @@ public final class RedactionPipeline { } } - // --------------------------------------------------------------------- // Rasterisation fallback - // --------------------------------------------------------------------- - /** - * Rasterise only the pages listed in {@code pagesToRaster} (all pages when null) at 150 DPI and - * replace those pages' content streams with the rendered image. Pages not in the set keep their - * original content streams verbatim so unrelated text remains searchable after the fallback. - */ + /** Rasterise the listed pages (all when null) at 150 DPI, replacing their content. */ private static PDDocument rasterisePages(byte[] sourceBytes, Set pagesToRaster) throws IOException { - // Load the document directly and mutate in place: rewriting only the affected pages' - // content streams is cheaper and guarantees untouched pages' xref entries survive - // byte-for-byte. + // Load the document directly and mutate in place: rewriting PDDocument source = org.apache.pdfbox.Loader.loadPDF(sourceBytes); try { PDFRenderer renderer = new PDFRenderer(source); @@ -740,7 +730,7 @@ public final class RedactionPipeline { PDImageXObject.createFromByteArray( source, imgOut.toByteArray(), "redacted-page-" + i); - // Drop all prior content / resources / annotations; the raster is the page now. + // Drop all prior content / resources / annotations; the raster is the page page.getCOSObject().removeItem(COSName.CONTENTS); page.setResources(new PDResources()); page.getCOSObject().removeItem(COSName.ANNOTS); @@ -754,8 +744,7 @@ public final class RedactionPipeline { PDPageContentStream.AppendMode.OVERWRITE, false, true)) { - // The rendered image already has the rotation baked in visually, so the - // resulting page is placed un-rotated against the media box. + // The rendered image already has the rotation baked in visually cs.drawImage( imageXObject, media.getLowerLeftX(), @@ -771,14 +760,9 @@ public final class RedactionPipeline { } } - // --------------------------------------------------------------------- // Pattern construction - // --------------------------------------------------------------------- - /** - * Build a set of regex patterns from user input. Returns an empty list if none of the entries - * produce a valid pattern. - */ + /** Build case-insensitive regex patterns from user input. */ public static List buildPatterns( String[] rawEntries, boolean useRegex, boolean wholeWordSearch) { List patterns = new ArrayList<>(); @@ -796,10 +780,9 @@ public final class RedactionPipeline { try { String core = useRegex ? trimmed : Pattern.quote(trimmed); if (wholeWordSearch) { - core = "\\b" + core + "\\b"; + // Shared with the finder so removal + verification use identical boundaries. + core = TextFinderUtils.applyWordBoundaries(trimmed, core); } - // Case-insensitive to mirror TextFinderUtils.createOptimizedSearchPatterns: what - // the finder matches, removal and verification must match too. patterns.add( Pattern.compile(core, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE)); } catch (PatternSyntaxException e) { @@ -809,9 +792,7 @@ public final class RedactionPipeline { return patterns; } - // --------------------------------------------------------------------- // Area capture - // --------------------------------------------------------------------- private static List captureTextInRects(PDPage page, List rects) throws IOException { @@ -819,8 +800,7 @@ public final class RedactionPipeline { stripper.setSortByPosition(true); for (int i = 0; i < rects.size(); i++) { PDRectangle rect = rects.get(i); - // PDFTextStripperByArea uses a Java2D rectangle in the same coordinate space that - // TextPosition reports, i.e. top-left origin with Y flipped relative to PDF user space. + // PDFTextStripperByArea uses a Java2D rectangle in the same coordinate float pdfY = page.getBBox().getHeight() - rect.getUpperRightY(); Rectangle2D.Float region = new Rectangle2D.Float( @@ -849,78 +829,16 @@ public final class RedactionPipeline { return new ArrayList<>(captured); } - // --------------------------------------------------------------------- // Content-stream rewriting (rect-driven glyph removal) - // --------------------------------------------------------------------- - private static void removeTokensIntersectingRects( + /** + * Blanks show-text operands whose glyphs fall in a rect. Returns false (surgical removal is + * unreliable, caller must rasterise) when text-bearing form XObjects skew the operator ordinals + * or an image sits under a rect - leaving the page stream untouched so the raster is correct. + */ + private static boolean removeTokensIntersectingRects( PDDocument document, PDPage page, List rects) throws IOException { - // Identify which show-text operators should be wiped by running a tiny engine that tracks - // each operator's rendered bounding box. - List dropTokenIndexes = identifyShowTextOperatorsInRects(document, page, rects); - - PDFStreamParser parser = new PDFStreamParser(page); - List tokens = new ArrayList<>(); - Object token; - while ((token = parser.parseNextToken()) != null) { - tokens.add(token); - } - - // Build set of token indexes to blank (the argument immediately preceding a flagged - // text-showing operator). - Set blankArgIndexes = new HashSet<>(); - int textOpCount = 0; - for (int i = 0; i < tokens.size(); i++) { - Object t = tokens.get(i); - if (t instanceof Operator op && TEXT_SHOWING_OPERATORS.contains(op.getName())) { - if (dropTokenIndexes.contains(textOpCount) && i > 0) { - blankArgIndexes.add(i - 1); - } - textOpCount++; - } - } - - for (Integer idx : blankArgIndexes) { - Object arg = tokens.get(idx); - if (arg instanceof COSString) { - tokens.set(idx, new COSString("")); - } else if (arg instanceof COSArray arr) { - COSArray empty = new COSArray(); - // Preserve numeric kerning entries so page layout doesn't shift wildly but drop - // every COSString. - for (COSBase element : arr) { - if (!(element instanceof COSString)) { - empty.add(element); - } - } - tokens.set(idx, empty); - } - } - - // Additionally wipe Do-drawn images whose bounding box intersects a rect. This needs a - // coordinate scan - for safety we drop every Do whose CTM origin sits inside a rect. - List doImageIndexes = identifyDoImagesInRects(page, rects); - int doCount = 0; - for (int i = tokens.size() - 1; i >= 0; i--) { - Object t = tokens.get(i); - if (t instanceof Operator op && "Do".equals(op.getName())) { - if (doImageIndexes.contains(doCount)) { - // Remove both the name argument and the operator. - if (i > 0) { - tokens.remove(i); - tokens.remove(i - 1); - } - } - doCount++; - } - } - - writePageTokens(document, page, tokens); - } - - private static List identifyShowTextOperatorsInRects( - PDDocument document, PDPage page, List rects) throws IOException { List areaRects = new ArrayList<>(); for (PDRectangle rect : rects) { float pdfY = page.getBBox().getHeight() - rect.getUpperRightY(); @@ -934,15 +852,75 @@ public final class RedactionPipeline { collector.setStartPage(pageIndex + 1); collector.setEndPage(pageIndex + 1); collector.getText(document); - return new ArrayList<>(collector.tokenIndexesToDrop); + Set dropTokenIndexes = collector.tokenIndexesToDrop; + int collectorTextOps = collector.totalTextOps(); + + List tokens = parseTokens(new PDFStreamParser(page)); + Set blankArgIndexes = new HashSet<>(); + int pageTextOps = 0; + for (int i = 0; i < tokens.size(); i++) { + Object t = tokens.get(i); + if (t instanceof Operator op && TEXT_SHOWING_OPERATORS.contains(op.getName())) { + if (dropTokenIndexes.contains(pageTextOps) && i > 0) { + blankArgIndexes.add(i - 1); + } + pageTextOps++; + } + } + + // Collector ordinals span form XObjects; if a form contributed text ops the page-stream + // ordinals no longer line up, so blanking would hit the wrong operator. + if (collectorTextOps != pageTextOps || imageIntersectsAnyRect(page, pageIndex, rects)) { + return false; + } + + for (Integer idx : blankArgIndexes) { + Object arg = tokens.get(idx); + if (arg instanceof COSString) { + tokens.set(idx, new COSString("")); + } else if (arg instanceof COSArray arr) { + COSArray empty = new COSArray(); + // Preserve numeric kerning entries so page layout doesn't shift wildly. + for (COSBase element : arr) { + if (!(element instanceof COSString)) { + empty.add(element); + } + } + tokens.set(idx, empty); + } + } + + writePageTokens(document, page, tokens); + return true; } - private static List identifyDoImagesInRects(PDPage page, List rects) { - // Conservative: we do not attempt to decode CTMs here; the overlay rectangle + Do removal - // is handled only via the rectangle overlay. Returning an empty list means Do operators - // are left alone. The overlay draw + image opaque box above the layer still hides image - // pixels in the rect. Full image physical removal is covered by convert-to-image mode. - return Collections.emptyList(); + /** + * True if any image on the page overlaps a redaction rect (would survive under the overlay). + */ + private static boolean imageIntersectsAnyRect( + PDPage page, int pageIndex, List rects) { + try { + PageImageLocator locator = new PageImageLocator(page, pageIndex); + locator.processPage(page); + for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) { + Rectangle2D.Float img = + new Rectangle2D.Float( + ib.x1(), ib.y1(), ib.x2() - ib.x1(), ib.y2() - ib.y1()); + for (PDRectangle rect : rects) { + if (img.intersects( + rect.getLowerLeftX(), + rect.getLowerLeftY(), + rect.getWidth(), + rect.getHeight())) { + return true; + } + } + } + return false; + } catch (Exception e) { + // Cannot prove no image sits under a box - treat as unsafe (force raster). + return true; + } } private static void writePageTokens(PDDocument document, PDPage page, List tokens) @@ -975,21 +953,9 @@ public final class RedactionPipeline { } } - // --------------------------------------------------------------------- // Font subsetting - // --------------------------------------------------------------------- - /** - * Warns when the document still carries embedded Type0/TrueType font programs after a - * redaction. PDFBox 3.0 has no stable API to surgically drop specific glyph ids from an - * already-loaded {@link PDType0Font} / {@link PDTrueTypeFont}, so we do not claim to subset - * them. - * - *

This is defence-in-depth only: the content-stream rewrite drops the byte codes that - * reference those glyphs, so the text is not extractable via a text stripper. Raw font-program - * inspection could still recover glyph outlines, so we log the limitation rather than pretend - * it is handled. The convert-to-image path removes this residue entirely. - */ + /** Warns when the document still carries embedded Type0/TrueType font */ private static void warnAboutEmbeddedFontGlyphs(PDDocument document) { boolean anyEmbedded = false; Set visited = new HashSet<>(); @@ -1023,9 +989,7 @@ public final class RedactionPipeline { } } - // --------------------------------------------------------------------- // Verification - // --------------------------------------------------------------------- private static void verify( byte[] bytes, @@ -1036,68 +1000,130 @@ public final class RedactionPipeline { && (patterns == null || patterns.isEmpty())) { return; } + Set pageSet = + (affectedPages == null || affectedPages.isEmpty()) + ? null + : new TreeSet<>(affectedPages); + // PDFBox pass, blind to /ActualText so a benign override can't mask real glyphs. + boolean needNativePass; try (PDDocument reopened = org.apache.pdfbox.Loader.loadPDF(bytes)) { - Set pageSet = - (affectedPages == null || affectedPages.isEmpty()) - ? null - : new TreeSet<>(affectedPages); - String extracted = extractText(reopened, pageSet); - if (extracted == null) { - return; - } - String normalised = extracted.toLowerCase(Locale.ROOT); - if (literalTargets != null) { - for (String target : literalTargets) { - if (target == null || target.isEmpty()) { - continue; - } - if (normalised.contains(target.toLowerCase(Locale.ROOT))) { - throw new RedactionVerificationFailedException( - "Redacted text still extractable: '" + target + "'"); - } - } - } - if (patterns != null) { - for (Pattern pattern : patterns) { - // Regex verification is security-critical: a pathological regex (stack - // overflow, - // catastrophic backtracking, illegal state) must NOT be treated as "no match". - // Treat any failure as a verification FAIL so the fallback path runs instead - // of silently returning clean bytes. - try { - if (pattern.matcher(extracted).find()) { - throw new RedactionVerificationFailedException( - "Redacted pattern still extractable: " + pattern.pattern()); - } - } catch (RedactionVerificationFailedException rvf) { - throw rvf; - } catch (RuntimeException | StackOverflowError e) { - log.warn( - "Verification regex '{}' threw {}; treating as verification FAIL", - pattern.pattern(), - e.toString()); - throw new RedactionVerificationFailedException( - "Verification regex failed (" - + pattern.pattern() - + "): " - + e.getMessage(), - e instanceof Exception ? (Exception) e : new Exception(e)); - } - } - } + assertNoTarget(extractText(reopened, pageSet), literalTargets, patterns); + needNativePass = documentHasUnreliableFont(reopened); } catch (IOException e) { throw new RedactionVerificationFailedException( "Failed to reopen redacted PDF for verification", e); } + // Additive producer-independent pass: native PDFium sees glyphs PDFBox may miss (fonts with + // no ToUnicode). Skipped when every font is Standard-14 or has /ToUnicode - there PDFBox + // extraction is authoritative, so the pass above already catches any leak and the (~10x + // slower) native extraction would be pure redundant cost. Never weakens the verdict. + if (needNativePass) { + assertNoTarget(extractTextJPDFium(bytes, pageSet), literalTargets, patterns); + } } /** - * Extract text from {@code pageIndexes} (1-based converted internally), or the whole document - * when {@code pageIndexes} is null. + * True if any font (on any page or nested form XObject) is not provably reliable for PDFBox + * glyph extraction - i.e. it is neither a built-in Standard-14 font nor carries a /ToUnicode + * map. These are exactly the fonts (CID/Type3/symbolic without ToUnicode) where the PDFBox pass + * can go blind, so the independent native pass earns its cost. Biased to {@code true} on any + * inspection failure so the native pass runs whenever reliability is uncertain. */ + static boolean documentHasUnreliableFont(PDDocument document) { + try { + Set visited = new HashSet<>(); + for (PDPage page : document.getPages()) { + if (resourcesHaveUnreliableFont(page.getResources(), visited, 0)) { + return true; + } + } + return false; + } catch (RuntimeException e) { + return true; + } + } + + private static boolean resourcesHaveUnreliableFont( + PDResources res, Set visited, int depth) { + if (res == null) { + return false; + } + if (depth > MAX_XOBJECT_DEPTH) { + return true; // too deeply nested to fully verify - run the native pass to be safe + } + if (!visited.add(res.getCOSObject())) { + return false; // already inspected this resource dictionary + } + for (COSName name : res.getFontNames()) { + PDFont font; + try { + font = res.getFont(name); + } catch (Exception e) { + return true; // font won't load for inspection - assume unreliable + } + if (font != null + && !font.isStandard14() + && !font.getCOSObject().containsKey(COSName.TO_UNICODE)) { + return true; + } + } + for (COSName name : res.getXObjectNames()) { + try { + if (res.getXObject(name) instanceof PDFormXObject form + && resourcesHaveUnreliableFont(form.getResources(), visited, depth + 1)) { + return true; + } + } catch (Exception e) { + return true; // can't inspect the XObject - assume unreliable + } + } + return false; + } + + /** Fail-closed match check with whitespace-normalised literals (F6b) and X2 regex semantics. */ + private static void assertNoTarget( + String extracted, Set literalTargets, List patterns) { + if (extracted == null) { + return; + } + String normalised = extracted.toLowerCase(Locale.ROOT).replaceAll("\\s+", " "); + if (literalTargets != null) { + for (String target : literalTargets) { + if (target == null || target.isEmpty()) { + continue; + } + String needle = target.toLowerCase(Locale.ROOT).replaceAll("\\s+", " "); + if (normalised.contains(needle)) { + throw new RedactionVerificationFailedException( + "Redacted text still extractable: '" + target + "'"); + } + } + } + if (patterns != null) { + for (Pattern pattern : patterns) { + try { + if (pattern.matcher(extracted).find()) { + throw new RedactionVerificationFailedException( + "Redacted pattern still extractable: " + pattern.pattern()); + } + } catch (RedactionVerificationFailedException rvf) { + throw rvf; + } catch (RuntimeException | StackOverflowError e) { + throw new RedactionVerificationFailedException( + "Verification regex failed (" + + pattern.pattern() + + "): " + + e.getMessage(), + e instanceof Exception ? (Exception) e : new Exception(e)); + } + } + } + } + + /** Extract text from pageIndexes (0-based) using an /ActualText-blind stripper. */ private static String extractText(PDDocument document, Set pageIndexes) throws IOException { - PDFTextStripper stripper = new PDFTextStripper(); + PDFTextStripper stripper = new GlyphOnlyTextStripper(); if (pageIndexes == null) { return stripper.getText(document); } @@ -1117,24 +1143,97 @@ public final class RedactionPipeline { } /** - * Scans each page of the saved bytes for surviving targets and returns the 0-based indexes of - * pages that still leak. Returns {@code null} (meaning "rasterise everything") when detection - * fails or finds nothing per-page despite the document-wide verification failure (e.g. a match - * spanning a page boundary in the concatenated extraction). + * Independent native (PDFium) extraction; null if the binding is unavailable (additive only). */ + private static String extractTextJPDFium(byte[] bytes, Set pageIndexes) { + if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) { + return null; + } + try (stirling.software.jpdfium.PdfDocument doc = + stirling.software.jpdfium.PdfDocument.open(bytes)) { + StringBuilder sb = new StringBuilder(); + int n = doc.pageCount(); + if (pageIndexes == null) { + for (int i = 0; i < n; i++) { + sb.append(jpdfiumPlainText(doc, i)).append('\n'); + } + } else { + for (Integer p : pageIndexes) { + if (p != null && p >= 0 && p < n) { + sb.append(jpdfiumPlainText(doc, p)).append('\n'); + } + } + } + return sb.toString(); + } catch (RuntimeException | Error e) { + onJpdfiumFailure(e); + return null; + } + } + + private static String jpdfiumPlainText(stirling.software.jpdfium.PdfDocument doc, int i) { + try { + return stirling.software.jpdfium.text.PdfTextExtractor.extractPage(doc, i).plainText(); + } catch (RuntimeException | Error e) { + return ""; + } + } + + /** One native open, all pages' plain text; null if the binding is unavailable. */ + private static List extractPagesJPDFium(byte[] bytes) { + if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) { + return null; + } + try (stirling.software.jpdfium.PdfDocument doc = + stirling.software.jpdfium.PdfDocument.open(bytes)) { + List pages = new ArrayList<>(); + int n = doc.pageCount(); + for (int i = 0; i < n; i++) { + pages.add(jpdfiumPlainText(doc, i)); + } + return pages; + } catch (RuntimeException | Error e) { + onJpdfiumFailure(e); + return null; + } + } + + /** + * A native-binding load error latches the pass off process-wide (warn once) so a host without + * the native stops retrying; a per-document error only skips this one document (debug). + */ + private static void onJpdfiumFailure(Throwable e) { + boolean nativeUnavailable = + e instanceof UnsatisfiedLinkError + || e instanceof NoClassDefFoundError + || e instanceof ExceptionInInitializerError + || e.getClass().getSimpleName().contains("NativeLoad"); + if (nativeUnavailable) { + jpdfiumAvailable = false; + log.warn( + "JPDFium native unavailable; redaction verification will use the PDFBox pass " + + "only: {}", + e.toString()); + } else { + log.debug("JPDFium verification skipped for this document: {}", e.toString()); + } + } + + /** Pages whose surviving text (PDFBox glyph-blind OR native PDFium) still matches a target. */ private static Set findLeakingPages( byte[] bytes, Set literalTargets, List patterns) { + List jpdfiumPages = extractPagesJPDFium(bytes); try (PDDocument reopened = org.apache.pdfbox.Loader.loadPDF(bytes)) { Set leaking = new TreeSet<>(); - PDFTextStripper stripper = new PDFTextStripper(); + GlyphOnlyTextStripper stripper = new GlyphOnlyTextStripper(); for (int i = 0; i < reopened.getNumberOfPages(); i++) { stripper.setStartPage(i + 1); stripper.setEndPage(i + 1); - String pageText = stripper.getText(reopened); - if (pageText == null || pageText.isEmpty()) { - continue; - } - if (pageLeaks(pageText, literalTargets, patterns)) { + String pdfboxText = stripper.getText(reopened); + String jpdfiumText = + jpdfiumPages != null && i < jpdfiumPages.size() ? jpdfiumPages.get(i) : ""; + String combined = (pdfboxText == null ? "" : pdfboxText) + "\n" + jpdfiumText; + if (pageLeaks(combined, literalTargets, patterns)) { leaking.add(i); } } @@ -1154,53 +1253,53 @@ public final class RedactionPipeline { private static boolean pageLeaks( String pageText, Set literalTargets, List patterns) { - String normalised = pageText.toLowerCase(Locale.ROOT); - if (literalTargets != null) { - for (String target : literalTargets) { - if (target != null - && !target.isEmpty() - && normalised.contains(target.toLowerCase(Locale.ROOT))) { - return true; - } - } + try { + assertNoTarget(pageText, literalTargets, patterns); + return false; + } catch (RedactionVerificationFailedException e) { + return true; } - if (patterns != null) { - for (Pattern pattern : patterns) { - // Mirror verify(): a throwing pattern counts as a leak so we fail closed. - try { - if (pattern.matcher(pageText).find()) { - return true; - } - } catch (RuntimeException | StackOverflowError e) { - return true; - } - } - } - return false; } - // --------------------------------------------------------------------- - // Helper types - // --------------------------------------------------------------------- + /** PDFTextStripper that strips /ActualText so verification sees the real glyph stream. */ + private static final class GlyphOnlyTextStripper extends PDFTextStripper { + private static final COSName ACTUAL_TEXT = COSName.getPDFName("ActualText"); - /** Result of a rect-driven redaction pass - text strings captured from within the rects. */ + GlyphOnlyTextStripper() throws IOException {} + + @Override + public void beginMarkedContentSequence(COSName tag, COSDictionary properties) { + COSDictionary safe = properties; + if (properties != null && properties.containsKey(ACTUAL_TEXT)) { + safe = new COSDictionary(properties); + safe.removeItem(ACTUAL_TEXT); + } + super.beginMarkedContentSequence(tag, safe); + } + } + + // Helper types + + /** Captured strings plus pages that must be rasterised (surgical removal was unreliable). */ public static final class RedactionResult { private final Set capturedStrings; + private final Set forceRasterPages; - public RedactionResult(Set capturedStrings) { + public RedactionResult(Set capturedStrings, Set forceRasterPages) { this.capturedStrings = capturedStrings == null ? Set.of() : capturedStrings; + this.forceRasterPages = forceRasterPages == null ? Set.of() : forceRasterPages; } public Set getCapturedStrings() { return capturedStrings; } + + public Set getForceRasterPages() { + return forceRasterPages; + } } - /** - * A {@link PDFTextStripper} subclass that records the ordinal index of every show-text operator - * whose glyphs intersect one of the target rects. The resulting indexes line up with the order - * in which {@link PDFStreamParser} emits text-showing operators from the same page. - */ + /** A PDFTextStripper subclass that records the ordinal index of every */ private static final class TokenIndexCollector extends PDFTextStripper { private final List rects; private final Set tokenIndexesToDrop = new HashSet<>(); @@ -1212,6 +1311,10 @@ public final class RedactionPipeline { setSortByPosition(false); } + int totalTextOps() { + return showTextOpCounter + 1; + } + @Override protected void processTextPosition(TextPosition text) { // PDFBox reports coordinates with top-left origin here. diff --git a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionVerificationFailedException.java b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionVerificationFailedException.java index c4320c7484..5a21b4a31c 100644 --- a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionVerificationFailedException.java +++ b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionVerificationFailedException.java @@ -1,10 +1,6 @@ package stirling.software.SPDF.pdf.redaction; -/** - * Thrown when a post-redaction verification pass still finds any of the target strings in the - * re-parsed PDF text. Indicates that the redaction pipeline did not fully remove the targeted - * content and the output must not be treated as safe to release. - */ +/** Thrown when post-redaction verification still finds a target in the saved PDF. */ public class RedactionVerificationFailedException extends RuntimeException { public RedactionVerificationFailedException(String message) { diff --git a/app/core/src/main/java/stirling/software/SPDF/utils/text/TextFinderUtils.java b/app/core/src/main/java/stirling/software/SPDF/utils/text/TextFinderUtils.java index 3126f90ac6..757005afb6 100644 --- a/app/core/src/main/java/stirling/software/SPDF/utils/text/TextFinderUtils.java +++ b/app/core/src/main/java/stirling/software/SPDF/utils/text/TextFinderUtils.java @@ -86,14 +86,21 @@ public class TextFinderUtils { return patterns; } - private static String applyWordBoundaries(String originalTerm, String patternString) { - if (originalTerm.length() == 1 && Character.isDigit(originalTerm.charAt(0))) { + /** Shared whole-word wrapper so the finder and RedactionPipeline never diverge (see F2). */ + public static String applyWordBoundaries(String originalTerm, String patternString) { + // Lookarounds for single/non-word-edge terms (\b no-ops on chars like '&'); \b otherwise. + boolean nonWordEdge = + originalTerm.length() == 1 + || !isWordChar(originalTerm.charAt(0)) + || !isWordChar(originalTerm.charAt(originalTerm.length() - 1)); + if (nonWordEdge) { return "(? { File f = inv.getArgument(0); - java.nio.file.Files.write(f.toPath(), "mock pdf".getBytes()); + java.nio.file.Files.write(f.toPath(), realPdfBytes); return null; }) .when(mockDocument) .save(any(File.class)); - // RedactionPipeline serialises via OutputStream before handing the bytes to TempFile, so - // the mock must emit a non-empty payload on that code path too. lenient() .doAnswer( inv -> { java.io.OutputStream os = inv.getArgument(0); - os.write("mock pdf".getBytes()); + os.write(realPdfBytes); return null; }) .when(mockDocument) diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactionPdfVarietyTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactionPdfVarietyTest.java index e3b330477b..9a7ab3525b 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactionPdfVarietyTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactionPdfVarietyTest.java @@ -48,13 +48,7 @@ import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; -/** - * Security matrix for redaction across PDF shapes: standard and embedded (subset and full) fonts, - * rotated pages, TJ splits, cross-operator splits, Form XObjects, CropBox offsets, multi-page and - * case variants. Every case asserts the target is unrecoverable from the output text layer, and - * (where removal should be surgical) that neighbouring text survives. Uses the LiberationSans TTF - * bundled inside the PDFBox jar so the embedded-font cases run on any CI platform. - */ +/** Security matrix for redaction across PDF shapes: standard and embedded */ @DisplayName("Redaction PDF-variety security matrix") class RedactionPdfVarietyTest { @@ -108,9 +102,7 @@ class RedactionPdfVarietyTest { } } - // ----------------------------------------------------------------------- // Matrix cases (all through the real /auto-redact controller path) - // ----------------------------------------------------------------------- @Test @DisplayName("standard Helvetica: target gone, neighbours survive") @@ -161,9 +153,7 @@ class RedactionPdfVarietyTest { @Test @DisplayName("real Type3 font (glyphs as content streams, no ToUnicode): target gone") void type3GlyphProcs() throws IOException { - // crop_test.pdf uses DejaVuSans embedded as a subset Type3 font with no ToUnicode map. - // Redacting it must not throw (Type3 font.encode() throws UnsupportedOperationException) - // and must remove the target from the extractable text layer. + // crop_test.pdf uses DejaVuSans embedded as a subset Type3 font with no byte[] input; try (InputStream in = getClass().getResourceAsStream("/redaction/type3_dejavu.pdf")) { input = in.readAllBytes(); @@ -220,7 +210,7 @@ class RedactionPdfVarietyTest { void crossOperatorSplit() throws IOException { byte[] out = autoRedact(crossOperatorPdf(), "SECRET"); assertGone(out, "SECRET"); - // Page 2 was never touched; whatever path handled page 1, page 2 text must survive. + // Page 2 was never touched; whatever path handled page 1, page 2 text must assertThat(pdfText(out)).contains("PUBLIC PAGE TWO"); } @@ -272,15 +262,12 @@ class RedactionPdfVarietyTest { assertThat(text).doesNotContainPattern("(?i)\\bcat\\b"); } - // ----------------------------------------------------------------------- // Page-scoped rasterisation (pipeline-level, deterministic) - // ----------------------------------------------------------------------- @Test @DisplayName("verification leak rasterises only the leaking page, others keep text") void leakRasterisesOnlyLeakingPage() throws IOException { - // Cross-operator split defeats the per-operand literal rewriter, so verification must - // catch the leak and rasterise page 1 only, leaving page 2 searchable. + // Cross-operator split defeats the per-operand literal rewriter byte[] input = crossOperatorPdf(); byte[] out; try (PDDocument doc = Loader.loadPDF(input)) { @@ -305,9 +292,7 @@ class RedactionPdfVarietyTest { } } - // ----------------------------------------------------------------------- // Drivers and assertions - // ----------------------------------------------------------------------- private byte[] autoRedact(byte[] pdfBytes, String target) throws IOException { factoryReturns(pdfBytes); @@ -356,9 +341,7 @@ class RedactionPdfVarietyTest { } } - // ----------------------------------------------------------------------- // PDF builders - // ----------------------------------------------------------------------- private static PDFont helvetica() { return new PDType1Font(Standard14Fonts.FontName.HELVETICA); @@ -464,7 +447,7 @@ class RedactionPdfVarietyTest { } } - /** One page whose text is emitted as a TJ array: ["public ", "SEC", -20, "RET", " end"]. */ + /** One page whose text is emitted as a TJ array: ["public ", "SEC", -20 */ private byte[] tjSplitPdf() throws IOException { try (PDDocument doc = new PDDocument()) { PDPage page = new PDPage(PDRectangle.LETTER); @@ -481,7 +464,7 @@ class RedactionPdfVarietyTest { } } - /** Page 1 shows "SEC" and "RET" as separate adjacent Tj operators; page 2 is clean. */ + /** Page 1 shows "SEC" and "RET" as separate adjacent Tj operators; page 2 */ private byte[] crossOperatorPdf() throws IOException { try (PDDocument doc = new PDDocument()) { PDPage page = new PDPage(PDRectangle.LETTER); @@ -546,8 +529,7 @@ class RedactionPdfVarietyTest { cs.beginText(); cs.setFont(font, FONT_SIZE); if (rotation != 0) { - // Real generators compensate the text matrix so text reads upright on rotated - // pages; anchor at page centre so every rotation keeps the line on-page. + // Real generators compensate the text matrix so text reads upright cs.setTextMatrix( Matrix.getRotateInstance( Math.toRadians(rotation), diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceExtraTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceExtraTest.java index 6c94dd3a24..2ee9984ff8 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceExtraTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceExtraTest.java @@ -287,8 +287,12 @@ class TextRedactionServiceExtraTest { service.createTokensWithoutTargetText(doc, page, Set.of("SECRET"), false, false); - // The deep traversal must have rewritten the inner form's content stream. - assertThat(inner.getCOSObject().containsKey(COSName.CONTENTS)).isTrue(); + // The deep traversal must have rewritten the inner form's own stream in place (F3). + String innerContent; + try (var in = inner.getContents()) { + innerContent = new String(in.readAllBytes(), StandardCharsets.ISO_8859_1); + } + assertThat(innerContent).doesNotContain("SECRET"); } } diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceMoreTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceMoreTest.java index ba0377a9e8..dab8360832 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceMoreTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceMoreTest.java @@ -226,19 +226,23 @@ class TextRedactionServiceMoreTest { } page.setContents(pageStream); - // Processing the page walks into the XObject graph; when a match is found inside - // the - // form, writeRedactedContentToXObject runs and sets a /Contents item on the form's - // COS dictionary. Asserting that item appears proves the XObject redaction path - // executed end-to-end without throwing. + // Processing the page walks into the XObject graph and rewrites the form's own + // stream body in place (F3), so the target is physically gone from the form. List tokens = service.createTokensWithoutTargetText( doc, page, Set.of("SECRET"), false, false); assertThat(tokens).isNotNull(); - assertThat(form.getCOSObject().containsKey(COSName.CONTENTS)) - .as("form XObject redaction path should have written a new content item") - .isTrue(); + String rewrittenForm; + try (var in = form.getContents()) { + rewrittenForm = + new String( + in.readAllBytes(), + java.nio.charset.StandardCharsets.ISO_8859_1); + } + assertThat(rewrittenForm) + .as("form XObject content should have the target removed in place") + .doesNotContain("SECRET"); } } } diff --git a/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineIntegrationTest.java b/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineIntegrationTest.java index e386397bda..431f11ce44 100644 --- a/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineIntegrationTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineIntegrationTest.java @@ -23,15 +23,7 @@ import org.apache.pdfbox.text.PDFTextStripper; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -/** - * Integration tests that exercise the real redaction pipeline against PDFs with characteristics - * that have tripped up the implementation on real user files: page rotation (/Rotate 90), entire - * phrases packed into one Tj operator, and standard Type1 fonts with WinAnsiEncoding. - * - *

The critical assertion for every test is that a fresh {@link PDFTextStripper} with default - * configuration run over the saved bytes does not contain the target search term - * (case-insensitive). - */ +/** Integration tests that exercise the real redaction pipeline against PDFs */ class RedactionPipelineIntegrationTest { private static byte[] loadFixture() throws Exception { @@ -52,10 +44,7 @@ class RedactionPipelineIntegrationTest { try (PDDocument doc = Loader.loadPDF(fixtureBytes)) { Set literalTargets = new LinkedHashSet<>(); literalTargets.add("Test"); - // No content-stream rewriting performed. The primary verification pass inside - // finalize must see the surviving target, switch to the rasterisation fallback, - // and return bytes that no longer extract as text. This proves the safety net is - // wired - without it the output would still contain the target. + // No content-stream rewriting performed. outputBytes = RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList()); } try (PDDocument reopened = Loader.loadPDF(outputBytes)) { @@ -73,9 +62,7 @@ class RedactionPipelineIntegrationTest { @DisplayName( "finalize with zero rewrite + synthetic upright PDF triggers RedactionVerificationFailedException only when fallback disabled") void verificationExceptionWiringSanityCheck() throws Exception { - // This test proves the verification hook itself still throws - when finalize cannot - // rasterise (we pass an already-closed document by letting finalize re-save an empty - // fresh document that still has the target text baked in). + // This test proves the verification hook itself still throws - try (PDDocument doc = new PDDocument()) { PDPage page = new PDPage(PDRectangle.A4); doc.addPage(page); @@ -88,8 +75,7 @@ class RedactionPipelineIntegrationTest { } Set literalTargets = new LinkedHashSet<>(); literalTargets.add("Smith"); - // Even with the rasterisation fallback active, the output must not contain the - // target. Either the primary pass or the fallback must succeed; both is a bug. + // Even with the rasterisation fallback active, the output must not contain byte[] outBytes = RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList()); try (PDDocument reopened = Loader.loadPDF(outBytes)) { @@ -153,8 +139,7 @@ class RedactionPipelineIntegrationTest { } ByteArrayOutputStream tmp = new ByteArrayOutputStream(); doc.save(tmp); - // reload from bytes so we are processing an already-saved PDF similar to the - // real upload flow. + // reload from bytes so we are processing an already-saved PDF similar try (PDDocument reloaded = Loader.loadPDF(tmp.toByteArray())) { RedactionPipeline.redactLiteralTerms(reloaded, literalTargets, patterns); outputBytes = RedactionPipeline.finalize(reloaded, literalTargets, patterns); diff --git a/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java b/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java index f6f9f35a36..b7734e5d5c 100644 --- a/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java @@ -2,11 +2,13 @@ package stirling.software.SPDF.pdf.redaction; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.awt.Color; import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -17,15 +19,24 @@ import java.util.Set; import java.util.regex.Pattern; import org.apache.pdfbox.Loader; +import org.apache.pdfbox.cos.COSDictionary; import org.apache.pdfbox.cos.COSName; import org.apache.pdfbox.cos.COSStream; import org.apache.pdfbox.cos.COSString; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.PDResources; import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.common.PDStream; +import org.apache.pdfbox.pdmodel.font.PDFont; +import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont; +import org.apache.pdfbox.pdmodel.font.PDType0Font; import org.apache.pdfbox.pdmodel.font.PDType1Font; import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.apache.pdfbox.pdmodel.font.encoding.WinAnsiEncoding; +import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject; +import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationHighlight; import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocumentOutline; import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem; @@ -54,7 +65,7 @@ class RedactionPipelineTest { } Map> rects = new HashMap<>(); - // The text above sits around y=700 with height ~12. Cover it fully. + // The text above sits around y=700 with height ~12. rects.put(0, List.of(new PDRectangle(90, 695, 260, 25))); RedactionPipeline.redactAreas(doc, rects, Color.BLACK); @@ -131,9 +142,7 @@ class RedactionPipelineTest { item.setTitle("See page on Smith case"); outline.addLast(item); - // AcroForm field carrying the redacted string. We set the V entry directly on the - // COS dictionary to avoid triggering AppearanceGeneratorHelper which requires a full - // /DA + /DR resource graph - CatalogScrubber is supposed to rewrite V regardless. + // AcroForm field carrying the redacted string. PDAcroForm form = new PDAcroForm(doc); doc.getDocumentCatalog().setAcroForm(form); PDTextField field = new PDTextField(form); @@ -197,8 +206,7 @@ class RedactionPipelineTest { Set targets = new LinkedHashSet<>(); targets.add("Smith"); - // No content-stream rewriting was done. The primary verification must trip and the - // rasterisation fallback must kick in so the final bytes still have no target. + // No content-stream rewriting was done. bytes = RedactionPipeline.finalize(doc, targets, Collections.emptyList()); } try (PDDocument reopened = Loader.loadPDF(bytes)) { @@ -213,10 +221,7 @@ class RedactionPipelineTest { @DisplayName( "Manual rect redaction feeds captured text into scoped verification - raster fallback kicks in when text survives") void manualRectVerificationUsesCapturedStrings() throws Exception { - // Simulate the failure mode: a rect is drawn over "LEAKED" but the content-stream rewrite - // is intentionally bypassed by only calling the CatalogScrubber path (finalize with - // targets=["LEAKED"] and affectedPages=[0]). Verification must see the surviving text and - // trigger rasterisation of page 0. Page 1 must remain text-searchable. + // Simulate the failure mode: a rect is drawn over "LEAKED" byte[] bytes; try (PDDocument doc = new PDDocument()) { PDPage p0 = new PDPage(PDRectangle.A4); @@ -316,17 +321,7 @@ class RedactionPipelineTest { @DisplayName( "Pathological verification regex triggers verification FAIL (not silent pass) and engages fallback") void pathologicalVerificationRegexFailsClosed() throws Exception { - // A regex that throws on .matcher(...).find() - build a pattern that causes a runtime - // exception by constructing a matcher over a degenerate character sequence. We simulate - // the pathological case by supplying a pattern whose matcher throws StackOverflowError via - // deep alternation. Because JVM reliably triggering SOE is tricky, we instead construct - // a pattern where .find() throws an unchecked exception using a custom Pattern subclass - // is not possible (Pattern is final). The realistic pathological case is catastrophic - // backtracking, but we cannot rely on timeouts in a unit test. Instead we verify the - // IMPORTANT invariant indirectly: when finalize is called with no content rewrite and a - // surviving target, verification must FAIL and the raster fallback must engage - this - // proves the verify path is not silently swallowing exceptions (which would return the - // unrasterised bytes). + // A regex that throws on .matcher(...).find() - build a pattern that causes byte[] bytes; try (PDDocument doc = new PDDocument()) { PDPage page = new PDPage(PDRectangle.A4); @@ -338,15 +333,13 @@ class RedactionPipelineTest { cs.showText("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!"); cs.endText(); } - // Pattern that matches the content - if regex exceptions were swallowed, verification - // would return unrasterised bytes with the match still present. + // Pattern that matches the content - if regex exceptions were swallowed List patterns = List.of(Pattern.compile("a{5,}")); bytes = RedactionPipeline.finalize(doc, Collections.emptySet(), patterns); } try (PDDocument reopened = Loader.loadPDF(bytes)) { String text = new PDFTextStripper().getText(reopened); - // Rasterisation should have eliminated text extractability entirely on the affected - // page (no affectedPages passed means whole-document rasterisation fallback). + // Rasterisation should have eliminated text extractability entirely assertFalse( Pattern.compile("a{5,}").matcher(text).find(), "Regex match must not survive verification fallback"); @@ -526,4 +519,288 @@ class RedactionPipelineTest { List regex = RedactionPipeline.buildPatterns(new String[] {"\\d{3}"}, true, false); assertTrue(regex.get(0).matcher("ID 123 issued").find()); } + + @Test + @DisplayName( + "whole-word single-char non-word target (&) matches; finder and pipeline agree (F2)") + void wholeWordSingleCharNonWordTarget() { + List pipeline = RedactionPipeline.buildPatterns(new String[] {"&"}, false, true); + assertEquals(1, pipeline.size()); + assertTrue( + pipeline.get(0).matcher("a & b").find(), + "\\b&\\b no-ops on '&'; the shared lookaround wrapper must match"); + List finder = + stirling.software.SPDF.utils.text.TextFinderUtils.createOptimizedSearchPatterns( + Set.of("&"), false, true); + assertTrue(finder.get(0).matcher("a & b").find(), "finder and pipeline must agree"); + } + + @Test + @DisplayName("form XObject text is physically removed, not left for rasterisation (F3)") + void formXObjectTextPhysicallyRemoved() throws Exception { + byte[] bytes; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + PDFormXObject form = new PDFormXObject(doc); + form.setResources(new PDResources()); + form.getResources() + .put( + COSName.getPDFName("F1"), + new PDType1Font(Standard14Fonts.FontName.HELVETICA)); + form.setBBox(new PDRectangle(0, 0, 300, 50)); + try (var out = form.getStream().createOutputStream()) { + out.write( + "BT /F1 12 Tf 0 10 Td (SECRET payload) Tj ET" + .getBytes(StandardCharsets.ISO_8859_1)); + } + PDResources pageRes = new PDResources(); + COSName fn = pageRes.add(form); + page.setResources(pageRes); + PDStream ps = new PDStream(doc); + try (var out = ps.createOutputStream()) { + out.write( + ("q 1 0 0 1 100 700 cm /" + fn.getName() + " Do Q") + .getBytes(StandardCharsets.ISO_8859_1)); + } + page.setContents(ps); + + Set targets = new LinkedHashSet<>(Set.of("SECRET")); + List pats = + RedactionPipeline.buildPatterns(new String[] {"SECRET"}, false, false); + RedactionPipeline.redactLiteralTerms(doc, targets, pats); + bytes = RedactionPipeline.finalize(doc, targets, pats); + } + try (PDDocument reopened = Loader.loadPDF(bytes)) { + assertFalse( + new PDFTextStripper().getText(reopened).contains("SECRET"), + "form XObject text must not be extractable"); + assertFalse( + pageHasImage(reopened.getPage(0)), + "removal should be surgical (form rewritten), not full rasterisation"); + } + } + + @Test + @DisplayName( + "verify() sees glyphs masked by /ActualText and rasterises the survivor (discovery)") + void actualTextMaskedSurvivorIsCaught() throws Exception { + byte[] bytes; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + PDResources res = new PDResources(); + COSName f = res.add(new PDType1Font(Standard14Fonts.FontName.HELVETICA)); + page.setResources(res); + // Real glyphs paint SECRET but /ActualText claims the run is SAFE. + String content = + "/Span <> BDC BT /" + + f.getName() + + " 12 Tf 100 700 Td (SECRET) Tj ET EMC"; + PDStream ps = new PDStream(doc); + try (var out = ps.createOutputStream()) { + out.write(content.getBytes(StandardCharsets.ISO_8859_1)); + } + page.setContents(ps); + + // No removal pass: rely on finalize's verify to catch the masked survivor. + bytes = + RedactionPipeline.finalize( + doc, new LinkedHashSet<>(Set.of("SECRET")), List.of()); + } + try (PDDocument reopened = Loader.loadPDF(bytes)) { + // A default stripper would read 'SAFE' and pass; the glyph-blind verify caught 'SECRET' + // and rasterised, so the page is now an image and nothing extractable survives. + assertTrue( + pageHasImage(reopened.getPage(0)), + "ActualText-masked survivor must be detected and rasterised"); + assertFalse(new PDFTextStripper().getText(reopened).contains("SECRET")); + } + } + + @Test + @DisplayName("scrub keeps /AP on form fields that do not contain the target (F7a)") + void untouchedFormFieldKeepsAppearance() throws Exception { + try (PDDocument doc = new PDDocument()) { + doc.addPage(new PDPage(PDRectangle.A4)); + PDAcroForm form = new PDAcroForm(doc); + doc.getDocumentCatalog().setAcroForm(form); + + PDTextField hit = new PDTextField(form); + hit.setPartialName("hit"); + hit.getCOSObject().setString(COSName.V, "paid by Smith"); + hit.getCOSObject().setItem(COSName.AP, new COSDictionary()); + form.getFields().add(hit); + + PDTextField clean = new PDTextField(form); + clean.setPartialName("clean"); + clean.getCOSObject().setString(COSName.V, "nothing sensitive"); + clean.getCOSObject().setItem(COSName.AP, new COSDictionary()); + form.getFields().add(clean); + + CatalogScrubber.scrub( + doc, new LinkedHashSet<>(Set.of("Smith")), Collections.emptyList()); + + assertNull( + hit.getCOSObject().getDictionaryObject(COSName.AP), + "matched field /AP should be dropped for regeneration"); + assertNotNull( + clean.getCOSObject().getDictionaryObject(COSName.AP), + "untouched field must keep its /AP (F7a)"); + } + } + + private static final String LIBERATION = + "/org/apache/pdfbox/resources/ttf/LiberationSans-Regular.ttf"; + + @Test + @DisplayName("verify gate skips the native pass when all fonts are Standard-14") + void gateSkipsForStandard14() throws Exception { + byte[] bytes; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(72, 700); + cs.showText("plain Helvetica text"); + cs.endText(); + } + ByteArrayOutputStream b = new ByteArrayOutputStream(); + doc.save(b); + bytes = b.toByteArray(); + } + try (PDDocument reopened = Loader.loadPDF(bytes)) { + assertFalse( + RedactionPipeline.documentHasUnreliableFont(reopened), + "Standard-14 fonts extract reliably in PDFBox; native pass is redundant"); + } + } + + @Test + @DisplayName("verify gate runs the native pass for an embedded font without /ToUnicode") + void gateRunsForNonStandardFontWithoutToUnicode() throws Exception { + byte[] bytes; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + PDFont font; + try (var ttf = PDDocument.class.getResourceAsStream(LIBERATION)) { + font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE); + } + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(font, 12); + cs.newLineAtOffset(72, 700); + cs.showText("embedded TrueType no ToUnicode"); + cs.endText(); + } + ByteArrayOutputStream b = new ByteArrayOutputStream(); + doc.save(b); + bytes = b.toByteArray(); + } + try (PDDocument reopened = Loader.loadPDF(bytes)) { + assertFalse( + reopened.getPage(0) + .getResources() + .getFontNames() + .iterator() + .next() + .getName() + .isEmpty()); + assertTrue( + RedactionPipeline.documentHasUnreliableFont(reopened), + "embedded non-Standard-14 font without /ToUnicode needs the native pass"); + } + } + + @Test + @DisplayName("verify gate skips the native pass when a Type0 font carries /ToUnicode") + void gateSkipsForType0WithToUnicode() throws Exception { + byte[] bytes; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + PDFont font; + try (var ttf = PDDocument.class.getResourceAsStream(LIBERATION)) { + font = PDType0Font.load(doc, ttf, false); + } + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(font, 12); + cs.newLineAtOffset(72, 700); + cs.showText("full Type0 with ToUnicode"); + cs.endText(); + } + ByteArrayOutputStream b = new ByteArrayOutputStream(); + doc.save(b); + bytes = b.toByteArray(); + } + try (PDDocument reopened = Loader.loadPDF(bytes)) { + assertFalse( + RedactionPipeline.documentHasUnreliableFont(reopened), + "a /ToUnicode map makes PDFBox extraction authoritative; native pass redundant"); + } + } + + @Test + @DisplayName("verify gate recurses into form XObjects to find an unreliable font") + void gateDetectsUnreliableFontInsideXObject() throws Exception { + byte[] bytes; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + // Page-level font is reliable (Standard-14); the risky font hides in a form XObject. + PDResources pageRes = new PDResources(); + pageRes.put( + COSName.getPDFName("PF"), + new PDType1Font(Standard14Fonts.FontName.HELVETICA)); + + PDFormXObject form = new PDFormXObject(doc); + PDResources formRes = new PDResources(); + PDFont embedded; + try (var ttf = PDDocument.class.getResourceAsStream(LIBERATION)) { + embedded = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE); + } + formRes.put(COSName.getPDFName("F1"), embedded); + form.setResources(formRes); + form.setBBox(new PDRectangle(0, 0, 200, 50)); + try (var out = form.getStream().createOutputStream()) { + out.write( + "BT /F1 12 Tf 0 10 Td (hidden) Tj ET" + .getBytes(StandardCharsets.ISO_8859_1)); + } + COSName formName = pageRes.add(form); + page.setResources(pageRes); + PDStream ps = new PDStream(doc); + try (var out = ps.createOutputStream()) { + out.write( + ("q 1 0 0 1 50 700 cm /" + formName.getName() + " Do Q") + .getBytes(StandardCharsets.ISO_8859_1)); + } + page.setContents(ps); + ByteArrayOutputStream b = new ByteArrayOutputStream(); + doc.save(b); + bytes = b.toByteArray(); + } + try (PDDocument reopened = Loader.loadPDF(bytes)) { + assertTrue( + RedactionPipeline.documentHasUnreliableFont(reopened), + "an unreliable font inside a form XObject must be detected by recursion"); + } + } + + private static boolean pageHasImage(PDPage page) throws Exception { + PDResources res = page.getResources(); + if (res == null) { + return false; + } + for (COSName n : res.getXObjectNames()) { + if (res.getXObject(n) instanceof PDImageXObject) { + return true; + } + } + return false; + } } diff --git a/testing/cucumber/features/redact.feature b/testing/cucumber/features/redact.feature index 8af3fb7a6b..5c1cb43f47 100644 --- a/testing/cucumber/features/redact.feature +++ b/testing/cucumber/features/redact.feature @@ -1,9 +1,6 @@ @security @redact Feature: PDF redaction physically removes text - Redaction must destroy the underlying text, not merely cover it with a box. - These scenarios push known text through the redaction endpoints and assert the - target is gone from the extracted text layer (and catalog carriers) while other - text survives. + Target text must be gone from the extracted text layer, not just covered by a box. Scenario: Auto-redact physically removes the target word Given I generate a PDF file as "fileInput" diff --git a/testing/cucumber/features/steps/step_definitions.py b/testing/cucumber/features/steps/step_definitions.py index c5a1005099..d9d8cd60a4 100644 --- a/testing/cucumber/features/steps/step_definitions.py +++ b/testing/cucumber/features/steps/step_definitions.py @@ -17,8 +17,7 @@ from PIL import Image, ImageDraw API_HEADERS = {"X-API-KEY": "123456789"} -# Base URL of the backend under test. Defaults to the CI value; override with -# STIRLING_BASE_URL to point at a backend on another port (e.g. a local sidecar run). +# Backend under test; override STIRLING_BASE_URL to target another port. BASE_URL = os.environ.get("STIRLING_BASE_URL", "http://localhost:8080") #########