From 4e8dd185947c7ed956e9c8ff925249d5211f0066 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:28:39 +0100 Subject: [PATCH] Fix redaction review leaks and clean up comments --- .../api/security/ManualRedactionService.java | 80 ++++++---- .../api/security/RedactController.java | 8 +- .../api/security/RedactExecuteService.java | 109 +++++--------- .../api/security/TextRedactionService.java | 15 +- .../SPDF/pdf/redaction/CatalogScrubber.java | 132 ++++++++++++++--- .../pdf/redaction/DeadlineCharSequence.java | 52 +++++++ .../SPDF/pdf/redaction/RedactionPipeline.java | 139 +++++++++++------- .../pdf/redaction/RedactionPipelineTest.java | 105 ++++++++++++- 8 files changed, 447 insertions(+), 193 deletions(-) create mode 100644 app/core/src/main/java/stirling/software/SPDF/pdf/redaction/DeadlineCharSequence.java diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java index 1aaa05edc5..1657a7dbf6 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java @@ -43,9 +43,7 @@ class ManualRedactionService { private final TempFileManager tempFileManager; - // ----------------------------------------------------------------------- // Area and page redaction - // ----------------------------------------------------------------------- AreaRedactionResult redactAreas( List redactionAreas, PDDocument document, PDPageTree allPages) @@ -56,7 +54,7 @@ class ManualRedactionService { Set forceRasterPages = new LinkedHashSet<>(); if (redactionAreas == null || redactionAreas.isEmpty()) { - return new AreaRedactionResult(rectsByPage, forceRasterPages); + return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings); } Map> redactionsByPage = new HashMap<>(); @@ -88,7 +86,7 @@ class ManualRedactionService { PDPage page = allPages.get(pageIndex); float pageHeight = page.getBBox().getHeight(); - // Group rects by their decoded colour so each area keeps its own overlay tint (F9a). + // Group rects by their decoded colour so each area keeps its own overlay tint. Map> byColor = new LinkedHashMap<>(); List allRects = new ArrayList<>(); for (RedactionArea area : areasForPage) { @@ -119,7 +117,7 @@ class ManualRedactionService { "Manual area redaction captured {} text run(s) across {} page(s)", capturedStrings.size(), rectsByPage.size()); - return new AreaRedactionResult(rectsByPage, forceRasterPages); + return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings); } List redactPages( @@ -134,9 +132,7 @@ class ManualRedactionService { return new ArrayList<>(pageIndexes); } - // ----------------------------------------------------------------------- // Overlay drawing - // ----------------------------------------------------------------------- void redactFoundText( PDDocument document, @@ -170,7 +166,10 @@ class ManualRedactionService { try { contentStream.setNonStrokingColor(redactColor); - PDRectangle pageBox = page.getBBox(); + // TextPosition coords are relative to the CropBox origin. + PDRectangle crop = page.getCropBox(); + float cropX = crop.getLowerLeftX(); + float cropY = crop.getLowerLeftY(); for (PDFText block : pageBlocks) { float padding = @@ -191,8 +190,8 @@ class ManualRedactionService { } contentStream.addRect( - boxX, - pageBox.getHeight() - block.getY2() - padding, + cropX + boxX, + cropY + crop.getHeight() - block.getY2() - padding, boxWidth, block.getY2() - block.getY1() + 2 * padding); } @@ -204,8 +203,7 @@ class ManualRedactionService { } } - // Remove annotations whose bounding rect overlaps a redacted block, to prevent - // users from hovering over redacted URLs and seeing the underlying destination. + // Remove annotations whose bounding rect overlaps a redacted block, to prevent users from hovering over redacted URLs and seeing the underlying destination. try { float pageH = page.getBBox().getHeight(); List kept = new ArrayList<>(); @@ -259,14 +257,18 @@ class ManualRedactionService { continue; } PDPage page = pages.get(pageIdx); + PDRectangle crop = page.getCropBox(); + float cropX = crop.getLowerLeftX(); + float cropY = crop.getLowerLeftY(); try (PDPageContentStream cs = new PDPageContentStream( document, page, PDPageContentStream.AppendMode.APPEND, true, true)) { cs.saveGraphicsState(); cs.setNonStrokingColor(color); for (float[] box : entry.getValue()) { + // Box coords are CropBox-relative; offset by the CropBox origin (cropbox-overlay). float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4]; - cs.addRect(x1, y1, x2 - x1, y2 - y1); + cs.addRect(cropX + x1, cropY + y1, x2 - x1, y2 - y1); } cs.fill(); cs.restoreGraphicsState(); @@ -274,14 +276,9 @@ class ManualRedactionService { } } - // ----------------------------------------------------------------------- // Page element extraction - // ----------------------------------------------------------------------- - /** - * Returns bounding boxes for every text line and image on {@code page} in PDF user-space - * coordinates: {@code [x1, y1, x2, y2]} (origin bottom-left, Y increases upward). - */ + /** Returns bounding boxes for every text line and image on {@code page} in PDF user-space coordinates: {@code [x1, y1, x2, y2]} (origin bottom-left, Y increases upward). */ List extractPageElementBoxes(PDDocument document, PDPage page, int pageIndex) throws IOException { List boxes = new ArrayList<>(); @@ -300,9 +297,7 @@ class ManualRedactionService { return boxes; } - // ----------------------------------------------------------------------- // Finalization - // ----------------------------------------------------------------------- TempFile finalizeRedaction( PDDocument document, @@ -314,6 +309,30 @@ class ManualRedactionService { Set literalTargets, List patterns) throws IOException { + return finalizeRedaction( + document, + allFoundTextsByPage, + colorString, + customPadding, + convertToImage, + isTextRemovalMode, + literalTargets, + patterns, + Collections.emptySet()); + } + + /** @param geometricRasterPages 0-based pages carrying a range / image-box redaction, whose covered content (text under an overlay, or an image) is not text-removable and so must be rasterised to guarantee removal. */ + TempFile finalizeRedaction( + PDDocument document, + Map> allFoundTextsByPage, + String colorString, + float customPadding, + Boolean convertToImage, + boolean isTextRemovalMode, + Set literalTargets, + List patterns, + Set geometricRasterPages) + throws IOException { List allFoundTexts = new ArrayList<>(); for (List pageTexts : allFoundTextsByPage.values()) { @@ -337,6 +356,10 @@ class ManualRedactionService { // Strip matched glyphs, then scrub/verify/rasterise via finalize. RedactionPipeline.redactLiteralTerms(document, literalTargets, patterns); outputBytes = RedactionPipeline.finalize(document, literalTargets, patterns); + // Geometric (range/image-box) redactions can't be text-removed; rasterise their pages. + outputBytes = + RedactionPipeline.rasteriseSpecificPages( + outputBytes, geometricRasterPages, literalTargets, patterns); } return writeBytes(outputBytes, document.getNumberOfPages()); @@ -357,7 +380,10 @@ class ManualRedactionService { } else { outputBytes = RedactionPipeline.finalizeAreas( - document, areaResult.rectsByPage, areaResult.forceRasterPages); + document, + areaResult.rectsByPage, + areaResult.forceRasterPages, + areaResult.capturedStrings); } return writeBytes(outputBytes, document.getNumberOfPages()); @@ -376,19 +402,21 @@ class ManualRedactionService { return tempOut; } - // ----------------------------------------------------------------------- // Utilities - // ----------------------------------------------------------------------- - /** Redaction rectangles + pages to force-rasterise, per 0-based page index. */ + /** Redaction rectangles, pages to force-rasterise, and the text captured under the rects. */ static final class AreaRedactionResult { final Map> rectsByPage; final Set forceRasterPages; + final Set capturedStrings; AreaRedactionResult( - Map> rectsByPage, Set forceRasterPages) { + Map> rectsByPage, + Set forceRasterPages, + Set capturedStrings) { this.rectsByPage = rectsByPage; this.forceRasterPages = forceRasterPages; + this.capturedStrings = capturedStrings; } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java index 8fe26a841c..d298e53bbc 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java @@ -191,8 +191,7 @@ public class RedactController { (useRegex || wholeWordSearchBool) ? Collections.emptySet() : literalTargets; if (allFoundTextsByPage.isEmpty()) { - // No page hit, but the target may live in a bookmark/annotation/form/JS carrier, so - // still run the scrub + verify path rather than just wiping metadata (F5). + // No page hit, but the target may live in a bookmark/annotation/form/JS carrier, so still run the scrub + verify path rather than just wiping metadata. log.info("No page text matched; scrubbing catalog carriers and verifying"); TempFile finalized = manualRedactionService.finalizeRedaction( @@ -266,11 +265,10 @@ public class RedactController { throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e); } finally { + // Both are distinct PDDocument handles. if (document != null) { try { - if (fallbackDocument == null) { - document.close(); - } + document.close(); } catch (IOException e) { log.warn("Failed to close main document: {}", e.getMessage()); } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java index 6bfa97abcd..4ece438479 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java @@ -7,6 +7,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -93,8 +94,7 @@ class RedactExecuteService { try { document = pdfDocumentFactory.load(request.getFileInput()); - // Single-pass text scan: collect all text-based targets so we run the PDF - // stripper only once across the entire execute() call rather than once per target. + // Single-pass text scan: collect all text-based targets so we run the PDF stripper only once across the entire execute() call rather than once per target. Map> foundTexts = hasTextOps ? collectTextMatches(document, request) : new HashMap<>(); @@ -104,13 +104,13 @@ class RedactExecuteService { totalMatches, foundTexts.size()); - // Text removal (content-stream rewriting) — skipped in overlay-only mode. + // Text removal (content-stream rewriting) - skipped in overlay-only mode. boolean needsOverlayOnly = overlayOnly; if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) { needsOverlayOnly = applyTextRemoval(document, request); } else if (overlayOnly) { log.info( - "[redact/execute] overlay-only mode requested — skipping content-stream rewriting"); + "[redact/execute] overlay-only mode requested - skipping content-stream rewriting"); } // Reload fresh document on fallback so we overlay onto clean content. @@ -131,12 +131,14 @@ class RedactExecuteService { applyPageWipe(document, wipePages, style); } + // Range + image-box redactions are geometric overlays. + Set geometricRasterPages = new HashSet<>(); for (TextRange range : ranges) { - applyRangeRedaction(document, range, style, layoutCache); + geometricRasterPages.addAll(applyRangeRedaction(document, range, style, layoutCache)); } for (ImageBox box : imageBoxes) { - applyImageBoxRedaction(document, box, style); + geometricRasterPages.addAll(applyImageBoxRedaction(document, box, style)); } if (request.getRedactImagePages() != null) { @@ -166,7 +168,8 @@ class RedactExecuteService { convertToImage, !needsOverlayOnly, finalizeTargets, - finalizePatterns); + finalizePatterns, + geometricRasterPages); } catch (Exception e) { log.error("Execute redaction failed: {}", e.getMessage(), e); @@ -182,14 +185,9 @@ class RedactExecuteService { } } - // ----------------------------------------------------------------------- // Single-pass text scan (one stripper pass per execute() call) - // ----------------------------------------------------------------------- - /** - * Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit - * map. - */ + /** Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit map. */ private Map> collectTextMatches( PDDocument document, RedactExecuteRequest request) { Map> found = new HashMap<>(); @@ -217,14 +215,9 @@ class RedactExecuteService { return found; } - // ----------------------------------------------------------------------- // Text removal (content-stream rewriting) - // ----------------------------------------------------------------------- - /** - * Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the - * document fell back to overlay-only mode. - */ + /** Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the document fell back to overlay-only mode. */ private boolean applyTextRemoval(PDDocument document, RedactExecuteRequest request) { try { boolean fallback = false; @@ -253,7 +246,7 @@ class RedactExecuteService { if (fallback) { log.warn( - "[redact/execute] font compatibility issue — falling back to overlay-only"); + "[redact/execute] font compatibility issue - falling back to overlay-only"); } else { log.info("[redact/execute] content-stream text removal applied successfully"); } @@ -266,9 +259,7 @@ class RedactExecuteService { } } - // ----------------------------------------------------------------------- // Per-operation dispatch methods - // ----------------------------------------------------------------------- private void applyPageWipe(PDDocument document, List pageNumbers, RedactStyle style) throws IOException { @@ -324,7 +315,8 @@ class RedactExecuteService { } } - private void applyRangeRedaction( + /** @return 0-based pages covered by the range overlay; those pages must be rasterised. */ + private Set applyRangeRedaction( PDDocument document, TextRange range, RedactStyle style, @@ -342,6 +334,11 @@ class RedactExecuteService { style.getPadding(), ManualRedactionService.decodeOrDefault(style.getColor()), false); + Set pages = new HashSet<>(); + for (PDFText block : blocks) { + pages.add(block.getPageIndex()); + } + return pages; } else { log.warn( "[redact/execute] range not found: start='{}' end='{}'", @@ -351,9 +348,11 @@ class RedactExecuteService { } catch (Exception e) { log.warn("[redact/execute] range redaction failed: {}", e.getMessage()); } + return Collections.emptySet(); } - private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style) + /** @return 0-based page covered by the image-box overlay; that page must be rasterised. */ + private Set applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style) throws IOException { List boxes = List.of( @@ -363,6 +362,7 @@ class RedactExecuteService { log.info("[redact/execute] image box overlay on page {}", box.pageIndex()); Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor()); manualRedactionService.redactImageBoxes(document, boxes, boxColor); + return Set.of(box.pageIndex()); } private void applyAllImagesRedaction( @@ -406,18 +406,9 @@ class RedactExecuteService { } } - // ----------------------------------------------------------------------- // Range collection helpers - // ----------------------------------------------------------------------- - /** - * Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text - * line and image from that point up to (but NOT including) the line where {@code endStr} - * begins. If {@code endStr} is blank, redacts from {@code startStr} to the end of the document. - * - *

Multi-column pages follow reading order: down the start column, jump to the top of the - * next column, continue to the end anchor. Single-column pages reduce to a plain Y-band check. - */ + /** Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text line and image from that point up to (but NOT including) the line where {@code endStr} begins. If {@code endStr} is blank, redacts from {@code startStr} to the end of the document.

Multi-column pages follow reading order: down the start column, jump to the top of the next column, continue to the end anchor. Single-column pages reduce to a plain Y-band check. */ List collectRangeBlocks( PDDocument document, String startStr, @@ -478,7 +469,7 @@ class RedactExecuteService { } if (end == null) { log.warn( - "[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping", + "[redact/execute] no end anchor after start at (page={}, col={}, y={}) - skipping", start.page + 1, start.col, start.y); @@ -499,19 +490,14 @@ class RedactExecuteService { } log.info( - "[redact/execute] range '{}'→'{}': {} total blocks", + "[redact/execute] range '{}'->'{}': {} total blocks", startStr, openEnded ? "" : endStr, blocks.size()); return blocks; } - /** - * Collects all redactable content (text line segments and images) between two anchor positions. - * - *

Line boxes are cached per page number in {@code lineBoxCache} and reused across range - * iterations within one execute() call, avoiding redundant {@link AllTextLineExtractor} passes. - */ + /** Collects all redactable content (text line segments and images) between two anchor positions.

Line boxes are cached per page number in {@code lineBoxCache} and reused across range iterations within one execute() call, avoiding redundant {@link AllTextLineExtractor} passes. */ private void collectBlocksForRange( PDDocument document, PDPageTree allPages, @@ -531,7 +517,6 @@ class RedactExecuteService { float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2(); // Line-box cache: populated lazily per page, reused across range iterations. - // Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException. Map> lineBoxCache = new HashMap<>(); for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) { @@ -601,10 +586,7 @@ class RedactExecuteService { blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, "")); } - /** - * Reading-order predicate: true when (col, yBottom) on page {@code pageIdx} sits between the - * start anchor (inclusive) and end anchor (inclusive). - */ + /** Reading-order predicate: true when (col, yBottom) on page {@code pageIdx} sits between the start anchor (inclusive) and end anchor (inclusive). */ static boolean inColumnZone( int pageIdx, int col, @@ -697,16 +679,13 @@ class RedactExecuteService { if (i > 0) sb.append(", "); sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y)); } - if (anchors.size() > max) sb.append(", …"); + if (anchors.size() > max) sb.append(", ..."); return sb.toString(); } private record Anchor(int page, int col, float y, PDFText text) {} - /** - * Tries progressively more permissive variants: raw (regex then literal), letter-spacing - * collapsed, then a punctuation-tolerant regex over alphanumeric runs. - */ + /** Tries progressively more permissive variants: raw (regex then literal), letter-spacing collapsed, then a punctuation-tolerant regex over alphanumeric runs. */ private Map> findWithFallbacks(PDDocument document, String raw) { String trimmed = raw.trim(); String collapsed = collapseLetterSpacing(trimmed); @@ -723,9 +702,7 @@ class RedactExecuteService { candidates.add(new Candidate(tolerant, true)); } - // If the anchor spans multiple lines (model provided entire paragraph instead of a short - // phrase), try just the first non-empty line — it's usually sufficient to locate the - // position and avoids mismatches from mid-paragraph text extraction artifacts. + // Long multi-line anchors mismatch on extraction artifacts, so try just the first non-empty line. if (trimmed.contains("\n")) { String firstLine = Arrays.stream(trimmed.split("\n")) @@ -753,7 +730,7 @@ class RedactExecuteService { if (!m.isEmpty()) { if (!c.pattern.equals(trimmed)) { log.info( - "[redact/execute] range boundary matched via fallback: '{}' → '{}'", + "[redact/execute] range boundary matched via fallback: '{}' -> '{}'", trimmed, c.pattern); } @@ -765,14 +742,9 @@ class RedactExecuteService { private record Candidate(String pattern, boolean useRegex) {} - // ----------------------------------------------------------------------- // Static helpers - // ----------------------------------------------------------------------- - /** - * Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation - * drift. Returns {@code null} when fewer than two tokens exist. - */ + /** Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation drift. Returns {@code null} when fewer than two tokens exist. */ private static String punctuationTolerantRegex(String raw) { List tokens = new ArrayList<>(); StringBuilder current = new StringBuilder(); @@ -795,13 +767,7 @@ class RedactExecuteService { return out.toString(); } - /** - * Collapses letter-spaced text produced by position-sorted text extraction. - * - *

When a PDF text stripper runs with {@code setSortByPosition(true)}, letter-spaced headings - * come out as {@code "T a b l e o f c o n t e n t s"}. This method converts the spaced form - * back to words. - */ + /** Collapses letter-spaced text produced by position-sorted text extraction.

When a PDF text stripper runs with {@code setSortByPosition(true)}, letter-spaced headings come out as {@code "T a b l e o f c o n t e n t s"}. This method converts the spaced form back to words. */ private static String collapseLetterSpacing(String text) { String[] tokens = text.split(" ", -1); StringBuilder result = new StringBuilder(); @@ -847,10 +813,7 @@ class RedactExecuteService { .toArray(String[]::new); } - /** - * Converts 1-based page numbers from the request to the 0-based indices used internally. - * Out-of-range and non-positive values are silently dropped. - */ + /** Converts 1-based page numbers from the request to the 0-based indices used internally. Out-of-range and non-positive values are silently dropped. */ private static List toZeroBasedIndices(List oneBasedPageNumbers) { if (oneBasedPageNumbers == null || oneBasedPageNumbers.isEmpty()) { return new ArrayList<>(); diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java index 5bb25a5ca7..9d6c91860f 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java @@ -49,9 +49,7 @@ class TextRedactionService { private static final Set TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\""); private static final COSString EMPTY_COS_STRING = new COSString(""); - // ----------------------------------------------------------------------- // Public API - // ----------------------------------------------------------------------- Map> findTextToRedact( PDDocument document, String[] listOfText, boolean useRegex, boolean wholeWordSearch) { @@ -135,9 +133,7 @@ class TextRedactionService { } } - // ----------------------------------------------------------------------- // Content stream manipulation - // ----------------------------------------------------------------------- List createTokensWithoutTargetText( PDDocument document, @@ -271,9 +267,7 @@ class TextRedactionService { } } - // ----------------------------------------------------------------------- // Placeholder creation - // ----------------------------------------------------------------------- String createPlaceholderWithFont(String originalWord, PDFont font) { if (originalWord == null || originalWord.isEmpty()) { @@ -410,9 +404,7 @@ class TextRedactionService { } } - // ----------------------------------------------------------------------- // Width calculation - // ----------------------------------------------------------------------- private float safeGetStringWidth(PDFont font, String text) { if (font == null || text == null || text.isEmpty()) { @@ -601,9 +593,7 @@ class TextRedactionService { } } - // ----------------------------------------------------------------------- // Token and segment operations - // ----------------------------------------------------------------------- private void processPageXObjects( PDDocument document, @@ -706,8 +696,7 @@ class TextRedactionService { private void writeRedactedContentToXObject( PDDocument document, PDFormXObject formXObject, List redactedTokens) throws IOException { - // A form XObject's content IS its own stream body; overwrite it in place. Attaching a - // /Contents key would be a no-op (its content is not read from /Contents). + // A form XObject's content IS its own stream body; overwrite it in place. PDStream formStream = new PDStream(formXObject.getCOSObject()); try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) { ContentStreamWriter writer = new ContentStreamWriter(out); @@ -1165,9 +1154,7 @@ class TextRedactionService { }; } - // ----------------------------------------------------------------------- // Inner data classes - // ----------------------------------------------------------------------- @Data private static class GraphicsState { diff --git a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java index 05996d86e0..86422bdee6 100644 --- a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java +++ b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/CatalogScrubber.java @@ -2,6 +2,7 @@ package stirling.software.SPDF.pdf.redaction; import java.util.Calendar; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Set; @@ -33,6 +34,9 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public final class CatalogScrubber { + // A literal shorter than this won't delete a whole carrier (JS/XFA/action/embedded file). + private static final int MIN_CARRIER_DROP_LITERAL = 3; + private CatalogScrubber() {} /** Scrub all catalog-level carriers of the given literal/regex targets. */ @@ -78,12 +82,26 @@ public final class CatalogScrubber { if (value == null) { return; } - if (containsTarget(value, targets, patterns, new HashSet<>())) { + if (containsTarget(value, carrierDropLiterals(targets), patterns, new HashSet<>())) { log.debug("Removing catalog {} due to target match", key.getName()); parent.removeItem(key); } } + /** Literals specific enough to justify DELETING an entire carrier (JS / XFA / action / embedded file). Sub-threshold literals (a single digit, a 2-char run) would nuke unrelated carriers, so they are excluded from whole-carrier drops - they are still removed in-string by stripMatches where the carrier is a plain string. */ + private static Set carrierDropLiterals(Set targets) { + if (targets == null) { + return java.util.Collections.emptySet(); + } + Set specific = new LinkedHashSet<>(); + for (String t : targets) { + if (t != null && t.trim().length() >= MIN_CARRIER_DROP_LITERAL) { + specific.add(t); + } + } + return specific; + } + private static boolean containsTarget( COSBase base, Set targets, List patterns, Set seen) { if (base == null) { @@ -112,7 +130,7 @@ public final class CatalogScrubber { if (matches(text, targets, patterns)) { return true; } - // Fail closed: content past the 2 MiB cap is unproven, so treat as a match (F7c). + // Fail closed: content past the 2 MiB cap is unproven, so treat as a match. return total >= buf.length && is.read() >= 0; } catch (Exception e) { log.debug("Failed to scan stream for targets: {}", e.getMessage()); @@ -153,6 +171,15 @@ public final class CatalogScrubber { info.setCreator(null); info.setProducer(null); info.setModificationDate(Calendar.getInstance()); + // Drop any non-standard custom /Info string entries (e.g. a client-added "CaseName") that the typed setters above don't cover. + COSDictionary infoDict = info.getCOSObject(); + if (infoDict != null) { + for (COSName key : new HashSet<>(infoDict.keySet())) { + if (infoDict.getDictionaryObject(key) instanceof COSString) { + infoDict.removeItem(key); + } + } + } } PDDocumentCatalog catalog = document.getDocumentCatalog(); if (catalog != null) { @@ -233,7 +260,8 @@ public final class CatalogScrubber { if (xfaBase == null) { return; } - boolean hit = containsTarget(xfaBase, targets, patterns, new HashSet<>()); + boolean hit = + containsTarget(xfaBase, carrierDropLiterals(targets), patterns, new HashSet<>()); if (hit) { // Simplest safe move: strip the XFA entry entirely. log.warn( @@ -253,13 +281,13 @@ public final class CatalogScrubber { } try { COSDictionary dict = field.getCOSObject(); - // Only touch fields whose own values actually contain a target (F7a): clearing /AP - // document-wide blanks unrelated fields in viewers that ignore /NeedAppearances. + // Only touch fields whose own values actually contain a target: clearing /AP document-wide blanks unrelated fields in viewers that ignore /NeedAppearances. boolean matched = dictValueMatches(dict, COSName.V, targets, patterns) || dictValueMatches(dict, COSName.DV, targets, patterns) || dictValueMatches(dict, COSName.getPDFName("RV"), targets, patterns) || dictValueMatches(dict, COSName.getPDFName("TU"), targets, patterns) + || buttonCaptionMatches(dict, targets, patterns) || fieldValueMatches(field, targets, patterns); if (!matched) { return; @@ -279,22 +307,74 @@ public final class CatalogScrubber { } catch (Exception e) { log.debug("Failed to rewrite field value via setValue: {}", e.getMessage()); } - // Button on-states (/Btn) cannot be rebuilt from /V by /NeedAppearances, so leave their - // /AP intact and rely on value scrubbing; drop /AP only for matched text fields. if (!isButtonField(dict)) { clearWidgetAppearances(dict); + } else if (scrubButtonCaption(dict, targets, patterns)) { + // A button caption (/MK /CA,/RC,/AC) that carried the target renders via /AP that /NeedAppearances can't rebuild, so drop the stale /AP too. + clearWidgetAppearances(dict); } } catch (Exception e) { log.debug("Failed to scrub field: {}", e.getMessage()); } } + private static final String[] MK_CAPTION_KEYS = {"CA", "RC", "AC"}; + + /** True if a button widget's /MK caption (down/rollover/alternate) contains a target. */ + private static boolean buttonCaptionMatches( + COSDictionary dict, Set targets, List patterns) { + for (COSDictionary mk : mkDicts(dict)) { + for (String k : MK_CAPTION_KEYS) { + if (dictValueMatches(mk, COSName.getPDFName(k), targets, patterns)) { + return true; + } + } + } + return false; + } + + /** Strip target substrings from a button widget's /MK captions; true if any matched. */ + private static boolean scrubButtonCaption( + COSDictionary dict, Set targets, List patterns) { + boolean hit = false; + for (COSDictionary mk : mkDicts(dict)) { + for (String k : MK_CAPTION_KEYS) { + COSName key = COSName.getPDFName(k); + if (dictValueMatches(mk, key, targets, patterns)) { + hit = true; + scrubDictStrings(mk, key, targets, patterns); + } + } + } + return hit; + } + + /** The /MK appearance-characteristics dicts on a field and any widget kids. */ + private static List mkDicts(COSDictionary dict) { + List out = new java.util.ArrayList<>(); + if (dict.getDictionaryObject(COSName.getPDFName("MK")) instanceof COSDictionary mk) { + out.add(mk); + } + if (dict.getDictionaryObject(COSName.KIDS) instanceof COSArray kids) { + for (int i = 0; i < kids.size(); i++) { + if (kids.getObject(i) instanceof COSDictionary kid + && kid.getDictionaryObject(COSName.getPDFName("MK")) + instanceof COSDictionary mk) { + out.add(mk); + } + } + } + return out; + } + private static boolean isButtonField(COSDictionary dict) { - COSName ft = (COSName) dict.getDictionaryObject(COSName.FT); + COSBase ftBase = dict.getDictionaryObject(COSName.FT); + COSName ft = ftBase instanceof COSName n ? n : null; if (ft == null) { COSBase parent = dict.getDictionaryObject(COSName.PARENT); - if (parent instanceof COSDictionary p) { - ft = (COSName) p.getDictionaryObject(COSName.FT); + if (parent instanceof COSDictionary p + && p.getDictionaryObject(COSName.FT) instanceof COSName pn) { + ft = pn; } } return COSName.getPDFName("Btn").equals(ft); @@ -350,6 +430,9 @@ public final class CatalogScrubber { PDDocument document, Set targets, List patterns) { try { for (PDPage page : document.getPages()) { + // Page-level additional actions (/AA - open/close JS) can carry the target too. + scrubActionIfMatching( + page.getCOSObject(), COSName.getPDFName("AA"), targets, patterns); List annotations; try { annotations = page.getAnnotations(); @@ -387,6 +470,9 @@ public final class CatalogScrubber { scrubDictStrings(dict, COSName.getPDFName("Subj"), targets, patterns); scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns); scrubDictStrings(dict, COSName.getPDFName("NM"), targets, patterns); + // Link/widget annotation actions can encode the target in a /URI or JS body. + scrubActionIfMatching(dict, COSName.A, targets, patterns); + scrubActionIfMatching(dict, COSName.getPDFName("AA"), targets, patterns); } catch (Exception e) { log.debug("Failed to scrub annotation: {}", e.getMessage()); } @@ -476,19 +562,21 @@ public final class CatalogScrubber { if (dict == null) { return; } - COSArray namesArray = (COSArray) dict.getDictionaryObject(COSName.NAMES); + // Guarded casts: a malformed /Names or /Kids must skip that carrier, not throw a swallowed ClassCastException. + COSArray namesArray = + dict.getDictionaryObject(COSName.NAMES) instanceof COSArray a ? a : null; if (namesArray != null) { for (int i = namesArray.size() - 2; i >= 0; i -= 2) { COSBase keyBase = namesArray.getObject(i); String key = keyBase instanceof COSString s ? s.getString() : null; - // Drop the pair when the KEY or the VALUE (JS /JS stream, embedded-file bytes) - // contains a target - not just the key (F7b). - boolean keyHit = key != null && matches(key, targets, patterns); + // Drop the pair when the KEY or the VALUE (JS /JS stream, embedded-file bytes) contains a target - not just the key. + Set dropLiterals = carrierDropLiterals(targets); + boolean keyHit = key != null && matches(key, dropLiterals, patterns); boolean valueHit = i + 1 < namesArray.size() && containsTarget( namesArray.getObject(i + 1), - targets, + dropLiterals, patterns, new HashSet<>()); if (keyHit || valueHit) { @@ -497,7 +585,7 @@ public final class CatalogScrubber { } } } - COSArray kids = (COSArray) dict.getDictionaryObject(COSName.KIDS); + COSArray kids = dict.getDictionaryObject(COSName.KIDS) instanceof COSArray a ? a : null; if (kids != null) { for (int i = 0; i < kids.size(); i++) { COSBase kid = kids.getObject(i); @@ -553,10 +641,9 @@ public final class CatalogScrubber { try { // Force case-insensitive matching for catalog carriers regardless Pattern ci = withCaseInsensitive(pattern); - result = ci.matcher(result).replaceAll(""); + result = ci.matcher(DeadlineCharSequence.of(result)).replaceAll(""); } catch (RuntimeException | StackOverflowError e) { - // Fail closed (X2): a throwing regex means we cannot prove the carrier clean, - // so drop the whole string rather than leaving it intact. + // Fail closed: a throwing regex means we cannot prove the carrier clean, so drop the whole string rather than leaving it intact. log.warn("Pattern replace failed for {}; dropping carrier text", pattern); return ""; } @@ -582,12 +669,13 @@ public final class CatalogScrubber { if (patterns != null) { for (Pattern pattern : patterns) { try { - if (withCaseInsensitive(pattern).matcher(source).find()) { + if (withCaseInsensitive(pattern) + .matcher(DeadlineCharSequence.of(source)) + .find()) { return true; } } catch (RuntimeException | StackOverflowError e) { - // Fail closed (X2): a throwing regex counts as a match so the carrier is - // scrubbed. + // Fail closed: a throwing regex counts as a match so the carrier is scrubbed. log.warn("Pattern match failed for {}; treating carrier as a match", pattern); return true; } diff --git a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/DeadlineCharSequence.java b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/DeadlineCharSequence.java new file mode 100644 index 0000000000..ba35336a36 --- /dev/null +++ b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/DeadlineCharSequence.java @@ -0,0 +1,52 @@ +package stirling.software.SPDF.pdf.redaction; + +/** Wraps a match subject so a runaway (catastrophic-backtracking) regex aborts instead of hanging the request thread: every {@link #charAt(int)} the matcher performs checks a wall-clock deadline and throws once it is exceeded. The thrown {@link RegexTimeoutException} is a {@link RuntimeException}, so the redaction match sites' existing fail-closed catches handle it. */ +final class DeadlineCharSequence implements CharSequence { + + /** Per-match wall-clock budget. A legitimate whole-document match finishes far inside this. */ + static final long DEFAULT_BUDGET_MILLIS = 2_000L; + + static final class RegexTimeoutException extends RuntimeException { + RegexTimeoutException(long budgetMillis) { + super("Regex evaluation exceeded " + budgetMillis + " ms (possible ReDoS)"); + } + } + + private final CharSequence inner; + private final long deadlineNanos; + private final long budgetMillis; + + private DeadlineCharSequence(CharSequence inner, long budgetMillis) { + this.inner = inner; + this.budgetMillis = budgetMillis; + this.deadlineNanos = System.nanoTime() + budgetMillis * 1_000_000L; + } + + /** Wrap {@code text} with the default budget; null becomes an empty sequence. */ + static DeadlineCharSequence of(String text) { + return new DeadlineCharSequence(text == null ? "" : text, DEFAULT_BUDGET_MILLIS); + } + + @Override + public char charAt(int index) { + if (System.nanoTime() > deadlineNanos) { + throw new RegexTimeoutException(budgetMillis); + } + return inner.charAt(index); + } + + @Override + public int length() { + return inner.length(); + } + + @Override + public CharSequence subSequence(int start, int end) { + return inner.subSequence(start, end); + } + + @Override + public String toString() { + return inner.toString(); + } +} diff --git a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java index 33bb91ad77..229a85c845 100644 --- a/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java +++ b/app/core/src/main/java/stirling/software/SPDF/pdf/redaction/RedactionPipeline.java @@ -59,16 +59,19 @@ public final class RedactionPipeline { private static final Set TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\""); private static final int MAX_XOBJECT_DEPTH = 10; - // Latched false if the native PDFium binding can't load, so a host without it (musl/exotic) - // logs once and falls back to the PDFBox pass instead of retrying the native open every redaction. + // Latched false when the native PDFium binding can't load, so the host falls back to the PDFBox pass. private static volatile boolean jpdfiumAvailable = true; - // Skip the additive native pass above this size to bound off-heap copy + native runtime on - // adversarial inputs; the PDFBox glyph-blind pass still verifies. 100 MB. + // Skip the additive native pass above this size to bound off-heap copy + native runtime on adversarial inputs; the PDFBox glyph-blind pass still verifies. private static final long MAX_JPDFIUM_VERIFY_BYTES = 100L * 1024 * 1024; private RedactionPipeline() {} + /** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */ + static void setJpdfiumAvailableForTest(boolean available) { + jpdfiumAvailable = available; + } + /** Redact page-local rects (PDF user-space), dropping intersecting glyphs + overlay. */ public static RedactionResult redactAreas( PDDocument document, @@ -88,9 +91,7 @@ public final class RedactionPipeline { PDPage page = document.getPage(pageIndex); capturedStrings.addAll(captureTextInRects(page, rects)); - // Rotation / non-zero CropBox origin break the coordinate flip, and text-bearing forms - // or in-rect images defeat page-stream ordinal removal. Draw the overlay but force the - // coordinate-independent rasterisation fallback for such pages (F1/F4/F12a). + // Rotation / non-zero CropBox origin break the coordinate flip, and text-bearing forms or in-rect images defeat page-stream ordinal removal. boolean surgical = isSurgicallySafe(page) && removeTokensIntersectingRects(document, page, rects); if (!surgical) { @@ -112,7 +113,7 @@ public final class RedactionPipeline { return crop.getLowerLeftX() == 0f && crop.getLowerLeftY() == 0f; } - /** Drop annotations whose rectangle overlaps any redaction rect (F6a). */ + /** Drop annotations whose rectangle overlaps any redaction rect. */ private static void removeOverlappingAnnotations(PDPage page, List rects) { try { List kept = @@ -162,6 +163,8 @@ public final class RedactionPipeline { page.getCOSObject().removeItem(COSName.CONTENTS); page.setResources(new PDResources()); page.getCOSObject().removeItem(COSName.ANNOTS); + // /Thumb is a rendered image of the original page - drop it too. + page.getCOSObject().removeItem(COSName.getPDFName("Thumb")); try (PDPageContentStream cs = new PDPageContentStream( @@ -257,9 +260,14 @@ public final class RedactionPipeline { public static byte[] finalizeAreas( PDDocument document, Map> rectsByPage, - Set forceRasterPages) + Set forceRasterPages, + Set capturedTargets) throws IOException { + // Text captured under the rects may also live in a bookmark / form value / annotation / JS carrier. + Set carrierTargets = + capturedTargets == null ? Collections.emptySet() : capturedTargets; + CatalogScrubber.scrub(document, carrierTargets, Collections.emptyList()); CatalogScrubber.wipeMetadata(document); warnAboutEmbeddedFontGlyphs(document); @@ -285,6 +293,23 @@ public final class RedactionPipeline { } } + /** Rasterise the given pages of already-finalized bytes to guarantee removal of geometric (range / image-box) redactions whose covered content - text under an overlay, or an image - is not removed by content-stream text redaction. Scrubs carriers and re-saves. */ + public static byte[] rasteriseSpecificPages( + byte[] bytes, Set pages, Set literalTargets, List patterns) + throws IOException { + if (pages == null || pages.isEmpty()) { + return bytes; + } + log.warn("Rasterising page(s) {} to guarantee geometric redaction removal", pages); + try (PDDocument rasterised = rasterisePages(bytes, new HashSet<>(pages))) { + CatalogScrubber.scrub(rasterised, literalTargets, patterns); + CatalogScrubber.wipeMetadata(rasterised); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + rasterised.save(out); + return out.toByteArray(); + } + } + /** Redacted pages that still show text inside a rect or carry an overlapping annotation. */ private static Set findLeakingRectPages( byte[] bytes, Map> rectsByPage) { @@ -407,7 +432,6 @@ public final class RedactionPipeline { boolean modified = rewriteTokens(tokens, formResources, patterns); if (modified) { // A form XObject's content IS its own stream body; overwrite it in place. - // Writing a /Contents key would be a no-op (F3). PDStream formStream = new PDStream(form.getCOSObject()); try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) { new ContentStreamWriter(out).writeTokens(tokens); @@ -619,7 +643,7 @@ public final class RedactionPipeline { for (Pattern pattern : patterns) { Matcher m; try { - m = pattern.matcher(text); + m = pattern.matcher(DeadlineCharSequence.of(text)); } catch (Exception ex) { continue; } @@ -650,7 +674,7 @@ public final class RedactionPipeline { dropStarts.add(decoded.codeStarts[i]); } } - // Map byte-start -> code length once (O(n)); findCodeLenAt was an O(n^2) linear scan (F10). + // Map byte-start -> code length once (O(n)); findCodeLenAt was an O(n^2) linear scan. Map lenByStart = new HashMap<>(); for (int j = 0; j < decoded.codeStarts.length; j++) { lenByStart.putIfAbsent(decoded.codeStarts[j], decoded.codeLens[j]); @@ -721,7 +745,8 @@ public final class RedactionPipeline { continue; } PDPage page = source.getPage(i); - PDRectangle media = page.getMediaBox(); + // PDFRenderer renders the CropBox region, so draw the raster over the CropBox (not the MediaBox) or a CropBox != MediaBox page is stretched/offset. + PDRectangle crop = page.getCropBox(); BufferedImage img = renderer.renderImageWithDPI(i, 150, ImageType.RGB); ByteArrayOutputStream imgOut = new ByteArrayOutputStream(); @@ -730,10 +755,11 @@ public final class RedactionPipeline { PDImageXObject.createFromByteArray( source, imgOut.toByteArray(), "redacted-page-" + i); - // Drop all prior content / resources / annotations; the raster is the page + // Drop all prior content / resources / annotations / thumbnail; the raster is the page. page.getCOSObject().removeItem(COSName.CONTENTS); page.setResources(new PDResources()); page.getCOSObject().removeItem(COSName.ANNOTS); + page.getCOSObject().removeItem(COSName.getPDFName("Thumb")); // Rotation is already baked into the rendered image, so reset it to zero. page.setRotation(0); @@ -747,10 +773,10 @@ public final class RedactionPipeline { // The rendered image already has the rotation baked in visually cs.drawImage( imageXObject, - media.getLowerLeftX(), - media.getLowerLeftY(), - media.getWidth(), - media.getHeight()); + crop.getLowerLeftX(), + crop.getLowerLeftY(), + crop.getWidth(), + crop.getHeight()); } } return source; @@ -831,11 +857,7 @@ public final class RedactionPipeline { // Content-stream rewriting (rect-driven glyph removal) - /** - * Blanks show-text operands whose glyphs fall in a rect. Returns false (surgical removal is - * unreliable, caller must rasterise) when text-bearing form XObjects skew the operator ordinals - * or an image sits under a rect - leaving the page stream untouched so the raster is correct. - */ + /** Blanks show-text operands whose glyphs fall in a rect. Returns false (surgical removal is unreliable, caller must rasterise) when text-bearing form XObjects skew the operator ordinals or an image sits under a rect - leaving the page stream untouched so the raster is correct. */ private static boolean removeTokensIntersectingRects( PDDocument document, PDPage page, List rects) throws IOException { @@ -868,8 +890,7 @@ public final class RedactionPipeline { } } - // Collector ordinals span form XObjects; if a form contributed text ops the page-stream - // ordinals no longer line up, so blanking would hit the wrong operator. + // Collector ordinals span form XObjects. if (collectorTextOps != pageTextOps || imageIntersectsAnyRect(page, pageIndex, rects)) { return false; } @@ -894,9 +915,7 @@ public final class RedactionPipeline { return true; } - /** - * True if any image on the page overlaps a redaction rect (would survive under the overlay). - */ + /** True if any image on the page overlaps a redaction rect (would survive under the overlay). */ private static boolean imageIntersectsAnyRect( PDPage page, int pageIndex, List rects) { try { @@ -1013,22 +1032,20 @@ public final class RedactionPipeline { throw new RedactionVerificationFailedException( "Failed to reopen redacted PDF for verification", e); } - // Additive producer-independent pass: native PDFium sees glyphs PDFBox may miss (fonts with - // no ToUnicode). Skipped when every font is Standard-14 or has /ToUnicode - there PDFBox - // extraction is authoritative, so the pass above already catches any leak and the (~10x - // slower) native extraction would be pure redundant cost. Never weakens the verdict. + // Additive producer-independent pass: native PDFium sees glyphs PDFBox may miss (fonts with no ToUnicode). if (needNativePass) { - assertNoTarget(extractTextJPDFium(bytes, pageSet), literalTargets, patterns); + String nativeText = extractTextJPDFium(bytes, pageSet); + if (nativeText == null) { + // Required independent pass could not run (native unavailable or doc over the size guard); fail closed. + throw new RedactionVerificationFailedException( + "Independent native verification could not run for a document whose fonts " + + "PDFBox cannot reliably extract; cannot confirm removal"); + } + assertNoTarget(nativeText, literalTargets, patterns); } } - /** - * True if any font (on any page or nested form XObject) is not provably reliable for PDFBox - * glyph extraction - i.e. it is neither a built-in Standard-14 font nor carries a /ToUnicode - * map. These are exactly the fonts (CID/Type3/symbolic without ToUnicode) where the PDFBox pass - * can go blind, so the independent native pass earns its cost. Biased to {@code true} on any - * inspection failure so the native pass runs whenever reliability is uncertain. - */ + /** True if any font (on any page or nested form XObject) is not provably reliable for PDFBox glyph extraction - i.e. it is neither a built-in Standard-14 font nor carries a /ToUnicode map. These are exactly the fonts (CID/Type3/symbolic without ToUnicode) where the PDFBox pass can go blind, so the independent native pass earns its cost. Biased to {@code true} on any inspection failure so the native pass runs whenever reliability is uncertain. */ static boolean documentHasUnreliableFont(PDDocument document) { try { Set visited = new HashSet<>(); @@ -1061,9 +1078,11 @@ public final class RedactionPipeline { } catch (Exception e) { return true; // font won't load for inspection - assume unreliable } + // Reliable only for built-in Standard-14 fonts or a /ToUnicode map we can trust. if (font != null && !font.isStandard14() - && !font.getCOSObject().containsKey(COSName.TO_UNICODE)) { + && (!font.getCOSObject().containsKey(COSName.TO_UNICODE) + || isSubsetFont(font))) { return true; } } @@ -1080,20 +1099,37 @@ public final class RedactionPipeline { return false; } - /** Fail-closed match check with whitespace-normalised literals (F6b) and X2 regex semantics. */ + /** A subset-embedded font carries a 6-uppercase-letter '+' BaseFont tag (e.g. {@code ABCDEF+}). Its /ToUnicode is custom-built and cannot be trusted for the reliability gate: a partial map silently loses text, and a crafted map can defeat text-based redaction entirely (only convert-to-image fully mitigates that adversarial case). */ + private static boolean isSubsetFont(PDFont font) { + String name = font.getName(); + if (name == null || name.length() < 8 || name.charAt(6) != '+') { + return false; + } + for (int i = 0; i < 6; i++) { + char c = name.charAt(i); + if (c < 'A' || c > 'Z') { + return false; + } + } + return true; + } + + /** Fail-closed match check with whitespace-normalised literals and X2 regex semantics. */ private static void assertNoTarget( String extracted, Set literalTargets, List patterns) { if (extracted == null) { return; } String normalised = extracted.toLowerCase(Locale.ROOT).replaceAll("\\s+", " "); + // De-hyphenated view catches a target split by a soft hyphen (U+00AD) or a line-break hyphen ("-" + space). + String dehyphenated = normalised.replace("\u00ad", "").replaceAll("-\\s+", ""); if (literalTargets != null) { for (String target : literalTargets) { if (target == null || target.isEmpty()) { continue; } String needle = target.toLowerCase(Locale.ROOT).replaceAll("\\s+", " "); - if (normalised.contains(needle)) { + if (normalised.contains(needle) || dehyphenated.contains(needle)) { throw new RedactionVerificationFailedException( "Redacted text still extractable: '" + target + "'"); } @@ -1102,7 +1138,7 @@ public final class RedactionPipeline { if (patterns != null) { for (Pattern pattern : patterns) { try { - if (pattern.matcher(extracted).find()) { + if (pattern.matcher(DeadlineCharSequence.of(extracted)).find()) { throw new RedactionVerificationFailedException( "Redacted pattern still extractable: " + pattern.pattern()); } @@ -1142,9 +1178,7 @@ public final class RedactionPipeline { return out.toString(); } - /** - * Independent native (PDFium) extraction; null if the binding is unavailable (additive only). - */ + /** Independent native (PDFium) extraction; null if the binding is unavailable (additive only). */ private static String extractTextJPDFium(byte[] bytes, Set pageIndexes) { if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) { return null; @@ -1198,10 +1232,7 @@ public final class RedactionPipeline { } } - /** - * A native-binding load error latches the pass off process-wide (warn once) so a host without - * the native stops retrying; a per-document error only skips this one document (debug). - */ + /** A native-binding load error latches the pass off process-wide (warn once) so a host without the native stops retrying; a per-document error only skips this one document (debug). */ private static void onJpdfiumFailure(Throwable e) { boolean nativeUnavailable = e instanceof UnsatisfiedLinkError @@ -1224,6 +1255,12 @@ public final class RedactionPipeline { byte[] bytes, Set literalTargets, List patterns) { List jpdfiumPages = extractPagesJPDFium(bytes); try (PDDocument reopened = org.apache.pdfbox.Loader.loadPDF(bytes)) { + // Native pass required but unavailable: PDFBox can't localise the leak, so rasterise every page. + if (jpdfiumPages == null && documentHasUnreliableFont(reopened)) { + log.warn("Independent native pass unavailable on an unreliable-font document; " + + "rasterising all pages to guarantee removal."); + return null; + } Set leaking = new TreeSet<>(); GlyphOnlyTextStripper stripper = new GlyphOnlyTextStripper(); for (int i = 0; i < reopened.getNumberOfPages(); i++) { diff --git a/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java b/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java index b7734e5d5c..a471c25e87 100644 --- a/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/pdf/redaction/RedactionPipelineTest.java @@ -609,8 +609,7 @@ class RedactionPipelineTest { doc, new LinkedHashSet<>(Set.of("SECRET")), List.of()); } try (PDDocument reopened = Loader.loadPDF(bytes)) { - // A default stripper would read 'SAFE' and pass; the glyph-blind verify caught 'SECRET' - // and rasterised, so the page is now an image and nothing extractable survives. + // A default stripper would read 'SAFE' and pass. assertTrue( pageHasImage(reopened.getPage(0)), "ActualText-masked survivor must be detected and rasterised"); @@ -791,6 +790,108 @@ class RedactionPipelineTest { } } + @Test + @DisplayName("finalizeAreas scrubs catalog carriers of the captured in-rect text (must-fix-1)") + void finalizeAreasScrubsCarriers() throws Exception { + byte[] out; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(72, 700); + cs.showText("Paid by Smith"); + cs.endText(); + } + PDDocumentOutline outline = new PDDocumentOutline(); + doc.getDocumentCatalog().setDocumentOutline(outline); + PDOutlineItem item = new PDOutlineItem(); + item.setTitle("Smith case notes"); + outline.addLast(item); + + Map> rects = new HashMap<>(); + rects.put(0, List.of(new PDRectangle(72, 695, 120, 15))); + out = + RedactionPipeline.finalizeAreas( + doc, rects, new HashSet<>(), new LinkedHashSet<>(Set.of("Smith"))); + } + try (PDDocument reopened = Loader.loadPDF(out)) { + PDDocumentOutline outline = reopened.getDocumentCatalog().getDocumentOutline(); + assertFalse( + outline.getFirstChild().getTitle().contains("Smith"), + "manual-area finalize must scrub the bookmark carrier (must-fix-1)"); + } + } + + @Test + @DisplayName("verify fails closed and rasterises when a required native pass can't run (must-fix-3)") + void failsClosedWhenNativeUnavailableForUnreliableFont() throws Exception { + RedactionPipeline.setJpdfiumAvailableForTest(false); + try { + byte[] out; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + PDFont font; + try (var ttf = PDDocument.class.getResourceAsStream(LIBERATION)) { + // embedded TrueType, no /ToUnicode -> PDFBox-unreliable -> native pass required + font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE); + } + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(font, 12); + cs.newLineAtOffset(72, 700); + cs.showText("keep me"); + cs.endText(); + } + out = RedactionPipeline.finalize(doc, new LinkedHashSet<>(Set.of("SECRET")), List.of()); + } + try (PDDocument reopened = Loader.loadPDF(out)) { + assertTrue( + pageHasImage(reopened.getPage(0)), + "native unavailable on an unreliable-font doc must fail closed -> rasterise"); + } + } finally { + RedactionPipeline.setJpdfiumAvailableForTest(true); + } + } + + @Test + @DisplayName("verify gate distrusts a subset-embedded font's /ToUnicode (must-fix-4)") + void gateRunsForSubsetFontEvenWithToUnicode() throws Exception { + byte[] bytes; + try (PDDocument doc = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + doc.addPage(page); + PDFont font; + try (var ttf = PDDocument.class.getResourceAsStream(LIBERATION)) { + font = PDType0Font.load(doc, ttf, true); // Type0 -> /ToUnicode present + } + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(font, 12); + cs.newLineAtOffset(72, 700); + cs.showText("subset text"); + cs.endText(); + } + // Force the subset tag so the test is deterministic regardless of save-time subsetting. + String tagged = "ABCDEF+" + font.getName(); + font.getCOSObject().setName(COSName.BASE_FONT, tagged); + if (font.getFontDescriptor() != null) { + font.getFontDescriptor().getCOSObject().setName(COSName.FONT_NAME, tagged); + } + ByteArrayOutputStream b = new ByteArrayOutputStream(); + doc.save(b); + bytes = b.toByteArray(); + } + try (PDDocument reopened = Loader.loadPDF(bytes)) { + assertTrue( + RedactionPipeline.documentHasUnreliableFont(reopened), + "a subset-embedded font's /ToUnicode must not be trusted (must-fix-4)"); + } + } + private static boolean pageHasImage(PDPage page) throws Exception { PDResources res = page.getResources(); if (res == null) {