Compare commits

...
Author SHA1 Message Date
Frooodle b764320e06 Merge branch 'main' into redact-true-removal 2026-07-13 09:40:58 +01:00
Frooodle 3feb757821 merge main 2026-07-10 11:17:51 +01:00
Frooodle faca7486c1 merge main 2026-07-06 19:18:04 +01:00
Anthony Stirling bc0ad7d0c3 Scrub XMP metadata surgically, keeping non-matching properties 2026-07-03 11:21:22 +01:00
Anthony Stirling dbe25402c1 Split RedactionPipeline into editor/verifier/rasteriser collaborators 2026-07-03 11:00:39 +01:00
Anthony Stirling 5400c02b81 Route redaction pattern compile through shared cache to clear regex-injection alert 2026-07-03 10:34:56 +01:00
Anthony Stirling 835863ebe9 Suppress by-design regex-injection alert on user-opted redaction regex 2026-07-03 10:28:29 +01:00
Anthony Stirling d0b4438c47 Ignore codespell false positive on Type1 ND operator 2026-07-03 10:23:37 +01:00
Anthony Stirling 7217e6ac15 Make manual-area redaction per-glyph precise and scope metadata scrubbing 2026-07-03 10:16:47 +01:00
Anthony Stirling e91d535c36 Fix no-ToUnicode redaction fixture and add font-matrix and cucumber e2e tests 2026-07-03 08:37:32 +01:00
Frooodle 7eaa54d633 fix format 2026-07-03 08:12:44 +01:00
Anthony Stirling 4e8dd18594 Fix redaction review leaks and clean up comments 2026-07-03 00:28:39 +01:00
Anthony Stirling b88cbe6bfc Harden true-removal redaction and gate independent verification 2026-07-02 22:59:58 +01:00
Anthony Stirling 18cc6dcdc8 Add true-removal redaction pipeline with verification and font-type coverage 2026-07-02 16:39:04 +01:00
30 changed files with 5435 additions and 207 deletions
@@ -5,8 +5,12 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
@@ -22,6 +26,7 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.model.PDFText;
import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest;
import stirling.software.SPDF.pdf.parser.PageImageLocator;
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
import stirling.software.common.model.api.security.RedactionArea;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.PdfUtils;
@@ -38,15 +43,18 @@ class ManualRedactionService {
private final TempFileManager tempFileManager;
// -----------------------------------------------------------------------
// Area and page redaction
// -----------------------------------------------------------------------
void redactAreas(List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
AreaRedactionResult redactAreas(
List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
throws IOException {
Set<String> capturedStrings = new LinkedHashSet<>();
Map<Integer, List<PDRectangle>> rectsByPage = new HashMap<>();
Set<Integer> forceRasterPages = new LinkedHashSet<>();
if (redactionAreas == null || redactionAreas.isEmpty()) {
return;
return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings);
}
Map<Integer, List<RedactionArea>> redactionsByPage = new HashMap<>();
@@ -74,57 +82,57 @@ class ManualRedactionService {
continue;
}
PDPage page = allPages.get(pageNumber - 1);
int pageIndex = pageNumber - 1;
PDPage page = allPages.get(pageIndex);
float pageHeight = page.getBBox().getHeight();
try (PDPageContentStream contentStream =
new PDPageContentStream(
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
contentStream.saveGraphicsState();
for (RedactionArea redactionArea : areasForPage) {
Color redactColor = decodeOrDefault(redactionArea.getColor());
contentStream.setNonStrokingColor(redactColor);
float x = redactionArea.getX().floatValue();
float y = redactionArea.getY().floatValue();
float width = redactionArea.getWidth().floatValue();
float height = redactionArea.getHeight().floatValue();
float pdfY = page.getBBox().getHeight() - y - height;
contentStream.addRect(x, pdfY, width, height);
contentStream.fill();
}
contentStream.restoreGraphicsState();
// Group rects by their decoded colour so each area keeps its own overlay tint.
Map<Color, List<PDRectangle>> byColor = new LinkedHashMap<>();
List<PDRectangle> allRects = new ArrayList<>();
for (RedactionArea area : areasForPage) {
float x = area.getX().floatValue();
float y = area.getY().floatValue();
float width = area.getWidth().floatValue();
float height = area.getHeight().floatValue();
// Request coords are top-left origin; convert to PDF user space (bottom-left).
float pdfY = pageHeight - y - height;
PDRectangle rect = new PDRectangle(x, pdfY, width, height);
allRects.add(rect);
byColor.computeIfAbsent(decodeOrDefault(area.getColor()), k -> new ArrayList<>())
.add(rect);
}
for (Map.Entry<Color, List<PDRectangle>> colorEntry : byColor.entrySet()) {
Map<Integer, List<PDRectangle>> singlePage = new HashMap<>();
singlePage.put(pageIndex, colorEntry.getValue());
RedactionPipeline.RedactionResult result =
RedactionPipeline.redactAreas(document, singlePage, colorEntry.getKey());
capturedStrings.addAll(result.getCapturedStrings());
forceRasterPages.addAll(result.getForceRasterPages());
}
rectsByPage.put(pageIndex, allRects);
}
log.debug(
"Manual area redaction captured {} text run(s) across {} page(s)",
capturedStrings.size(),
rectsByPage.size());
return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings);
}
void redactPages(ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
List<Integer> redactPages(
ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
throws IOException {
Color redactColor = decodeOrDefault(request.getPageRedactionColor());
List<Integer> pageNumbers = getPageNumbers(request, allPages.getCount());
List<Integer> pageIndexes = getPageNumbers(request, allPages.getCount());
for (Integer pageNumber : pageNumbers) {
PDPage page = allPages.get(pageNumber);
try (PDPageContentStream contentStream =
new PDPageContentStream(
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
contentStream.setNonStrokingColor(redactColor);
PDRectangle box = page.getBBox();
contentStream.addRect(0, 0, box.getWidth(), box.getHeight());
contentStream.fill();
}
}
// Whole-page wipe: drop the content stream, resources and annotations, then fill.
RedactionPipeline.redactWholePages(document, pageIndexes, redactColor);
return new ArrayList<>(pageIndexes);
}
// -----------------------------------------------------------------------
// Overlay drawing
// -----------------------------------------------------------------------
void redactFoundText(
PDDocument document,
@@ -158,7 +166,10 @@ class ManualRedactionService {
try {
contentStream.setNonStrokingColor(redactColor);
PDRectangle pageBox = page.getBBox();
// TextPosition coords are relative to the CropBox origin.
PDRectangle crop = page.getCropBox();
float cropX = crop.getLowerLeftX();
float cropY = crop.getLowerLeftY();
for (PDFText block : pageBlocks) {
float padding =
@@ -179,8 +190,8 @@ class ManualRedactionService {
}
contentStream.addRect(
boxX,
pageBox.getHeight() - block.getY2() - padding,
cropX + boxX,
cropY + crop.getHeight() - block.getY2() - padding,
boxWidth,
block.getY2() - block.getY1() + 2 * padding);
}
@@ -192,8 +203,8 @@ class ManualRedactionService {
}
}
// Remove annotations whose bounding rect overlaps a redacted block, to prevent
// users from hovering over redacted URLs and seeing the underlying destination.
// Remove annotations whose bounding rect overlaps a redacted block, to prevent users
// from hovering over redacted URLs and seeing the underlying destination.
try {
float pageH = page.getBBox().getHeight();
List<PDAnnotation> kept = new ArrayList<>();
@@ -247,14 +258,19 @@ class ManualRedactionService {
continue;
}
PDPage page = pages.get(pageIdx);
PDRectangle crop = page.getCropBox();
float cropX = crop.getLowerLeftX();
float cropY = crop.getLowerLeftY();
try (PDPageContentStream cs =
new PDPageContentStream(
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
cs.saveGraphicsState();
cs.setNonStrokingColor(color);
for (float[] box : entry.getValue()) {
// Box coords are CropBox-relative; offset by the CropBox origin
// (cropbox-overlay).
float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4];
cs.addRect(x1, y1, x2 - x1, y2 - y1);
cs.addRect(cropX + x1, cropY + y1, x2 - x1, y2 - y1);
}
cs.fill();
cs.restoreGraphicsState();
@@ -262,9 +278,7 @@ class ManualRedactionService {
}
}
// -----------------------------------------------------------------------
// Page element extraction
// -----------------------------------------------------------------------
/**
* Returns bounding boxes for every text line and image on {@code page} in PDF user-space
@@ -288,9 +302,7 @@ class ManualRedactionService {
return boxes;
}
// -----------------------------------------------------------------------
// Finalization
// -----------------------------------------------------------------------
TempFile finalizeRedaction(
PDDocument document,
@@ -298,7 +310,37 @@ class ManualRedactionService {
String colorString,
float customPadding,
Boolean convertToImage,
boolean isTextRemovalMode)
boolean isTextRemovalMode,
Set<String> literalTargets,
List<Pattern> patterns)
throws IOException {
return finalizeRedaction(
document,
allFoundTextsByPage,
colorString,
customPadding,
convertToImage,
isTextRemovalMode,
literalTargets,
patterns,
Collections.emptySet());
}
/**
* @param geometricRasterPages 0-based pages carrying a range / image-box redaction, whose
* covered content (text under an overlay, or an image) is not text-removable and so must be
* rasterised to guarantee removal.
*/
TempFile finalizeRedaction(
PDDocument document,
Map<Integer, List<PDFText>> allFoundTextsByPage,
String colorString,
float customPadding,
Boolean convertToImage,
boolean isTextRemovalMode,
Set<String> literalTargets,
List<Pattern> patterns,
Set<Integer> geometricRasterPages)
throws IOException {
List<PDFText> allFoundTexts = new ArrayList<>();
@@ -309,74 +351,84 @@ class ManualRedactionService {
if (!allFoundTexts.isEmpty()) {
Color redactColor = decodeOrDefault(colorString);
redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode);
cleanDocumentMetadata(document);
}
byte[] outputBytes;
if (Boolean.TRUE.equals(convertToImage)) {
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
cleanDocumentMetadata(convertedPdf);
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
try {
convertedPdf.save(tempOut.getFile());
} catch (IOException e) {
tempOut.close();
throw e;
}
log.info(
"Redaction finalized (image mode): {} pages ➜ {} KB",
convertedPdf.getNumberOfPages(),
tempOut.getFile().length() / 1024);
return tempOut;
// Convert-to-image physically removes all text, so verification is a plain save.
outputBytes =
RedactionPipeline.finalize(
convertedPdf, Collections.emptySet(), Collections.emptyList());
}
} else {
// Strip matched glyphs, then scrub/verify/rasterise via finalize.
RedactionPipeline.redactLiteralTerms(document, literalTargets, patterns);
outputBytes = RedactionPipeline.finalize(document, literalTargets, patterns);
// Geometric (range/image-box) redactions can't be text-removed; rasterise their pages.
outputBytes =
RedactionPipeline.rasteriseSpecificPages(
outputBytes, geometricRasterPages, literalTargets, patterns);
}
return writeBytes(outputBytes, document.getNumberOfPages());
}
/** Finalize a manual redaction; forced + verified-leaking pages are rasterised. */
TempFile finalizeManual(
PDDocument document, AreaRedactionResult areaResult, Boolean convertToImage)
throws IOException {
byte[] outputBytes;
if (Boolean.TRUE.equals(convertToImage)) {
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
outputBytes =
RedactionPipeline.finalize(
convertedPdf, Collections.emptySet(), Collections.emptyList());
}
} else {
outputBytes =
RedactionPipeline.finalizeAreas(
document,
areaResult.rectsByPage,
areaResult.forceRasterPages,
areaResult.capturedStrings);
}
return writeBytes(outputBytes, document.getNumberOfPages());
}
private TempFile writeBytes(byte[] outputBytes, int pageCount) throws IOException {
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
try {
document.save(tempOut.getFile());
java.nio.file.Files.write(tempOut.getFile().toPath(), outputBytes);
} catch (IOException e) {
tempOut.close();
throw e;
}
log.info(
"Redaction finalized: {} pages ➜ {} KB",
document.getNumberOfPages(),
tempOut.getFile().length() / 1024);
log.info("Redaction finalized: {} pages -> {} KB", pageCount, outputBytes.length / 1024);
return tempOut;
}
private void cleanDocumentMetadata(PDDocument document) {
try {
var documentInfo = document.getDocumentInformation();
if (documentInfo != null) {
documentInfo.setAuthor(null);
documentInfo.setSubject(null);
documentInfo.setKeywords(null);
documentInfo.setModificationDate(java.util.Calendar.getInstance());
log.debug("Cleaned document metadata for security");
}
// Utilities
if (document.getDocumentCatalog() != null) {
try {
document.getDocumentCatalog().setMetadata(null);
} catch (Exception e) {
log.debug("Could not clear XMP metadata: {}", e.getMessage());
}
}
/** Redaction rectangles, pages to force-rasterise, and the text captured under the rects. */
static final class AreaRedactionResult {
final Map<Integer, List<PDRectangle>> rectsByPage;
final Set<Integer> forceRasterPages;
final Set<String> capturedStrings;
} catch (Exception e) {
log.warn("Failed to clean document metadata: {}", e.getMessage());
AreaRedactionResult(
Map<Integer, List<PDRectangle>> rectsByPage,
Set<Integer> forceRasterPages,
Set<String> capturedStrings) {
this.rectsByPage = rectsByPage;
this.forceRasterPages = forceRasterPages;
this.capturedStrings = capturedStrings;
}
}
// -----------------------------------------------------------------------
// Utilities
// -----------------------------------------------------------------------
static Color decodeOrDefault(String hex) {
if (hex == null) {
return Color.BLACK;
@@ -1,9 +1,15 @@
package stirling.software.SPDF.controller.api.security;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPageTree;
@@ -29,13 +35,13 @@ import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox;
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle;
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.SecurityApi;
import stirling.software.common.enumeration.ResourceWeight;
import stirling.software.common.model.api.security.RedactionArea;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.PdfUtils;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
@@ -93,33 +99,24 @@ public class RedactController {
throws IOException {
MultipartFile file = request.getFileInput();
String filename =
removeFileExtension(
Objects.requireNonNull(
Filenames.toSimpleFileName(file.getOriginalFilename())))
+ "_redacted.pdf";
try (PDDocument document = pdfDocumentFactory.load(file)) {
PDPageTree allPages = document.getDocumentCatalog().getPages();
// Whole-page wipes drop content; areas drop glyphs + overlay, verified later.
manualRedactionService.redactPages(request, document, allPages);
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
ManualRedactionService.AreaRedactionResult areaResult =
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
if (Boolean.TRUE.equals(request.getConvertPDFToImage())) {
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
return WebResponseUtils.pdfDocToWebResponse(
convertedPdf,
removeFileExtension(
Objects.requireNonNull(
Filenames.toSimpleFileName(
file.getOriginalFilename())))
+ "_redacted.pdf",
tempFileManager);
}
}
return WebResponseUtils.pdfDocToWebResponse(
document,
removeFileExtension(
Objects.requireNonNull(
Filenames.toSimpleFileName(file.getOriginalFilename())))
+ "_redacted.pdf",
tempFileManager);
TempFile out =
manualRedactionService.finalizeManual(
document, areaResult, request.getConvertPDFToImage());
return WebResponseUtils.pdfFileToWebResponse(out, filename);
}
}
@@ -188,9 +185,32 @@ public class RedactController {
request.getFileInput().getOriginalFilename())))
+ "_redacted.pdf";
Set<String> literalTargets =
Arrays.stream(listOfText)
.map(String::trim)
.filter(s -> !s.isEmpty())
.collect(Collectors.toCollection(LinkedHashSet::new));
List<Pattern> compiledPatterns =
RedactionPipeline.buildPatterns(listOfText, useRegex, wholeWordSearchBool);
// Bare literals match substrings; regex/whole-word rely on compiled patterns.
Set<String> verificationTargets =
(useRegex || wholeWordSearchBool) ? Collections.emptySet() : literalTargets;
if (allFoundTextsByPage.isEmpty()) {
log.info("No text found matching redaction patterns");
return WebResponseUtils.pdfDocToWebResponse(document, filename, tempFileManager);
// No page hit, but the target may live in a bookmark/annotation/form/JS carrier, so
// still run the scrub + verify path rather than just wiping metadata.
log.info("No page text matched; scrubbing catalog carriers and verifying");
TempFile finalized =
manualRedactionService.finalizeRedaction(
document,
Collections.emptyMap(),
request.getRedactColor(),
request.getCustomPadding(),
request.getConvertPDFToImage(),
true,
verificationTargets,
compiledPatterns);
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
}
boolean fallbackToBoxOnlyMode;
@@ -211,7 +231,8 @@ public class RedactController {
if (fallbackToBoxOnlyMode) {
log.warn(
"Font compatibility issues detected. Using box-only redaction mode for better reliability.");
"Font compatibility issue in placeholder pass; the true-removal pass and "
+ "verification still guarantee the target is gone.");
fallbackDocument = pdfDocumentFactory.load(request.getFileInput());
@@ -226,7 +247,9 @@ public class RedactController {
request.getRedactColor(),
request.getCustomPadding(),
request.getConvertPDFToImage(),
false);
false,
verificationTargets,
compiledPatterns);
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
}
@@ -238,7 +261,9 @@ public class RedactController {
request.getRedactColor(),
request.getCustomPadding(),
request.getConvertPDFToImage(),
true);
true,
verificationTargets,
compiledPatterns);
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
@@ -247,11 +272,10 @@ public class RedactController {
throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e);
} finally {
// Both are distinct PDDocument handles.
if (document != null) {
try {
if (fallbackDocument == null) {
document.close();
}
document.close();
} catch (IOException e) {
log.warn("Failed to close main document: {}", e.getMessage());
}
@@ -7,8 +7,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.pdfbox.cos.COSName;
@@ -30,6 +33,7 @@ import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyl
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
import stirling.software.SPDF.pdf.parser.PageImageLocator;
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.TempFile;
@@ -90,8 +94,8 @@ class RedactExecuteService {
try {
document = pdfDocumentFactory.load(request.getFileInput());
// Single-pass text scan: collect all text-based targets so we run the PDF
// stripper only once across the entire execute() call rather than once per target.
// Single-pass text scan: collect all text-based targets so we run the PDF stripper only
// once across the entire execute() call rather than once per target.
Map<Integer, List<PDFText>> foundTexts =
hasTextOps ? collectTextMatches(document, request) : new HashMap<>();
@@ -101,13 +105,13 @@ class RedactExecuteService {
totalMatches,
foundTexts.size());
// Text removal (content-stream rewriting) — skipped in overlay-only mode.
// Text removal (content-stream rewriting) - skipped in overlay-only mode.
boolean needsOverlayOnly = overlayOnly;
if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) {
needsOverlayOnly = applyTextRemoval(document, request);
} else if (overlayOnly) {
log.info(
"[redact/execute] overlay-only mode requested — skipping content-stream rewriting");
"[redact/execute] overlay-only mode requested - skipping content-stream rewriting");
}
// Reload fresh document on fallback so we overlay onto clean content.
@@ -128,25 +132,46 @@ class RedactExecuteService {
applyPageWipe(document, wipePages, style);
}
// Range + image-box redactions are geometric overlays.
Set<Integer> geometricRasterPages = new HashSet<>();
for (TextRange range : ranges) {
applyRangeRedaction(document, range, style, layoutCache);
geometricRasterPages.addAll(
applyRangeRedaction(document, range, style, layoutCache));
}
for (ImageBox box : imageBoxes) {
applyImageBoxRedaction(document, box, style);
geometricRasterPages.addAll(applyImageBoxRedaction(document, box, style));
}
if (request.getRedactImagePages() != null) {
applyAllImagesRedaction(document, request.getRedactImagePages(), style);
}
// Overlay-only skips removal/verify; font-fallback overlays still pass targets.
Set<String> literalTargets = new LinkedHashSet<>();
for (String value : textValues) {
String trimmed = value == null ? "" : value.trim();
if (!trimmed.isEmpty()) {
literalTargets.add(trimmed);
}
}
List<Pattern> verificationPatterns =
RedactionPipeline.buildPatterns(
regexPatterns.toArray(new String[0]), true, false);
Set<String> finalizeTargets = overlayOnly ? Collections.emptySet() : literalTargets;
List<Pattern> finalizePatterns =
overlayOnly ? Collections.emptyList() : verificationPatterns;
return manualRedactionService.finalizeRedaction(
document,
foundTexts,
style.getColor(),
style.getPadding(),
convertToImage,
!needsOverlayOnly);
!needsOverlayOnly,
finalizeTargets,
finalizePatterns,
geometricRasterPages);
} catch (Exception e) {
log.error("Execute redaction failed: {}", e.getMessage(), e);
@@ -162,9 +187,7 @@ class RedactExecuteService {
}
}
// -----------------------------------------------------------------------
// Single-pass text scan (one stripper pass per execute() call)
// -----------------------------------------------------------------------
/**
* Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit
@@ -197,9 +220,7 @@ class RedactExecuteService {
return found;
}
// -----------------------------------------------------------------------
// Text removal (content-stream rewriting)
// -----------------------------------------------------------------------
/**
* Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the
@@ -233,7 +254,7 @@ class RedactExecuteService {
if (fallback) {
log.warn(
"[redact/execute] font compatibility issue — falling back to overlay-only");
"[redact/execute] font compatibility issue - falling back to overlay-only");
} else {
log.info("[redact/execute] content-stream text removal applied successfully");
}
@@ -246,9 +267,7 @@ class RedactExecuteService {
}
}
// -----------------------------------------------------------------------
// Per-operation dispatch methods
// -----------------------------------------------------------------------
private void applyPageWipe(PDDocument document, List<Integer> pageNumbers, RedactStyle style)
throws IOException {
@@ -304,7 +323,10 @@ class RedactExecuteService {
}
}
private void applyRangeRedaction(
/**
* @return 0-based pages covered by the range overlay; those pages must be rasterised.
*/
private Set<Integer> applyRangeRedaction(
PDDocument document,
TextRange range,
RedactStyle style,
@@ -322,6 +344,11 @@ class RedactExecuteService {
style.getPadding(),
ManualRedactionService.decodeOrDefault(style.getColor()),
false);
Set<Integer> pages = new HashSet<>();
for (PDFText block : blocks) {
pages.add(block.getPageIndex());
}
return pages;
} else {
log.warn(
"[redact/execute] range not found: start='{}' end='{}'",
@@ -331,10 +358,14 @@ class RedactExecuteService {
} catch (Exception e) {
log.warn("[redact/execute] range redaction failed: {}", e.getMessage());
}
return Collections.emptySet();
}
private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style)
throws IOException {
/**
* @return 0-based page covered by the image-box overlay; that page must be rasterised.
*/
private Set<Integer> applyImageBoxRedaction(
PDDocument document, ImageBox box, RedactStyle style) throws IOException {
List<float[]> boxes =
List.of(
new float[] {
@@ -343,6 +374,7 @@ class RedactExecuteService {
log.info("[redact/execute] image box overlay on page {}", box.pageIndex());
Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor());
manualRedactionService.redactImageBoxes(document, boxes, boxColor);
return Set.of(box.pageIndex());
}
private void applyAllImagesRedaction(
@@ -386,9 +418,7 @@ class RedactExecuteService {
}
}
// -----------------------------------------------------------------------
// Range collection helpers
// -----------------------------------------------------------------------
/**
* Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text
@@ -458,7 +488,7 @@ class RedactExecuteService {
}
if (end == null) {
log.warn(
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping",
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) - skipping",
start.page + 1,
start.col,
start.y);
@@ -479,7 +509,7 @@ class RedactExecuteService {
}
log.info(
"[redact/execute] range '{}'→'{}': {} total blocks",
"[redact/execute] range '{}'->'{}': {} total blocks",
startStr,
openEnded ? "<end of document>" : endStr,
blocks.size());
@@ -511,7 +541,6 @@ class RedactExecuteService {
float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2();
// Line-box cache: populated lazily per page, reused across range iterations.
// Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException.
Map<Integer, List<float[]>> lineBoxCache = new HashMap<>();
for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) {
@@ -677,7 +706,7 @@ class RedactExecuteService {
if (i > 0) sb.append(", ");
sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y));
}
if (anchors.size() > max) sb.append(", …");
if (anchors.size() > max) sb.append(", ...");
return sb.toString();
}
@@ -703,9 +732,8 @@ class RedactExecuteService {
candidates.add(new Candidate(tolerant, true));
}
// If the anchor spans multiple lines (model provided entire paragraph instead of a short
// phrase), try just the first non-empty line — it's usually sufficient to locate the
// position and avoids mismatches from mid-paragraph text extraction artifacts.
// Long multi-line anchors mismatch on extraction artifacts, so try just the first non-empty
// line.
if (trimmed.contains("\n")) {
String firstLine =
Arrays.stream(trimmed.split("\n"))
@@ -733,7 +761,7 @@ class RedactExecuteService {
if (!m.isEmpty()) {
if (!c.pattern.equals(trimmed)) {
log.info(
"[redact/execute] range boundary matched via fallback: '{}' → '{}'",
"[redact/execute] range boundary matched via fallback: '{}' -> '{}'",
trimmed,
c.pattern);
}
@@ -745,9 +773,7 @@ class RedactExecuteService {
private record Candidate(String pattern, boolean useRegex) {}
// -----------------------------------------------------------------------
// Static helpers
// -----------------------------------------------------------------------
/**
* Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation
@@ -46,12 +46,11 @@ class TextRedactionService {
private static final int MAX_XOBJECT_DEPTH = 10;
private static final float PRECISION_THRESHOLD = 1e-3f;
private static final int FONT_SCALE_FACTOR = 1000;
private static final Set<String> TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\"");
private static final Set<String> TEXT_SHOWING_OPERATORS =
stirling.software.SPDF.pdf.redaction.RedactionPipeline.TEXT_SHOWING_OPERATORS;
private static final COSString EMPTY_COS_STRING = new COSString("");
// -----------------------------------------------------------------------
// Public API
// -----------------------------------------------------------------------
Map<Integer, List<PDFText>> findTextToRedact(
PDDocument document, String[] listOfText, boolean useRegex, boolean wholeWordSearch) {
@@ -135,9 +134,7 @@ class TextRedactionService {
}
}
// -----------------------------------------------------------------------
// Content stream manipulation
// -----------------------------------------------------------------------
List<Object> createTokensWithoutTargetText(
PDDocument document,
@@ -271,9 +268,7 @@ class TextRedactionService {
}
}
// -----------------------------------------------------------------------
// Placeholder creation
// -----------------------------------------------------------------------
String createPlaceholderWithFont(String originalWord, PDFont font) {
if (originalWord == null || originalWord.isEmpty()) {
@@ -410,9 +405,7 @@ class TextRedactionService {
}
}
// -----------------------------------------------------------------------
// Width calculation
// -----------------------------------------------------------------------
private float safeGetStringWidth(PDFont font, String text) {
if (font == null || text == null || text.isEmpty()) {
@@ -601,9 +594,7 @@ class TextRedactionService {
}
}
// -----------------------------------------------------------------------
// Token and segment operations
// -----------------------------------------------------------------------
private void processPageXObjects(
PDDocument document,
@@ -706,16 +697,12 @@ class TextRedactionService {
private void writeRedactedContentToXObject(
PDDocument document, PDFormXObject formXObject, List<Object> redactedTokens)
throws IOException {
PDStream newStream = new PDStream(document);
try (var out = newStream.createOutputStream()) {
// A form XObject's content IS its own stream body; overwrite it in place.
PDStream formStream = new PDStream(formXObject.getCOSObject());
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
ContentStreamWriter writer = new ContentStreamWriter(out);
writer.writeTokens(redactedTokens);
}
formXObject.getCOSObject().removeItem(COSName.CONTENTS);
formXObject.getCOSObject().setItem(COSName.CONTENTS, newStream.getCOSObject());
}
private List<TextSegment> extractTextSegments(PDPage page, List<Object> tokens) {
@@ -1168,9 +1155,7 @@ class TextRedactionService {
};
}
// -----------------------------------------------------------------------
// Inner data classes
// -----------------------------------------------------------------------
@Data
private static class GraphicsState {
@@ -0,0 +1,755 @@
package stirling.software.SPDF.pdf.redaction;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSObject;
import org.apache.pdfbox.cos.COSStream;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
import org.apache.pdfbox.pdmodel.PDDocumentNameDictionary;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDMetadata;
import org.apache.pdfbox.pdmodel.common.PDNameTreeNode;
import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureTreeRoot;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocumentOutline;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
import lombok.extern.slf4j.Slf4j;
/** Removes/rewrites every catalog carrier that could leak redacted text. */
@Slf4j
public final class CatalogScrubber {
// A literal shorter than this won't delete a whole carrier (JS/XFA/action/embedded file).
private static final int MIN_CARRIER_DROP_LITERAL = 3;
private CatalogScrubber() {}
/** Scrub all catalog-level carriers of the given literal/regex targets. */
public static void scrub(
PDDocument document, Set<String> literalTargets, List<Pattern> patterns) {
if (document == null) {
return;
}
PDDocumentCatalog catalog = document.getDocumentCatalog();
if (catalog == null) {
return;
}
scrubOutline(catalog.getDocumentOutline(), literalTargets, patterns);
scrubAcroForm(catalog.getAcroForm(), literalTargets, patterns);
scrubAnnotations(document, literalTargets, patterns);
scrubStructTree(catalog.getStructureTreeRoot(), literalTargets, patterns);
scrubNames(catalog.getNames(), literalTargets, patterns);
scrubCatalogActions(catalog, literalTargets, patterns);
scrubDocumentInfo(document, literalTargets, patterns);
}
// Catalog actions: OpenAction, AA, and any JavaScript / URI payloads
private static void scrubCatalogActions(
PDDocumentCatalog catalog, Set<String> targets, List<Pattern> patterns) {
COSDictionary root = catalog.getCOSObject();
if (root == null) {
return;
}
// OpenAction may be either an action dict (with /URI or /JS) or an explicit
scrubActionIfMatching(root, COSName.getPDFName("OpenAction"), targets, patterns);
scrubActionIfMatching(root, COSName.getPDFName("AA"), targets, patterns);
}
/** Drop the action at key if any target appears in its URI/JS payload. */
private static void scrubActionIfMatching(
COSDictionary parent, COSName key, Set<String> targets, List<Pattern> patterns) {
if (parent == null || key == null) {
return;
}
COSBase value = parent.getDictionaryObject(key);
if (value == null) {
return;
}
if (containsTarget(value, carrierDropLiterals(targets), patterns, new HashSet<>())) {
log.debug("Removing catalog {} due to target match", key.getName());
parent.removeItem(key);
}
}
/**
* Literals specific enough to justify DELETING an entire carrier (JS / XFA / action / embedded
* file). Sub-threshold literals (a single digit, a 2-char run) would nuke unrelated carriers,
* so they are excluded from whole-carrier drops - they are still removed in-string by
* stripMatches where the carrier is a plain string.
*/
private static Set<String> carrierDropLiterals(Set<String> targets) {
if (targets == null) {
return java.util.Collections.emptySet();
}
Set<String> specific = new LinkedHashSet<>();
for (String t : targets) {
if (t != null && t.trim().length() >= MIN_CARRIER_DROP_LITERAL) {
specific.add(t);
}
}
return specific;
}
private static boolean containsTarget(
COSBase base, Set<String> targets, List<Pattern> patterns, Set<COSBase> seen) {
if (base == null) {
return false;
}
COSBase resolved = base instanceof COSObject obj ? obj.getObject() : base;
if (resolved == null || !seen.add(resolved)) {
return false;
}
if (resolved instanceof COSString cs) {
return matches(cs.getString(), targets, patterns);
}
if (resolved instanceof COSStream stream) {
// Streams in XFA / OpenAction contexts are text (XML, JavaScript).
try (java.io.InputStream is = stream.createInputStream()) {
byte[] buf = new byte[2 * 1024 * 1024];
int total = 0;
int n;
while ((n = is.read(buf, total, buf.length - total)) > 0) {
total += n;
if (total >= buf.length) {
break;
}
}
String text = new String(buf, 0, total, java.nio.charset.StandardCharsets.UTF_8);
if (matches(text, targets, patterns)) {
return true;
}
// Fail closed: content past the 2 MiB cap is unproven, so treat as a match.
return total >= buf.length && is.read() >= 0;
} catch (Exception e) {
log.debug("Failed to scan stream for targets: {}", e.getMessage());
// Fail closed: if we cannot read it we cannot prove it is clean, so treat
return true;
}
}
if (resolved instanceof COSDictionary dict) {
for (COSName k : new HashSet<>(dict.keySet())) {
if (containsTarget(dict.getItem(k), targets, patterns, seen)) {
return true;
}
}
return false;
}
if (resolved instanceof COSArray array) {
for (int i = 0; i < array.size(); i++) {
if (containsTarget(array.getObject(i), targets, patterns, seen)) {
return true;
}
}
return false;
}
return false;
}
/**
* Remove the redaction target from /Info string entries and drop the XMP packet only if it
* carries a target. Non-matching document properties (Title, Author, dates) are left intact -
* redaction removes the target, it does not blanket-wipe metadata.
*/
private static void scrubDocumentInfo(
PDDocument document, Set<String> targets, List<Pattern> patterns) {
PDDocumentInformation info = document.getDocumentInformation();
if (info != null && info.getCOSObject() != null) {
COSDictionary infoDict = info.getCOSObject();
for (COSName key : new HashSet<>(infoDict.keySet())) {
if (infoDict.getDictionaryObject(key) instanceof COSString cs) {
String stripped = stripMatches(cs.getString(), targets, patterns);
if (!stripped.equals(cs.getString())) {
if (stripped.isEmpty()) {
infoDict.removeItem(key);
} else {
infoDict.setString(key, stripped);
}
}
}
}
}
PDDocumentCatalog catalog = document.getDocumentCatalog();
if (catalog != null && catalog.getMetadata() != null) {
scrubXmp(document, catalog, targets, patterns);
}
}
/**
* XMP is RDF/XML that usually mirrors /Info. Strip only the target occurrences so non-matching
* properties (dates, rights, custom schema) survive; fall back to dropping the whole packet if
* the edit can't be proven to have removed the target (e.g. entity-encoded) or anything throws.
*/
private static void scrubXmp(
PDDocument document,
PDDocumentCatalog catalog,
Set<String> targets,
List<Pattern> patterns) {
String xmp;
try {
xmp = new String(catalog.getMetadata().toByteArray(), StandardCharsets.UTF_8);
} catch (Exception e) {
log.debug("Could not read XMP metadata: {}", e.getMessage());
return;
}
if (!matches(xmp, targets, patterns)) {
return;
}
try {
String stripped = stripMatches(xmp, targets, patterns);
// Only keep the edited packet if the target is provably gone from it.
if (!matches(stripped, targets, patterns)) {
catalog.setMetadata(
new PDMetadata(
document,
new ByteArrayInputStream(
stripped.getBytes(StandardCharsets.UTF_8))));
return;
}
} catch (Exception e) {
log.debug("Surgical XMP scrub failed; dropping packet: {}", e.getMessage());
}
catalog.setMetadata(null);
}
// Outline
private static void scrubOutline(
PDDocumentOutline outline, Set<String> targets, List<Pattern> patterns) {
if (outline == null) {
return;
}
scrubOutlineItems(outline.children(), targets, patterns);
}
private static void scrubOutlineItems(
Iterable<PDOutlineItem> items, Set<String> targets, List<Pattern> patterns) {
if (items == null) {
return;
}
for (PDOutlineItem item : items) {
try {
String title = item.getTitle();
if (title != null) {
String stripped = stripMatches(title, targets, patterns);
if (!stripped.equals(title)) {
item.setTitle(stripped);
}
}
// Bookmark actions: /A is an action dict which may carry a /URI or /JS
COSDictionary itemDict = item.getCOSObject();
if (itemDict != null) {
scrubActionIfMatching(itemDict, COSName.A, targets, patterns);
scrubActionIfMatching(itemDict, COSName.getPDFName("AA"), targets, patterns);
}
scrubOutlineItems(item.children(), targets, patterns);
} catch (Exception e) {
log.debug("Failed to scrub outline item: {}", e.getMessage());
}
}
}
// AcroForm
private static void scrubAcroForm(
PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
if (form == null) {
return;
}
// XFA forms: scrubbed separately because the XFA XML packet carries
scrubXfa(form, targets, patterns);
try {
for (PDField field : form.getFieldTree()) {
scrubField(field, targets, patterns);
}
} catch (Exception e) {
log.debug("Failed to walk AcroForm field tree: {}", e.getMessage());
}
// Force viewers to regenerate appearance streams from the (scrubbed) /V
try {
form.setNeedAppearances(true);
} catch (Exception e) {
log.debug("Failed to set /NeedAppearances on AcroForm: {}", e.getMessage());
}
}
private static void scrubXfa(PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
try {
COSBase xfaBase = form.getCOSObject().getDictionaryObject(COSName.XFA);
if (xfaBase == null) {
return;
}
boolean hit =
containsTarget(
xfaBase, carrierDropLiterals(targets), patterns, new HashSet<>());
if (hit) {
// Simplest safe move: strip the XFA entry entirely.
log.warn(
"Removing XFA form packet from AcroForm - XFA XML contained a redaction "
+ "target and has been dropped so viewers render AcroForm widgets "
+ "instead.");
form.getCOSObject().removeItem(COSName.XFA);
}
} catch (Exception e) {
log.debug("Failed to scrub XFA: {}", e.getMessage());
}
}
private static void scrubField(PDField field, Set<String> targets, List<Pattern> patterns) {
if (field == null) {
return;
}
try {
COSDictionary dict = field.getCOSObject();
// Only touch fields whose own values actually contain a target: clearing /AP
// document-wide blanks unrelated fields in viewers that ignore /NeedAppearances.
boolean matched =
dictValueMatches(dict, COSName.V, targets, patterns)
|| dictValueMatches(dict, COSName.DV, targets, patterns)
|| dictValueMatches(dict, COSName.getPDFName("RV"), targets, patterns)
|| dictValueMatches(dict, COSName.getPDFName("TU"), targets, patterns)
|| buttonCaptionMatches(dict, targets, patterns)
|| fieldValueMatches(field, targets, patterns);
if (!matched) {
return;
}
scrubDictStrings(dict, COSName.V, targets, patterns);
scrubDictStrings(dict, COSName.DV, targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("RV"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("TU"), targets, patterns);
// Keep field appearance streams in sync with value where possible.
try {
if (field.getValueAsString() != null) {
String stripped = stripMatches(field.getValueAsString(), targets, patterns);
if (!stripped.equals(field.getValueAsString())) {
field.setValue(stripped);
}
}
} catch (Exception e) {
log.debug("Failed to rewrite field value via setValue: {}", e.getMessage());
}
if (!isButtonField(dict)) {
clearWidgetAppearances(dict);
} else if (scrubButtonCaption(dict, targets, patterns)) {
// A button caption (/MK /CA,/RC,/AC) that carried the target renders via /AP that
// /NeedAppearances can't rebuild, so drop the stale /AP too.
clearWidgetAppearances(dict);
}
} catch (Exception e) {
log.debug("Failed to scrub field: {}", e.getMessage());
}
}
private static final String[] MK_CAPTION_KEYS = {"CA", "RC", "AC"};
/** True if a button widget's /MK caption (down/rollover/alternate) contains a target. */
private static boolean buttonCaptionMatches(
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
for (COSDictionary mk : mkDicts(dict)) {
for (String k : MK_CAPTION_KEYS) {
if (dictValueMatches(mk, COSName.getPDFName(k), targets, patterns)) {
return true;
}
}
}
return false;
}
/** Strip target substrings from a button widget's /MK captions; true if any matched. */
private static boolean scrubButtonCaption(
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
boolean hit = false;
for (COSDictionary mk : mkDicts(dict)) {
for (String k : MK_CAPTION_KEYS) {
COSName key = COSName.getPDFName(k);
if (dictValueMatches(mk, key, targets, patterns)) {
hit = true;
scrubDictStrings(mk, key, targets, patterns);
}
}
}
return hit;
}
/** The /MK appearance-characteristics dicts on a field and any widget kids. */
private static List<COSDictionary> mkDicts(COSDictionary dict) {
List<COSDictionary> out = new java.util.ArrayList<>();
if (dict.getDictionaryObject(COSName.getPDFName("MK")) instanceof COSDictionary mk) {
out.add(mk);
}
if (dict.getDictionaryObject(COSName.KIDS) instanceof COSArray kids) {
for (int i = 0; i < kids.size(); i++) {
if (kids.getObject(i) instanceof COSDictionary kid
&& kid.getDictionaryObject(COSName.getPDFName("MK"))
instanceof COSDictionary mk) {
out.add(mk);
}
}
}
return out;
}
private static boolean isButtonField(COSDictionary dict) {
COSBase ftBase = dict.getDictionaryObject(COSName.FT);
COSName ft = ftBase instanceof COSName n ? n : null;
if (ft == null) {
COSBase parent = dict.getDictionaryObject(COSName.PARENT);
if (parent instanceof COSDictionary p
&& p.getDictionaryObject(COSName.FT) instanceof COSName pn) {
ft = pn;
}
}
return COSName.getPDFName("Btn").equals(ft);
}
private static boolean dictValueMatches(
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
COSBase value = dict.getDictionaryObject(key);
if (value instanceof COSString cs) {
return matches(cs.getString(), targets, patterns);
}
if (value instanceof COSArray array) {
for (int i = 0; i < array.size(); i++) {
if (array.getObject(i) instanceof COSString element
&& matches(element.getString(), targets, patterns)) {
return true;
}
}
}
return false;
}
private static boolean fieldValueMatches(
PDField field, Set<String> targets, List<Pattern> patterns) {
try {
String value = field.getValueAsString();
return value != null && matches(value, targets, patterns);
} catch (Exception e) {
return false;
}
}
private static void clearWidgetAppearances(COSDictionary fieldDict) {
if (fieldDict == null) {
return;
}
// The field itself may be a widget (single-widget field) and/or have Kids.
fieldDict.removeItem(COSName.AP);
COSBase kids = fieldDict.getDictionaryObject(COSName.KIDS);
if (kids instanceof COSArray arr) {
for (int i = 0; i < arr.size(); i++) {
COSBase kidBase = arr.getObject(i);
if (kidBase instanceof COSDictionary kidDict) {
kidDict.removeItem(COSName.AP);
}
}
}
}
// Annotations
private static void scrubAnnotations(
PDDocument document, Set<String> targets, List<Pattern> patterns) {
try {
for (PDPage page : document.getPages()) {
// Page-level additional actions (/AA - open/close JS) can carry the target too.
scrubActionIfMatching(
page.getCOSObject(), COSName.getPDFName("AA"), targets, patterns);
List<PDAnnotation> annotations;
try {
annotations = page.getAnnotations();
} catch (Exception e) {
log.debug("Failed to load annotations for page: {}", e.getMessage());
continue;
}
if (annotations == null) {
continue;
}
for (PDAnnotation annotation : annotations) {
scrubAnnotation(annotation, targets, patterns);
}
}
} catch (Exception e) {
log.debug("Annotation scrub walk failed: {}", e.getMessage());
}
}
private static void scrubAnnotation(
PDAnnotation annotation, Set<String> targets, List<Pattern> patterns) {
if (annotation == null) {
return;
}
try {
String contents = annotation.getContents();
if (contents != null) {
String stripped = stripMatches(contents, targets, patterns);
if (!stripped.equals(contents)) {
annotation.setContents(stripped);
}
}
COSDictionary dict = annotation.getCOSObject();
scrubDictStrings(dict, COSName.getPDFName("RC"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("Subj"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("NM"), targets, patterns);
// Link/widget annotation actions can encode the target in a /URI or JS body.
scrubActionIfMatching(dict, COSName.A, targets, patterns);
scrubActionIfMatching(dict, COSName.getPDFName("AA"), targets, patterns);
} catch (Exception e) {
log.debug("Failed to scrub annotation: {}", e.getMessage());
}
}
// Structure tree
private static void scrubStructTree(
PDStructureTreeRoot root, Set<String> targets, List<Pattern> patterns) {
if (root == null) {
return;
}
try {
scrubStructDict(root.getCOSObject(), targets, patterns, new HashSet<>());
} catch (Exception e) {
log.debug("Structure tree scrub failed: {}", e.getMessage());
}
}
private static void scrubStructDict(
COSBase base, Set<String> targets, List<Pattern> patterns, Set<COSBase> seen) {
if (base == null) {
return;
}
COSBase resolved = base instanceof COSObject obj ? obj.getObject() : base;
if (resolved == null || !seen.add(resolved)) {
return;
}
if (resolved instanceof COSDictionary dict) {
// Do not walk into content streams - those are handled by content-stream
if (resolved instanceof COSStream) {
return;
}
scrubDictStrings(dict, COSName.getPDFName("ActualText"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("Alt"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("E"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns);
scrubDictStrings(dict, COSName.getPDFName("Lang"), targets, patterns);
for (COSName key : new HashSet<>(dict.keySet())) {
COSBase value = dict.getItem(key);
if (value instanceof COSDictionary
|| value instanceof COSArray
|| value instanceof COSObject) {
scrubStructDict(value, targets, patterns, seen);
}
}
} else if (resolved instanceof COSArray array) {
for (int i = 0; i < array.size(); i++) {
scrubStructDict(array.getObject(i), targets, patterns, seen);
}
}
}
// Names tree (JavaScript + embedded files)
private static void scrubNames(
PDDocumentNameDictionary names, Set<String> targets, List<Pattern> patterns) {
if (names == null) {
return;
}
try {
dropMatchingNames(names.getJavaScript(), targets, patterns);
} catch (Exception e) {
log.debug("Failed to scrub JavaScript names: {}", e.getMessage());
}
try {
dropMatchingNames(names.getEmbeddedFiles(), targets, patterns);
} catch (Exception e) {
log.debug("Failed to scrub embedded-file names: {}", e.getMessage());
}
}
private static void dropMatchingNames(
PDNameTreeNode<?> node, Set<String> targets, List<Pattern> patterns) {
if (node == null) {
return;
}
COSDictionary dict = node.getCOSObject();
if (dict == null) {
return;
}
scrubNameTreeDict(dict, targets, patterns);
}
private static void scrubNameTreeDict(
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
if (dict == null) {
return;
}
// Guarded casts: a malformed /Names or /Kids must skip that carrier, not throw a swallowed
// ClassCastException.
COSArray namesArray =
dict.getDictionaryObject(COSName.NAMES) instanceof COSArray a ? a : null;
if (namesArray != null) {
Set<String> dropLiterals = carrierDropLiterals(targets);
for (int i = namesArray.size() - 2; i >= 0; i -= 2) {
COSBase keyBase = namesArray.getObject(i);
String key = keyBase instanceof COSString s ? s.getString() : null;
// Drop the pair when the KEY or the VALUE (JS /JS stream, embedded-file bytes)
// contains a target - not just the key.
boolean keyHit = key != null && matches(key, dropLiterals, patterns);
boolean valueHit =
i + 1 < namesArray.size()
&& containsTarget(
namesArray.getObject(i + 1),
dropLiterals,
patterns,
new HashSet<>());
if (keyHit || valueHit) {
namesArray.remove(i + 1);
namesArray.remove(i);
}
}
}
COSArray kids = dict.getDictionaryObject(COSName.KIDS) instanceof COSArray a ? a : null;
if (kids != null) {
for (int i = 0; i < kids.size(); i++) {
COSBase kid = kids.getObject(i);
if (kid instanceof COSDictionary kidDict) {
scrubNameTreeDict(kidDict, targets, patterns);
}
}
}
}
// Helpers
private static void scrubDictStrings(
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
if (dict == null || key == null) {
return;
}
COSBase value = dict.getDictionaryObject(key);
if (value instanceof COSString cosString) {
String stripped = stripMatches(cosString.getString(), targets, patterns);
if (!stripped.equals(cosString.getString())) {
dict.setString(key, stripped);
}
} else if (value instanceof COSArray array) {
for (int i = 0; i < array.size(); i++) {
COSBase element = array.getObject(i);
if (element instanceof COSString elementString) {
String stripped = stripMatches(elementString.getString(), targets, patterns);
if (!stripped.equals(elementString.getString())) {
array.set(i, new COSString(stripped));
}
}
}
}
}
static String stripMatches(String source, Set<String> literalTargets, List<Pattern> patterns) {
if (source == null || source.isEmpty()) {
return source;
}
String result = source;
if (literalTargets != null) {
for (String target : literalTargets) {
if (target == null || target.isEmpty()) {
continue;
}
// Case-insensitive literal removal. Verification is case-insensitive
result = caseInsensitiveReplaceAll(result, target);
}
}
if (patterns != null) {
for (Pattern pattern : patterns) {
try {
// Force case-insensitive matching for catalog carriers regardless
Pattern ci = withCaseInsensitive(pattern);
result = ci.matcher(DeadlineCharSequence.of(result)).replaceAll("");
} catch (RuntimeException | StackOverflowError e) {
// Fail closed: a throwing regex means we cannot prove the carrier clean, so
// drop the whole string rather than leaving it intact.
log.warn("Pattern replace failed for {}; dropping carrier text", pattern);
return "";
}
}
}
return result;
}
static boolean matches(String source, Set<String> literalTargets, List<Pattern> patterns) {
if (source == null || source.isEmpty()) {
return false;
}
String lower = source.toLowerCase(Locale.ROOT);
if (literalTargets != null) {
for (String target : literalTargets) {
if (target != null
&& !target.isEmpty()
&& lower.contains(target.toLowerCase(Locale.ROOT))) {
return true;
}
}
}
if (patterns != null) {
for (Pattern pattern : patterns) {
try {
if (withCaseInsensitive(pattern)
.matcher(DeadlineCharSequence.of(source))
.find()) {
return true;
}
} catch (RuntimeException | StackOverflowError e) {
// Fail closed: a throwing regex counts as a match so the carrier is scrubbed.
log.warn("Pattern match failed for {}; treating carrier as a match", pattern);
return true;
}
}
}
return false;
}
private static String caseInsensitiveReplaceAll(String source, String target) {
if (target.isEmpty()) {
return source;
}
Pattern literal =
Pattern.compile(
Pattern.quote(target), Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
return literal.matcher(source).replaceAll("");
}
private static Pattern withCaseInsensitive(Pattern pattern) {
if ((pattern.flags() & Pattern.CASE_INSENSITIVE) != 0) {
return pattern;
}
try {
return Pattern.compile(
pattern.pattern(),
pattern.flags() | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
} catch (Exception e) {
return pattern;
}
}
}
@@ -0,0 +1,57 @@
package stirling.software.SPDF.pdf.redaction;
/**
* Wraps a match subject so a runaway (catastrophic-backtracking) regex aborts instead of hanging
* the request thread: every {@link #charAt(int)} the matcher performs checks a wall-clock deadline
* and throws once it is exceeded. The thrown {@link RegexTimeoutException} is a {@link
* RuntimeException}, so the redaction match sites' existing fail-closed catches handle it.
*/
final class DeadlineCharSequence implements CharSequence {
/** Per-match wall-clock budget. A legitimate whole-document match finishes far inside this. */
static final long DEFAULT_BUDGET_MILLIS = 2_000L;
static final class RegexTimeoutException extends RuntimeException {
RegexTimeoutException(long budgetMillis) {
super("Regex evaluation exceeded " + budgetMillis + " ms (possible ReDoS)");
}
}
private final CharSequence inner;
private final long deadlineNanos;
private final long budgetMillis;
private DeadlineCharSequence(CharSequence inner, long budgetMillis) {
this.inner = inner;
this.budgetMillis = budgetMillis;
this.deadlineNanos = System.nanoTime() + budgetMillis * 1_000_000L;
}
/** Wrap {@code text} with the default budget; null becomes an empty sequence. */
static DeadlineCharSequence of(String text) {
return new DeadlineCharSequence(text == null ? "" : text, DEFAULT_BUDGET_MILLIS);
}
@Override
public char charAt(int index) {
if (System.nanoTime() > deadlineNanos) {
throw new RegexTimeoutException(budgetMillis);
}
return inner.charAt(index);
}
@Override
public int length() {
return inner.length();
}
@Override
public CharSequence subSequence(int start, int end) {
return inner.subSequence(start, end);
}
@Override
public String toString() {
return inner.toString();
}
}
@@ -0,0 +1,748 @@
package stirling.software.SPDF.pdf.redaction;
import java.awt.Color;
import java.awt.geom.Rectangle2D;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.pdfbox.contentstream.operator.Operator;
import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSFloat;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdfparser.PDFStreamParser;
import org.apache.pdfbox.pdfwriter.ContentStreamWriter;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.common.PDStream;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.text.PDFTextStripperByArea;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.pdf.parser.PageImageLocator;
/**
* Content-stream editing: physically removes glyphs from page and form-XObject streams, both by
* literal/regex match (auto-word) and by rect intersection (manual area), plus the area-capture and
* overlay/annotation helpers those paths rely on.
*/
@Slf4j
final class RedactionContentEditor {
private static final int MAX_XOBJECT_DEPTH = 10;
private RedactionContentEditor() {}
/** True only for upright pages whose CropBox origin is (0,0), where the rect flip is exact. */
static boolean isSurgicallySafe(PDPage page) {
if (page.getRotation() != 0) {
return false;
}
PDRectangle crop = page.getCropBox();
return crop.getLowerLeftX() == 0f && crop.getLowerLeftY() == 0f;
}
/** Drop annotations whose rectangle overlaps any redaction rect. */
static void removeOverlappingAnnotations(PDPage page, List<PDRectangle> rects) {
try {
List<PDAnnotation> kept = new ArrayList<>();
for (var ann : page.getAnnotations()) {
PDRectangle ar = ann.getRectangle();
boolean overlaps = false;
if (ar != null) {
Rectangle2D.Float a =
new Rectangle2D.Float(
ar.getLowerLeftX(),
ar.getLowerLeftY(),
ar.getWidth(),
ar.getHeight());
for (PDRectangle rect : rects) {
if (a.intersects(
rect.getLowerLeftX(),
rect.getLowerLeftY(),
rect.getWidth(),
rect.getHeight())) {
overlaps = true;
break;
}
}
}
if (!overlaps) {
kept.add(ann);
}
}
page.setAnnotations(kept);
} catch (Exception e) {
log.debug("Could not strip overlapping annotations: {}", e.getMessage());
}
}
static List<Pattern> effectivePatterns(Set<String> literalTargets, List<Pattern> patterns) {
List<Pattern> result = new ArrayList<>();
if (literalTargets != null) {
for (String target : literalTargets) {
if (target == null || target.isEmpty()) {
continue;
}
// Case-insensitive to match TextFinderUtils; removal must not miss finder hits.
result.add(
Pattern.compile(
Pattern.quote(target),
Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE));
}
}
if (patterns != null) {
result.addAll(patterns);
}
return result;
}
// Per-page content-stream rewrite (literal/regex based)
static void rewritePageContent(PDDocument document, PDPage page, List<Pattern> patterns)
throws IOException {
PDResources resources = page.getResources();
if (resources == null) {
return;
}
List<Object> tokens = parseTokens(new PDFStreamParser(page));
boolean modified = rewriteTokens(tokens, resources, patterns);
if (modified) {
writePageTokens(document, page, tokens);
}
// Recurse into form XObjects referenced by this page (shared visited set + depth cap).
rewriteFormXObjects(resources, resources, patterns, new HashSet<>(), 0);
}
private static void rewriteFormXObjects(
PDResources resources,
PDResources parentResources,
List<Pattern> patterns,
Set<COSBase> visited,
int depth)
throws IOException {
if (depth > MAX_XOBJECT_DEPTH) {
log.warn("XObject nesting exceeded {}; stopping recursion", MAX_XOBJECT_DEPTH);
return;
}
for (COSName name : resources.getXObjectNames()) {
try {
var xobj = resources.getXObject(name);
if (!(xobj instanceof PDFormXObject form)) {
continue;
}
if (!visited.add(form.getCOSObject())) {
continue;
}
List<Object> tokens = parseTokens(new PDFStreamParser(form));
// Forms may inherit fonts from the parent when they carry no own /Resources.
PDResources formResources =
form.getResources() != null ? form.getResources() : parentResources;
boolean modified = rewriteTokens(tokens, formResources, patterns);
if (modified) {
// A form XObject's content IS its own stream body; overwrite it in place.
PDStream formStream = new PDStream(form.getCOSObject());
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
new ContentStreamWriter(out).writeTokens(tokens);
}
}
rewriteFormXObjects(formResources, formResources, patterns, visited, depth + 1);
} catch (IOException e) {
log.debug("Failed to rewrite XObject {}: {}", name.getName(), e.getMessage());
}
}
}
private static List<Object> parseTokens(PDFStreamParser parser) throws IOException {
List<Object> tokens = new ArrayList<>();
Object t;
while ((t = parser.parseNextToken()) != null) {
tokens.add(t);
}
return tokens;
}
/** Walk tokens keeping a tiny text state (current font). */
private static boolean rewriteTokens(
List<Object> tokens, PDResources resources, List<Pattern> patterns) {
boolean modified = false;
PDFont currentFont = null;
for (int i = 0; i < tokens.size(); i++) {
Object tok = tokens.get(i);
if (!(tok instanceof Operator op)) {
continue;
}
String name = op.getName();
if ("Tf".equals(name) && i >= 2) {
Object fontNameTok = tokens.get(i - 2);
if (fontNameTok instanceof COSName fontName) {
try {
currentFont = resources.getFont(fontName);
} catch (IOException ex) {
log.debug(
"Could not resolve font {}: {}",
fontName.getName(),
ex.getMessage());
currentFont = null;
}
}
} else if (RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name) && i >= 1) {
int operandIdx = i - 1;
Object operand = tokens.get(operandIdx);
if (operand instanceof COSString cosString) {
COSString replacement = rewriteCosString(cosString, currentFont, patterns);
if (replacement != null) {
tokens.set(operandIdx, replacement);
modified = true;
}
} else if (operand instanceof COSArray arr) {
COSArray newArr = rewriteCosArray(arr, currentFont, patterns);
if (newArr != null) {
tokens.set(operandIdx, newArr);
modified = true;
}
}
}
}
return modified;
}
private static COSString rewriteCosString(
COSString cosString, PDFont font, List<Pattern> patterns) {
if (font == null) {
// Without a font we cannot decode safely.
return rewriteRawLatin(cosString, patterns);
}
DecodeResult decoded = decodeCosString(cosString, font);
if (decoded == null) {
return rewriteRawLatin(cosString, patterns);
}
boolean[] drop = findDroppedCharsMask(decoded.text, patterns);
if (drop == null) {
return null;
}
return buildFilteredCosString(decoded, drop, font);
}
private static COSArray rewriteCosArray(COSArray arr, PDFont font, List<Pattern> patterns) {
// Build a concatenated decode across all COSString elements so that matches
List<DecodeResult> parts = new ArrayList<>();
StringBuilder concat = new StringBuilder();
for (int i = 0; i < arr.size(); i++) {
COSBase elem = arr.get(i);
if (elem instanceof COSString cs) {
DecodeResult decoded = font != null ? decodeCosString(cs, font) : null;
if (decoded == null) {
decoded = decodeAsLatin(cs);
}
parts.add(decoded);
concat.append(decoded.text);
} else {
parts.add(null);
}
}
boolean[] fullDrop = findDroppedCharsMask(concat.toString(), patterns);
if (fullDrop == null) {
return null;
}
COSArray out = new COSArray();
int cursor = 0;
for (int i = 0; i < arr.size(); i++) {
COSBase elem = arr.get(i);
if (elem instanceof COSString cs) {
DecodeResult decoded = parts.get(i);
int partLen = decoded.text.length();
boolean[] partDrop = new boolean[partLen];
System.arraycopy(fullDrop, cursor, partDrop, 0, partLen);
cursor += partLen;
COSString rebuilt =
buildFilteredCosStringRaw(decoded, partDrop, font, cs.getBytes());
// A zero-length COSString is valid; PDFBox emits it as () producing no
out.add(rebuilt);
} else {
out.add(elem);
}
}
return out;
}
/** For cases where we have no font - treat the bytes as latin-1 characters. */
private static COSString rewriteRawLatin(COSString cosString, List<Pattern> patterns) {
DecodeResult decoded = decodeAsLatin(cosString);
boolean[] drop = findDroppedCharsMask(decoded.text, patterns);
if (drop == null) {
return null;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
for (int i = 0; i < decoded.text.length(); i++) {
if (drop[i]) continue;
out.write(decoded.text.charAt(i) & 0xFF);
}
return new COSString(out.toByteArray());
}
private static DecodeResult decodeAsLatin(COSString cosString) {
byte[] bytes = cosString.getBytes();
StringBuilder sb = new StringBuilder(bytes.length);
int[] codeStart = new int[bytes.length];
int[] codeLen = new int[bytes.length];
for (int i = 0; i < bytes.length; i++) {
sb.append((char) (bytes[i] & 0xFF));
codeStart[i] = i;
codeLen[i] = 1;
}
return new DecodeResult(sb.toString(), bytes, codeStart, codeLen);
}
/** Decode a COSString to Unicode via the font; null if decoding fails. */
private static DecodeResult decodeCosString(COSString cosString, PDFont font) {
byte[] bytes = cosString.getBytes();
StringBuilder text = new StringBuilder();
List<Integer> starts = new ArrayList<>();
List<Integer> lens = new ArrayList<>();
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes)) {
int pos = 0;
while (in.available() > 0) {
int before = in.available();
int code;
try {
code = font.readCode(in);
} catch (IOException | RuntimeException ex) {
log.debug(
"Font {} failed to decode byte sequence: {}",
font.getName(),
ex.getMessage());
return null;
}
int consumed = before - in.available();
String unicode;
try {
unicode = font.toUnicode(code);
} catch (Exception ex) {
unicode = null;
}
if (unicode == null) {
// If the font has no ToUnicode mapping we cannot match reliably - return
return null;
}
text.append(unicode);
// Associate every Unicode character produced with the same code byte range
for (int c = 0; c < unicode.length(); c++) {
starts.add(pos);
lens.add(consumed);
}
if (consumed == 0) {
// Defensive: avoid infinite loop on malformed fonts.
break;
}
pos += consumed;
}
} catch (IOException e) {
return null;
}
int[] startArr = starts.stream().mapToInt(Integer::intValue).toArray();
int[] lenArr = lens.stream().mapToInt(Integer::intValue).toArray();
return new DecodeResult(text.toString(), bytes, startArr, lenArr);
}
/** Null if no pattern matches; else a per-char drop mask over the text. */
private static boolean[] findDroppedCharsMask(String text, List<Pattern> patterns) {
boolean any = false;
boolean[] mask = new boolean[text.length()];
for (Pattern pattern : patterns) {
Matcher m;
try {
m = pattern.matcher(DeadlineCharSequence.of(text));
} catch (Exception ex) {
continue;
}
while (m.find()) {
int s = m.start();
int e = m.end();
if (e <= s) continue;
for (int i = s; i < e; i++) {
mask[i] = true;
}
any = true;
}
}
return any ? mask : null;
}
private static COSString buildFilteredCosString(
DecodeResult decoded, boolean[] drop, PDFont font) {
return buildFilteredCosStringRaw(decoded, drop, font, decoded.bytes);
}
private static COSString buildFilteredCosStringRaw(
DecodeResult decoded, boolean[] drop, PDFont font, byte[] originalBytes) {
// Collect code byte-ranges to drop.
Set<Integer> dropStarts = new HashSet<>();
for (int i = 0; i < drop.length; i++) {
if (drop[i]) {
dropStarts.add(decoded.codeStarts[i]);
}
}
// Map byte-start -> code length once (O(n)); findCodeLenAt was an O(n^2) linear scan.
Map<Integer, Integer> lenByStart = new HashMap<>();
for (int j = 0; j < decoded.codeStarts.length; j++) {
lenByStart.putIfAbsent(decoded.codeStarts[j], decoded.codeLens[j]);
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
int i = 0;
while (i < originalBytes.length) {
int start = i;
int len = lenByStart.getOrDefault(start, -1);
if (len <= 0) {
// Unknown - keep the byte verbatim.
out.write(originalBytes[i] & 0xFF);
i += 1;
continue;
}
if (dropStarts.contains(start)) {
// Replace dropped code with encoded space if possible.
byte[] spaceBytes = tryEncodeSpace(font);
if (spaceBytes != null) {
out.write(spaceBytes, 0, spaceBytes.length);
}
} else {
out.write(originalBytes, start, len);
}
i += len;
}
return new COSString(out.toByteArray());
}
private static byte[] tryEncodeSpace(PDFont font) {
if (font == null) {
return new byte[] {0x20};
}
try {
return font.encode(" ");
} catch (Exception e) {
// Some fonts cannot encode a space at all (e.g. Type3 throws
return null;
}
}
private static final class DecodeResult {
final String text;
final byte[] bytes;
final int[] codeStarts;
final int[] codeLens;
DecodeResult(String text, byte[] bytes, int[] codeStarts, int[] codeLens) {
this.text = text;
this.bytes = bytes;
this.codeStarts = codeStarts;
this.codeLens = codeLens;
}
}
// Area capture
static List<String> captureTextInRects(PDPage page, List<PDRectangle> rects)
throws IOException {
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
stripper.setSortByPosition(true);
for (int i = 0; i < rects.size(); i++) {
PDRectangle rect = rects.get(i);
// PDFTextStripperByArea uses a Java2D rectangle in the same coordinate
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
Rectangle2D.Float region =
new Rectangle2D.Float(
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight());
stripper.addRegion("r" + i, region);
}
try {
stripper.extractRegions(page);
} catch (Exception e) {
log.debug("Failed to extract text in rects: {}", e.getMessage());
return Collections.emptyList();
}
Set<String> captured = new LinkedHashSet<>();
for (int i = 0; i < rects.size(); i++) {
String text = stripper.getTextForRegion("r" + i);
if (text == null) {
continue;
}
for (String token : text.split("\\s+")) {
String trimmed = token.trim();
if (!trimmed.isEmpty()) {
captured.add(trimmed);
}
}
}
return new ArrayList<>(captured);
}
// Content-stream rewriting (rect-driven glyph removal)
/**
* Blanks show-text operands whose glyphs fall in a rect. Returns false (surgical removal is
* unreliable, caller must rasterise) when text-bearing form XObjects skew the operator ordinals
* or an image sits under a rect - leaving the page stream untouched so the raster is correct.
*/
static boolean removeTokensIntersectingRects(
PDDocument document, PDPage page, List<PDRectangle> rects) throws IOException {
List<Rectangle2D.Float> areaRects = new ArrayList<>();
for (PDRectangle rect : rects) {
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
areaRects.add(
new Rectangle2D.Float(
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight()));
}
int pageIndex = document.getPages().indexOf(page);
TokenIndexCollector collector = new TokenIndexCollector(areaRects);
collector.setStartPage(pageIndex + 1);
collector.setEndPage(pageIndex + 1);
collector.getText(document);
int collectorTextOps = collector.totalTextOps();
List<Object> tokens = parseTokens(new PDFStreamParser(page));
int pageTextOps = 0;
for (Object t : tokens) {
if (t instanceof Operator op
&& RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(op.getName())) {
pageTextOps++;
}
}
// Collector ordinals span form XObjects; images under a rect can't be surgically removed.
if (collectorTextOps != pageTextOps || imageIntersectsAnyRect(page, pageIndex, rects)) {
return false;
}
// Drop ONLY the in-rect glyphs of each operand (not the whole run), tracking the active
// font so codes decode. Bail to rasterise if the font can't decode or glyph!=code counts.
PDResources resources = page.getResources();
PDFont currentFont = null;
int opCounter = -1;
boolean modified = false;
for (int i = 0; i < tokens.size(); i++) {
Object tok = tokens.get(i);
if (!(tok instanceof Operator op)) {
continue;
}
String name = op.getName();
if ("Tf".equals(name) && i >= 2 && tokens.get(i - 2) instanceof COSName fn) {
try {
currentFont = resources != null ? resources.getFont(fn) : null;
} catch (IOException e) {
currentFont = null;
}
} else if (RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name)) {
opCounter++;
Set<Integer> dropGlyphs = collector.dropGlyphsByOp.get(opCounter);
if (dropGlyphs == null || dropGlyphs.isEmpty() || i < 1) {
continue;
}
// ' and " also move to the next line; converting them to TJ would lose that -
// raster.
if ("'".equals(name) || "\"".equals(name)) {
return false;
}
int expected = collector.glyphCountByOp.getOrDefault(opCounter, -1);
COSArray rebuilt =
rebuildAdvancePreserving(
tokens.get(i - 1), currentFont, dropGlyphs, expected);
if (rebuilt == null) {
return false; // undecodable / glyph-count mismatch - rasterise instead
}
// Emit as TJ so the removed glyphs' advance is preserved (no reflow).
tokens.set(i - 1, rebuilt);
tokens.set(i, Operator.getOperator("TJ"));
modified = true;
}
}
if (modified) {
writePageTokens(document, page, tokens);
}
return true;
}
/** Distinct code byte-starts in order (one per glyph/code) from a per-char DecodeResult. */
private static int[] distinctCodeStarts(DecodeResult d) {
List<Integer> starts = new ArrayList<>();
int last = -1;
for (int s : d.codeStarts) {
if (s != last) {
starts.add(s);
last = s;
}
}
return starts.stream().mapToInt(Integer::intValue).toArray();
}
private static int codeAt(PDFont font, byte[] bytes, int start, int len) throws IOException {
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes, start, len)) {
return font.readCode(in);
}
}
/**
* Rebuild a Tj/TJ operand as a TJ array that keeps every glyph EXCEPT the given 0-based
* indexes, replacing each dropped run with a numeric adjustment equal to its advance so the
* surviving text does not reflow into (or out of) the box. Returns null (caller rasterises) if
* anything can't be decoded or the glyph count doesn't line up with the position pass.
*/
private static COSArray rebuildAdvancePreserving(
Object operand, PDFont font, Set<Integer> dropGlyphs, int expectedGlyphs) {
if (font == null) {
return null;
}
List<COSBase> elements = new ArrayList<>();
if (operand instanceof COSString cs) {
elements.add(cs);
} else if (operand instanceof COSArray arr) {
for (COSBase b : arr) {
elements.add(b);
}
} else {
return null;
}
COSArray out = new COSArray();
ByteArrayOutputStream seg = new ByteArrayOutputStream();
float pendingAdvance = 0f; // accumulated width of dropped codes, in 1/1000 text units
int globalCode = 0;
for (COSBase el : elements) {
if (!(el instanceof COSString s)) {
// Existing numeric adjustment: flush whichever run is pending, then preserve it.
if (seg.size() > 0) {
out.add(new COSString(seg.toByteArray()));
seg.reset();
} else if (pendingAdvance != 0f) {
out.add(new COSFloat(-pendingAdvance));
pendingAdvance = 0f;
}
out.add(el);
continue;
}
DecodeResult d = decodeCosString(s, font);
if (d == null) {
return null;
}
byte[] b = s.getBytes();
int[] starts = distinctCodeStarts(d);
Map<Integer, Integer> lenByStart = new HashMap<>();
for (int j = 0; j < d.codeStarts.length; j++) {
lenByStart.putIfAbsent(d.codeStarts[j], d.codeLens[j]);
}
for (int k = 0; k < starts.length; k++) {
int bs = starts[k];
int len = lenByStart.getOrDefault(bs, -1);
if (len <= 0) {
return null;
}
if (dropGlyphs.contains(globalCode)) {
if (seg.size() > 0) { // close the kept run before accumulating advance
out.add(new COSString(seg.toByteArray()));
seg.reset();
}
try {
pendingAdvance += font.getWidth(codeAt(font, b, bs, len));
} catch (Exception e) {
return null;
}
} else {
if (pendingAdvance != 0f) { // emit the removed run's advance, then keep glyphs
out.add(new COSFloat(-pendingAdvance));
pendingAdvance = 0f;
}
seg.write(b, bs, len);
}
globalCode++;
}
}
if (pendingAdvance != 0f) {
out.add(new COSFloat(-pendingAdvance));
}
if (seg.size() > 0) {
out.add(new COSString(seg.toByteArray()));
}
if (expectedGlyphs >= 0 && globalCode != expectedGlyphs) {
return null;
}
return out;
}
/**
* True if any image on the page overlaps a redaction rect (would survive under the overlay).
*/
private static boolean imageIntersectsAnyRect(
PDPage page, int pageIndex, List<PDRectangle> rects) {
try {
PageImageLocator locator = new PageImageLocator(page, pageIndex);
locator.processPage(page);
for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) {
Rectangle2D.Float img =
new Rectangle2D.Float(
ib.x1(), ib.y1(), ib.x2() - ib.x1(), ib.y2() - ib.y1());
for (PDRectangle rect : rects) {
if (img.intersects(
rect.getLowerLeftX(),
rect.getLowerLeftY(),
rect.getWidth(),
rect.getHeight())) {
return true;
}
}
}
return false;
} catch (Exception e) {
// Cannot prove no image sits under a box - treat as unsafe (force raster).
return true;
}
}
private static void writePageTokens(PDDocument document, PDPage page, List<Object> tokens)
throws IOException {
PDStream stream = new PDStream(document);
try (var out = stream.createOutputStream(COSName.FLATE_DECODE)) {
ContentStreamWriter writer = new ContentStreamWriter(out);
writer.writeTokens(tokens);
}
page.setContents(stream);
}
static void drawOverlay(
PDDocument document, PDPage page, List<PDRectangle> rects, Color overlayColor)
throws IOException {
try (PDPageContentStream cs =
new PDPageContentStream(
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
cs.saveGraphicsState();
cs.setNonStrokingColor(overlayColor);
for (PDRectangle rect : rects) {
cs.addRect(
rect.getLowerLeftX(),
rect.getLowerLeftY(),
rect.getWidth(),
rect.getHeight());
}
cs.fill();
cs.restoreGraphicsState();
}
}
}
@@ -0,0 +1,301 @@
package stirling.software.SPDF.pdf.redaction;
import java.awt.Color;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.utils.text.TextFinderUtils;
import stirling.software.common.util.RegexPatternUtils;
/**
* Public facade for the manual-area, whole-page and auto-word redaction paths. Orchestrates the
* collaborators that do the work: {@link RedactionContentEditor} (content-stream glyph removal),
* {@link RedactionVerifier} (fail-closed independent verification) and {@link RedactionRasteriser}
* (last-resort rasterisation), with {@link CatalogScrubber} for out-of-page carriers.
*/
@Slf4j
public final class RedactionPipeline {
public static final Set<String> TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\"");
private RedactionPipeline() {}
/** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */
static void setJpdfiumAvailableForTest(boolean available) {
RedactionVerifier.setJpdfiumAvailableForTest(available);
}
/** Redact page-local rects (PDF user-space), dropping intersecting glyphs + overlay. */
public static RedactionResult redactAreas(
PDDocument document,
Map<Integer, List<PDRectangle>> rectsByPageIndex,
Color overlayColor)
throws IOException {
Set<String> capturedStrings = new LinkedHashSet<>();
Set<Integer> forceRasterPages = new LinkedHashSet<>();
for (Map.Entry<Integer, List<PDRectangle>> entry : rectsByPageIndex.entrySet()) {
int pageIndex = entry.getKey();
List<PDRectangle> rects = entry.getValue();
if (pageIndex < 0 || pageIndex >= document.getNumberOfPages() || rects.isEmpty()) {
continue;
}
PDPage page = document.getPage(pageIndex);
capturedStrings.addAll(RedactionContentEditor.captureTextInRects(page, rects));
// Rotation / non-zero CropBox origin break the coordinate flip, and text-bearing forms
// or in-rect images defeat page-stream ordinal removal.
boolean surgical =
RedactionContentEditor.isSurgicallySafe(page)
&& RedactionContentEditor.removeTokensIntersectingRects(
document, page, rects);
if (!surgical) {
forceRasterPages.add(pageIndex);
}
RedactionContentEditor.removeOverlappingAnnotations(page, rects);
RedactionContentEditor.drawOverlay(document, page, rects, overlayColor);
}
return new RedactionResult(capturedStrings, forceRasterPages);
}
/** Wipe whole pages: drop all content/resources, fill with a rectangle. */
public static void redactWholePages(
PDDocument document, List<Integer> pageIndexes, Color overlayColor) throws IOException {
for (Integer pageIndex : pageIndexes) {
if (pageIndex == null || pageIndex < 0 || pageIndex >= document.getNumberOfPages()) {
continue;
}
PDPage page = document.getPage(pageIndex);
PDRectangle media = page.getMediaBox();
// Drop existing content streams and page resources outright.
page.getCOSObject().removeItem(COSName.CONTENTS);
page.setResources(new PDResources());
page.getCOSObject().removeItem(COSName.ANNOTS);
// /Thumb is a rendered image of the original page - drop it too.
page.getCOSObject().removeItem(COSName.getPDFName("Thumb"));
try (PDPageContentStream cs =
new PDPageContentStream(
document, page, PDPageContentStream.AppendMode.OVERWRITE, true, true)) {
cs.setNonStrokingColor(overlayColor);
cs.addRect(
media.getLowerLeftX(),
media.getLowerLeftY(),
media.getWidth(),
media.getHeight());
cs.fill();
}
}
}
/** Physically remove every literal/regex match from all page content streams. */
public static void redactLiteralTerms(
PDDocument document, Set<String> literalTargets, List<Pattern> patterns)
throws IOException {
List<Pattern> effectivePatterns =
RedactionContentEditor.effectivePatterns(literalTargets, patterns);
if (effectivePatterns.isEmpty()) {
return;
}
int pageIndex = 0;
for (PDPage page : document.getPages()) {
try {
RedactionContentEditor.rewritePageContent(document, page, effectivePatterns);
} catch (IOException | RuntimeException e) {
// Never let one page's font quirk (e.g. Type3 encode, damaged program)
log.warn(
"Content-stream rewrite failed on page {} ({}); leaving it for the "
+ "verification/rasterisation pass.",
pageIndex + 1,
e.toString());
}
pageIndex++;
}
}
/** Finalize with document-wide verification: scrub, wipe metadata, save, verify. */
public static byte[] finalize(
PDDocument document, Set<String> literalTargets, List<Pattern> patterns)
throws IOException {
return finalize(document, literalTargets, patterns, null);
}
/** Finalize with verification scoped to a specific set of page indexes. */
public static byte[] finalize(
PDDocument document,
Set<String> literalTargets,
List<Pattern> patterns,
Set<Integer> affectedPages)
throws IOException {
CatalogScrubber.scrub(document, literalTargets, patterns);
RedactionVerifier.warnAboutEmbeddedFontGlyphs(document);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
document.save(baos);
byte[] bytes = baos.toByteArray();
try {
RedactionVerifier.verify(bytes, literalTargets, patterns, affectedPages);
return bytes;
} catch (RedactionVerificationFailedException primaryFailure) {
// Rewriter could not guarantee removal; rasterise as a last resort.
log.warn(
"Primary redaction verification failed ({}); falling back to page-scoped "
+ "rasterisation to guarantee removal.",
primaryFailure.getMessage());
// With no page scope (auto-word), rasterise only the pages that still leak.
Set<Integer> pagesToRaster =
(affectedPages == null || affectedPages.isEmpty())
? RedactionVerifier.findLeakingPages(bytes, literalTargets, patterns)
: new HashSet<>(affectedPages);
try (PDDocument rasterised = RedactionRasteriser.rasterisePages(bytes, pagesToRaster)) {
CatalogScrubber.scrub(rasterised, literalTargets, patterns);
ByteArrayOutputStream rasterOut = new ByteArrayOutputStream();
rasterised.save(rasterOut);
byte[] rasterBytes = rasterOut.toByteArray();
RedactionVerifier.verify(rasterBytes, literalTargets, patterns, affectedPages);
return rasterBytes;
} catch (IOException e) {
throw new RedactionVerificationFailedException(
"Rasterisation fallback failed after primary redaction leak", e);
}
}
}
/** Finalize a manual area redaction; rasterises forced + verified-leaking pages. */
public static byte[] finalizeAreas(
PDDocument document,
Map<Integer, List<PDRectangle>> rectsByPage,
Set<Integer> forceRasterPages,
Set<String> capturedTargets)
throws IOException {
// Text captured under the rects may also live in a bookmark / form value / annotation / JS
// carrier.
Set<String> carrierTargets =
capturedTargets == null ? Collections.emptySet() : capturedTargets;
CatalogScrubber.scrub(document, carrierTargets, Collections.emptyList());
RedactionVerifier.warnAboutEmbeddedFontGlyphs(document);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
document.save(baos);
byte[] bytes = baos.toByteArray();
Set<Integer> toRaster =
new HashSet<>(forceRasterPages == null ? Set.of() : forceRasterPages);
toRaster.addAll(RedactionVerifier.findLeakingRectPages(bytes, rectsByPage));
if (toRaster.isEmpty()) {
return bytes;
}
log.warn("Manual redaction rasterising page(s) {} to guarantee removal", toRaster);
try (PDDocument rasterised = RedactionRasteriser.rasterisePages(bytes, toRaster)) {
ByteArrayOutputStream rasterOut = new ByteArrayOutputStream();
rasterised.save(rasterOut);
return rasterOut.toByteArray();
} catch (IOException e) {
throw new RedactionVerificationFailedException(
"Rasterisation fallback failed after manual redaction leak", e);
}
}
/**
* Rasterise the given pages of already-finalized bytes to guarantee removal of geometric (range
* / image-box) redactions whose covered content - text under an overlay, or an image - is not
* removed by content-stream text redaction. Scrubs carriers and re-saves.
*/
public static byte[] rasteriseSpecificPages(
byte[] bytes, Set<Integer> pages, Set<String> literalTargets, List<Pattern> patterns)
throws IOException {
if (pages == null || pages.isEmpty()) {
return bytes;
}
log.warn("Rasterising page(s) {} to guarantee geometric redaction removal", pages);
try (PDDocument rasterised =
RedactionRasteriser.rasterisePages(bytes, new HashSet<>(pages))) {
CatalogScrubber.scrub(rasterised, literalTargets, patterns);
ByteArrayOutputStream out = new ByteArrayOutputStream();
rasterised.save(out);
return out.toByteArray();
}
}
/** Build case-insensitive regex patterns from user input. */
public static List<Pattern> buildPatterns(
String[] rawEntries, boolean useRegex, boolean wholeWordSearch) {
List<Pattern> patterns = new ArrayList<>();
if (rawEntries == null) {
return patterns;
}
for (String raw : rawEntries) {
if (raw == null) {
continue;
}
String trimmed = raw.trim();
if (trimmed.isEmpty()) {
continue;
}
try {
String core = useRegex ? trimmed : Pattern.quote(trimmed);
if (wholeWordSearch) {
// Shared with the finder so removal + verification use identical boundaries.
core = TextFinderUtils.applyWordBoundaries(trimmed, core);
}
// Compile via the shared cache (same path as the finder) for consistent
// handling of user-supplied regex; ReDoS is bounded by DeadlineCharSequence.
patterns.add(RegexPatternUtils.getInstance().createSearchPattern(core, true));
} catch (PatternSyntaxException e) {
log.debug("Skipping invalid regex '{}': {}", trimmed, e.getMessage());
}
}
return patterns;
}
/**
* True if any font is not provably reliable for PDFBox glyph extraction (delegates to {@link
* RedactionVerifier}); exposed here for the pipeline's callers and tests.
*/
static boolean documentHasUnreliableFont(PDDocument document) {
return RedactionVerifier.documentHasUnreliableFont(document);
}
/** Captured strings plus pages that must be rasterised (surgical removal was unreliable). */
public static final class RedactionResult {
private final Set<String> capturedStrings;
private final Set<Integer> forceRasterPages;
public RedactionResult(Set<String> capturedStrings, Set<Integer> forceRasterPages) {
this.capturedStrings = capturedStrings == null ? Set.of() : capturedStrings;
this.forceRasterPages = forceRasterPages == null ? Set.of() : forceRasterPages;
}
public Set<String> getCapturedStrings() {
return capturedStrings;
}
public Set<Integer> getForceRasterPages() {
return forceRasterPages;
}
}
}
@@ -0,0 +1,84 @@
package stirling.software.SPDF.pdf.redaction;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Set;
import javax.imageio.ImageIO;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
import lombok.extern.slf4j.Slf4j;
/** Rasterisation fallback: renders whole pages to an image so no text/vector content survives. */
@Slf4j
final class RedactionRasteriser {
private RedactionRasteriser() {}
/** Rasterise the listed pages (all when null) at 150 DPI, replacing their content. */
static PDDocument rasterisePages(byte[] sourceBytes, Set<Integer> pagesToRaster)
throws IOException {
// Load the document directly and mutate in place: rewriting
PDDocument source = Loader.loadPDF(sourceBytes);
try {
PDFRenderer renderer = new PDFRenderer(source);
int pageCount = source.getNumberOfPages();
for (int i = 0; i < pageCount; i++) {
if (pagesToRaster != null && !pagesToRaster.contains(i)) {
continue;
}
PDPage page = source.getPage(i);
// PDFRenderer renders the CropBox region, so draw the raster over the CropBox (not
// the MediaBox) or a CropBox != MediaBox page is stretched/offset.
PDRectangle crop = page.getCropBox();
BufferedImage img = renderer.renderImageWithDPI(i, 150, ImageType.RGB);
ByteArrayOutputStream imgOut = new ByteArrayOutputStream();
ImageIO.write(img, "png", imgOut);
PDImageXObject imageXObject =
PDImageXObject.createFromByteArray(
source, imgOut.toByteArray(), "redacted-page-" + i);
// Drop all prior content / resources / annotations / thumbnail; the raster is the
// page.
page.getCOSObject().removeItem(COSName.CONTENTS);
page.setResources(new PDResources());
page.getCOSObject().removeItem(COSName.ANNOTS);
page.getCOSObject().removeItem(COSName.getPDFName("Thumb"));
// Rotation is already baked into the rendered image, so reset it to zero.
page.setRotation(0);
try (PDPageContentStream cs =
new PDPageContentStream(
source,
page,
PDPageContentStream.AppendMode.OVERWRITE,
false,
true)) {
// The rendered image already has the rotation baked in visually
cs.drawImage(
imageXObject,
crop.getLowerLeftX(),
crop.getLowerLeftY(),
crop.getWidth(),
crop.getHeight());
}
}
return source;
} catch (IOException | RuntimeException e) {
source.close();
throw e;
}
}
}
@@ -0,0 +1,13 @@
package stirling.software.SPDF.pdf.redaction;
/** Thrown when post-redaction verification still finds a target in the saved PDF. */
public class RedactionVerificationFailedException extends RuntimeException {
public RedactionVerificationFailedException(String message) {
super(message);
}
public RedactionVerificationFailedException(String message, Throwable cause) {
super(message, cause);
}
}
@@ -0,0 +1,495 @@
package stirling.software.SPDF.pdf.redaction;
import java.awt.geom.Rectangle2D;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
import org.apache.pdfbox.text.PDFTextStripper;
import lombok.extern.slf4j.Slf4j;
import stirling.software.jpdfium.PdfDocument;
import stirling.software.jpdfium.text.PdfTextExtractor;
/**
* Independent, fail-closed verification that redacted text is truly gone: an /ActualText-blind
* PDFBox pass plus an additive native (PDFium) pass for fonts PDFBox cannot reliably extract, with
* per-page leak localisation for the rasterisation fallback.
*/
@Slf4j
final class RedactionVerifier {
private static final int MAX_XOBJECT_DEPTH = 10;
// Latched false when the native PDFium binding can't load, so the host falls back to the PDFBox
// pass.
private static volatile boolean jpdfiumAvailable = true;
// Skip the additive native pass above this size to bound off-heap copy + native runtime on
// adversarial inputs; the PDFBox glyph-blind pass still verifies.
private static final long MAX_JPDFIUM_VERIFY_BYTES = 100L * 1024 * 1024;
private RedactionVerifier() {}
/** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */
static void setJpdfiumAvailableForTest(boolean available) {
jpdfiumAvailable = available;
}
/** Warns when the document still carries embedded Type0/TrueType font */
static void warnAboutEmbeddedFontGlyphs(PDDocument document) {
boolean anyEmbedded = false;
Set<PDFont> visited = new HashSet<>();
for (PDPage page : document.getPages()) {
PDResources resources = page.getResources();
if (resources == null) {
continue;
}
for (COSName name : resources.getFontNames()) {
PDFont font;
try {
font = resources.getFont(name);
} catch (IOException ioe) {
continue;
}
if (font == null || !visited.add(font)) {
continue;
}
if (font instanceof PDType0Font || font instanceof PDTrueTypeFont) {
anyEmbedded = true;
}
}
}
if (anyEmbedded) {
log.warn(
"Redacted document contains embedded Type0/TrueType fonts; glyph outlines for "
+ "redacted characters may remain in the font program. Text is not "
+ "extractable via content-stream reading, but raw font inspection can "
+ "still recover glyph shapes. Use the convert-to-image fallback for "
+ "maximum assurance.");
}
}
static void verify(
byte[] bytes,
Set<String> literalTargets,
List<Pattern> patterns,
Set<Integer> affectedPages) {
if ((literalTargets == null || literalTargets.isEmpty())
&& (patterns == null || patterns.isEmpty())) {
return;
}
Set<Integer> pageSet =
(affectedPages == null || affectedPages.isEmpty())
? null
: new TreeSet<>(affectedPages);
// PDFBox pass, blind to /ActualText so a benign override can't mask real glyphs.
boolean needNativePass;
try (PDDocument reopened = Loader.loadPDF(bytes)) {
assertNoTarget(extractText(reopened, pageSet), literalTargets, patterns);
needNativePass = documentHasUnreliableFont(reopened);
} catch (IOException e) {
throw new RedactionVerificationFailedException(
"Failed to reopen redacted PDF for verification", e);
}
// Additive producer-independent pass: native PDFium sees glyphs PDFBox may miss (fonts with
// no ToUnicode).
if (needNativePass) {
String nativeText = extractTextJPDFium(bytes, pageSet);
if (nativeText == null) {
// Required independent pass could not run (native unavailable or doc over the size
// guard); fail closed.
throw new RedactionVerificationFailedException(
"Independent native verification could not run for a document whose fonts "
+ "PDFBox cannot reliably extract; cannot confirm removal");
}
assertNoTarget(nativeText, literalTargets, patterns);
}
}
/**
* True if any font (on any page or nested form XObject) is not provably reliable for PDFBox
* glyph extraction - i.e. it is neither a built-in Standard-14 font nor carries a /ToUnicode
* map. These are exactly the fonts (CID/Type3/symbolic without ToUnicode) where the PDFBox pass
* can go blind, so the independent native pass earns its cost. Biased to {@code true} on any
* inspection failure so the native pass runs whenever reliability is uncertain.
*/
static boolean documentHasUnreliableFont(PDDocument document) {
try {
Set<COSBase> visited = new HashSet<>();
for (PDPage page : document.getPages()) {
if (resourcesHaveUnreliableFont(page.getResources(), visited, 0)) {
return true;
}
}
return false;
} catch (RuntimeException e) {
return true;
}
}
private static boolean resourcesHaveUnreliableFont(
PDResources res, Set<COSBase> visited, int depth) {
if (res == null) {
return false;
}
if (depth > MAX_XOBJECT_DEPTH) {
return true; // too deeply nested to fully verify - run the native pass to be safe
}
if (!visited.add(res.getCOSObject())) {
return false; // already inspected this resource dictionary
}
for (COSName name : res.getFontNames()) {
PDFont font;
try {
font = res.getFont(name);
} catch (Exception e) {
return true; // font won't load for inspection - assume unreliable
}
// Reliable only for built-in Standard-14 fonts or a /ToUnicode map we can trust.
if (font != null
&& !font.isStandard14()
&& (!font.getCOSObject().containsKey(COSName.TO_UNICODE)
|| isSubsetFont(font))) {
return true;
}
}
for (COSName name : res.getXObjectNames()) {
try {
if (res.getXObject(name) instanceof PDFormXObject form
&& resourcesHaveUnreliableFont(form.getResources(), visited, depth + 1)) {
return true;
}
} catch (Exception e) {
return true; // can't inspect the XObject - assume unreliable
}
}
return false;
}
/**
* A subset-embedded font carries a 6-uppercase-letter '+' BaseFont tag (e.g. {@code ABCDEF+}).
* Its /ToUnicode is custom-built and cannot be trusted for the reliability gate: a partial map
* silently loses text, and a crafted map can defeat text-based redaction entirely (only
* convert-to-image fully mitigates that adversarial case).
*/
private static boolean isSubsetFont(PDFont font) {
String name = font.getName();
if (name == null || name.length() < 8 || name.charAt(6) != '+') {
return false;
}
for (int i = 0; i < 6; i++) {
char c = name.charAt(i);
if (c < 'A' || c > 'Z') {
return false;
}
}
return true;
}
/** Fail-closed match check with whitespace-normalised literals and X2 regex semantics. */
private static void assertNoTarget(
String extracted, Set<String> literalTargets, List<Pattern> patterns) {
if (extracted == null) {
return;
}
String normalised = extracted.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
// De-hyphenated view catches a target split by a soft hyphen (U+00AD) or a line-break
// hyphen ("-" + space).
String dehyphenated = normalised.replace("\u00ad", "").replaceAll("-\\s+", "");
if (literalTargets != null) {
for (String target : literalTargets) {
if (target == null || target.isEmpty()) {
continue;
}
String needle = target.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
if (normalised.contains(needle) || dehyphenated.contains(needle)) {
throw new RedactionVerificationFailedException(
"Redacted text still extractable: '" + target + "'");
}
}
}
if (patterns != null) {
for (Pattern pattern : patterns) {
try {
if (pattern.matcher(DeadlineCharSequence.of(extracted)).find()) {
throw new RedactionVerificationFailedException(
"Redacted pattern still extractable: " + pattern.pattern());
}
} catch (RedactionVerificationFailedException rvf) {
throw rvf;
} catch (RuntimeException | StackOverflowError e) {
throw new RedactionVerificationFailedException(
"Verification regex failed ("
+ pattern.pattern()
+ "): "
+ e.getMessage(),
e instanceof Exception ? (Exception) e : new Exception(e));
}
}
}
}
/** Extract text from pageIndexes (0-based) using an /ActualText-blind stripper. */
private static String extractText(PDDocument document, Set<Integer> pageIndexes)
throws IOException {
PDFTextStripper stripper = new GlyphOnlyTextStripper();
if (pageIndexes == null) {
return stripper.getText(document);
}
StringBuilder out = new StringBuilder();
for (Integer pageIdx : pageIndexes) {
if (pageIdx == null || pageIdx < 0 || pageIdx >= document.getNumberOfPages()) {
continue;
}
stripper.setStartPage(pageIdx + 1);
stripper.setEndPage(pageIdx + 1);
String pageText = stripper.getText(document);
if (pageText != null) {
out.append(pageText);
}
}
return out.toString();
}
/**
* Independent native (PDFium) extraction; null if the binding is unavailable (additive only).
*/
private static String extractTextJPDFium(byte[] bytes, Set<Integer> pageIndexes) {
if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) {
return null;
}
try (PdfDocument doc = PdfDocument.open(bytes)) {
StringBuilder sb = new StringBuilder();
int n = doc.pageCount();
if (pageIndexes == null) {
for (int i = 0; i < n; i++) {
sb.append(jpdfiumPlainText(doc, i)).append('\n');
}
} else {
for (Integer p : pageIndexes) {
if (p != null && p >= 0 && p < n) {
sb.append(jpdfiumPlainText(doc, p)).append('\n');
}
}
}
return sb.toString();
} catch (RuntimeException | Error e) {
onJpdfiumFailure(e);
return null;
}
}
private static String jpdfiumPlainText(PdfDocument doc, int i) {
try {
return PdfTextExtractor.extractPage(doc, i).plainText();
} catch (RuntimeException | Error e) {
return "";
}
}
/** One native open, all pages' plain text; null if the binding is unavailable. */
private static List<String> extractPagesJPDFium(byte[] bytes) {
if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) {
return null;
}
try (PdfDocument doc = PdfDocument.open(bytes)) {
List<String> pages = new ArrayList<>();
int n = doc.pageCount();
for (int i = 0; i < n; i++) {
pages.add(jpdfiumPlainText(doc, i));
}
return pages;
} catch (RuntimeException | Error e) {
onJpdfiumFailure(e);
return null;
}
}
/**
* A native-binding load error latches the pass off process-wide (warn once) so a host without
* the native stops retrying; a per-document error only skips this one document (debug).
*/
private static void onJpdfiumFailure(Throwable e) {
boolean nativeUnavailable =
e instanceof UnsatisfiedLinkError
|| e instanceof NoClassDefFoundError
|| e instanceof ExceptionInInitializerError
|| e.getClass().getSimpleName().contains("NativeLoad");
if (nativeUnavailable) {
jpdfiumAvailable = false;
log.warn(
"JPDFium native unavailable; redaction verification will use the PDFBox pass "
+ "only: {}",
e.toString());
} else {
log.debug("JPDFium verification skipped for this document: {}", e.toString());
}
}
/** Pages whose surviving text (PDFBox glyph-blind OR native PDFium) still matches a target. */
static Set<Integer> findLeakingPages(
byte[] bytes, Set<String> literalTargets, List<Pattern> patterns) {
List<String> jpdfiumPages = extractPagesJPDFium(bytes);
try (PDDocument reopened = Loader.loadPDF(bytes)) {
// Native pass required but unavailable: PDFBox can't localise the leak, so rasterise
// every page.
if (jpdfiumPages == null && documentHasUnreliableFont(reopened)) {
log.warn(
"Independent native pass unavailable on an unreliable-font document; "
+ "rasterising all pages to guarantee removal.");
return null;
}
Set<Integer> leaking = new TreeSet<>();
GlyphOnlyTextStripper stripper = new GlyphOnlyTextStripper();
for (int i = 0; i < reopened.getNumberOfPages(); i++) {
stripper.setStartPage(i + 1);
stripper.setEndPage(i + 1);
String pdfboxText = stripper.getText(reopened);
String jpdfiumText =
jpdfiumPages != null && i < jpdfiumPages.size() ? jpdfiumPages.get(i) : "";
String combined = (pdfboxText == null ? "" : pdfboxText) + "\n" + jpdfiumText;
if (pageLeaks(combined, literalTargets, patterns)) {
leaking.add(i);
}
}
if (leaking.isEmpty()) {
log.warn(
"Verification failed but no single page leaks in isolation; rasterising "
+ "all pages to be safe.");
return null;
}
log.info("Leak detection: rasterising only page(s) {}", leaking);
return leaking;
} catch (Exception e) {
log.warn("Per-page leak detection failed ({}); rasterising all pages.", e.toString());
return null;
}
}
private static boolean pageLeaks(
String pageText, Set<String> literalTargets, List<Pattern> patterns) {
try {
assertNoTarget(pageText, literalTargets, patterns);
return false;
} catch (RedactionVerificationFailedException e) {
return true;
}
}
/** Redacted pages that still show text inside a rect or carry an overlapping annotation. */
static Set<Integer> findLeakingRectPages(
byte[] bytes, Map<Integer, List<PDRectangle>> rectsByPage) {
Set<Integer> leaking = new HashSet<>();
if (rectsByPage == null || rectsByPage.isEmpty()) {
return leaking;
}
try (PDDocument reopened = Loader.loadPDF(bytes)) {
for (Map.Entry<Integer, List<PDRectangle>> entry : rectsByPage.entrySet()) {
int pageIndex = entry.getKey();
if (pageIndex < 0 || pageIndex >= reopened.getNumberOfPages()) {
continue;
}
PDPage page = reopened.getPage(pageIndex);
if (glyphStillInRects(reopened, page, pageIndex, entry.getValue())
|| annotationOverlapsRect(page, entry.getValue())) {
leaking.add(pageIndex);
}
}
} catch (IOException e) {
// Cannot verify - rasterise every redacted page to be safe.
return new HashSet<>(rectsByPage.keySet());
}
return leaking;
}
/**
* True if any non-blank glyph is still painted inside a rect. Position-based (not ToUnicode),
* so it catches residual CID/Type3/no-ToUnicode glyphs the text stripper would miss; fails
* closed.
*/
private static boolean glyphStillInRects(
PDDocument doc, PDPage page, int pageIndex, List<PDRectangle> rects) {
try {
List<Rectangle2D.Float> areaRects = new ArrayList<>();
for (PDRectangle rect : rects) {
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
areaRects.add(
new Rectangle2D.Float(
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight()));
}
TokenIndexCollector collector = new TokenIndexCollector(areaRects);
collector.setStartPage(pageIndex + 1);
collector.setEndPage(pageIndex + 1);
collector.getText(doc);
return collector.anyGlyphInRect();
} catch (Exception e) {
return true; // cannot prove the rect is clean - rasterise to be safe
}
}
private static boolean annotationOverlapsRect(PDPage page, List<PDRectangle> rects) {
try {
for (var ann : page.getAnnotations()) {
PDRectangle ar = ann.getRectangle();
if (ar == null) {
continue;
}
Rectangle2D.Float a =
new Rectangle2D.Float(
ar.getLowerLeftX(),
ar.getLowerLeftY(),
ar.getWidth(),
ar.getHeight());
for (PDRectangle rect : rects) {
if (a.intersects(
rect.getLowerLeftX(),
rect.getLowerLeftY(),
rect.getWidth(),
rect.getHeight())) {
return true;
}
}
}
} catch (Exception e) {
return true;
}
return false;
}
/** PDFTextStripper that strips /ActualText so verification sees the real glyph stream. */
private static final class GlyphOnlyTextStripper extends PDFTextStripper {
private static final COSName ACTUAL_TEXT = COSName.getPDFName("ActualText");
GlyphOnlyTextStripper() throws IOException {}
@Override
public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {
COSDictionary safe = properties;
if (properties != null && properties.containsKey(ACTUAL_TEXT)) {
safe = new COSDictionary(properties);
safe.removeItem(ACTUAL_TEXT);
}
super.beginMarkedContentSequence(tag, safe);
}
}
}
@@ -0,0 +1,75 @@
package stirling.software.SPDF.pdf.redaction;
import java.awt.geom.Rectangle2D;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.pdfbox.contentstream.operator.Operator;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
/**
* Position-based (ToUnicode-independent) glyph locator: records, per text-showing operator, the
* 0-based indexes of the non-blank glyphs whose box intersects a rect, plus each operator's total
* glyph count. Blank glyphs are ignored so a space left behind by removal is not a "leak".
*/
final class TokenIndexCollector extends PDFTextStripper {
private final List<Rectangle2D.Float> rects;
final Map<Integer, Set<Integer>> dropGlyphsByOp = new HashMap<>();
final Map<Integer, Integer> glyphCountByOp = new HashMap<>();
private int showTextOpCounter = -1;
private int glyphInOp = 0;
TokenIndexCollector(List<Rectangle2D.Float> rects) throws IOException {
this.rects = rects;
setSortByPosition(false);
}
int totalTextOps() {
return showTextOpCounter + 1;
}
boolean anyGlyphInRect() {
return !dropGlyphsByOp.isEmpty();
}
@Override
protected void processTextPosition(TextPosition text) {
int op = showTextOpCounter;
int idx = glyphInOp++;
String u = text.getUnicode();
if (u != null && !u.isBlank()) {
// PDFBox reports coordinates with top-left origin here.
float x = text.getX();
float y = text.getY() - text.getHeight();
Rectangle2D.Float glyph =
new Rectangle2D.Float(x, y, text.getWidth(), text.getHeight());
for (Rectangle2D.Float rect : rects) {
if (rect.intersects(glyph)) {
dropGlyphsByOp.computeIfAbsent(op, k -> new HashSet<>()).add(idx);
return;
}
}
}
super.processTextPosition(text);
}
@Override
protected void processOperator(Operator operator, List<COSBase> operands) throws IOException {
String name = operator.getName();
boolean textOp = RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name);
if (textOp) {
showTextOpCounter++;
glyphInOp = 0;
}
super.processOperator(operator, operands);
if (textOp) {
glyphCountByOp.put(showTextOpCounter, glyphInOp);
}
}
}
@@ -86,14 +86,21 @@ public class TextFinderUtils {
return patterns;
}
private static String applyWordBoundaries(String originalTerm, String patternString) {
if (originalTerm.length() == 1 && Character.isDigit(originalTerm.charAt(0))) {
/** Shared whole-word wrapper so the finder and RedactionPipeline never diverge (see F2). */
public static String applyWordBoundaries(String originalTerm, String patternString) {
// Lookarounds for single/non-word-edge terms (\b no-ops on chars like '&'); \b otherwise.
boolean nonWordEdge =
originalTerm.length() == 1
|| !isWordChar(originalTerm.charAt(0))
|| !isWordChar(originalTerm.charAt(originalTerm.length() - 1));
if (nonWordEdge) {
return "(?<![\\w])" + patternString + "(?![\\w])";
} else if (originalTerm.length() == 1) {
return "(?<![\\w])" + patternString + "(?![\\w])";
} else {
return "\\b" + patternString + "\\b";
}
return "\\b" + patternString + "\\b";
}
private static boolean isWordChar(char c) {
return Character.isLetterOrDigit(c) || c == '_';
}
public static boolean hasProblematicFonts(PDPage page) {
@@ -19,6 +19,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -568,7 +569,15 @@ class ManualRedactionServiceTest {
byPage.put(0, new ArrayList<>(Arrays.asList(text(0, 72, 690, 300, 710))));
TempFile result =
service.finalizeRedaction(doc, byPage, "#000000", 1.0f, false, false);
service.finalizeRedaction(
doc,
byPage,
"#000000",
1.0f,
false,
false,
Collections.emptySet(),
Collections.emptyList());
assertNotNull(result);
assertNotNull(result.getFile());
@@ -590,7 +599,15 @@ class ManualRedactionServiceTest {
Map<Integer, List<PDFText>> byPage = new HashMap<>();
TempFile result =
service.finalizeRedaction(doc, byPage, "#000000", 0.0f, false, false);
service.finalizeRedaction(
doc,
byPage,
"#000000",
0.0f,
false,
false,
Collections.emptySet(),
Collections.emptyList());
assertNotNull(result);
assertTrue(result.getFile().exists());
@@ -610,7 +627,15 @@ class ManualRedactionServiceTest {
Map<Integer, List<PDFText>> byPage = new HashMap<>();
TempFile result =
service.finalizeRedaction(doc, byPage, "#000000", 0.0f, null, false);
service.finalizeRedaction(
doc,
byPage,
"#000000",
0.0f,
null,
false,
Collections.emptySet(),
Collections.emptyList());
assertNotNull(result);
assertTrue(result.getFile().exists());
@@ -630,7 +655,15 @@ class ManualRedactionServiceTest {
byPage.put(0, new ArrayList<>(Arrays.asList(text(0, 72, 690, 300, 710))));
TempFile result =
service.finalizeRedaction(doc, byPage, "#FF0000", 2.0f, false, true);
service.finalizeRedaction(
doc,
byPage,
"#FF0000",
2.0f,
false,
true,
Collections.emptySet(),
Collections.emptyList());
assertNotNull(result);
try (PDDocument reloaded = Loader.loadPDF(result.getFile())) {
@@ -658,7 +691,14 @@ class ManualRedactionServiceTest {
IOException.class,
() ->
service.finalizeRedaction(
doc, byPage, "#000000", 0.0f, false, false));
doc,
byPage,
"#000000",
0.0f,
false,
false,
Collections.emptySet(),
Collections.emptyList()));
// The failing temp file is closed on the error path.
verify(failing).close();
} finally {
@@ -192,15 +192,27 @@ class RedactControllerTest {
when(mockCOSStream.createOutputStream()).thenReturn(mockOutputStream);
when(mockCOSStream.createOutputStream(any())).thenReturn(mockOutputStream);
// The pipeline saves then reloads the bytes to verify, so the mock must emit a real,
// reloadable PDF (not a placeholder) or verification throws on reopen.
byte[] realPdfBytes = createSimplePdfContent();
lenient()
.doAnswer(
inv -> {
File f = inv.getArgument(0);
java.nio.file.Files.write(f.toPath(), "mock pdf".getBytes());
java.nio.file.Files.write(f.toPath(), realPdfBytes);
return null;
})
.when(mockDocument)
.save(any(File.class));
lenient()
.doAnswer(
inv -> {
java.io.OutputStream os = inv.getArgument(0);
os.write(realPdfBytes);
return null;
})
.when(mockDocument)
.save(any(java.io.OutputStream.class));
doNothing().when(mockDocument).close();
// Build real service instances so tests exercise actual logic
@@ -347,7 +359,7 @@ class RedactControllerTest {
assertNotNull(response);
assertEquals(200, response.getStatusCode().value());
verify(mockDocument).save(any(File.class));
verify(mockDocument).save(any(java.io.OutputStream.class));
verify(mockDocument).close();
}
}
@@ -753,7 +765,7 @@ class RedactControllerTest {
assertEquals(200, response.getStatusCode().value());
assertNotNull(response.getBody());
assertTrue(drainBody(response).length > 0);
verify(mockDocument, times(1)).save(any(File.class));
verify(mockDocument, times(1)).save(any(java.io.OutputStream.class));
verify(mockDocument, times(1)).close();
}
} catch (Exception e) {
@@ -776,7 +788,7 @@ class RedactControllerTest {
if (response != null) {
assertNotNull(response);
assertEquals(200, response.getStatusCode().value());
verify(mockDocument, times(1)).save(any(File.class));
verify(mockDocument, times(1)).save(any(java.io.OutputStream.class));
}
} catch (Exception e) {
log.info("Manual redaction test completed with graceful handling: {}", e.getMessage());
@@ -0,0 +1,755 @@
package stirling.software.SPDF.controller.api.security;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSInteger;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSStream;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDFormContentStream;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.common.PDStream;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDFontDescriptor;
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.apache.pdfbox.pdmodel.font.encoding.MacRomanEncoding;
import org.apache.pdfbox.pdmodel.font.encoding.WinAnsiEncoding;
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.util.Matrix;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.Resource;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
/** Security matrix for redaction across PDF shapes: standard and embedded */
@DisplayName("Redaction PDF-variety security matrix")
class RedactionPdfVarietyTest {
private static final String LIBERATION =
"/org/apache/pdfbox/resources/ttf/LiberationSans-Regular.ttf";
private static final float FONT_SIZE = 12f;
private static final float LEFT_X = 72f;
private static final float TOP_Y = PDRectangle.LETTER.getHeight() - 80f;
private CustomPDFDocumentFactory pdfDocumentFactory;
private TempFileManager tempFileManager;
private RedactController controller;
private final List<File> createdTempFiles = new ArrayList<>();
@BeforeEach
void setUp() throws IOException {
pdfDocumentFactory = mock(CustomPDFDocumentFactory.class);
tempFileManager = mock(TempFileManager.class);
lenient()
.when(tempFileManager.createManagedTempFile(anyString()))
.thenAnswer(
inv -> {
File f =
Files.createTempFile(
"redact-variety", inv.<String>getArgument(0))
.toFile();
createdTempFiles.add(f);
TempFile tf = mock(TempFile.class);
lenient().when(tf.getFile()).thenReturn(f);
lenient().when(tf.getPath()).thenReturn(f.toPath());
return tf;
});
controller =
new RedactController(
pdfDocumentFactory,
tempFileManager,
new ManualRedactionService(tempFileManager),
new TextRedactionService(),
mock(RedactExecuteService.class));
}
@AfterEach
void tearDown() {
for (File f : createdTempFiles) {
if (f != null && f.exists()) {
f.delete();
}
}
}
// Matrix cases (all through the real /auto-redact controller path)
@Test
@DisplayName("standard Helvetica: target gone, neighbours survive")
void standard14Helvetica() throws IOException {
byte[] out = autoRedact(helveticaPdf("alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("alpha").contains("omega");
}
@Test
@DisplayName("standard-14 Times and Courier: target gone, neighbours survive")
void standard14TimesAndCourier() throws IOException {
for (Standard14Fonts.FontName fn :
new Standard14Fonts.FontName[] {
Standard14Fonts.FontName.TIMES_ROMAN, Standard14Fonts.FontName.COURIER
}) {
byte[] pdf = std14Pdf(fn, "alpha SECRET omega");
byte[] out = autoRedact(pdf, "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).as("%s keeps neighbours", fn).contains("alpha");
}
}
@Test
@DisplayName("simple TrueType with MacRoman encoding: target gone, neighbours survive")
void simpleTrueTypeMacRoman() throws IOException {
byte[] out = autoRedact(macRomanTtfPdf("alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("alpha").contains("omega");
}
@Test
@DisplayName("embedded Type0 with /ToUnicode stripped: target still gone, neighbours survive")
void type0WithoutToUnicode() throws IOException {
byte[] out = autoRedact(type0NoToUnicodePdf("alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("alpha").contains("omega");
}
@Test
@DisplayName("symbolic TrueType (no encoding, no ToUnicode): target not extractable")
void symbolicTrueType() throws IOException {
// PDFBox extraction is glyph-blind here, so removal is guaranteed by the native pass +
// rasterise fallback rather than surgical decode; this pins that the pipeline handles a
// symbolic font without error and leaves no extractable target.
byte[] out = autoRedact(symbolicTtfPdf("alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
}
@Test
@DisplayName("subset-tagged font (ABCDEF+): target gone, neighbours survive")
void subsetTaggedFont() throws IOException {
byte[] out = autoRedact(subsetTaggedPdf("alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("alpha").contains("omega");
}
@Test
@DisplayName("real Type3 font (glyphs as content streams, no ToUnicode): target gone")
void type3GlyphProcs() throws IOException {
// crop_test.pdf uses DejaVuSans embedded as a subset Type3 font with no
byte[] input;
try (InputStream in = getClass().getResourceAsStream("/redaction/type3_dejavu.pdf")) {
input = in.readAllBytes();
}
byte[] out = autoRedact(input, "EXAMPLE");
assertGone(out, "EXAMPLE");
assertThat(pdfText(out)).contains("CROP");
}
@Test
@DisplayName("embedded subset Type0 (CID) font: target gone, neighbours survive")
void embeddedSubsetType0() throws IOException {
byte[] out = autoRedact(type0Pdf(true, "alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("alpha").contains("omega");
}
@Test
@DisplayName("embedded full Type0 (CID) font: target gone, neighbours survive")
void embeddedFullType0() throws IOException {
byte[] out = autoRedact(type0Pdf(false, "alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("alpha").contains("omega");
}
@Test
@DisplayName("embedded Type1 (PFB) font: target gone, neighbours survive")
void embeddedType1() throws IOException {
byte[] out = autoRedact(type1Pdf("PUBLICA SECRETWORD KEEP"), "SECRETWORD");
assertGone(out, "SECRETWORD");
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
}
@Test
@DisplayName("embedded CJK Type0 (CIDFontType2) font: CJK target gone, neighbours survive")
void embeddedCjkType0() throws IOException {
byte[] out = autoRedact(cjkPdf("PUBLICA \u79D8\u5BC6 KEEP"), "\u79D8\u5BC6");
assertGone(out, "\u79D8\u5BC6");
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
}
@Test
@DisplayName("embedded CIDFontType0 (CFF) font: target gone, neighbours survive")
void embeddedCidFontType0() throws IOException {
byte[] out = autoRedact(cidFontType0Pdf("PUBLICA SECRETWORD KEEP"), "SECRETWORD");
assertGone(out, "SECRETWORD");
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
}
@Test
@DisplayName("simple embedded TrueType (WinAnsi) font: target gone, neighbours survive")
void simpleTrueTypeWinAnsi() throws IOException {
byte[] out = autoRedact(simpleTtfPdf("alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("alpha").contains("omega");
}
@Test
@DisplayName("rotated pages (90/180/270): target gone on every rotation")
void rotatedPages() throws IOException {
for (int rotation : new int[] {90, 180, 270}) {
byte[] out = autoRedact(rotatedPdf(rotation, "alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).as("rotation %d keeps neighbours", rotation).contains("alpha");
}
}
@Test
@DisplayName("target split across TJ array operands is removed")
void tjArraySplit() throws IOException {
byte[] out = autoRedact(tjSplitPdf(), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("public");
}
@Test
@DisplayName("target split across separate Tj operators is removed, other pages keep text")
void crossOperatorSplit() throws IOException {
byte[] out = autoRedact(crossOperatorPdf(), "SECRET");
assertGone(out, "SECRET");
// Page 2 was never touched; whatever path handled page 1, page 2 text must
assertThat(pdfText(out)).contains("PUBLIC PAGE TWO");
}
@Test
@DisplayName("target inside a Form XObject is removed")
void formXObjectText() throws IOException {
byte[] out = autoRedact(formXObjectPdf(), "SECRET");
assertGone(out, "SECRET");
}
@Test
@DisplayName("CropBox smaller than MediaBox: target gone, neighbours survive")
void cropBoxOffset() throws IOException {
byte[] out = autoRedact(cropBoxPdf("alpha SECRET omega"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("alpha");
}
@Test
@DisplayName("multi-page: target on middle page only, other pages keep their text")
void multiPageMiddleTarget() throws IOException {
byte[] out =
autoRedact(
multiPagePdf("PUBLIC ONE", "middle SECRET line", "PUBLIC THREE"), "SECRET");
assertGone(out, "SECRET");
assertThat(pdfText(out)).contains("PUBLIC ONE").contains("PUBLIC THREE");
}
@Test
@DisplayName("case variant: doc contains 'Secret', target 'SECRET' removes it")
void caseVariantRemoved() throws IOException {
byte[] out = autoRedact(helveticaPdf("alpha Secret omega"), "SECRET");
assertGone(out, "Secret");
assertThat(pdfText(out)).contains("alpha").contains("omega");
}
@Test
@DisplayName("whole-word with case variant: 'Cat' removed, substrings survive")
void wholeWordCaseVariant() throws IOException {
byte[] bytes = helveticaPdf("Cat classification scatter");
factoryReturns(bytes);
RedactPdfRequest request = baseRequest(bytes, "cat");
request.setWholeWordSearch(true);
byte[] out = drainBody(controller.redactPdf(request));
String text = pdfText(out);
assertThat(text).contains("classification").contains("scatter");
// The standalone word must be gone in any case variant.
assertThat(text).doesNotContainPattern("(?i)\\bcat\\b");
}
// Page-scoped rasterisation (pipeline-level, deterministic)
@Test
@DisplayName("verification leak rasterises only the leaking page, others keep text")
void leakRasterisesOnlyLeakingPage() throws IOException {
// Cross-operator split defeats the per-operand literal rewriter
byte[] input = crossOperatorPdf();
byte[] out;
try (PDDocument doc = Loader.loadPDF(input)) {
Set<String> targets = Set.of("SECRET");
List<Pattern> patterns =
RedactionPipeline.buildPatterns(new String[] {"SECRET"}, false, false);
RedactionPipeline.redactLiteralTerms(doc, targets, patterns);
out = RedactionPipeline.finalize(doc, targets, patterns);
}
try (PDDocument reopened = Loader.loadPDF(out)) {
PDFTextStripper stripper = new PDFTextStripper();
stripper.setStartPage(1);
stripper.setEndPage(1);
String page1 = stripper.getText(reopened);
stripper.setStartPage(2);
stripper.setEndPage(2);
String page2 = stripper.getText(reopened);
assertThat(page1.toLowerCase()).doesNotContain("secret");
assertThat(page2).contains("PUBLIC PAGE TWO");
}
}
// Drivers and assertions
private byte[] autoRedact(byte[] pdfBytes, String target) throws IOException {
factoryReturns(pdfBytes);
RedactPdfRequest request = baseRequest(pdfBytes, target);
ResponseEntity<Resource> response = controller.redactPdf(request);
assertThat(response.getStatusCode().value()).isEqualTo(200);
return drainBody(response);
}
private RedactPdfRequest baseRequest(byte[] pdfBytes, String target) {
RedactPdfRequest request = new RedactPdfRequest();
request.setFileInput(pdfFile(pdfBytes));
request.setListOfText(target);
request.setUseRegex(false);
request.setWholeWordSearch(false);
request.setRedactColor("#000000");
request.setConvertPDFToImage(false);
return request;
}
private void assertGone(byte[] out, String target) throws IOException {
assertThat(pdfText(out).toLowerCase()).doesNotContain(target.toLowerCase());
}
private void factoryReturns(byte[] pdfBytes) throws IOException {
lenient()
.when(pdfDocumentFactory.load(any(MultipartFile.class)))
.thenAnswer(inv -> Loader.loadPDF(pdfBytes));
}
private MockMultipartFile pdfFile(byte[] bytes) {
return new MockMultipartFile("fileInput", "doc.pdf", "application/pdf", bytes);
}
private byte[] drainBody(ResponseEntity<Resource> response) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (InputStream in = response.getBody().getInputStream()) {
in.transferTo(baos);
}
return baos.toByteArray();
}
private String pdfText(byte[] pdfBytes) throws IOException {
try (PDDocument doc = Loader.loadPDF(pdfBytes)) {
return new PDFTextStripper().getText(doc);
}
}
// PDF builders
private static PDFont helvetica() {
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
}
private byte[] helveticaPdf(String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
addTextPage(doc, helvetica(), line, 0, null);
return save(doc);
}
}
private byte[] std14Pdf(Standard14Fonts.FontName fontName, String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
addTextPage(doc, new PDType1Font(fontName), line, 0, null);
return save(doc);
}
}
private byte[] macRomanTtfPdf(String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDFont font;
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
font = PDTrueTypeFont.load(doc, ttf, MacRomanEncoding.INSTANCE);
}
addTextPage(doc, font, line, 0, null);
return save(doc);
}
}
// GID order emitted by generators/gen_cff_cidfont.py: 0=.notdef 1=space then the letters.
private static final String CFF_GLYPHS = "\0 ABCDEIKLOPRSTUW";
private static int cffGid(char c) {
int i = CFF_GLYPHS.indexOf(c);
return i < 0 ? 0 : i;
}
/**
* PDFBox 3.0.7 has no CFF embedder, so hand-assemble a Type0 -> CIDFontType0 (CFF in
* /FontFile3, Identity-H, CID==GID, explicit /ToUnicode) from the generated CFF program.
*/
private byte[] cidFontType0Pdf(String line) throws IOException {
byte[] cff;
try (InputStream in = getClass().getResourceAsStream("/redaction/cidfont0_cff.bin")) {
cff = in.readAllBytes();
}
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
PDStream fontFile = new PDStream(doc);
try (var os = fontFile.createOutputStream()) {
os.write(cff);
}
COSStream ff = fontFile.getCOSObject();
ff.setItem(COSName.SUBTYPE, COSName.getPDFName("CIDFontType0C"));
COSDictionary fdesc = new COSDictionary();
fdesc.setItem(COSName.TYPE, COSName.FONT_DESC);
fdesc.setName(COSName.FONT_NAME, "MinimalCFFCID");
fdesc.setInt(COSName.FLAGS, 4);
COSArray bbox = new COSArray();
for (int v : new int[] {0, -200, 600, 800}) {
bbox.add(COSInteger.get(v));
}
fdesc.setItem(COSName.FONT_BBOX, bbox);
fdesc.setInt(COSName.ITALIC_ANGLE, 0);
fdesc.setInt(COSName.ASCENT, 800);
fdesc.setInt(COSName.DESCENT, -200);
fdesc.setInt(COSName.CAP_HEIGHT, 700);
fdesc.setInt(COSName.STEM_V, 80);
fdesc.setItem(COSName.getPDFName("FontFile3"), ff);
COSDictionary cidSystemInfo = new COSDictionary();
cidSystemInfo.setString(COSName.REGISTRY, "Adobe");
cidSystemInfo.setString(COSName.ORDERING, "Identity");
cidSystemInfo.setInt(COSName.SUPPLEMENT, 0);
COSDictionary cidFont = new COSDictionary();
cidFont.setItem(COSName.TYPE, COSName.FONT);
cidFont.setItem(COSName.SUBTYPE, COSName.CID_FONT_TYPE0);
cidFont.setName(COSName.BASE_FONT, "MinimalCFFCID");
cidFont.setItem(COSName.CIDSYSTEMINFO, cidSystemInfo);
cidFont.setItem(COSName.FONT_DESC, fdesc);
cidFont.setInt(COSName.DW, 600);
COSStream toUni = new PDStream(doc).getCOSObject();
try (var os = toUni.createOutputStream()) {
os.write(cffToUnicode().getBytes(java.nio.charset.StandardCharsets.US_ASCII));
}
COSDictionary type0 = new COSDictionary();
type0.setItem(COSName.TYPE, COSName.FONT);
type0.setItem(COSName.SUBTYPE, COSName.TYPE0);
type0.setName(COSName.BASE_FONT, "MinimalCFFCID");
type0.setItem(COSName.ENCODING, COSName.IDENTITY_H);
COSArray descendants = new COSArray();
descendants.add(cidFont);
type0.setItem(COSName.DESCENDANT_FONTS, descendants);
type0.setItem(COSName.TO_UNICODE, toUni);
PDFont font = org.apache.pdfbox.pdmodel.font.PDFontFactory.createFont(type0);
PDResources res = new PDResources();
COSName fName = res.add(font);
page.setResources(res);
StringBuilder csb = new StringBuilder();
csb.append("BT\n/").append(fName.getName()).append(" 24 Tf\n50 700 Td\n<");
for (int i = 0; i < line.length(); i++) {
csb.append(String.format("%04X", cffGid(line.charAt(i))));
}
csb.append("> Tj\nET\n");
PDStream content = new PDStream(doc);
try (var os = content.createOutputStream()) {
os.write(csb.toString().getBytes(java.nio.charset.StandardCharsets.US_ASCII));
}
page.setContents(content);
return save(doc);
}
}
private static String cffToUnicode() {
StringBuilder sb = new StringBuilder();
sb.append("/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n");
sb.append("/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n");
sb.append("/CMapName /Adobe-Identity-UCS def\n/CMapType 2 def\n");
sb.append("1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n");
StringBuilder body = new StringBuilder();
int count = 0;
for (int gid = 1; gid < CFF_GLYPHS.length(); gid++) {
body.append(String.format("<%04X> <%04X>\n", gid, (int) CFF_GLYPHS.charAt(gid)));
count++;
}
sb.append(count).append(" beginbfchar\n").append(body).append("endbfchar\n");
sb.append("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n");
return sb.toString();
}
private byte[] type1Pdf(String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDFont font;
try (InputStream pfb = getClass().getResourceAsStream("/redaction/type1_minimal.pfb")) {
font = new PDType1Font(doc, pfb, WinAnsiEncoding.INSTANCE);
}
addTextPage(doc, font, line, 0, null);
return save(doc);
}
}
private byte[] cjkPdf(String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDFont font;
try (InputStream ttf = getClass().getResourceAsStream("/redaction/cjk_minimal.ttf")) {
font = PDType0Font.load(doc, ttf, true);
}
addTextPage(doc, font, line, 0, null);
return save(doc);
}
}
private byte[] symbolicTtfPdf(String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDTrueTypeFont font;
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
}
addTextPage(doc, font, line, 0, null);
// Strip /Encoding and flag the descriptor Symbolic so extraction must use the cmap.
font.getCOSObject().removeItem(org.apache.pdfbox.cos.COSName.ENCODING);
PDFontDescriptor fd = font.getFontDescriptor();
fd.setFlags((fd.getFlags() | 4) & ~32);
return save(doc);
}
}
private byte[] type0NoToUnicodePdf(String line) throws IOException {
byte[] pdf;
try (PDDocument doc = new PDDocument()) {
PDFont font;
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
// Full embed, not subset: the subsetter regenerates /ToUnicode at save, undoing the
// strip.
font = PDType0Font.load(doc, ttf, false);
}
addTextPage(doc, font, line, 0, null);
font.getCOSObject().removeItem(org.apache.pdfbox.cos.COSName.TO_UNICODE);
pdf = save(doc);
}
// Guard: the strip must survive save so this fixture genuinely exercises the no-mapping
// path.
try (PDDocument reopened = Loader.loadPDF(pdf)) {
PDResources res = reopened.getPage(0).getResources();
for (org.apache.pdfbox.cos.COSName n : res.getFontNames()) {
assertThat(
res.getFont(n)
.getCOSObject()
.containsKey(org.apache.pdfbox.cos.COSName.TO_UNICODE))
.as("no-ToUnicode fixture must not ship /ToUnicode")
.isFalse();
}
}
return pdf;
}
private byte[] subsetTaggedPdf(String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDFont font;
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
font = PDType0Font.load(doc, ttf, true);
}
addTextPage(doc, font, line, 0, null);
String tagged = "ABCDEF+" + font.getName();
font.getCOSObject().setName(org.apache.pdfbox.cos.COSName.BASE_FONT, tagged);
if (font.getFontDescriptor() != null) {
font.getFontDescriptor()
.getCOSObject()
.setName(org.apache.pdfbox.cos.COSName.FONT_NAME, tagged);
}
return save(doc);
}
}
private byte[] type0Pdf(boolean subset, String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDFont font;
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
font = PDType0Font.load(doc, ttf, subset);
}
addTextPage(doc, font, line, 0, null);
return save(doc);
}
}
private byte[] simpleTtfPdf(String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDFont font;
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
}
addTextPage(doc, font, line, 0, null);
return save(doc);
}
}
private byte[] rotatedPdf(int rotation, String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
addTextPage(doc, helvetica(), line, rotation, null);
return save(doc);
}
}
private byte[] cropBoxPdf(String line) throws IOException {
try (PDDocument doc = new PDDocument()) {
addTextPage(doc, helvetica(), line, 0, new PDRectangle(40, 40, 500, 700));
return save(doc);
}
}
private byte[] multiPagePdf(String... pageLines) throws IOException {
try (PDDocument doc = new PDDocument()) {
for (String line : pageLines) {
addTextPage(doc, helvetica(), line, 0, null);
}
return save(doc);
}
}
/** One page whose text is emitted as a TJ array: ["public ", "SEC", -20 */
private byte[] tjSplitPdf() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.beginText();
cs.setFont(helvetica(), FONT_SIZE);
cs.newLineAtOffset(LEFT_X, TOP_Y);
cs.showTextWithPositioning(
new Object[] {"public ", "SEC", Float.valueOf(-20f), "RET", " end"});
cs.endText();
}
return save(doc);
}
}
/** Page 1 shows "SEC" and "RET" as separate adjacent Tj operators; page 2 */
private byte[] crossOperatorPdf() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
PDFont font = helvetica();
float secWidth = font.getStringWidth("SEC") / 1000f * FONT_SIZE;
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.beginText();
cs.setFont(font, FONT_SIZE);
cs.newLineAtOffset(LEFT_X, TOP_Y);
cs.showText("SEC");
cs.endText();
cs.beginText();
cs.setFont(font, FONT_SIZE);
cs.newLineAtOffset(LEFT_X + secWidth, TOP_Y);
cs.showText("RET");
cs.endText();
}
addTextPage(doc, helvetica(), "PUBLIC PAGE TWO", 0, null);
return save(doc);
}
}
private byte[] formXObjectPdf() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
PDFormXObject form = new PDFormXObject(doc);
form.setBBox(new PDRectangle(0, 0, 400, 60));
form.setResources(new PDResources());
try (PDFormContentStream fcs = new PDFormContentStream(form)) {
fcs.beginText();
fcs.setFont(helvetica(), FONT_SIZE);
fcs.newLineAtOffset(10, 20);
fcs.showText("xobj SECRET payload");
fcs.endText();
}
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.saveGraphicsState();
cs.transform(Matrix.getTranslateInstance(LEFT_X, TOP_Y - 60));
cs.drawForm(form);
cs.restoreGraphicsState();
}
return save(doc);
}
}
private void addTextPage(
PDDocument doc, PDFont font, String line, int rotation, PDRectangle cropBox)
throws IOException {
PDPage page = new PDPage(PDRectangle.LETTER);
if (rotation != 0) {
page.setRotation(rotation);
}
if (cropBox != null) {
page.setCropBox(cropBox);
}
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.beginText();
cs.setFont(font, FONT_SIZE);
if (rotation != 0) {
// Real generators compensate the text matrix so text reads upright
cs.setTextMatrix(
Matrix.getRotateInstance(
Math.toRadians(rotation),
PDRectangle.LETTER.getWidth() / 2,
PDRectangle.LETTER.getHeight() / 2));
} else {
cs.newLineAtOffset(LEFT_X, TOP_Y);
}
cs.showText(line);
cs.endText();
}
}
private static byte[] save(PDDocument doc) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
doc.save(baos);
return baos.toByteArray();
}
}
@@ -287,8 +287,12 @@ class TextRedactionServiceExtraTest {
service.createTokensWithoutTargetText(doc, page, Set.of("SECRET"), false, false);
// The deep traversal must have rewritten the inner form's content stream.
assertThat(inner.getCOSObject().containsKey(COSName.CONTENTS)).isTrue();
// The deep traversal must have rewritten the inner form's own stream in place (F3).
String innerContent;
try (var in = inner.getContents()) {
innerContent = new String(in.readAllBytes(), StandardCharsets.ISO_8859_1);
}
assertThat(innerContent).doesNotContain("SECRET");
}
}
@@ -226,19 +226,23 @@ class TextRedactionServiceMoreTest {
}
page.setContents(pageStream);
// Processing the page walks into the XObject graph; when a match is found inside
// the
// form, writeRedactedContentToXObject runs and sets a /Contents item on the form's
// COS dictionary. Asserting that item appears proves the XObject redaction path
// executed end-to-end without throwing.
// Processing the page walks into the XObject graph and rewrites the form's own
// stream body in place (F3), so the target is physically gone from the form.
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
assertThat(tokens).isNotNull();
assertThat(form.getCOSObject().containsKey(COSName.CONTENTS))
.as("form XObject redaction path should have written a new content item")
.isTrue();
String rewrittenForm;
try (var in = form.getContents()) {
rewrittenForm =
new String(
in.readAllBytes(),
java.nio.charset.StandardCharsets.ISO_8859_1);
}
assertThat(rewrittenForm)
.as("form XObject content should have the target removed in place")
.doesNotContain("SECRET");
}
}
}
@@ -0,0 +1,159 @@
package stirling.software.SPDF.pdf.redaction;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.apache.pdfbox.text.PDFTextStripper;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
/** Integration tests that exercise the real redaction pipeline against PDFs */
class RedactionPipelineIntegrationTest {
private static byte[] loadFixture() throws Exception {
try (InputStream in =
RedactionPipelineIntegrationTest.class.getResourceAsStream(
"/redaction/test_pdf_1.pdf")) {
assertNotNull(in, "fixture resource must exist on classpath");
return in.readAllBytes();
}
}
@Test
@DisplayName(
"finalize on rotated ReportLab PDF without a rewrite pass falls back to rasterisation so target disappears")
void finaliseFallsBackToRasterisationWhenTargetWouldSurvive() throws Exception {
byte[] fixtureBytes = loadFixture();
byte[] outputBytes;
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
Set<String> literalTargets = new LinkedHashSet<>();
literalTargets.add("Test");
// No content-stream rewriting performed.
outputBytes = RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
}
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
String extracted = new PDFTextStripper().getText(reopened);
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
assertFalse(
lower.contains("test"),
"Rasterisation fallback must have removed target. Extracted: '"
+ extracted
+ "'");
}
}
@Test
@DisplayName(
"finalize with zero rewrite + synthetic upright PDF triggers RedactionVerificationFailedException only when fallback disabled")
void verificationExceptionWiringSanityCheck() throws Exception {
// This test proves the verification hook itself still throws -
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.beginText();
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
cs.newLineAtOffset(100, 700);
cs.showText("Top Smith Classified");
cs.endText();
}
Set<String> literalTargets = new LinkedHashSet<>();
literalTargets.add("Smith");
// Even with the rasterisation fallback active, the output must not contain
byte[] outBytes =
RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
try (PDDocument reopened = Loader.loadPDF(outBytes)) {
String extracted = new PDFTextStripper().getText(reopened);
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
assertFalse(
lower.contains("smith"),
"Neither primary pass nor rasterisation removed target. Extracted: '"
+ extracted
+ "'");
}
}
}
@Test
@DisplayName(
"auto-word redact on rotated PDF with target packed in single Tj removes the word from text stream")
void autoWordRedactRemovesTargetOnRotatedSingleTjPdf() throws Exception {
byte[] fixtureBytes = loadFixture();
byte[] outputBytes;
Set<String> literalTargets = new LinkedHashSet<>();
literalTargets.add("Test");
List<Pattern> patterns =
RedactionPipeline.buildPatterns(new String[] {"Test"}, false, false);
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
RedactionPipeline.redactLiteralTerms(doc, literalTargets, patterns);
outputBytes = RedactionPipeline.finalize(doc, literalTargets, patterns);
}
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
PDFTextStripper stripper = new PDFTextStripper();
String extracted = stripper.getText(reopened);
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
assertFalse(
lower.contains("test"),
"Target term must not be extractable after redact. Extracted: '"
+ extracted
+ "'");
}
}
@Test
@DisplayName("simple upright Helvetica fixture still gets word redacted via the pipeline")
void autoWordRedactRemovesTargetOnSyntheticPdf() throws Exception {
byte[] outputBytes;
Set<String> literalTargets = new LinkedHashSet<>();
literalTargets.add("Secret");
List<Pattern> patterns =
RedactionPipeline.buildPatterns(new String[] {"Secret"}, false, false);
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.beginText();
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
cs.newLineAtOffset(100, 700);
cs.showText("Top Secret Classified");
cs.endText();
}
ByteArrayOutputStream tmp = new ByteArrayOutputStream();
doc.save(tmp);
// reload from bytes so we are processing an already-saved PDF similar
try (PDDocument reloaded = Loader.loadPDF(tmp.toByteArray())) {
RedactionPipeline.redactLiteralTerms(reloaded, literalTargets, patterns);
outputBytes = RedactionPipeline.finalize(reloaded, literalTargets, patterns);
}
}
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
String extracted = new PDFTextStripper().getText(reopened);
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
assertFalse(
lower.contains("secret"),
"Target term must not be extractable after redact on synthetic PDF. Extracted: '"
+ extracted
+ "'");
}
}
}
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
"""Generate a minimal OpenType-CFF (.otf) font for a CIDFontType0 redaction fixture.
Why this exists
---------------
Real-world Adobe/InDesign PDFs embed CFF-based CID fonts (Type0 -> CIDFontType0,
program in /FontFile3 /Subtype /CIDFontType0C). PDFBox 3.0.7 can *read* those but
ships NO CFF embedder: every PDType0Font.load(...) overload routes CFF outlines
through the TrueType path (which either rejects "TTF using CFF outlines" or, via the
OTFParser+TrueTypeFont overload, emits a CIDFontType2 and then fails to subset CFF
because there is no glyf table). So to build a genuine CIDFontType0 fixture you must:
1. generate the CFF .otf with fontTools (this script), and
2. assemble the Type0/CIDFontType0 dictionary by hand, attaching the raw CFF table
as /FontFile3 /Subtype /CIDFontType0C (see the companion Java harness).
This script emits BOTH minimal_cff.otf and cff_raw.bin (the raw CFF table bytes,
ready to drop into /FontFile3).
Glyph coverage: only the ASCII letters in "PUBLICA SECRETWORD KEEP"
(P U B L I C A S E R T W O D K) plus .notdef and space. Outlines are trivial filled
boxes; only the cmap / encoding correctness matters so the drawn text extracts.
"""
import sys
from fontTools.fontBuilder import FontBuilder
from fontTools.pens.t2CharStringPen import T2CharStringPen
from fontTools.ttLib import TTFont
UPM = 1000
LETTERS = sorted(set("PUBLICASECRETWORDKEEP")) # A B C D E I K L O P R S T U W
def box_charstring(width=600, height=700):
"""A simple filled rectangle as a Type2 (CFF) charstring."""
pen = T2CharStringPen(width, None)
pen.moveTo((50, 0))
pen.lineTo((width - 50, 0))
pen.lineTo((width - 50, height))
pen.lineTo((50, height))
pen.closePath()
return pen.getCharString()
def build_otf(out_otf):
# GID order: .notdef (GID 0), space (GID 1), then the letters. The companion
# harness relies on this exact order for its Identity-H CID==GID mapping.
glyph_order = [".notdef", "space"] + list(LETTERS)
fb = FontBuilder(unitsPerEm=UPM, isTTF=False) # isTTF=False -> CFF/OTF
fb.setupGlyphOrder(glyph_order)
# Unicode cmap: codepoint -> glyph name. PDFBox reads this to build /ToUnicode,
# and it also lets any consumer resolve the glyphs by character.
cmap = {0x20: "space"}
for c in LETTERS:
cmap[ord(c)] = c
fb.setupCharacterMap(cmap)
charstrings = {
".notdef": box_charstring(),
"space": box_charstring(width=250, height=0), # empty advance-only glyph
}
for c in LETTERS:
charstrings[c] = box_charstring()
fb.setupCFF(
psName="MinimalCFFCID",
charStringsDict=charstrings,
fontInfo={
"FullName": "Minimal CFF CID",
"FamilyName": "MinimalCFFCID",
"Weight": "Regular",
},
privateDict={},
)
metrics = {".notdef": (600, 50), "space": (250, 0)}
for c in LETTERS:
metrics[c] = (600, 50)
fb.setupHorizontalMetrics(metrics)
fb.setupHorizontalHeader(ascent=800, descent=-200)
fb.setupNameTable(
{
"familyName": "MinimalCFFCID",
"styleName": "Regular",
"psName": "MinimalCFFCID",
"fullName": "Minimal CFF CID",
"version": "Version 1.000",
"uniqueFontIdentifier": "MinimalCFFCID;1.000",
}
)
fb.setupOS2(
sTypoAscender=800,
sTypoDescender=-200,
usWinAscent=800,
usWinDescent=200,
sCapHeight=700,
)
fb.setupPost(keepGlyphNames=True)
fb.save(out_otf)
print("wrote", out_otf, "(", len(glyph_order), "glyphs )")
def dump_cff(out_otf, out_cff):
"""Extract the raw CFF table bytes for use as PDF /FontFile3."""
f = TTFont(out_otf)
cff = f["CFF "].compile(f)
with open(out_cff, "wb") as fh:
fh.write(cff)
print("wrote", out_cff, "(", len(cff), "bytes CFF )")
if __name__ == "__main__":
otf = sys.argv[1] if len(sys.argv) > 1 else "minimal_cff.otf"
cffbin = sys.argv[2] if len(sys.argv) > 2 else "cff_raw.bin"
build_otf(otf)
dump_cff(otf, cffbin)
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""
Generate a minimal TrueType font (.ttf) that covers exactly the characters
needed for a CJK Type0 (Identity-H) redaction test:
- ASCII letters in "PUBLICA SECRETWORD KEEP": P U B L I C A S E R T W O D K + space
- Two CJK ideographs used as the redaction target:
U+79D8 (secret1) and U+5BC6 (secret2) -> "秘密"
- .notdef
Glyph outlines are trivial boxes; the point of the test is that the cmap /
encoding is correct so PDFBox can build a working /ToUnicode and the text is
EXTRACTABLE. When PDFBox loads this via PDType0Font.load(doc, in, true) it
becomes an Identity-H Type0 font with a CIDFontType2 descendant.
Usage: python gen_cjk_font.py <output.ttf>
"""
import sys
from fontTools.fontBuilder import FontBuilder
from fontTools.pens.ttGlyphPen import TTGlyphPen
UPM = 1000
def box_glyph(width_units):
"""A trivial filled box outline so the glyph has real contours."""
pen = TTGlyphPen(None)
x0, y0, x1, y1 = 80, 0, max(120, width_units - 80), 700
pen.moveTo((x0, y0))
pen.lineTo((x0, y1))
pen.lineTo((x1, y1))
pen.lineTo((x1, y0))
pen.closePath()
return pen.glyph()
def empty_glyph():
return TTGlyphPen(None).glyph()
def main(out_path):
# Characters we must cover. Order defines glyph order (after .notdef).
ascii_chars = list("PUBLICA SECRETWORD KEEP")
ascii_unique = []
for ch in ascii_chars:
if ch not in ascii_unique:
ascii_unique.append(ch)
cjk = ["秘", "密"] # 秘 密
# glyph name -> codepoint mapping
def gname(cp):
if cp == 0x20:
return "space"
return "uni%04X" % cp
codepoints = []
for ch in ascii_unique:
codepoints.append(ord(ch))
for ch in cjk:
codepoints.append(ord(ch))
glyph_order = [".notdef"] + [gname(cp) for cp in codepoints]
fb = FontBuilder(UPM, isTTF=True)
fb.setupGlyphOrder(glyph_order)
# cmap: map every codepoint to its glyph. FontBuilder builds format-4
# (BMP) + format-12 as needed. Both CJK codepoints are BMP so format-4
# covers them.
cmap = {cp: gname(cp) for cp in codepoints}
fb.setupCharacterMap(cmap)
# Build glyf table with box outlines; advance widths.
glyphs = {".notdef": empty_glyph()}
metrics = {".notdef": (600, 0)}
for cp in codepoints:
name = gname(cp)
if cp == 0x20:
glyphs[name] = empty_glyph()
metrics[name] = (300, 0) # space advance, no outline
elif cp > 0x2000: # CJK -> full-width
glyphs[name] = box_glyph(900)
metrics[name] = (1000, 80)
else:
glyphs[name] = box_glyph(560)
metrics[name] = (600, 80)
fb.setupGlyf(glyphs)
fb.setupHorizontalMetrics(metrics)
fb.setupHorizontalHeader(ascent=800, descent=-200)
# Names + the rest of the required tables.
fb.setupNameTable({
"familyName": "CJKRedactTest",
"styleName": "Regular",
"psName": "CJKRedactTest-Regular",
"fullName": "CJKRedactTest Regular",
})
fb.setupOS2(sTypoAscender=800, sTypoDescender=-200, usWinAscent=800,
usWinDescent=200)
fb.setupPost()
fb.save(out_path)
print("Wrote", out_path)
print("Glyph order:", glyph_order)
print("cmap entries:", {("U+%04X" % k): v for k, v in cmap.items()})
if __name__ == "__main__":
if len(sys.argv) != 2:
print("usage: gen_cjk_font.py <output.ttf>", file=sys.stderr)
sys.exit(2)
main(sys.argv[1])
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
"""Generate a minimal real Type1 font as a classic PFB (0x80 0x01 segment format).
Pure-python, uses fontTools.t1Lib.T1Font.createData() + writePFB. The font contains
.notdef plus the ASCII letters needed for "PUBLICA SECRETWORD KEEP", with a proper
Encoding array + standard PostScript glyph names. Because the glyph names are the
standard ones and the PDF font uses WinAnsiEncoding, PDFBox extracts text via the
base encoding (code -> glyphname -> Unicode); no /ToUnicode CMap is required.
Usage: python gen_type1.py <out.pfb>
"""
import sys
from fontTools import t1Lib
from fontTools.misc import psCharStrings
# Glyph name for each character we must render. PostScript standard glyph names.
CHARS = {
"P": "P", "U": "U", "B": "B", "L": "L", "I": "I", "C": "C", "A": "A",
"S": "S", "E": "E", "R": "R", "T": "T", "W": "W", "O": "O", "D": "D",
"K": "K", " ": "space",
}
EM = 1000
WIDTH = 600
BOX_L, BOX_B, BOX_R, BOX_T = 50, 0, 550, 700
def box_charstring(width):
"""A trivial filled box outline. hsbw sets sidebearing+width; then a closed rect."""
prog = [
0, width, "hsbw", # sb=0, width
BOX_L, BOX_B, "rmoveto", # move to bottom-left (relative from 0,0)
(BOX_R - BOX_L), "hlineto", # right
(BOX_T - BOX_B), "vlineto", # up
-(BOX_R - BOX_L), "hlineto", # left
"closepath",
"endchar",
]
cs = psCharStrings.T1CharString(program=prog)
return cs
def space_charstring(width):
prog = [0, width, "hsbw", "endchar"]
return psCharStrings.T1CharString(program=prog)
def build_font(font_name="MinType1Test"):
# 256-slot encoding, default .notdef, filled at each char's codepoint.
encoding = [".notdef"] * 256
for ch, gname in CHARS.items():
encoding[ord(ch)] = gname
charstrings = {".notdef": box_charstring(WIDTH)}
for ch, gname in CHARS.items():
if gname == "space":
charstrings[gname] = space_charstring(WIDTH)
else:
charstrings[gname] = box_charstring(WIDTH)
# Subrs: the 5 standard Type1 subrs (division/hint machinery). Provide as
# T1CharString objects so createData can compile+encrypt them.
subrs = [psCharStrings.T1CharString(bytecode=s) for s in t1Lib.std_subrs]
# Order matters. fontTools emits a stray "put" after the Subrs array; fontbox's
# Type1Parser breaks its Private loop on the first non-LITERAL after Subrs, so
# every key AFTER Subrs is dropped. Keep Subrs LAST so nothing gets skipped.
private = {
"RD": t1Lib.RD_value, # ('string','currentfile','exch','readstring','pop')
"ND": t1Lib.ND_values[0], # ('def',) codespell:ignore
"NP": t1Lib.PD_values[0], # ('put',)
"MinFeature": [16, 16],
# PDFBox reads BlueValues[2] for CapHeight, so >=3 entries required.
# Pairs: baseline (0 0), then cap-height zone (700 712).
"BlueValues": [0, 0, BOX_T, BOX_T + 12],
"StdHW": [60],
"StdVW": [80],
"lenIV": 4,
"password": 5839,
"Subrs": subrs,
}
font_info = {
"version": "001.001",
"FullName": font_name,
"FamilyName": font_name,
"Weight": "Regular",
"ItalicAngle": 0,
"isFixedPitch": False,
"UnderlinePosition": -100,
"UnderlineThickness": 50,
}
font = {
"FontName": font_name,
"FontInfo": font_info,
"FontType": 1,
"PaintType": 0,
"FontMatrix": [0.001, 0, 0, 0.001, 0, 0],
"FontBBox": [0, 0, WIDTH, BOX_T],
"Encoding": encoding,
"Private": private,
"CharStrings": charstrings,
}
return font
def main():
out = sys.argv[1] if len(sys.argv) > 1 else "min.pfb"
# Construct a T1Font without reading a file, then hand-build .font.
t1 = t1Lib.T1Font.__new__(t1Lib.T1Font)
t1.encoding = "ascii"
t1.font = build_font()
data = t1.createData() # full ASCII+eexec Type1 program (binary eexec inline)
t1.data = data
t1.saveAs(out, "PFB") # split into 0x80 0x01 / 0x80 0x02 / 0x80 0x03 segments
# Report first bytes to confirm PFB marker.
with open(out, "rb") as f:
head = f.read(6)
print("WROTE", out)
print("PFB head bytes:", " ".join(f"{b:02x}" for b in head))
assert head[0] == 0x80 and head[1] == 0x01, "not a classic PFB 0x80 0x01 header"
print("OK: classic PFB 0x80 0x01 ASCII segment marker present")
if __name__ == "__main__":
main()
Binary file not shown.
+143
View File
@@ -0,0 +1,143 @@
@security @redact
Feature: PDF redaction physically removes text
Target text must be gone from the extracted text layer, not just covered by a box.
Scenario: Auto-redact physically removes the target word
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "PUBLIC alpha SECRET99 omega PUBLIC"
And the request data includes
| parameter | value |
| listOfText | SECRET99 |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response content type should be "application/pdf"
And the response PDF should not contain the text "SECRET99"
And the response PDF should contain the text "omega"
Scenario: Auto-redact leaves substrings when whole-word search is on
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "cat classification scatter"
And the request data includes
| parameter | value |
| listOfText | cat |
| wholeWordSearch | true |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should contain the text "classification"
And the response PDF should contain the text "scatter"
Scenario: Auto-redact with a regex pattern removes matches
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "call 123-45-6789 today"
And the request data includes
| parameter | value |
| listOfText | \d{3}-\d{2}-\d{4} |
| useRegex | true |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should not contain the text "123-45-6789"
And the response PDF should contain the text "today"
Scenario: Auto-redact convert-to-image drops the entire text layer
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "keep SECRET77 hidden"
And the request data includes
| parameter | value |
| listOfText | SECRET77 |
| convertPDFToImage | true |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should not contain the text "SECRET77"
And the response PDF should not contain the text "keep"
Scenario: Auto-redact also scrubs the target from bookmark titles
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "body SECRET55 text"
And the pdf has a bookmark titled "Chapter SECRET55 overview"
And the request data includes
| parameter | value |
| listOfText | SECRET55 |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should not contain the text "SECRET55"
And the response PDF bookmarks should not contain "SECRET55"
Scenario: Manual whole-page redaction wipes every word on the page
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "TOP SECRET material"
And the request data includes
| parameter | value |
| pageNumbers | 1 |
When I send the API request to the endpoint "/api/v1/security/redact"
Then the response status code should be 200
And the response PDF should not contain the text "SECRET"
Scenario: Auto-redact removes case variants of the target
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "alpha Secret99x omega"
And the request data includes
| parameter | value |
| listOfText | SECRET99X |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should not contain the text "Secret99x"
And the response PDF should contain the text "omega"
Scenario: Auto-redact only touches pages containing the target
Given I generate a PDF file as "fileInput"
And the pdf contains 3 pages
And the request data includes
| parameter | value |
| listOfText | Page 2 |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should not contain the text "Page 2"
And the response PDF should contain the text "Page 1"
And the response PDF should contain the text "Page 3"
Scenario: Auto-redact handles a Type3 font PDF (glyphs as content streams)
Given I use an example file at "../crop_test.pdf" as parameter "fileInput"
And the request data includes
| parameter | value |
| listOfText | EXAMPLE |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should not contain the text "EXAMPLE"
And the response PDF should contain the text "CROP"
Scenario: Auto-redact removes the target from an embedded TrueType font PDF
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "keepA SECRET88 keepB" in an embedded font
And the request data includes
| parameter | value |
| listOfText | SECRET88 |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response content type should be "application/pdf"
And the response PDF should not contain the text "SECRET88"
And the response PDF should contain the text "keepA"
And the response PDF should contain the text "keepB"
Scenario: Auto-redact with a regex pattern on an embedded-font PDF
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "invoice 987-65-4321 total" in an embedded font
And the request data includes
| parameter | value |
| listOfText | \d{3}-\d{2}-\d{4} |
| useRegex | true |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should not contain the text "987-65-4321"
And the response PDF should contain the text "invoice"
Scenario: Auto-redact convert-to-image on an embedded-font PDF drops the text layer
Given I generate a PDF file as "fileInput"
And the pdf pages all contain the text "keep SECRET66 hidden" in an embedded font
And the request data includes
| parameter | value |
| listOfText | SECRET66 |
| convertPDFToImage | true |
When I send the API request to the endpoint "/api/v1/security/auto-redact"
Then the response status code should be 200
And the response PDF should not contain the text "SECRET66"
And the response PDF should not contain the text "keep"
@@ -17,6 +17,9 @@ from PIL import Image, ImageDraw
API_HEADERS = {"X-API-KEY": "123456789"}
# Backend under test; override STIRLING_BASE_URL to target another port.
BASE_URL = os.environ.get("STIRLING_BASE_URL", "http://localhost:8080")
#########
# GIVEN #
#########
@@ -277,6 +280,47 @@ def step_pdf_pages_contain_text(context, text):
context.files[context.param_name] = open(context.file_name, "rb")
# DejaVuSans is a repo asset; embedding it exercises redaction over a real embedded (Type0/CID,
# non Standard-14) font whose text pypdf can still extract to verify removal end-to-end.
_DEJAVU_TTF = os.path.normpath(
os.path.join(
os.path.dirname(__file__),
"..", "..", "..", "..",
"app", "core", "src", "main", "resources", "static", "fonts", "DejaVuSans.ttf",
)
)
_embedded_font_registered = False
def _ensure_embedded_font():
global _embedded_font_registered
if not _embedded_font_registered:
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.pdfbase import pdfmetrics
pdfmetrics.registerFont(TTFont("DejaVuEmbedded", _DEJAVU_TTF))
_embedded_font_registered = True
@given('the pdf pages all contain the text "{text}" in an embedded font')
def step_pdf_pages_contain_text_embedded(context, text):
_ensure_embedded_font()
buffer = io.BytesIO()
c = canvas.Canvas(buffer, pagesize=letter)
width, height = letter
for _ in range(len(PdfReader(context.file_name).pages)):
c.setFont("DejaVuEmbedded", 14)
c.drawString(100, height - 100, text)
c.showPage()
c.save()
with open(context.file_name, "wb") as f:
f.write(buffer.getvalue())
context.files[context.param_name].close()
context.files[context.param_name] = open(context.file_name, "rb")
@given('the pdf is encrypted with password "{password}"')
def step_encrypt_pdf(context, password):
writer = PdfWriter()
@@ -581,7 +625,7 @@ def step_request_json_part(context, part_name, json_content):
@when('I send a GET request to "{endpoint}"')
def step_send_get_request(context, endpoint):
base_url = "http://localhost:8080"
base_url = BASE_URL
full_url = f"{base_url}{endpoint}"
response = requests.get(full_url, headers=API_HEADERS, timeout=60)
context.response = response
@@ -589,7 +633,7 @@ def step_send_get_request(context, endpoint):
@when('I send a GET request to "{endpoint}" with parameters')
def step_send_get_request_with_params(context, endpoint):
base_url = "http://localhost:8080"
base_url = BASE_URL
params = {row["parameter"]: row["value"] for row in context.table}
full_url = f"{base_url}{endpoint}"
response = requests.get(full_url, params=params, headers=API_HEADERS, timeout=60)
@@ -598,7 +642,7 @@ def step_send_get_request_with_params(context, endpoint):
@when('I send the API request to the endpoint "{endpoint}"')
def step_send_api_request(context, endpoint):
url = f"http://localhost:8080{endpoint}"
url = f"{BASE_URL}{endpoint}"
files = context.files if hasattr(context, "files") else {}
if not hasattr(context, "request_data") or context.request_data is None:
@@ -803,3 +847,69 @@ def step_response_matches_regex(context, pattern):
assert re.match(
pattern, response_text
), f"Response '{response_text}' does not match the expected pattern '{pattern}'"
# ---------------------------------------------------------------------------
# Redaction: text-layer and catalog assertions
# ---------------------------------------------------------------------------
def _extract_response_pdf_text(context):
reader = PdfReader(io.BytesIO(context.response.content))
return "\n".join((page.extract_text() or "") for page in reader.pages)
@then('the response PDF should contain the text "{text}"')
def step_response_pdf_contains_text(context, text):
extracted = _extract_response_pdf_text(context)
assert text in extracted, (
f"Expected redacted PDF to still contain '{text}', but it was missing. "
f"Extracted text: {extracted!r}"
)
@then('the response PDF should not contain the text "{text}"')
def step_response_pdf_not_contains_text(context, text):
extracted = _extract_response_pdf_text(context)
assert text not in extracted, (
f"Redacted PDF still contains '{text}' - redaction did not remove it. "
f"Extracted text: {extracted!r}"
)
def _collect_outline_titles(outline, titles):
for item in outline:
if isinstance(item, list):
_collect_outline_titles(item, titles)
else:
title = getattr(item, "title", None)
if title:
titles.append(title)
@then('the response PDF bookmarks should not contain "{text}"')
def step_response_pdf_bookmarks_not_contain(context, text):
reader = PdfReader(io.BytesIO(context.response.content))
titles = []
try:
_collect_outline_titles(reader.outline, titles)
except Exception:
titles = []
joined = " ".join(titles)
assert text not in joined, (
f"Redacted PDF bookmark titles still contain '{text}': {titles!r}"
)
@given('the pdf has a bookmark titled "{title}"')
def step_pdf_has_bookmark_titled(context, title):
"""Add a single top-level outline entry with an explicit title."""
reader = PdfReader(context.file_name)
writer = PdfWriter()
for page in reader.pages:
writer.add_page(page)
writer.add_outline_item(title, 0)
with open(context.file_name, "wb") as f:
writer.write(f)
context.files[context.param_name].close()
context.files[context.param_name] = open(context.file_name, "rb")