Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b764320e06 | ||
|
|
3feb757821 | ||
|
|
faca7486c1 | ||
|
|
bc0ad7d0c3 | ||
|
|
dbe25402c1 | ||
|
|
5400c02b81 | ||
|
|
835863ebe9 | ||
|
|
d0b4438c47 | ||
|
|
7217e6ac15 | ||
|
|
e91d535c36 | ||
|
|
7eaa54d633 | ||
|
|
4e8dd18594 | ||
|
|
b88cbe6bfc | ||
|
|
18cc6dcdc8 |
+152
-100
@@ -5,8 +5,12 @@ import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
@@ -22,6 +26,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
@@ -38,15 +43,18 @@ class ManualRedactionService {
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Area and page redaction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactAreas(List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
AreaRedactionResult redactAreas(
|
||||
List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Set<String> capturedStrings = new LinkedHashSet<>();
|
||||
Map<Integer, List<PDRectangle>> rectsByPage = new HashMap<>();
|
||||
Set<Integer> forceRasterPages = new LinkedHashSet<>();
|
||||
|
||||
if (redactionAreas == null || redactionAreas.isEmpty()) {
|
||||
return;
|
||||
return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings);
|
||||
}
|
||||
|
||||
Map<Integer, List<RedactionArea>> redactionsByPage = new HashMap<>();
|
||||
@@ -74,57 +82,57 @@ class ManualRedactionService {
|
||||
continue;
|
||||
}
|
||||
|
||||
PDPage page = allPages.get(pageNumber - 1);
|
||||
int pageIndex = pageNumber - 1;
|
||||
PDPage page = allPages.get(pageIndex);
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
for (RedactionArea redactionArea : areasForPage) {
|
||||
Color redactColor = decodeOrDefault(redactionArea.getColor());
|
||||
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
float x = redactionArea.getX().floatValue();
|
||||
float y = redactionArea.getY().floatValue();
|
||||
float width = redactionArea.getWidth().floatValue();
|
||||
float height = redactionArea.getHeight().floatValue();
|
||||
|
||||
float pdfY = page.getBBox().getHeight() - y - height;
|
||||
|
||||
contentStream.addRect(x, pdfY, width, height);
|
||||
contentStream.fill();
|
||||
}
|
||||
contentStream.restoreGraphicsState();
|
||||
// Group rects by their decoded colour so each area keeps its own overlay tint.
|
||||
Map<Color, List<PDRectangle>> byColor = new LinkedHashMap<>();
|
||||
List<PDRectangle> allRects = new ArrayList<>();
|
||||
for (RedactionArea area : areasForPage) {
|
||||
float x = area.getX().floatValue();
|
||||
float y = area.getY().floatValue();
|
||||
float width = area.getWidth().floatValue();
|
||||
float height = area.getHeight().floatValue();
|
||||
// Request coords are top-left origin; convert to PDF user space (bottom-left).
|
||||
float pdfY = pageHeight - y - height;
|
||||
PDRectangle rect = new PDRectangle(x, pdfY, width, height);
|
||||
allRects.add(rect);
|
||||
byColor.computeIfAbsent(decodeOrDefault(area.getColor()), k -> new ArrayList<>())
|
||||
.add(rect);
|
||||
}
|
||||
|
||||
for (Map.Entry<Color, List<PDRectangle>> colorEntry : byColor.entrySet()) {
|
||||
Map<Integer, List<PDRectangle>> singlePage = new HashMap<>();
|
||||
singlePage.put(pageIndex, colorEntry.getValue());
|
||||
RedactionPipeline.RedactionResult result =
|
||||
RedactionPipeline.redactAreas(document, singlePage, colorEntry.getKey());
|
||||
capturedStrings.addAll(result.getCapturedStrings());
|
||||
forceRasterPages.addAll(result.getForceRasterPages());
|
||||
}
|
||||
rectsByPage.put(pageIndex, allRects);
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Manual area redaction captured {} text run(s) across {} page(s)",
|
||||
capturedStrings.size(),
|
||||
rectsByPage.size());
|
||||
return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings);
|
||||
}
|
||||
|
||||
void redactPages(ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
List<Integer> redactPages(
|
||||
ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Color redactColor = decodeOrDefault(request.getPageRedactionColor());
|
||||
List<Integer> pageNumbers = getPageNumbers(request, allPages.getCount());
|
||||
List<Integer> pageIndexes = getPageNumbers(request, allPages.getCount());
|
||||
|
||||
for (Integer pageNumber : pageNumbers) {
|
||||
PDPage page = allPages.get(pageNumber);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
PDRectangle box = page.getBBox();
|
||||
contentStream.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
contentStream.fill();
|
||||
}
|
||||
}
|
||||
// Whole-page wipe: drop the content stream, resources and annotations, then fill.
|
||||
RedactionPipeline.redactWholePages(document, pageIndexes, redactColor);
|
||||
return new ArrayList<>(pageIndexes);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Overlay drawing
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactFoundText(
|
||||
PDDocument document,
|
||||
@@ -158,7 +166,10 @@ class ManualRedactionService {
|
||||
|
||||
try {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
PDRectangle pageBox = page.getBBox();
|
||||
// TextPosition coords are relative to the CropBox origin.
|
||||
PDRectangle crop = page.getCropBox();
|
||||
float cropX = crop.getLowerLeftX();
|
||||
float cropY = crop.getLowerLeftY();
|
||||
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
@@ -179,8 +190,8 @@ class ManualRedactionService {
|
||||
}
|
||||
|
||||
contentStream.addRect(
|
||||
boxX,
|
||||
pageBox.getHeight() - block.getY2() - padding,
|
||||
cropX + boxX,
|
||||
cropY + crop.getHeight() - block.getY2() - padding,
|
||||
boxWidth,
|
||||
block.getY2() - block.getY1() + 2 * padding);
|
||||
}
|
||||
@@ -192,8 +203,8 @@ class ManualRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent
|
||||
// users from hovering over redacted URLs and seeing the underlying destination.
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent users
|
||||
// from hovering over redacted URLs and seeing the underlying destination.
|
||||
try {
|
||||
float pageH = page.getBBox().getHeight();
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
@@ -247,14 +258,19 @@ class ManualRedactionService {
|
||||
continue;
|
||||
}
|
||||
PDPage page = pages.get(pageIdx);
|
||||
PDRectangle crop = page.getCropBox();
|
||||
float cropX = crop.getLowerLeftX();
|
||||
float cropY = crop.getLowerLeftY();
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(color);
|
||||
for (float[] box : entry.getValue()) {
|
||||
// Box coords are CropBox-relative; offset by the CropBox origin
|
||||
// (cropbox-overlay).
|
||||
float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4];
|
||||
cs.addRect(x1, y1, x2 - x1, y2 - y1);
|
||||
cs.addRect(cropX + x1, cropY + y1, x2 - x1, y2 - y1);
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
@@ -262,9 +278,7 @@ class ManualRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Page element extraction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns bounding boxes for every text line and image on {@code page} in PDF user-space
|
||||
@@ -288,9 +302,7 @@ class ManualRedactionService {
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Finalization
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
@@ -298,7 +310,37 @@ class ManualRedactionService {
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode)
|
||||
boolean isTextRemovalMode,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns)
|
||||
throws IOException {
|
||||
return finalizeRedaction(
|
||||
document,
|
||||
allFoundTextsByPage,
|
||||
colorString,
|
||||
customPadding,
|
||||
convertToImage,
|
||||
isTextRemovalMode,
|
||||
literalTargets,
|
||||
patterns,
|
||||
Collections.emptySet());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param geometricRasterPages 0-based pages carrying a range / image-box redaction, whose
|
||||
* covered content (text under an overlay, or an image) is not text-removable and so must be
|
||||
* rasterised to guarantee removal.
|
||||
*/
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> allFoundTextsByPage,
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> geometricRasterPages)
|
||||
throws IOException {
|
||||
|
||||
List<PDFText> allFoundTexts = new ArrayList<>();
|
||||
@@ -309,74 +351,84 @@ class ManualRedactionService {
|
||||
if (!allFoundTexts.isEmpty()) {
|
||||
Color redactColor = decodeOrDefault(colorString);
|
||||
redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode);
|
||||
cleanDocumentMetadata(document);
|
||||
}
|
||||
|
||||
byte[] outputBytes;
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
cleanDocumentMetadata(convertedPdf);
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
convertedPdf.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized (image mode): {} pages ➜ {} KB",
|
||||
convertedPdf.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
// Convert-to-image physically removes all text, so verification is a plain save.
|
||||
outputBytes =
|
||||
RedactionPipeline.finalize(
|
||||
convertedPdf, Collections.emptySet(), Collections.emptyList());
|
||||
}
|
||||
} else {
|
||||
// Strip matched glyphs, then scrub/verify/rasterise via finalize.
|
||||
RedactionPipeline.redactLiteralTerms(document, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(document, literalTargets, patterns);
|
||||
// Geometric (range/image-box) redactions can't be text-removed; rasterise their pages.
|
||||
outputBytes =
|
||||
RedactionPipeline.rasteriseSpecificPages(
|
||||
outputBytes, geometricRasterPages, literalTargets, patterns);
|
||||
}
|
||||
|
||||
return writeBytes(outputBytes, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
/** Finalize a manual redaction; forced + verified-leaking pages are rasterised. */
|
||||
TempFile finalizeManual(
|
||||
PDDocument document, AreaRedactionResult areaResult, Boolean convertToImage)
|
||||
throws IOException {
|
||||
|
||||
byte[] outputBytes;
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
outputBytes =
|
||||
RedactionPipeline.finalize(
|
||||
convertedPdf, Collections.emptySet(), Collections.emptyList());
|
||||
}
|
||||
} else {
|
||||
outputBytes =
|
||||
RedactionPipeline.finalizeAreas(
|
||||
document,
|
||||
areaResult.rectsByPage,
|
||||
areaResult.forceRasterPages,
|
||||
areaResult.capturedStrings);
|
||||
}
|
||||
|
||||
return writeBytes(outputBytes, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
private TempFile writeBytes(byte[] outputBytes, int pageCount) throws IOException {
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
document.save(tempOut.getFile());
|
||||
java.nio.file.Files.write(tempOut.getFile().toPath(), outputBytes);
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized: {} pages ➜ {} KB",
|
||||
document.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
log.info("Redaction finalized: {} pages -> {} KB", pageCount, outputBytes.length / 1024);
|
||||
return tempOut;
|
||||
}
|
||||
|
||||
private void cleanDocumentMetadata(PDDocument document) {
|
||||
try {
|
||||
var documentInfo = document.getDocumentInformation();
|
||||
if (documentInfo != null) {
|
||||
documentInfo.setAuthor(null);
|
||||
documentInfo.setSubject(null);
|
||||
documentInfo.setKeywords(null);
|
||||
documentInfo.setModificationDate(java.util.Calendar.getInstance());
|
||||
log.debug("Cleaned document metadata for security");
|
||||
}
|
||||
// Utilities
|
||||
|
||||
if (document.getDocumentCatalog() != null) {
|
||||
try {
|
||||
document.getDocumentCatalog().setMetadata(null);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not clear XMP metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
/** Redaction rectangles, pages to force-rasterise, and the text captured under the rects. */
|
||||
static final class AreaRedactionResult {
|
||||
final Map<Integer, List<PDRectangle>> rectsByPage;
|
||||
final Set<Integer> forceRasterPages;
|
||||
final Set<String> capturedStrings;
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to clean document metadata: {}", e.getMessage());
|
||||
AreaRedactionResult(
|
||||
Map<Integer, List<PDRectangle>> rectsByPage,
|
||||
Set<Integer> forceRasterPages,
|
||||
Set<String> capturedStrings) {
|
||||
this.rectsByPage = rectsByPage;
|
||||
this.forceRasterPages = forceRasterPages;
|
||||
this.capturedStrings = capturedStrings;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Utilities
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
static Color decodeOrDefault(String hex) {
|
||||
if (hex == null) {
|
||||
return Color.BLACK;
|
||||
|
||||
+54
-30
@@ -1,9 +1,15 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
@@ -29,13 +35,13 @@ import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.SecurityApi;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
@@ -93,33 +99,24 @@ public class RedactController {
|
||||
throws IOException {
|
||||
|
||||
MultipartFile file = request.getFileInput();
|
||||
String filename =
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(file.getOriginalFilename())))
|
||||
+ "_redacted.pdf";
|
||||
|
||||
try (PDDocument document = pdfDocumentFactory.load(file)) {
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
|
||||
// Whole-page wipes drop content; areas drop glyphs + overlay, verified later.
|
||||
manualRedactionService.redactPages(request, document, allPages);
|
||||
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
|
||||
ManualRedactionService.AreaRedactionResult areaResult =
|
||||
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
|
||||
|
||||
if (Boolean.TRUE.equals(request.getConvertPDFToImage())) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
convertedPdf,
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(
|
||||
file.getOriginalFilename())))
|
||||
+ "_redacted.pdf",
|
||||
tempFileManager);
|
||||
}
|
||||
}
|
||||
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
document,
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(file.getOriginalFilename())))
|
||||
+ "_redacted.pdf",
|
||||
tempFileManager);
|
||||
TempFile out =
|
||||
manualRedactionService.finalizeManual(
|
||||
document, areaResult, request.getConvertPDFToImage());
|
||||
return WebResponseUtils.pdfFileToWebResponse(out, filename);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,9 +185,32 @@ public class RedactController {
|
||||
request.getFileInput().getOriginalFilename())))
|
||||
+ "_redacted.pdf";
|
||||
|
||||
Set<String> literalTargets =
|
||||
Arrays.stream(listOfText)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
List<Pattern> compiledPatterns =
|
||||
RedactionPipeline.buildPatterns(listOfText, useRegex, wholeWordSearchBool);
|
||||
// Bare literals match substrings; regex/whole-word rely on compiled patterns.
|
||||
Set<String> verificationTargets =
|
||||
(useRegex || wholeWordSearchBool) ? Collections.emptySet() : literalTargets;
|
||||
|
||||
if (allFoundTextsByPage.isEmpty()) {
|
||||
log.info("No text found matching redaction patterns");
|
||||
return WebResponseUtils.pdfDocToWebResponse(document, filename, tempFileManager);
|
||||
// No page hit, but the target may live in a bookmark/annotation/form/JS carrier, so
|
||||
// still run the scrub + verify path rather than just wiping metadata.
|
||||
log.info("No page text matched; scrubbing catalog carriers and verifying");
|
||||
TempFile finalized =
|
||||
manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
Collections.emptyMap(),
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
true,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
}
|
||||
|
||||
boolean fallbackToBoxOnlyMode;
|
||||
@@ -211,7 +231,8 @@ public class RedactController {
|
||||
|
||||
if (fallbackToBoxOnlyMode) {
|
||||
log.warn(
|
||||
"Font compatibility issues detected. Using box-only redaction mode for better reliability.");
|
||||
"Font compatibility issue in placeholder pass; the true-removal pass and "
|
||||
+ "verification still guarantee the target is gone.");
|
||||
|
||||
fallbackDocument = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
@@ -226,7 +247,9 @@ public class RedactController {
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
false);
|
||||
false,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
}
|
||||
@@ -238,7 +261,9 @@ public class RedactController {
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
true);
|
||||
true,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
|
||||
@@ -247,11 +272,10 @@ public class RedactController {
|
||||
throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e);
|
||||
|
||||
} finally {
|
||||
// Both are distinct PDDocument handles.
|
||||
if (document != null) {
|
||||
try {
|
||||
if (fallbackDocument == null) {
|
||||
document.close();
|
||||
}
|
||||
document.close();
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to close main document: {}", e.getMessage());
|
||||
}
|
||||
|
||||
+55
-29
@@ -7,8 +7,11 @@ import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
@@ -30,6 +33,7 @@ import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyl
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
@@ -90,8 +94,8 @@ class RedactExecuteService {
|
||||
try {
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF
|
||||
// stripper only once across the entire execute() call rather than once per target.
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF stripper only
|
||||
// once across the entire execute() call rather than once per target.
|
||||
Map<Integer, List<PDFText>> foundTexts =
|
||||
hasTextOps ? collectTextMatches(document, request) : new HashMap<>();
|
||||
|
||||
@@ -101,13 +105,13 @@ class RedactExecuteService {
|
||||
totalMatches,
|
||||
foundTexts.size());
|
||||
|
||||
// Text removal (content-stream rewriting) — skipped in overlay-only mode.
|
||||
// Text removal (content-stream rewriting) - skipped in overlay-only mode.
|
||||
boolean needsOverlayOnly = overlayOnly;
|
||||
if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) {
|
||||
needsOverlayOnly = applyTextRemoval(document, request);
|
||||
} else if (overlayOnly) {
|
||||
log.info(
|
||||
"[redact/execute] overlay-only mode requested — skipping content-stream rewriting");
|
||||
"[redact/execute] overlay-only mode requested - skipping content-stream rewriting");
|
||||
}
|
||||
|
||||
// Reload fresh document on fallback so we overlay onto clean content.
|
||||
@@ -128,25 +132,46 @@ class RedactExecuteService {
|
||||
applyPageWipe(document, wipePages, style);
|
||||
}
|
||||
|
||||
// Range + image-box redactions are geometric overlays.
|
||||
Set<Integer> geometricRasterPages = new HashSet<>();
|
||||
for (TextRange range : ranges) {
|
||||
applyRangeRedaction(document, range, style, layoutCache);
|
||||
geometricRasterPages.addAll(
|
||||
applyRangeRedaction(document, range, style, layoutCache));
|
||||
}
|
||||
|
||||
for (ImageBox box : imageBoxes) {
|
||||
applyImageBoxRedaction(document, box, style);
|
||||
geometricRasterPages.addAll(applyImageBoxRedaction(document, box, style));
|
||||
}
|
||||
|
||||
if (request.getRedactImagePages() != null) {
|
||||
applyAllImagesRedaction(document, request.getRedactImagePages(), style);
|
||||
}
|
||||
|
||||
// Overlay-only skips removal/verify; font-fallback overlays still pass targets.
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
for (String value : textValues) {
|
||||
String trimmed = value == null ? "" : value.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
literalTargets.add(trimmed);
|
||||
}
|
||||
}
|
||||
List<Pattern> verificationPatterns =
|
||||
RedactionPipeline.buildPatterns(
|
||||
regexPatterns.toArray(new String[0]), true, false);
|
||||
Set<String> finalizeTargets = overlayOnly ? Collections.emptySet() : literalTargets;
|
||||
List<Pattern> finalizePatterns =
|
||||
overlayOnly ? Collections.emptyList() : verificationPatterns;
|
||||
|
||||
return manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
foundTexts,
|
||||
style.getColor(),
|
||||
style.getPadding(),
|
||||
convertToImage,
|
||||
!needsOverlayOnly);
|
||||
!needsOverlayOnly,
|
||||
finalizeTargets,
|
||||
finalizePatterns,
|
||||
geometricRasterPages);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Execute redaction failed: {}", e.getMessage(), e);
|
||||
@@ -162,9 +187,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Single-pass text scan (one stripper pass per execute() call)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit
|
||||
@@ -197,9 +220,7 @@ class RedactExecuteService {
|
||||
return found;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Text removal (content-stream rewriting)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the
|
||||
@@ -233,7 +254,7 @@ class RedactExecuteService {
|
||||
|
||||
if (fallback) {
|
||||
log.warn(
|
||||
"[redact/execute] font compatibility issue — falling back to overlay-only");
|
||||
"[redact/execute] font compatibility issue - falling back to overlay-only");
|
||||
} else {
|
||||
log.info("[redact/execute] content-stream text removal applied successfully");
|
||||
}
|
||||
@@ -246,9 +267,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Per-operation dispatch methods
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void applyPageWipe(PDDocument document, List<Integer> pageNumbers, RedactStyle style)
|
||||
throws IOException {
|
||||
@@ -304,7 +323,10 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
private void applyRangeRedaction(
|
||||
/**
|
||||
* @return 0-based pages covered by the range overlay; those pages must be rasterised.
|
||||
*/
|
||||
private Set<Integer> applyRangeRedaction(
|
||||
PDDocument document,
|
||||
TextRange range,
|
||||
RedactStyle style,
|
||||
@@ -322,6 +344,11 @@ class RedactExecuteService {
|
||||
style.getPadding(),
|
||||
ManualRedactionService.decodeOrDefault(style.getColor()),
|
||||
false);
|
||||
Set<Integer> pages = new HashSet<>();
|
||||
for (PDFText block : blocks) {
|
||||
pages.add(block.getPageIndex());
|
||||
}
|
||||
return pages;
|
||||
} else {
|
||||
log.warn(
|
||||
"[redact/execute] range not found: start='{}' end='{}'",
|
||||
@@ -331,10 +358,14 @@ class RedactExecuteService {
|
||||
} catch (Exception e) {
|
||||
log.warn("[redact/execute] range redaction failed: {}", e.getMessage());
|
||||
}
|
||||
return Collections.emptySet();
|
||||
}
|
||||
|
||||
private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style)
|
||||
throws IOException {
|
||||
/**
|
||||
* @return 0-based page covered by the image-box overlay; that page must be rasterised.
|
||||
*/
|
||||
private Set<Integer> applyImageBoxRedaction(
|
||||
PDDocument document, ImageBox box, RedactStyle style) throws IOException {
|
||||
List<float[]> boxes =
|
||||
List.of(
|
||||
new float[] {
|
||||
@@ -343,6 +374,7 @@ class RedactExecuteService {
|
||||
log.info("[redact/execute] image box overlay on page {}", box.pageIndex());
|
||||
Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
manualRedactionService.redactImageBoxes(document, boxes, boxColor);
|
||||
return Set.of(box.pageIndex());
|
||||
}
|
||||
|
||||
private void applyAllImagesRedaction(
|
||||
@@ -386,9 +418,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range collection helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text
|
||||
@@ -458,7 +488,7 @@ class RedactExecuteService {
|
||||
}
|
||||
if (end == null) {
|
||||
log.warn(
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping",
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) - skipping",
|
||||
start.page + 1,
|
||||
start.col,
|
||||
start.y);
|
||||
@@ -479,7 +509,7 @@ class RedactExecuteService {
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range '{}'→'{}': {} total blocks",
|
||||
"[redact/execute] range '{}'->'{}': {} total blocks",
|
||||
startStr,
|
||||
openEnded ? "<end of document>" : endStr,
|
||||
blocks.size());
|
||||
@@ -511,7 +541,6 @@ class RedactExecuteService {
|
||||
float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2();
|
||||
|
||||
// Line-box cache: populated lazily per page, reused across range iterations.
|
||||
// Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException.
|
||||
Map<Integer, List<float[]>> lineBoxCache = new HashMap<>();
|
||||
|
||||
for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) {
|
||||
@@ -677,7 +706,7 @@ class RedactExecuteService {
|
||||
if (i > 0) sb.append(", ");
|
||||
sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y));
|
||||
}
|
||||
if (anchors.size() > max) sb.append(", …");
|
||||
if (anchors.size() > max) sb.append(", ...");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@@ -703,9 +732,8 @@ class RedactExecuteService {
|
||||
candidates.add(new Candidate(tolerant, true));
|
||||
}
|
||||
|
||||
// If the anchor spans multiple lines (model provided entire paragraph instead of a short
|
||||
// phrase), try just the first non-empty line — it's usually sufficient to locate the
|
||||
// position and avoids mismatches from mid-paragraph text extraction artifacts.
|
||||
// Long multi-line anchors mismatch on extraction artifacts, so try just the first non-empty
|
||||
// line.
|
||||
if (trimmed.contains("\n")) {
|
||||
String firstLine =
|
||||
Arrays.stream(trimmed.split("\n"))
|
||||
@@ -733,7 +761,7 @@ class RedactExecuteService {
|
||||
if (!m.isEmpty()) {
|
||||
if (!c.pattern.equals(trimmed)) {
|
||||
log.info(
|
||||
"[redact/execute] range boundary matched via fallback: '{}' → '{}'",
|
||||
"[redact/execute] range boundary matched via fallback: '{}' -> '{}'",
|
||||
trimmed,
|
||||
c.pattern);
|
||||
}
|
||||
@@ -745,9 +773,7 @@ class RedactExecuteService {
|
||||
|
||||
private record Candidate(String pattern, boolean useRegex) {}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Static helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation
|
||||
|
||||
+5
-20
@@ -46,12 +46,11 @@ class TextRedactionService {
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
private static final float PRECISION_THRESHOLD = 1e-3f;
|
||||
private static final int FONT_SCALE_FACTOR = 1000;
|
||||
private static final Set<String> TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\"");
|
||||
private static final Set<String> TEXT_SHOWING_OPERATORS =
|
||||
stirling.software.SPDF.pdf.redaction.RedactionPipeline.TEXT_SHOWING_OPERATORS;
|
||||
private static final COSString EMPTY_COS_STRING = new COSString("");
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Public API
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
Map<Integer, List<PDFText>> findTextToRedact(
|
||||
PDDocument document, String[] listOfText, boolean useRegex, boolean wholeWordSearch) {
|
||||
@@ -135,9 +134,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Content stream manipulation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
List<Object> createTokensWithoutTargetText(
|
||||
PDDocument document,
|
||||
@@ -271,9 +268,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Placeholder creation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
String createPlaceholderWithFont(String originalWord, PDFont font) {
|
||||
if (originalWord == null || originalWord.isEmpty()) {
|
||||
@@ -410,9 +405,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Width calculation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private float safeGetStringWidth(PDFont font, String text) {
|
||||
if (font == null || text == null || text.isEmpty()) {
|
||||
@@ -601,9 +594,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Token and segment operations
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void processPageXObjects(
|
||||
PDDocument document,
|
||||
@@ -706,16 +697,12 @@ class TextRedactionService {
|
||||
private void writeRedactedContentToXObject(
|
||||
PDDocument document, PDFormXObject formXObject, List<Object> redactedTokens)
|
||||
throws IOException {
|
||||
|
||||
PDStream newStream = new PDStream(document);
|
||||
|
||||
try (var out = newStream.createOutputStream()) {
|
||||
// A form XObject's content IS its own stream body; overwrite it in place.
|
||||
PDStream formStream = new PDStream(formXObject.getCOSObject());
|
||||
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
ContentStreamWriter writer = new ContentStreamWriter(out);
|
||||
writer.writeTokens(redactedTokens);
|
||||
}
|
||||
|
||||
formXObject.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
formXObject.getCOSObject().setItem(COSName.CONTENTS, newStream.getCOSObject());
|
||||
}
|
||||
|
||||
private List<TextSegment> extractTextSegments(PDPage page, List<Object> tokens) {
|
||||
@@ -1168,9 +1155,7 @@ class TextRedactionService {
|
||||
};
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Inner data classes
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@Data
|
||||
private static class GraphicsState {
|
||||
|
||||
@@ -0,0 +1,755 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSObject;
|
||||
import org.apache.pdfbox.cos.COSStream;
|
||||
import org.apache.pdfbox.cos.COSString;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentNameDictionary;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDMetadata;
|
||||
import org.apache.pdfbox.pdmodel.common.PDNameTreeNode;
|
||||
import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureTreeRoot;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocumentOutline;
|
||||
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/** Removes/rewrites every catalog carrier that could leak redacted text. */
|
||||
@Slf4j
|
||||
public final class CatalogScrubber {
|
||||
|
||||
// A literal shorter than this won't delete a whole carrier (JS/XFA/action/embedded file).
|
||||
private static final int MIN_CARRIER_DROP_LITERAL = 3;
|
||||
|
||||
private CatalogScrubber() {}
|
||||
|
||||
/** Scrub all catalog-level carriers of the given literal/regex targets. */
|
||||
public static void scrub(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (document == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
PDDocumentCatalog catalog = document.getDocumentCatalog();
|
||||
if (catalog == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
scrubOutline(catalog.getDocumentOutline(), literalTargets, patterns);
|
||||
scrubAcroForm(catalog.getAcroForm(), literalTargets, patterns);
|
||||
scrubAnnotations(document, literalTargets, patterns);
|
||||
scrubStructTree(catalog.getStructureTreeRoot(), literalTargets, patterns);
|
||||
scrubNames(catalog.getNames(), literalTargets, patterns);
|
||||
scrubCatalogActions(catalog, literalTargets, patterns);
|
||||
scrubDocumentInfo(document, literalTargets, patterns);
|
||||
}
|
||||
|
||||
// Catalog actions: OpenAction, AA, and any JavaScript / URI payloads
|
||||
|
||||
private static void scrubCatalogActions(
|
||||
PDDocumentCatalog catalog, Set<String> targets, List<Pattern> patterns) {
|
||||
COSDictionary root = catalog.getCOSObject();
|
||||
if (root == null) {
|
||||
return;
|
||||
}
|
||||
// OpenAction may be either an action dict (with /URI or /JS) or an explicit
|
||||
scrubActionIfMatching(root, COSName.getPDFName("OpenAction"), targets, patterns);
|
||||
scrubActionIfMatching(root, COSName.getPDFName("AA"), targets, patterns);
|
||||
}
|
||||
|
||||
/** Drop the action at key if any target appears in its URI/JS payload. */
|
||||
private static void scrubActionIfMatching(
|
||||
COSDictionary parent, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
if (parent == null || key == null) {
|
||||
return;
|
||||
}
|
||||
COSBase value = parent.getDictionaryObject(key);
|
||||
if (value == null) {
|
||||
return;
|
||||
}
|
||||
if (containsTarget(value, carrierDropLiterals(targets), patterns, new HashSet<>())) {
|
||||
log.debug("Removing catalog {} due to target match", key.getName());
|
||||
parent.removeItem(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Literals specific enough to justify DELETING an entire carrier (JS / XFA / action / embedded
|
||||
* file). Sub-threshold literals (a single digit, a 2-char run) would nuke unrelated carriers,
|
||||
* so they are excluded from whole-carrier drops - they are still removed in-string by
|
||||
* stripMatches where the carrier is a plain string.
|
||||
*/
|
||||
private static Set<String> carrierDropLiterals(Set<String> targets) {
|
||||
if (targets == null) {
|
||||
return java.util.Collections.emptySet();
|
||||
}
|
||||
Set<String> specific = new LinkedHashSet<>();
|
||||
for (String t : targets) {
|
||||
if (t != null && t.trim().length() >= MIN_CARRIER_DROP_LITERAL) {
|
||||
specific.add(t);
|
||||
}
|
||||
}
|
||||
return specific;
|
||||
}
|
||||
|
||||
private static boolean containsTarget(
|
||||
COSBase base, Set<String> targets, List<Pattern> patterns, Set<COSBase> seen) {
|
||||
if (base == null) {
|
||||
return false;
|
||||
}
|
||||
COSBase resolved = base instanceof COSObject obj ? obj.getObject() : base;
|
||||
if (resolved == null || !seen.add(resolved)) {
|
||||
return false;
|
||||
}
|
||||
if (resolved instanceof COSString cs) {
|
||||
return matches(cs.getString(), targets, patterns);
|
||||
}
|
||||
if (resolved instanceof COSStream stream) {
|
||||
// Streams in XFA / OpenAction contexts are text (XML, JavaScript).
|
||||
try (java.io.InputStream is = stream.createInputStream()) {
|
||||
byte[] buf = new byte[2 * 1024 * 1024];
|
||||
int total = 0;
|
||||
int n;
|
||||
while ((n = is.read(buf, total, buf.length - total)) > 0) {
|
||||
total += n;
|
||||
if (total >= buf.length) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
String text = new String(buf, 0, total, java.nio.charset.StandardCharsets.UTF_8);
|
||||
if (matches(text, targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
// Fail closed: content past the 2 MiB cap is unproven, so treat as a match.
|
||||
return total >= buf.length && is.read() >= 0;
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scan stream for targets: {}", e.getMessage());
|
||||
// Fail closed: if we cannot read it we cannot prove it is clean, so treat
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (resolved instanceof COSDictionary dict) {
|
||||
for (COSName k : new HashSet<>(dict.keySet())) {
|
||||
if (containsTarget(dict.getItem(k), targets, patterns, seen)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (resolved instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
if (containsTarget(array.getObject(i), targets, patterns, seen)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the redaction target from /Info string entries and drop the XMP packet only if it
|
||||
* carries a target. Non-matching document properties (Title, Author, dates) are left intact -
|
||||
* redaction removes the target, it does not blanket-wipe metadata.
|
||||
*/
|
||||
private static void scrubDocumentInfo(
|
||||
PDDocument document, Set<String> targets, List<Pattern> patterns) {
|
||||
PDDocumentInformation info = document.getDocumentInformation();
|
||||
if (info != null && info.getCOSObject() != null) {
|
||||
COSDictionary infoDict = info.getCOSObject();
|
||||
for (COSName key : new HashSet<>(infoDict.keySet())) {
|
||||
if (infoDict.getDictionaryObject(key) instanceof COSString cs) {
|
||||
String stripped = stripMatches(cs.getString(), targets, patterns);
|
||||
if (!stripped.equals(cs.getString())) {
|
||||
if (stripped.isEmpty()) {
|
||||
infoDict.removeItem(key);
|
||||
} else {
|
||||
infoDict.setString(key, stripped);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PDDocumentCatalog catalog = document.getDocumentCatalog();
|
||||
if (catalog != null && catalog.getMetadata() != null) {
|
||||
scrubXmp(document, catalog, targets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* XMP is RDF/XML that usually mirrors /Info. Strip only the target occurrences so non-matching
|
||||
* properties (dates, rights, custom schema) survive; fall back to dropping the whole packet if
|
||||
* the edit can't be proven to have removed the target (e.g. entity-encoded) or anything throws.
|
||||
*/
|
||||
private static void scrubXmp(
|
||||
PDDocument document,
|
||||
PDDocumentCatalog catalog,
|
||||
Set<String> targets,
|
||||
List<Pattern> patterns) {
|
||||
String xmp;
|
||||
try {
|
||||
xmp = new String(catalog.getMetadata().toByteArray(), StandardCharsets.UTF_8);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not read XMP metadata: {}", e.getMessage());
|
||||
return;
|
||||
}
|
||||
if (!matches(xmp, targets, patterns)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String stripped = stripMatches(xmp, targets, patterns);
|
||||
// Only keep the edited packet if the target is provably gone from it.
|
||||
if (!matches(stripped, targets, patterns)) {
|
||||
catalog.setMetadata(
|
||||
new PDMetadata(
|
||||
document,
|
||||
new ByteArrayInputStream(
|
||||
stripped.getBytes(StandardCharsets.UTF_8))));
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Surgical XMP scrub failed; dropping packet: {}", e.getMessage());
|
||||
}
|
||||
catalog.setMetadata(null);
|
||||
}
|
||||
|
||||
// Outline
|
||||
|
||||
private static void scrubOutline(
|
||||
PDDocumentOutline outline, Set<String> targets, List<Pattern> patterns) {
|
||||
if (outline == null) {
|
||||
return;
|
||||
}
|
||||
scrubOutlineItems(outline.children(), targets, patterns);
|
||||
}
|
||||
|
||||
private static void scrubOutlineItems(
|
||||
Iterable<PDOutlineItem> items, Set<String> targets, List<Pattern> patterns) {
|
||||
if (items == null) {
|
||||
return;
|
||||
}
|
||||
for (PDOutlineItem item : items) {
|
||||
try {
|
||||
String title = item.getTitle();
|
||||
if (title != null) {
|
||||
String stripped = stripMatches(title, targets, patterns);
|
||||
if (!stripped.equals(title)) {
|
||||
item.setTitle(stripped);
|
||||
}
|
||||
}
|
||||
// Bookmark actions: /A is an action dict which may carry a /URI or /JS
|
||||
COSDictionary itemDict = item.getCOSObject();
|
||||
if (itemDict != null) {
|
||||
scrubActionIfMatching(itemDict, COSName.A, targets, patterns);
|
||||
scrubActionIfMatching(itemDict, COSName.getPDFName("AA"), targets, patterns);
|
||||
}
|
||||
scrubOutlineItems(item.children(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub outline item: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AcroForm
|
||||
|
||||
private static void scrubAcroForm(
|
||||
PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
|
||||
if (form == null) {
|
||||
return;
|
||||
}
|
||||
// XFA forms: scrubbed separately because the XFA XML packet carries
|
||||
scrubXfa(form, targets, patterns);
|
||||
|
||||
try {
|
||||
for (PDField field : form.getFieldTree()) {
|
||||
scrubField(field, targets, patterns);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to walk AcroForm field tree: {}", e.getMessage());
|
||||
}
|
||||
|
||||
// Force viewers to regenerate appearance streams from the (scrubbed) /V
|
||||
try {
|
||||
form.setNeedAppearances(true);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to set /NeedAppearances on AcroForm: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubXfa(PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
COSBase xfaBase = form.getCOSObject().getDictionaryObject(COSName.XFA);
|
||||
if (xfaBase == null) {
|
||||
return;
|
||||
}
|
||||
boolean hit =
|
||||
containsTarget(
|
||||
xfaBase, carrierDropLiterals(targets), patterns, new HashSet<>());
|
||||
if (hit) {
|
||||
// Simplest safe move: strip the XFA entry entirely.
|
||||
log.warn(
|
||||
"Removing XFA form packet from AcroForm - XFA XML contained a redaction "
|
||||
+ "target and has been dropped so viewers render AcroForm widgets "
|
||||
+ "instead.");
|
||||
form.getCOSObject().removeItem(COSName.XFA);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub XFA: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubField(PDField field, Set<String> targets, List<Pattern> patterns) {
|
||||
if (field == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
COSDictionary dict = field.getCOSObject();
|
||||
// Only touch fields whose own values actually contain a target: clearing /AP
|
||||
// document-wide blanks unrelated fields in viewers that ignore /NeedAppearances.
|
||||
boolean matched =
|
||||
dictValueMatches(dict, COSName.V, targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.DV, targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.getPDFName("RV"), targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.getPDFName("TU"), targets, patterns)
|
||||
|| buttonCaptionMatches(dict, targets, patterns)
|
||||
|| fieldValueMatches(field, targets, patterns);
|
||||
if (!matched) {
|
||||
return;
|
||||
}
|
||||
scrubDictStrings(dict, COSName.V, targets, patterns);
|
||||
scrubDictStrings(dict, COSName.DV, targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("RV"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("TU"), targets, patterns);
|
||||
// Keep field appearance streams in sync with value where possible.
|
||||
try {
|
||||
if (field.getValueAsString() != null) {
|
||||
String stripped = stripMatches(field.getValueAsString(), targets, patterns);
|
||||
if (!stripped.equals(field.getValueAsString())) {
|
||||
field.setValue(stripped);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to rewrite field value via setValue: {}", e.getMessage());
|
||||
}
|
||||
if (!isButtonField(dict)) {
|
||||
clearWidgetAppearances(dict);
|
||||
} else if (scrubButtonCaption(dict, targets, patterns)) {
|
||||
// A button caption (/MK /CA,/RC,/AC) that carried the target renders via /AP that
|
||||
// /NeedAppearances can't rebuild, so drop the stale /AP too.
|
||||
clearWidgetAppearances(dict);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub field: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static final String[] MK_CAPTION_KEYS = {"CA", "RC", "AC"};
|
||||
|
||||
/** True if a button widget's /MK caption (down/rollover/alternate) contains a target. */
|
||||
private static boolean buttonCaptionMatches(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
for (COSDictionary mk : mkDicts(dict)) {
|
||||
for (String k : MK_CAPTION_KEYS) {
|
||||
if (dictValueMatches(mk, COSName.getPDFName(k), targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Strip target substrings from a button widget's /MK captions; true if any matched. */
|
||||
private static boolean scrubButtonCaption(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
boolean hit = false;
|
||||
for (COSDictionary mk : mkDicts(dict)) {
|
||||
for (String k : MK_CAPTION_KEYS) {
|
||||
COSName key = COSName.getPDFName(k);
|
||||
if (dictValueMatches(mk, key, targets, patterns)) {
|
||||
hit = true;
|
||||
scrubDictStrings(mk, key, targets, patterns);
|
||||
}
|
||||
}
|
||||
}
|
||||
return hit;
|
||||
}
|
||||
|
||||
/** The /MK appearance-characteristics dicts on a field and any widget kids. */
|
||||
private static List<COSDictionary> mkDicts(COSDictionary dict) {
|
||||
List<COSDictionary> out = new java.util.ArrayList<>();
|
||||
if (dict.getDictionaryObject(COSName.getPDFName("MK")) instanceof COSDictionary mk) {
|
||||
out.add(mk);
|
||||
}
|
||||
if (dict.getDictionaryObject(COSName.KIDS) instanceof COSArray kids) {
|
||||
for (int i = 0; i < kids.size(); i++) {
|
||||
if (kids.getObject(i) instanceof COSDictionary kid
|
||||
&& kid.getDictionaryObject(COSName.getPDFName("MK"))
|
||||
instanceof COSDictionary mk) {
|
||||
out.add(mk);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static boolean isButtonField(COSDictionary dict) {
|
||||
COSBase ftBase = dict.getDictionaryObject(COSName.FT);
|
||||
COSName ft = ftBase instanceof COSName n ? n : null;
|
||||
if (ft == null) {
|
||||
COSBase parent = dict.getDictionaryObject(COSName.PARENT);
|
||||
if (parent instanceof COSDictionary p
|
||||
&& p.getDictionaryObject(COSName.FT) instanceof COSName pn) {
|
||||
ft = pn;
|
||||
}
|
||||
}
|
||||
return COSName.getPDFName("Btn").equals(ft);
|
||||
}
|
||||
|
||||
private static boolean dictValueMatches(
|
||||
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
COSBase value = dict.getDictionaryObject(key);
|
||||
if (value instanceof COSString cs) {
|
||||
return matches(cs.getString(), targets, patterns);
|
||||
}
|
||||
if (value instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
if (array.getObject(i) instanceof COSString element
|
||||
&& matches(element.getString(), targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static boolean fieldValueMatches(
|
||||
PDField field, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
String value = field.getValueAsString();
|
||||
return value != null && matches(value, targets, patterns);
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static void clearWidgetAppearances(COSDictionary fieldDict) {
|
||||
if (fieldDict == null) {
|
||||
return;
|
||||
}
|
||||
// The field itself may be a widget (single-widget field) and/or have Kids.
|
||||
fieldDict.removeItem(COSName.AP);
|
||||
COSBase kids = fieldDict.getDictionaryObject(COSName.KIDS);
|
||||
if (kids instanceof COSArray arr) {
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase kidBase = arr.getObject(i);
|
||||
if (kidBase instanceof COSDictionary kidDict) {
|
||||
kidDict.removeItem(COSName.AP);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Annotations
|
||||
|
||||
private static void scrubAnnotations(
|
||||
PDDocument document, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
for (PDPage page : document.getPages()) {
|
||||
// Page-level additional actions (/AA - open/close JS) can carry the target too.
|
||||
scrubActionIfMatching(
|
||||
page.getCOSObject(), COSName.getPDFName("AA"), targets, patterns);
|
||||
List<PDAnnotation> annotations;
|
||||
try {
|
||||
annotations = page.getAnnotations();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to load annotations for page: {}", e.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (annotations == null) {
|
||||
continue;
|
||||
}
|
||||
for (PDAnnotation annotation : annotations) {
|
||||
scrubAnnotation(annotation, targets, patterns);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Annotation scrub walk failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubAnnotation(
|
||||
PDAnnotation annotation, Set<String> targets, List<Pattern> patterns) {
|
||||
if (annotation == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String contents = annotation.getContents();
|
||||
if (contents != null) {
|
||||
String stripped = stripMatches(contents, targets, patterns);
|
||||
if (!stripped.equals(contents)) {
|
||||
annotation.setContents(stripped);
|
||||
}
|
||||
}
|
||||
COSDictionary dict = annotation.getCOSObject();
|
||||
scrubDictStrings(dict, COSName.getPDFName("RC"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Subj"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("NM"), targets, patterns);
|
||||
// Link/widget annotation actions can encode the target in a /URI or JS body.
|
||||
scrubActionIfMatching(dict, COSName.A, targets, patterns);
|
||||
scrubActionIfMatching(dict, COSName.getPDFName("AA"), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub annotation: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Structure tree
|
||||
|
||||
private static void scrubStructTree(
|
||||
PDStructureTreeRoot root, Set<String> targets, List<Pattern> patterns) {
|
||||
if (root == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
scrubStructDict(root.getCOSObject(), targets, patterns, new HashSet<>());
|
||||
} catch (Exception e) {
|
||||
log.debug("Structure tree scrub failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubStructDict(
|
||||
COSBase base, Set<String> targets, List<Pattern> patterns, Set<COSBase> seen) {
|
||||
if (base == null) {
|
||||
return;
|
||||
}
|
||||
COSBase resolved = base instanceof COSObject obj ? obj.getObject() : base;
|
||||
if (resolved == null || !seen.add(resolved)) {
|
||||
return;
|
||||
}
|
||||
if (resolved instanceof COSDictionary dict) {
|
||||
// Do not walk into content streams - those are handled by content-stream
|
||||
if (resolved instanceof COSStream) {
|
||||
return;
|
||||
}
|
||||
scrubDictStrings(dict, COSName.getPDFName("ActualText"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Alt"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("E"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Lang"), targets, patterns);
|
||||
for (COSName key : new HashSet<>(dict.keySet())) {
|
||||
COSBase value = dict.getItem(key);
|
||||
if (value instanceof COSDictionary
|
||||
|| value instanceof COSArray
|
||||
|| value instanceof COSObject) {
|
||||
scrubStructDict(value, targets, patterns, seen);
|
||||
}
|
||||
}
|
||||
} else if (resolved instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
scrubStructDict(array.getObject(i), targets, patterns, seen);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Names tree (JavaScript + embedded files)
|
||||
|
||||
private static void scrubNames(
|
||||
PDDocumentNameDictionary names, Set<String> targets, List<Pattern> patterns) {
|
||||
if (names == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
dropMatchingNames(names.getJavaScript(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub JavaScript names: {}", e.getMessage());
|
||||
}
|
||||
try {
|
||||
dropMatchingNames(names.getEmbeddedFiles(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub embedded-file names: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void dropMatchingNames(
|
||||
PDNameTreeNode<?> node, Set<String> targets, List<Pattern> patterns) {
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
COSDictionary dict = node.getCOSObject();
|
||||
if (dict == null) {
|
||||
return;
|
||||
}
|
||||
scrubNameTreeDict(dict, targets, patterns);
|
||||
}
|
||||
|
||||
private static void scrubNameTreeDict(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
if (dict == null) {
|
||||
return;
|
||||
}
|
||||
// Guarded casts: a malformed /Names or /Kids must skip that carrier, not throw a swallowed
|
||||
// ClassCastException.
|
||||
COSArray namesArray =
|
||||
dict.getDictionaryObject(COSName.NAMES) instanceof COSArray a ? a : null;
|
||||
if (namesArray != null) {
|
||||
Set<String> dropLiterals = carrierDropLiterals(targets);
|
||||
for (int i = namesArray.size() - 2; i >= 0; i -= 2) {
|
||||
COSBase keyBase = namesArray.getObject(i);
|
||||
String key = keyBase instanceof COSString s ? s.getString() : null;
|
||||
// Drop the pair when the KEY or the VALUE (JS /JS stream, embedded-file bytes)
|
||||
// contains a target - not just the key.
|
||||
boolean keyHit = key != null && matches(key, dropLiterals, patterns);
|
||||
boolean valueHit =
|
||||
i + 1 < namesArray.size()
|
||||
&& containsTarget(
|
||||
namesArray.getObject(i + 1),
|
||||
dropLiterals,
|
||||
patterns,
|
||||
new HashSet<>());
|
||||
if (keyHit || valueHit) {
|
||||
namesArray.remove(i + 1);
|
||||
namesArray.remove(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
COSArray kids = dict.getDictionaryObject(COSName.KIDS) instanceof COSArray a ? a : null;
|
||||
if (kids != null) {
|
||||
for (int i = 0; i < kids.size(); i++) {
|
||||
COSBase kid = kids.getObject(i);
|
||||
if (kid instanceof COSDictionary kidDict) {
|
||||
scrubNameTreeDict(kidDict, targets, patterns);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helpers
|
||||
|
||||
private static void scrubDictStrings(
|
||||
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
if (dict == null || key == null) {
|
||||
return;
|
||||
}
|
||||
COSBase value = dict.getDictionaryObject(key);
|
||||
if (value instanceof COSString cosString) {
|
||||
String stripped = stripMatches(cosString.getString(), targets, patterns);
|
||||
if (!stripped.equals(cosString.getString())) {
|
||||
dict.setString(key, stripped);
|
||||
}
|
||||
} else if (value instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
COSBase element = array.getObject(i);
|
||||
if (element instanceof COSString elementString) {
|
||||
String stripped = stripMatches(elementString.getString(), targets, patterns);
|
||||
if (!stripped.equals(elementString.getString())) {
|
||||
array.set(i, new COSString(stripped));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static String stripMatches(String source, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (source == null || source.isEmpty()) {
|
||||
return source;
|
||||
}
|
||||
String result = source;
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// Case-insensitive literal removal. Verification is case-insensitive
|
||||
result = caseInsensitiveReplaceAll(result, target);
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
// Force case-insensitive matching for catalog carriers regardless
|
||||
Pattern ci = withCaseInsensitive(pattern);
|
||||
result = ci.matcher(DeadlineCharSequence.of(result)).replaceAll("");
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
// Fail closed: a throwing regex means we cannot prove the carrier clean, so
|
||||
// drop the whole string rather than leaving it intact.
|
||||
log.warn("Pattern replace failed for {}; dropping carrier text", pattern);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static boolean matches(String source, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (source == null || source.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
String lower = source.toLowerCase(Locale.ROOT);
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target != null
|
||||
&& !target.isEmpty()
|
||||
&& lower.contains(target.toLowerCase(Locale.ROOT))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
if (withCaseInsensitive(pattern)
|
||||
.matcher(DeadlineCharSequence.of(source))
|
||||
.find()) {
|
||||
return true;
|
||||
}
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
// Fail closed: a throwing regex counts as a match so the carrier is scrubbed.
|
||||
log.warn("Pattern match failed for {}; treating carrier as a match", pattern);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String caseInsensitiveReplaceAll(String source, String target) {
|
||||
if (target.isEmpty()) {
|
||||
return source;
|
||||
}
|
||||
Pattern literal =
|
||||
Pattern.compile(
|
||||
Pattern.quote(target), Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
|
||||
return literal.matcher(source).replaceAll("");
|
||||
}
|
||||
|
||||
private static Pattern withCaseInsensitive(Pattern pattern) {
|
||||
if ((pattern.flags() & Pattern.CASE_INSENSITIVE) != 0) {
|
||||
return pattern;
|
||||
}
|
||||
try {
|
||||
return Pattern.compile(
|
||||
pattern.pattern(),
|
||||
pattern.flags() | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
|
||||
} catch (Exception e) {
|
||||
return pattern;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
/**
|
||||
* Wraps a match subject so a runaway (catastrophic-backtracking) regex aborts instead of hanging
|
||||
* the request thread: every {@link #charAt(int)} the matcher performs checks a wall-clock deadline
|
||||
* and throws once it is exceeded. The thrown {@link RegexTimeoutException} is a {@link
|
||||
* RuntimeException}, so the redaction match sites' existing fail-closed catches handle it.
|
||||
*/
|
||||
final class DeadlineCharSequence implements CharSequence {
|
||||
|
||||
/** Per-match wall-clock budget. A legitimate whole-document match finishes far inside this. */
|
||||
static final long DEFAULT_BUDGET_MILLIS = 2_000L;
|
||||
|
||||
static final class RegexTimeoutException extends RuntimeException {
|
||||
RegexTimeoutException(long budgetMillis) {
|
||||
super("Regex evaluation exceeded " + budgetMillis + " ms (possible ReDoS)");
|
||||
}
|
||||
}
|
||||
|
||||
private final CharSequence inner;
|
||||
private final long deadlineNanos;
|
||||
private final long budgetMillis;
|
||||
|
||||
private DeadlineCharSequence(CharSequence inner, long budgetMillis) {
|
||||
this.inner = inner;
|
||||
this.budgetMillis = budgetMillis;
|
||||
this.deadlineNanos = System.nanoTime() + budgetMillis * 1_000_000L;
|
||||
}
|
||||
|
||||
/** Wrap {@code text} with the default budget; null becomes an empty sequence. */
|
||||
static DeadlineCharSequence of(String text) {
|
||||
return new DeadlineCharSequence(text == null ? "" : text, DEFAULT_BUDGET_MILLIS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public char charAt(int index) {
|
||||
if (System.nanoTime() > deadlineNanos) {
|
||||
throw new RegexTimeoutException(budgetMillis);
|
||||
}
|
||||
return inner.charAt(index);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int length() {
|
||||
return inner.length();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharSequence subSequence(int start, int end) {
|
||||
return inner.subSequence(start, end);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return inner.toString();
|
||||
}
|
||||
}
|
||||
+748
@@ -0,0 +1,748 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSFloat;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSString;
|
||||
import org.apache.pdfbox.pdfparser.PDFStreamParser;
|
||||
import org.apache.pdfbox.pdfwriter.ContentStreamWriter;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.common.PDStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.text.PDFTextStripperByArea;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
|
||||
/**
|
||||
* Content-stream editing: physically removes glyphs from page and form-XObject streams, both by
|
||||
* literal/regex match (auto-word) and by rect intersection (manual area), plus the area-capture and
|
||||
* overlay/annotation helpers those paths rely on.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RedactionContentEditor {
|
||||
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
|
||||
private RedactionContentEditor() {}
|
||||
|
||||
/** True only for upright pages whose CropBox origin is (0,0), where the rect flip is exact. */
|
||||
static boolean isSurgicallySafe(PDPage page) {
|
||||
if (page.getRotation() != 0) {
|
||||
return false;
|
||||
}
|
||||
PDRectangle crop = page.getCropBox();
|
||||
return crop.getLowerLeftX() == 0f && crop.getLowerLeftY() == 0f;
|
||||
}
|
||||
|
||||
/** Drop annotations whose rectangle overlaps any redaction rect. */
|
||||
static void removeOverlappingAnnotations(PDPage page, List<PDRectangle> rects) {
|
||||
try {
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
for (var ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
boolean overlaps = false;
|
||||
if (ar != null) {
|
||||
Rectangle2D.Float a =
|
||||
new Rectangle2D.Float(
|
||||
ar.getLowerLeftX(),
|
||||
ar.getLowerLeftY(),
|
||||
ar.getWidth(),
|
||||
ar.getHeight());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (a.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
overlaps = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!overlaps) {
|
||||
kept.add(ann);
|
||||
}
|
||||
}
|
||||
page.setAnnotations(kept);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not strip overlapping annotations: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
static List<Pattern> effectivePatterns(Set<String> literalTargets, List<Pattern> patterns) {
|
||||
List<Pattern> result = new ArrayList<>();
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// Case-insensitive to match TextFinderUtils; removal must not miss finder hits.
|
||||
result.add(
|
||||
Pattern.compile(
|
||||
Pattern.quote(target),
|
||||
Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE));
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
result.addAll(patterns);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Per-page content-stream rewrite (literal/regex based)
|
||||
|
||||
static void rewritePageContent(PDDocument document, PDPage page, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
PDResources resources = page.getResources();
|
||||
if (resources == null) {
|
||||
return;
|
||||
}
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(page));
|
||||
boolean modified = rewriteTokens(tokens, resources, patterns);
|
||||
if (modified) {
|
||||
writePageTokens(document, page, tokens);
|
||||
}
|
||||
// Recurse into form XObjects referenced by this page (shared visited set + depth cap).
|
||||
rewriteFormXObjects(resources, resources, patterns, new HashSet<>(), 0);
|
||||
}
|
||||
|
||||
private static void rewriteFormXObjects(
|
||||
PDResources resources,
|
||||
PDResources parentResources,
|
||||
List<Pattern> patterns,
|
||||
Set<COSBase> visited,
|
||||
int depth)
|
||||
throws IOException {
|
||||
if (depth > MAX_XOBJECT_DEPTH) {
|
||||
log.warn("XObject nesting exceeded {}; stopping recursion", MAX_XOBJECT_DEPTH);
|
||||
return;
|
||||
}
|
||||
for (COSName name : resources.getXObjectNames()) {
|
||||
try {
|
||||
var xobj = resources.getXObject(name);
|
||||
if (!(xobj instanceof PDFormXObject form)) {
|
||||
continue;
|
||||
}
|
||||
if (!visited.add(form.getCOSObject())) {
|
||||
continue;
|
||||
}
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(form));
|
||||
// Forms may inherit fonts from the parent when they carry no own /Resources.
|
||||
PDResources formResources =
|
||||
form.getResources() != null ? form.getResources() : parentResources;
|
||||
boolean modified = rewriteTokens(tokens, formResources, patterns);
|
||||
if (modified) {
|
||||
// A form XObject's content IS its own stream body; overwrite it in place.
|
||||
PDStream formStream = new PDStream(form.getCOSObject());
|
||||
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
new ContentStreamWriter(out).writeTokens(tokens);
|
||||
}
|
||||
}
|
||||
rewriteFormXObjects(formResources, formResources, patterns, visited, depth + 1);
|
||||
} catch (IOException e) {
|
||||
log.debug("Failed to rewrite XObject {}: {}", name.getName(), e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Object> parseTokens(PDFStreamParser parser) throws IOException {
|
||||
List<Object> tokens = new ArrayList<>();
|
||||
Object t;
|
||||
while ((t = parser.parseNextToken()) != null) {
|
||||
tokens.add(t);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/** Walk tokens keeping a tiny text state (current font). */
|
||||
private static boolean rewriteTokens(
|
||||
List<Object> tokens, PDResources resources, List<Pattern> patterns) {
|
||||
boolean modified = false;
|
||||
PDFont currentFont = null;
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
Object tok = tokens.get(i);
|
||||
if (!(tok instanceof Operator op)) {
|
||||
continue;
|
||||
}
|
||||
String name = op.getName();
|
||||
if ("Tf".equals(name) && i >= 2) {
|
||||
Object fontNameTok = tokens.get(i - 2);
|
||||
if (fontNameTok instanceof COSName fontName) {
|
||||
try {
|
||||
currentFont = resources.getFont(fontName);
|
||||
} catch (IOException ex) {
|
||||
log.debug(
|
||||
"Could not resolve font {}: {}",
|
||||
fontName.getName(),
|
||||
ex.getMessage());
|
||||
currentFont = null;
|
||||
}
|
||||
}
|
||||
} else if (RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name) && i >= 1) {
|
||||
int operandIdx = i - 1;
|
||||
Object operand = tokens.get(operandIdx);
|
||||
if (operand instanceof COSString cosString) {
|
||||
COSString replacement = rewriteCosString(cosString, currentFont, patterns);
|
||||
if (replacement != null) {
|
||||
tokens.set(operandIdx, replacement);
|
||||
modified = true;
|
||||
}
|
||||
} else if (operand instanceof COSArray arr) {
|
||||
COSArray newArr = rewriteCosArray(arr, currentFont, patterns);
|
||||
if (newArr != null) {
|
||||
tokens.set(operandIdx, newArr);
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return modified;
|
||||
}
|
||||
|
||||
private static COSString rewriteCosString(
|
||||
COSString cosString, PDFont font, List<Pattern> patterns) {
|
||||
if (font == null) {
|
||||
// Without a font we cannot decode safely.
|
||||
return rewriteRawLatin(cosString, patterns);
|
||||
}
|
||||
DecodeResult decoded = decodeCosString(cosString, font);
|
||||
if (decoded == null) {
|
||||
return rewriteRawLatin(cosString, patterns);
|
||||
}
|
||||
boolean[] drop = findDroppedCharsMask(decoded.text, patterns);
|
||||
if (drop == null) {
|
||||
return null;
|
||||
}
|
||||
return buildFilteredCosString(decoded, drop, font);
|
||||
}
|
||||
|
||||
private static COSArray rewriteCosArray(COSArray arr, PDFont font, List<Pattern> patterns) {
|
||||
// Build a concatenated decode across all COSString elements so that matches
|
||||
List<DecodeResult> parts = new ArrayList<>();
|
||||
StringBuilder concat = new StringBuilder();
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase elem = arr.get(i);
|
||||
if (elem instanceof COSString cs) {
|
||||
DecodeResult decoded = font != null ? decodeCosString(cs, font) : null;
|
||||
if (decoded == null) {
|
||||
decoded = decodeAsLatin(cs);
|
||||
}
|
||||
parts.add(decoded);
|
||||
concat.append(decoded.text);
|
||||
} else {
|
||||
parts.add(null);
|
||||
}
|
||||
}
|
||||
boolean[] fullDrop = findDroppedCharsMask(concat.toString(), patterns);
|
||||
if (fullDrop == null) {
|
||||
return null;
|
||||
}
|
||||
COSArray out = new COSArray();
|
||||
int cursor = 0;
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase elem = arr.get(i);
|
||||
if (elem instanceof COSString cs) {
|
||||
DecodeResult decoded = parts.get(i);
|
||||
int partLen = decoded.text.length();
|
||||
boolean[] partDrop = new boolean[partLen];
|
||||
System.arraycopy(fullDrop, cursor, partDrop, 0, partLen);
|
||||
cursor += partLen;
|
||||
COSString rebuilt =
|
||||
buildFilteredCosStringRaw(decoded, partDrop, font, cs.getBytes());
|
||||
// A zero-length COSString is valid; PDFBox emits it as () producing no
|
||||
out.add(rebuilt);
|
||||
} else {
|
||||
out.add(elem);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** For cases where we have no font - treat the bytes as latin-1 characters. */
|
||||
private static COSString rewriteRawLatin(COSString cosString, List<Pattern> patterns) {
|
||||
DecodeResult decoded = decodeAsLatin(cosString);
|
||||
boolean[] drop = findDroppedCharsMask(decoded.text, patterns);
|
||||
if (drop == null) {
|
||||
return null;
|
||||
}
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
for (int i = 0; i < decoded.text.length(); i++) {
|
||||
if (drop[i]) continue;
|
||||
out.write(decoded.text.charAt(i) & 0xFF);
|
||||
}
|
||||
return new COSString(out.toByteArray());
|
||||
}
|
||||
|
||||
private static DecodeResult decodeAsLatin(COSString cosString) {
|
||||
byte[] bytes = cosString.getBytes();
|
||||
StringBuilder sb = new StringBuilder(bytes.length);
|
||||
int[] codeStart = new int[bytes.length];
|
||||
int[] codeLen = new int[bytes.length];
|
||||
for (int i = 0; i < bytes.length; i++) {
|
||||
sb.append((char) (bytes[i] & 0xFF));
|
||||
codeStart[i] = i;
|
||||
codeLen[i] = 1;
|
||||
}
|
||||
return new DecodeResult(sb.toString(), bytes, codeStart, codeLen);
|
||||
}
|
||||
|
||||
/** Decode a COSString to Unicode via the font; null if decoding fails. */
|
||||
private static DecodeResult decodeCosString(COSString cosString, PDFont font) {
|
||||
byte[] bytes = cosString.getBytes();
|
||||
StringBuilder text = new StringBuilder();
|
||||
List<Integer> starts = new ArrayList<>();
|
||||
List<Integer> lens = new ArrayList<>();
|
||||
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes)) {
|
||||
int pos = 0;
|
||||
while (in.available() > 0) {
|
||||
int before = in.available();
|
||||
int code;
|
||||
try {
|
||||
code = font.readCode(in);
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
log.debug(
|
||||
"Font {} failed to decode byte sequence: {}",
|
||||
font.getName(),
|
||||
ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
int consumed = before - in.available();
|
||||
String unicode;
|
||||
try {
|
||||
unicode = font.toUnicode(code);
|
||||
} catch (Exception ex) {
|
||||
unicode = null;
|
||||
}
|
||||
if (unicode == null) {
|
||||
// If the font has no ToUnicode mapping we cannot match reliably - return
|
||||
return null;
|
||||
}
|
||||
text.append(unicode);
|
||||
// Associate every Unicode character produced with the same code byte range
|
||||
for (int c = 0; c < unicode.length(); c++) {
|
||||
starts.add(pos);
|
||||
lens.add(consumed);
|
||||
}
|
||||
if (consumed == 0) {
|
||||
// Defensive: avoid infinite loop on malformed fonts.
|
||||
break;
|
||||
}
|
||||
pos += consumed;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
}
|
||||
int[] startArr = starts.stream().mapToInt(Integer::intValue).toArray();
|
||||
int[] lenArr = lens.stream().mapToInt(Integer::intValue).toArray();
|
||||
return new DecodeResult(text.toString(), bytes, startArr, lenArr);
|
||||
}
|
||||
|
||||
/** Null if no pattern matches; else a per-char drop mask over the text. */
|
||||
private static boolean[] findDroppedCharsMask(String text, List<Pattern> patterns) {
|
||||
boolean any = false;
|
||||
boolean[] mask = new boolean[text.length()];
|
||||
for (Pattern pattern : patterns) {
|
||||
Matcher m;
|
||||
try {
|
||||
m = pattern.matcher(DeadlineCharSequence.of(text));
|
||||
} catch (Exception ex) {
|
||||
continue;
|
||||
}
|
||||
while (m.find()) {
|
||||
int s = m.start();
|
||||
int e = m.end();
|
||||
if (e <= s) continue;
|
||||
for (int i = s; i < e; i++) {
|
||||
mask[i] = true;
|
||||
}
|
||||
any = true;
|
||||
}
|
||||
}
|
||||
return any ? mask : null;
|
||||
}
|
||||
|
||||
private static COSString buildFilteredCosString(
|
||||
DecodeResult decoded, boolean[] drop, PDFont font) {
|
||||
return buildFilteredCosStringRaw(decoded, drop, font, decoded.bytes);
|
||||
}
|
||||
|
||||
private static COSString buildFilteredCosStringRaw(
|
||||
DecodeResult decoded, boolean[] drop, PDFont font, byte[] originalBytes) {
|
||||
// Collect code byte-ranges to drop.
|
||||
Set<Integer> dropStarts = new HashSet<>();
|
||||
for (int i = 0; i < drop.length; i++) {
|
||||
if (drop[i]) {
|
||||
dropStarts.add(decoded.codeStarts[i]);
|
||||
}
|
||||
}
|
||||
// Map byte-start -> code length once (O(n)); findCodeLenAt was an O(n^2) linear scan.
|
||||
Map<Integer, Integer> lenByStart = new HashMap<>();
|
||||
for (int j = 0; j < decoded.codeStarts.length; j++) {
|
||||
lenByStart.putIfAbsent(decoded.codeStarts[j], decoded.codeLens[j]);
|
||||
}
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
int i = 0;
|
||||
while (i < originalBytes.length) {
|
||||
int start = i;
|
||||
int len = lenByStart.getOrDefault(start, -1);
|
||||
if (len <= 0) {
|
||||
// Unknown - keep the byte verbatim.
|
||||
out.write(originalBytes[i] & 0xFF);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (dropStarts.contains(start)) {
|
||||
// Replace dropped code with encoded space if possible.
|
||||
byte[] spaceBytes = tryEncodeSpace(font);
|
||||
if (spaceBytes != null) {
|
||||
out.write(spaceBytes, 0, spaceBytes.length);
|
||||
}
|
||||
} else {
|
||||
out.write(originalBytes, start, len);
|
||||
}
|
||||
i += len;
|
||||
}
|
||||
return new COSString(out.toByteArray());
|
||||
}
|
||||
|
||||
private static byte[] tryEncodeSpace(PDFont font) {
|
||||
if (font == null) {
|
||||
return new byte[] {0x20};
|
||||
}
|
||||
try {
|
||||
return font.encode(" ");
|
||||
} catch (Exception e) {
|
||||
// Some fonts cannot encode a space at all (e.g. Type3 throws
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class DecodeResult {
|
||||
final String text;
|
||||
final byte[] bytes;
|
||||
final int[] codeStarts;
|
||||
final int[] codeLens;
|
||||
|
||||
DecodeResult(String text, byte[] bytes, int[] codeStarts, int[] codeLens) {
|
||||
this.text = text;
|
||||
this.bytes = bytes;
|
||||
this.codeStarts = codeStarts;
|
||||
this.codeLens = codeLens;
|
||||
}
|
||||
}
|
||||
|
||||
// Area capture
|
||||
|
||||
static List<String> captureTextInRects(PDPage page, List<PDRectangle> rects)
|
||||
throws IOException {
|
||||
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
|
||||
stripper.setSortByPosition(true);
|
||||
for (int i = 0; i < rects.size(); i++) {
|
||||
PDRectangle rect = rects.get(i);
|
||||
// PDFTextStripperByArea uses a Java2D rectangle in the same coordinate
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
Rectangle2D.Float region =
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight());
|
||||
stripper.addRegion("r" + i, region);
|
||||
}
|
||||
try {
|
||||
stripper.extractRegions(page);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to extract text in rects: {}", e.getMessage());
|
||||
return Collections.emptyList();
|
||||
}
|
||||
Set<String> captured = new LinkedHashSet<>();
|
||||
for (int i = 0; i < rects.size(); i++) {
|
||||
String text = stripper.getTextForRegion("r" + i);
|
||||
if (text == null) {
|
||||
continue;
|
||||
}
|
||||
for (String token : text.split("\\s+")) {
|
||||
String trimmed = token.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
captured.add(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ArrayList<>(captured);
|
||||
}
|
||||
|
||||
// Content-stream rewriting (rect-driven glyph removal)
|
||||
|
||||
/**
|
||||
* Blanks show-text operands whose glyphs fall in a rect. Returns false (surgical removal is
|
||||
* unreliable, caller must rasterise) when text-bearing form XObjects skew the operator ordinals
|
||||
* or an image sits under a rect - leaving the page stream untouched so the raster is correct.
|
||||
*/
|
||||
static boolean removeTokensIntersectingRects(
|
||||
PDDocument document, PDPage page, List<PDRectangle> rects) throws IOException {
|
||||
|
||||
List<Rectangle2D.Float> areaRects = new ArrayList<>();
|
||||
for (PDRectangle rect : rects) {
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
areaRects.add(
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight()));
|
||||
}
|
||||
|
||||
int pageIndex = document.getPages().indexOf(page);
|
||||
TokenIndexCollector collector = new TokenIndexCollector(areaRects);
|
||||
collector.setStartPage(pageIndex + 1);
|
||||
collector.setEndPage(pageIndex + 1);
|
||||
collector.getText(document);
|
||||
int collectorTextOps = collector.totalTextOps();
|
||||
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(page));
|
||||
int pageTextOps = 0;
|
||||
for (Object t : tokens) {
|
||||
if (t instanceof Operator op
|
||||
&& RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(op.getName())) {
|
||||
pageTextOps++;
|
||||
}
|
||||
}
|
||||
// Collector ordinals span form XObjects; images under a rect can't be surgically removed.
|
||||
if (collectorTextOps != pageTextOps || imageIntersectsAnyRect(page, pageIndex, rects)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Drop ONLY the in-rect glyphs of each operand (not the whole run), tracking the active
|
||||
// font so codes decode. Bail to rasterise if the font can't decode or glyph!=code counts.
|
||||
PDResources resources = page.getResources();
|
||||
PDFont currentFont = null;
|
||||
int opCounter = -1;
|
||||
boolean modified = false;
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
Object tok = tokens.get(i);
|
||||
if (!(tok instanceof Operator op)) {
|
||||
continue;
|
||||
}
|
||||
String name = op.getName();
|
||||
if ("Tf".equals(name) && i >= 2 && tokens.get(i - 2) instanceof COSName fn) {
|
||||
try {
|
||||
currentFont = resources != null ? resources.getFont(fn) : null;
|
||||
} catch (IOException e) {
|
||||
currentFont = null;
|
||||
}
|
||||
} else if (RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name)) {
|
||||
opCounter++;
|
||||
Set<Integer> dropGlyphs = collector.dropGlyphsByOp.get(opCounter);
|
||||
if (dropGlyphs == null || dropGlyphs.isEmpty() || i < 1) {
|
||||
continue;
|
||||
}
|
||||
// ' and " also move to the next line; converting them to TJ would lose that -
|
||||
// raster.
|
||||
if ("'".equals(name) || "\"".equals(name)) {
|
||||
return false;
|
||||
}
|
||||
int expected = collector.glyphCountByOp.getOrDefault(opCounter, -1);
|
||||
COSArray rebuilt =
|
||||
rebuildAdvancePreserving(
|
||||
tokens.get(i - 1), currentFont, dropGlyphs, expected);
|
||||
if (rebuilt == null) {
|
||||
return false; // undecodable / glyph-count mismatch - rasterise instead
|
||||
}
|
||||
// Emit as TJ so the removed glyphs' advance is preserved (no reflow).
|
||||
tokens.set(i - 1, rebuilt);
|
||||
tokens.set(i, Operator.getOperator("TJ"));
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
writePageTokens(document, page, tokens);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Distinct code byte-starts in order (one per glyph/code) from a per-char DecodeResult. */
|
||||
private static int[] distinctCodeStarts(DecodeResult d) {
|
||||
List<Integer> starts = new ArrayList<>();
|
||||
int last = -1;
|
||||
for (int s : d.codeStarts) {
|
||||
if (s != last) {
|
||||
starts.add(s);
|
||||
last = s;
|
||||
}
|
||||
}
|
||||
return starts.stream().mapToInt(Integer::intValue).toArray();
|
||||
}
|
||||
|
||||
private static int codeAt(PDFont font, byte[] bytes, int start, int len) throws IOException {
|
||||
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes, start, len)) {
|
||||
return font.readCode(in);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild a Tj/TJ operand as a TJ array that keeps every glyph EXCEPT the given 0-based
|
||||
* indexes, replacing each dropped run with a numeric adjustment equal to its advance so the
|
||||
* surviving text does not reflow into (or out of) the box. Returns null (caller rasterises) if
|
||||
* anything can't be decoded or the glyph count doesn't line up with the position pass.
|
||||
*/
|
||||
private static COSArray rebuildAdvancePreserving(
|
||||
Object operand, PDFont font, Set<Integer> dropGlyphs, int expectedGlyphs) {
|
||||
if (font == null) {
|
||||
return null;
|
||||
}
|
||||
List<COSBase> elements = new ArrayList<>();
|
||||
if (operand instanceof COSString cs) {
|
||||
elements.add(cs);
|
||||
} else if (operand instanceof COSArray arr) {
|
||||
for (COSBase b : arr) {
|
||||
elements.add(b);
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
COSArray out = new COSArray();
|
||||
ByteArrayOutputStream seg = new ByteArrayOutputStream();
|
||||
float pendingAdvance = 0f; // accumulated width of dropped codes, in 1/1000 text units
|
||||
int globalCode = 0;
|
||||
for (COSBase el : elements) {
|
||||
if (!(el instanceof COSString s)) {
|
||||
// Existing numeric adjustment: flush whichever run is pending, then preserve it.
|
||||
if (seg.size() > 0) {
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
seg.reset();
|
||||
} else if (pendingAdvance != 0f) {
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
pendingAdvance = 0f;
|
||||
}
|
||||
out.add(el);
|
||||
continue;
|
||||
}
|
||||
DecodeResult d = decodeCosString(s, font);
|
||||
if (d == null) {
|
||||
return null;
|
||||
}
|
||||
byte[] b = s.getBytes();
|
||||
int[] starts = distinctCodeStarts(d);
|
||||
Map<Integer, Integer> lenByStart = new HashMap<>();
|
||||
for (int j = 0; j < d.codeStarts.length; j++) {
|
||||
lenByStart.putIfAbsent(d.codeStarts[j], d.codeLens[j]);
|
||||
}
|
||||
for (int k = 0; k < starts.length; k++) {
|
||||
int bs = starts[k];
|
||||
int len = lenByStart.getOrDefault(bs, -1);
|
||||
if (len <= 0) {
|
||||
return null;
|
||||
}
|
||||
if (dropGlyphs.contains(globalCode)) {
|
||||
if (seg.size() > 0) { // close the kept run before accumulating advance
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
seg.reset();
|
||||
}
|
||||
try {
|
||||
pendingAdvance += font.getWidth(codeAt(font, b, bs, len));
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
if (pendingAdvance != 0f) { // emit the removed run's advance, then keep glyphs
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
pendingAdvance = 0f;
|
||||
}
|
||||
seg.write(b, bs, len);
|
||||
}
|
||||
globalCode++;
|
||||
}
|
||||
}
|
||||
if (pendingAdvance != 0f) {
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
}
|
||||
if (seg.size() > 0) {
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
}
|
||||
if (expectedGlyphs >= 0 && globalCode != expectedGlyphs) {
|
||||
return null;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any image on the page overlaps a redaction rect (would survive under the overlay).
|
||||
*/
|
||||
private static boolean imageIntersectsAnyRect(
|
||||
PDPage page, int pageIndex, List<PDRectangle> rects) {
|
||||
try {
|
||||
PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
locator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) {
|
||||
Rectangle2D.Float img =
|
||||
new Rectangle2D.Float(
|
||||
ib.x1(), ib.y1(), ib.x2() - ib.x1(), ib.y2() - ib.y1());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (img.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
// Cannot prove no image sits under a box - treat as unsafe (force raster).
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static void writePageTokens(PDDocument document, PDPage page, List<Object> tokens)
|
||||
throws IOException {
|
||||
PDStream stream = new PDStream(document);
|
||||
try (var out = stream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
ContentStreamWriter writer = new ContentStreamWriter(out);
|
||||
writer.writeTokens(tokens);
|
||||
}
|
||||
page.setContents(stream);
|
||||
}
|
||||
|
||||
static void drawOverlay(
|
||||
PDDocument document, PDPage page, List<PDRectangle> rects, Color overlayColor)
|
||||
throws IOException {
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(overlayColor);
|
||||
for (PDRectangle rect : rects) {
|
||||
cs.addRect(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight());
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.utils.text.TextFinderUtils;
|
||||
import stirling.software.common.util.RegexPatternUtils;
|
||||
|
||||
/**
|
||||
* Public facade for the manual-area, whole-page and auto-word redaction paths. Orchestrates the
|
||||
* collaborators that do the work: {@link RedactionContentEditor} (content-stream glyph removal),
|
||||
* {@link RedactionVerifier} (fail-closed independent verification) and {@link RedactionRasteriser}
|
||||
* (last-resort rasterisation), with {@link CatalogScrubber} for out-of-page carriers.
|
||||
*/
|
||||
@Slf4j
|
||||
public final class RedactionPipeline {
|
||||
|
||||
public static final Set<String> TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\"");
|
||||
|
||||
private RedactionPipeline() {}
|
||||
|
||||
/** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */
|
||||
static void setJpdfiumAvailableForTest(boolean available) {
|
||||
RedactionVerifier.setJpdfiumAvailableForTest(available);
|
||||
}
|
||||
|
||||
/** Redact page-local rects (PDF user-space), dropping intersecting glyphs + overlay. */
|
||||
public static RedactionResult redactAreas(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDRectangle>> rectsByPageIndex,
|
||||
Color overlayColor)
|
||||
throws IOException {
|
||||
|
||||
Set<String> capturedStrings = new LinkedHashSet<>();
|
||||
Set<Integer> forceRasterPages = new LinkedHashSet<>();
|
||||
|
||||
for (Map.Entry<Integer, List<PDRectangle>> entry : rectsByPageIndex.entrySet()) {
|
||||
int pageIndex = entry.getKey();
|
||||
List<PDRectangle> rects = entry.getValue();
|
||||
if (pageIndex < 0 || pageIndex >= document.getNumberOfPages() || rects.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
capturedStrings.addAll(RedactionContentEditor.captureTextInRects(page, rects));
|
||||
|
||||
// Rotation / non-zero CropBox origin break the coordinate flip, and text-bearing forms
|
||||
// or in-rect images defeat page-stream ordinal removal.
|
||||
boolean surgical =
|
||||
RedactionContentEditor.isSurgicallySafe(page)
|
||||
&& RedactionContentEditor.removeTokensIntersectingRects(
|
||||
document, page, rects);
|
||||
if (!surgical) {
|
||||
forceRasterPages.add(pageIndex);
|
||||
}
|
||||
RedactionContentEditor.removeOverlappingAnnotations(page, rects);
|
||||
RedactionContentEditor.drawOverlay(document, page, rects, overlayColor);
|
||||
}
|
||||
|
||||
return new RedactionResult(capturedStrings, forceRasterPages);
|
||||
}
|
||||
|
||||
/** Wipe whole pages: drop all content/resources, fill with a rectangle. */
|
||||
public static void redactWholePages(
|
||||
PDDocument document, List<Integer> pageIndexes, Color overlayColor) throws IOException {
|
||||
for (Integer pageIndex : pageIndexes) {
|
||||
if (pageIndex == null || pageIndex < 0 || pageIndex >= document.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PDRectangle media = page.getMediaBox();
|
||||
|
||||
// Drop existing content streams and page resources outright.
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
page.getCOSObject().removeItem(COSName.ANNOTS);
|
||||
// /Thumb is a rendered image of the original page - drop it too.
|
||||
page.getCOSObject().removeItem(COSName.getPDFName("Thumb"));
|
||||
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.OVERWRITE, true, true)) {
|
||||
cs.setNonStrokingColor(overlayColor);
|
||||
cs.addRect(
|
||||
media.getLowerLeftX(),
|
||||
media.getLowerLeftY(),
|
||||
media.getWidth(),
|
||||
media.getHeight());
|
||||
cs.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Physically remove every literal/regex match from all page content streams. */
|
||||
public static void redactLiteralTerms(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
List<Pattern> effectivePatterns =
|
||||
RedactionContentEditor.effectivePatterns(literalTargets, patterns);
|
||||
if (effectivePatterns.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int pageIndex = 0;
|
||||
for (PDPage page : document.getPages()) {
|
||||
try {
|
||||
RedactionContentEditor.rewritePageContent(document, page, effectivePatterns);
|
||||
} catch (IOException | RuntimeException e) {
|
||||
// Never let one page's font quirk (e.g. Type3 encode, damaged program)
|
||||
log.warn(
|
||||
"Content-stream rewrite failed on page {} ({}); leaving it for the "
|
||||
+ "verification/rasterisation pass.",
|
||||
pageIndex + 1,
|
||||
e.toString());
|
||||
}
|
||||
pageIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalize with document-wide verification: scrub, wipe metadata, save, verify. */
|
||||
public static byte[] finalize(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
return finalize(document, literalTargets, patterns, null);
|
||||
}
|
||||
|
||||
/** Finalize with verification scoped to a specific set of page indexes. */
|
||||
public static byte[] finalize(
|
||||
PDDocument document,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> affectedPages)
|
||||
throws IOException {
|
||||
|
||||
CatalogScrubber.scrub(document, literalTargets, patterns);
|
||||
RedactionVerifier.warnAboutEmbeddedFontGlyphs(document);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
try {
|
||||
RedactionVerifier.verify(bytes, literalTargets, patterns, affectedPages);
|
||||
return bytes;
|
||||
} catch (RedactionVerificationFailedException primaryFailure) {
|
||||
// Rewriter could not guarantee removal; rasterise as a last resort.
|
||||
log.warn(
|
||||
"Primary redaction verification failed ({}); falling back to page-scoped "
|
||||
+ "rasterisation to guarantee removal.",
|
||||
primaryFailure.getMessage());
|
||||
// With no page scope (auto-word), rasterise only the pages that still leak.
|
||||
Set<Integer> pagesToRaster =
|
||||
(affectedPages == null || affectedPages.isEmpty())
|
||||
? RedactionVerifier.findLeakingPages(bytes, literalTargets, patterns)
|
||||
: new HashSet<>(affectedPages);
|
||||
try (PDDocument rasterised = RedactionRasteriser.rasterisePages(bytes, pagesToRaster)) {
|
||||
CatalogScrubber.scrub(rasterised, literalTargets, patterns);
|
||||
ByteArrayOutputStream rasterOut = new ByteArrayOutputStream();
|
||||
rasterised.save(rasterOut);
|
||||
byte[] rasterBytes = rasterOut.toByteArray();
|
||||
RedactionVerifier.verify(rasterBytes, literalTargets, patterns, affectedPages);
|
||||
return rasterBytes;
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Rasterisation fallback failed after primary redaction leak", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalize a manual area redaction; rasterises forced + verified-leaking pages. */
|
||||
public static byte[] finalizeAreas(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDRectangle>> rectsByPage,
|
||||
Set<Integer> forceRasterPages,
|
||||
Set<String> capturedTargets)
|
||||
throws IOException {
|
||||
|
||||
// Text captured under the rects may also live in a bookmark / form value / annotation / JS
|
||||
// carrier.
|
||||
Set<String> carrierTargets =
|
||||
capturedTargets == null ? Collections.emptySet() : capturedTargets;
|
||||
CatalogScrubber.scrub(document, carrierTargets, Collections.emptyList());
|
||||
RedactionVerifier.warnAboutEmbeddedFontGlyphs(document);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
Set<Integer> toRaster =
|
||||
new HashSet<>(forceRasterPages == null ? Set.of() : forceRasterPages);
|
||||
toRaster.addAll(RedactionVerifier.findLeakingRectPages(bytes, rectsByPage));
|
||||
if (toRaster.isEmpty()) {
|
||||
return bytes;
|
||||
}
|
||||
log.warn("Manual redaction rasterising page(s) {} to guarantee removal", toRaster);
|
||||
try (PDDocument rasterised = RedactionRasteriser.rasterisePages(bytes, toRaster)) {
|
||||
ByteArrayOutputStream rasterOut = new ByteArrayOutputStream();
|
||||
rasterised.save(rasterOut);
|
||||
return rasterOut.toByteArray();
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Rasterisation fallback failed after manual redaction leak", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rasterise the given pages of already-finalized bytes to guarantee removal of geometric (range
|
||||
* / image-box) redactions whose covered content - text under an overlay, or an image - is not
|
||||
* removed by content-stream text redaction. Scrubs carriers and re-saves.
|
||||
*/
|
||||
public static byte[] rasteriseSpecificPages(
|
||||
byte[] bytes, Set<Integer> pages, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
if (pages == null || pages.isEmpty()) {
|
||||
return bytes;
|
||||
}
|
||||
log.warn("Rasterising page(s) {} to guarantee geometric redaction removal", pages);
|
||||
try (PDDocument rasterised =
|
||||
RedactionRasteriser.rasterisePages(bytes, new HashSet<>(pages))) {
|
||||
CatalogScrubber.scrub(rasterised, literalTargets, patterns);
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
rasterised.save(out);
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
/** Build case-insensitive regex patterns from user input. */
|
||||
public static List<Pattern> buildPatterns(
|
||||
String[] rawEntries, boolean useRegex, boolean wholeWordSearch) {
|
||||
List<Pattern> patterns = new ArrayList<>();
|
||||
if (rawEntries == null) {
|
||||
return patterns;
|
||||
}
|
||||
for (String raw : rawEntries) {
|
||||
if (raw == null) {
|
||||
continue;
|
||||
}
|
||||
String trimmed = raw.trim();
|
||||
if (trimmed.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String core = useRegex ? trimmed : Pattern.quote(trimmed);
|
||||
if (wholeWordSearch) {
|
||||
// Shared with the finder so removal + verification use identical boundaries.
|
||||
core = TextFinderUtils.applyWordBoundaries(trimmed, core);
|
||||
}
|
||||
// Compile via the shared cache (same path as the finder) for consistent
|
||||
// handling of user-supplied regex; ReDoS is bounded by DeadlineCharSequence.
|
||||
patterns.add(RegexPatternUtils.getInstance().createSearchPattern(core, true));
|
||||
} catch (PatternSyntaxException e) {
|
||||
log.debug("Skipping invalid regex '{}': {}", trimmed, e.getMessage());
|
||||
}
|
||||
}
|
||||
return patterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any font is not provably reliable for PDFBox glyph extraction (delegates to {@link
|
||||
* RedactionVerifier}); exposed here for the pipeline's callers and tests.
|
||||
*/
|
||||
static boolean documentHasUnreliableFont(PDDocument document) {
|
||||
return RedactionVerifier.documentHasUnreliableFont(document);
|
||||
}
|
||||
|
||||
/** Captured strings plus pages that must be rasterised (surgical removal was unreliable). */
|
||||
public static final class RedactionResult {
|
||||
private final Set<String> capturedStrings;
|
||||
private final Set<Integer> forceRasterPages;
|
||||
|
||||
public RedactionResult(Set<String> capturedStrings, Set<Integer> forceRasterPages) {
|
||||
this.capturedStrings = capturedStrings == null ? Set.of() : capturedStrings;
|
||||
this.forceRasterPages = forceRasterPages == null ? Set.of() : forceRasterPages;
|
||||
}
|
||||
|
||||
public Set<String> getCapturedStrings() {
|
||||
return capturedStrings;
|
||||
}
|
||||
|
||||
public Set<Integer> getForceRasterPages() {
|
||||
return forceRasterPages;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.rendering.ImageType;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/** Rasterisation fallback: renders whole pages to an image so no text/vector content survives. */
|
||||
@Slf4j
|
||||
final class RedactionRasteriser {
|
||||
|
||||
private RedactionRasteriser() {}
|
||||
|
||||
/** Rasterise the listed pages (all when null) at 150 DPI, replacing their content. */
|
||||
static PDDocument rasterisePages(byte[] sourceBytes, Set<Integer> pagesToRaster)
|
||||
throws IOException {
|
||||
// Load the document directly and mutate in place: rewriting
|
||||
PDDocument source = Loader.loadPDF(sourceBytes);
|
||||
try {
|
||||
PDFRenderer renderer = new PDFRenderer(source);
|
||||
int pageCount = source.getNumberOfPages();
|
||||
for (int i = 0; i < pageCount; i++) {
|
||||
if (pagesToRaster != null && !pagesToRaster.contains(i)) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = source.getPage(i);
|
||||
// PDFRenderer renders the CropBox region, so draw the raster over the CropBox (not
|
||||
// the MediaBox) or a CropBox != MediaBox page is stretched/offset.
|
||||
PDRectangle crop = page.getCropBox();
|
||||
|
||||
BufferedImage img = renderer.renderImageWithDPI(i, 150, ImageType.RGB);
|
||||
ByteArrayOutputStream imgOut = new ByteArrayOutputStream();
|
||||
ImageIO.write(img, "png", imgOut);
|
||||
PDImageXObject imageXObject =
|
||||
PDImageXObject.createFromByteArray(
|
||||
source, imgOut.toByteArray(), "redacted-page-" + i);
|
||||
|
||||
// Drop all prior content / resources / annotations / thumbnail; the raster is the
|
||||
// page.
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
page.getCOSObject().removeItem(COSName.ANNOTS);
|
||||
page.getCOSObject().removeItem(COSName.getPDFName("Thumb"));
|
||||
// Rotation is already baked into the rendered image, so reset it to zero.
|
||||
page.setRotation(0);
|
||||
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
source,
|
||||
page,
|
||||
PDPageContentStream.AppendMode.OVERWRITE,
|
||||
false,
|
||||
true)) {
|
||||
// The rendered image already has the rotation baked in visually
|
||||
cs.drawImage(
|
||||
imageXObject,
|
||||
crop.getLowerLeftX(),
|
||||
crop.getLowerLeftY(),
|
||||
crop.getWidth(),
|
||||
crop.getHeight());
|
||||
}
|
||||
}
|
||||
return source;
|
||||
} catch (IOException | RuntimeException e) {
|
||||
source.close();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
/** Thrown when post-redaction verification still finds a target in the saved PDF. */
|
||||
public class RedactionVerificationFailedException extends RuntimeException {
|
||||
|
||||
public RedactionVerificationFailedException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public RedactionVerificationFailedException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,495 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.text.PdfTextExtractor;
|
||||
|
||||
/**
|
||||
* Independent, fail-closed verification that redacted text is truly gone: an /ActualText-blind
|
||||
* PDFBox pass plus an additive native (PDFium) pass for fonts PDFBox cannot reliably extract, with
|
||||
* per-page leak localisation for the rasterisation fallback.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RedactionVerifier {
|
||||
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
|
||||
// Latched false when the native PDFium binding can't load, so the host falls back to the PDFBox
|
||||
// pass.
|
||||
private static volatile boolean jpdfiumAvailable = true;
|
||||
|
||||
// Skip the additive native pass above this size to bound off-heap copy + native runtime on
|
||||
// adversarial inputs; the PDFBox glyph-blind pass still verifies.
|
||||
private static final long MAX_JPDFIUM_VERIFY_BYTES = 100L * 1024 * 1024;
|
||||
|
||||
private RedactionVerifier() {}
|
||||
|
||||
/** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */
|
||||
static void setJpdfiumAvailableForTest(boolean available) {
|
||||
jpdfiumAvailable = available;
|
||||
}
|
||||
|
||||
/** Warns when the document still carries embedded Type0/TrueType font */
|
||||
static void warnAboutEmbeddedFontGlyphs(PDDocument document) {
|
||||
boolean anyEmbedded = false;
|
||||
Set<PDFont> visited = new HashSet<>();
|
||||
for (PDPage page : document.getPages()) {
|
||||
PDResources resources = page.getResources();
|
||||
if (resources == null) {
|
||||
continue;
|
||||
}
|
||||
for (COSName name : resources.getFontNames()) {
|
||||
PDFont font;
|
||||
try {
|
||||
font = resources.getFont(name);
|
||||
} catch (IOException ioe) {
|
||||
continue;
|
||||
}
|
||||
if (font == null || !visited.add(font)) {
|
||||
continue;
|
||||
}
|
||||
if (font instanceof PDType0Font || font instanceof PDTrueTypeFont) {
|
||||
anyEmbedded = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (anyEmbedded) {
|
||||
log.warn(
|
||||
"Redacted document contains embedded Type0/TrueType fonts; glyph outlines for "
|
||||
+ "redacted characters may remain in the font program. Text is not "
|
||||
+ "extractable via content-stream reading, but raw font inspection can "
|
||||
+ "still recover glyph shapes. Use the convert-to-image fallback for "
|
||||
+ "maximum assurance.");
|
||||
}
|
||||
}
|
||||
|
||||
static void verify(
|
||||
byte[] bytes,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> affectedPages) {
|
||||
if ((literalTargets == null || literalTargets.isEmpty())
|
||||
&& (patterns == null || patterns.isEmpty())) {
|
||||
return;
|
||||
}
|
||||
Set<Integer> pageSet =
|
||||
(affectedPages == null || affectedPages.isEmpty())
|
||||
? null
|
||||
: new TreeSet<>(affectedPages);
|
||||
// PDFBox pass, blind to /ActualText so a benign override can't mask real glyphs.
|
||||
boolean needNativePass;
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
assertNoTarget(extractText(reopened, pageSet), literalTargets, patterns);
|
||||
needNativePass = documentHasUnreliableFont(reopened);
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Failed to reopen redacted PDF for verification", e);
|
||||
}
|
||||
// Additive producer-independent pass: native PDFium sees glyphs PDFBox may miss (fonts with
|
||||
// no ToUnicode).
|
||||
if (needNativePass) {
|
||||
String nativeText = extractTextJPDFium(bytes, pageSet);
|
||||
if (nativeText == null) {
|
||||
// Required independent pass could not run (native unavailable or doc over the size
|
||||
// guard); fail closed.
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Independent native verification could not run for a document whose fonts "
|
||||
+ "PDFBox cannot reliably extract; cannot confirm removal");
|
||||
}
|
||||
assertNoTarget(nativeText, literalTargets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any font (on any page or nested form XObject) is not provably reliable for PDFBox
|
||||
* glyph extraction - i.e. it is neither a built-in Standard-14 font nor carries a /ToUnicode
|
||||
* map. These are exactly the fonts (CID/Type3/symbolic without ToUnicode) where the PDFBox pass
|
||||
* can go blind, so the independent native pass earns its cost. Biased to {@code true} on any
|
||||
* inspection failure so the native pass runs whenever reliability is uncertain.
|
||||
*/
|
||||
static boolean documentHasUnreliableFont(PDDocument document) {
|
||||
try {
|
||||
Set<COSBase> visited = new HashSet<>();
|
||||
for (PDPage page : document.getPages()) {
|
||||
if (resourcesHaveUnreliableFont(page.getResources(), visited, 0)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} catch (RuntimeException e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean resourcesHaveUnreliableFont(
|
||||
PDResources res, Set<COSBase> visited, int depth) {
|
||||
if (res == null) {
|
||||
return false;
|
||||
}
|
||||
if (depth > MAX_XOBJECT_DEPTH) {
|
||||
return true; // too deeply nested to fully verify - run the native pass to be safe
|
||||
}
|
||||
if (!visited.add(res.getCOSObject())) {
|
||||
return false; // already inspected this resource dictionary
|
||||
}
|
||||
for (COSName name : res.getFontNames()) {
|
||||
PDFont font;
|
||||
try {
|
||||
font = res.getFont(name);
|
||||
} catch (Exception e) {
|
||||
return true; // font won't load for inspection - assume unreliable
|
||||
}
|
||||
// Reliable only for built-in Standard-14 fonts or a /ToUnicode map we can trust.
|
||||
if (font != null
|
||||
&& !font.isStandard14()
|
||||
&& (!font.getCOSObject().containsKey(COSName.TO_UNICODE)
|
||||
|| isSubsetFont(font))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for (COSName name : res.getXObjectNames()) {
|
||||
try {
|
||||
if (res.getXObject(name) instanceof PDFormXObject form
|
||||
&& resourcesHaveUnreliableFont(form.getResources(), visited, depth + 1)) {
|
||||
return true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return true; // can't inspect the XObject - assume unreliable
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* A subset-embedded font carries a 6-uppercase-letter '+' BaseFont tag (e.g. {@code ABCDEF+}).
|
||||
* Its /ToUnicode is custom-built and cannot be trusted for the reliability gate: a partial map
|
||||
* silently loses text, and a crafted map can defeat text-based redaction entirely (only
|
||||
* convert-to-image fully mitigates that adversarial case).
|
||||
*/
|
||||
private static boolean isSubsetFont(PDFont font) {
|
||||
String name = font.getName();
|
||||
if (name == null || name.length() < 8 || name.charAt(6) != '+') {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < 6; i++) {
|
||||
char c = name.charAt(i);
|
||||
if (c < 'A' || c > 'Z') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Fail-closed match check with whitespace-normalised literals and X2 regex semantics. */
|
||||
private static void assertNoTarget(
|
||||
String extracted, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (extracted == null) {
|
||||
return;
|
||||
}
|
||||
String normalised = extracted.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
// De-hyphenated view catches a target split by a soft hyphen (U+00AD) or a line-break
|
||||
// hyphen ("-" + space).
|
||||
String dehyphenated = normalised.replace("\u00ad", "").replaceAll("-\\s+", "");
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
String needle = target.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
if (normalised.contains(needle) || dehyphenated.contains(needle)) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Redacted text still extractable: '" + target + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
if (pattern.matcher(DeadlineCharSequence.of(extracted)).find()) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Redacted pattern still extractable: " + pattern.pattern());
|
||||
}
|
||||
} catch (RedactionVerificationFailedException rvf) {
|
||||
throw rvf;
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Verification regex failed ("
|
||||
+ pattern.pattern()
|
||||
+ "): "
|
||||
+ e.getMessage(),
|
||||
e instanceof Exception ? (Exception) e : new Exception(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Extract text from pageIndexes (0-based) using an /ActualText-blind stripper. */
|
||||
private static String extractText(PDDocument document, Set<Integer> pageIndexes)
|
||||
throws IOException {
|
||||
PDFTextStripper stripper = new GlyphOnlyTextStripper();
|
||||
if (pageIndexes == null) {
|
||||
return stripper.getText(document);
|
||||
}
|
||||
StringBuilder out = new StringBuilder();
|
||||
for (Integer pageIdx : pageIndexes) {
|
||||
if (pageIdx == null || pageIdx < 0 || pageIdx >= document.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
stripper.setStartPage(pageIdx + 1);
|
||||
stripper.setEndPage(pageIdx + 1);
|
||||
String pageText = stripper.getText(document);
|
||||
if (pageText != null) {
|
||||
out.append(pageText);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Independent native (PDFium) extraction; null if the binding is unavailable (additive only).
|
||||
*/
|
||||
private static String extractTextJPDFium(byte[] bytes, Set<Integer> pageIndexes) {
|
||||
if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) {
|
||||
return null;
|
||||
}
|
||||
try (PdfDocument doc = PdfDocument.open(bytes)) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int n = doc.pageCount();
|
||||
if (pageIndexes == null) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
sb.append(jpdfiumPlainText(doc, i)).append('\n');
|
||||
}
|
||||
} else {
|
||||
for (Integer p : pageIndexes) {
|
||||
if (p != null && p >= 0 && p < n) {
|
||||
sb.append(jpdfiumPlainText(doc, p)).append('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (RuntimeException | Error e) {
|
||||
onJpdfiumFailure(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String jpdfiumPlainText(PdfDocument doc, int i) {
|
||||
try {
|
||||
return PdfTextExtractor.extractPage(doc, i).plainText();
|
||||
} catch (RuntimeException | Error e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** One native open, all pages' plain text; null if the binding is unavailable. */
|
||||
private static List<String> extractPagesJPDFium(byte[] bytes) {
|
||||
if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) {
|
||||
return null;
|
||||
}
|
||||
try (PdfDocument doc = PdfDocument.open(bytes)) {
|
||||
List<String> pages = new ArrayList<>();
|
||||
int n = doc.pageCount();
|
||||
for (int i = 0; i < n; i++) {
|
||||
pages.add(jpdfiumPlainText(doc, i));
|
||||
}
|
||||
return pages;
|
||||
} catch (RuntimeException | Error e) {
|
||||
onJpdfiumFailure(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A native-binding load error latches the pass off process-wide (warn once) so a host without
|
||||
* the native stops retrying; a per-document error only skips this one document (debug).
|
||||
*/
|
||||
private static void onJpdfiumFailure(Throwable e) {
|
||||
boolean nativeUnavailable =
|
||||
e instanceof UnsatisfiedLinkError
|
||||
|| e instanceof NoClassDefFoundError
|
||||
|| e instanceof ExceptionInInitializerError
|
||||
|| e.getClass().getSimpleName().contains("NativeLoad");
|
||||
if (nativeUnavailable) {
|
||||
jpdfiumAvailable = false;
|
||||
log.warn(
|
||||
"JPDFium native unavailable; redaction verification will use the PDFBox pass "
|
||||
+ "only: {}",
|
||||
e.toString());
|
||||
} else {
|
||||
log.debug("JPDFium verification skipped for this document: {}", e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
/** Pages whose surviving text (PDFBox glyph-blind OR native PDFium) still matches a target. */
|
||||
static Set<Integer> findLeakingPages(
|
||||
byte[] bytes, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
List<String> jpdfiumPages = extractPagesJPDFium(bytes);
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
// Native pass required but unavailable: PDFBox can't localise the leak, so rasterise
|
||||
// every page.
|
||||
if (jpdfiumPages == null && documentHasUnreliableFont(reopened)) {
|
||||
log.warn(
|
||||
"Independent native pass unavailable on an unreliable-font document; "
|
||||
+ "rasterising all pages to guarantee removal.");
|
||||
return null;
|
||||
}
|
||||
Set<Integer> leaking = new TreeSet<>();
|
||||
GlyphOnlyTextStripper stripper = new GlyphOnlyTextStripper();
|
||||
for (int i = 0; i < reopened.getNumberOfPages(); i++) {
|
||||
stripper.setStartPage(i + 1);
|
||||
stripper.setEndPage(i + 1);
|
||||
String pdfboxText = stripper.getText(reopened);
|
||||
String jpdfiumText =
|
||||
jpdfiumPages != null && i < jpdfiumPages.size() ? jpdfiumPages.get(i) : "";
|
||||
String combined = (pdfboxText == null ? "" : pdfboxText) + "\n" + jpdfiumText;
|
||||
if (pageLeaks(combined, literalTargets, patterns)) {
|
||||
leaking.add(i);
|
||||
}
|
||||
}
|
||||
if (leaking.isEmpty()) {
|
||||
log.warn(
|
||||
"Verification failed but no single page leaks in isolation; rasterising "
|
||||
+ "all pages to be safe.");
|
||||
return null;
|
||||
}
|
||||
log.info("Leak detection: rasterising only page(s) {}", leaking);
|
||||
return leaking;
|
||||
} catch (Exception e) {
|
||||
log.warn("Per-page leak detection failed ({}); rasterising all pages.", e.toString());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean pageLeaks(
|
||||
String pageText, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
try {
|
||||
assertNoTarget(pageText, literalTargets, patterns);
|
||||
return false;
|
||||
} catch (RedactionVerificationFailedException e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/** Redacted pages that still show text inside a rect or carry an overlapping annotation. */
|
||||
static Set<Integer> findLeakingRectPages(
|
||||
byte[] bytes, Map<Integer, List<PDRectangle>> rectsByPage) {
|
||||
Set<Integer> leaking = new HashSet<>();
|
||||
if (rectsByPage == null || rectsByPage.isEmpty()) {
|
||||
return leaking;
|
||||
}
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
for (Map.Entry<Integer, List<PDRectangle>> entry : rectsByPage.entrySet()) {
|
||||
int pageIndex = entry.getKey();
|
||||
if (pageIndex < 0 || pageIndex >= reopened.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = reopened.getPage(pageIndex);
|
||||
if (glyphStillInRects(reopened, page, pageIndex, entry.getValue())
|
||||
|| annotationOverlapsRect(page, entry.getValue())) {
|
||||
leaking.add(pageIndex);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Cannot verify - rasterise every redacted page to be safe.
|
||||
return new HashSet<>(rectsByPage.keySet());
|
||||
}
|
||||
return leaking;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any non-blank glyph is still painted inside a rect. Position-based (not ToUnicode),
|
||||
* so it catches residual CID/Type3/no-ToUnicode glyphs the text stripper would miss; fails
|
||||
* closed.
|
||||
*/
|
||||
private static boolean glyphStillInRects(
|
||||
PDDocument doc, PDPage page, int pageIndex, List<PDRectangle> rects) {
|
||||
try {
|
||||
List<Rectangle2D.Float> areaRects = new ArrayList<>();
|
||||
for (PDRectangle rect : rects) {
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
areaRects.add(
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight()));
|
||||
}
|
||||
TokenIndexCollector collector = new TokenIndexCollector(areaRects);
|
||||
collector.setStartPage(pageIndex + 1);
|
||||
collector.setEndPage(pageIndex + 1);
|
||||
collector.getText(doc);
|
||||
return collector.anyGlyphInRect();
|
||||
} catch (Exception e) {
|
||||
return true; // cannot prove the rect is clean - rasterise to be safe
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean annotationOverlapsRect(PDPage page, List<PDRectangle> rects) {
|
||||
try {
|
||||
for (var ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
if (ar == null) {
|
||||
continue;
|
||||
}
|
||||
Rectangle2D.Float a =
|
||||
new Rectangle2D.Float(
|
||||
ar.getLowerLeftX(),
|
||||
ar.getLowerLeftY(),
|
||||
ar.getWidth(),
|
||||
ar.getHeight());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (a.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** PDFTextStripper that strips /ActualText so verification sees the real glyph stream. */
|
||||
private static final class GlyphOnlyTextStripper extends PDFTextStripper {
|
||||
private static final COSName ACTUAL_TEXT = COSName.getPDFName("ActualText");
|
||||
|
||||
GlyphOnlyTextStripper() throws IOException {}
|
||||
|
||||
@Override
|
||||
public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {
|
||||
COSDictionary safe = properties;
|
||||
if (properties != null && properties.containsKey(ACTUAL_TEXT)) {
|
||||
safe = new COSDictionary(properties);
|
||||
safe.removeItem(ACTUAL_TEXT);
|
||||
}
|
||||
super.beginMarkedContentSequence(tag, safe);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* Position-based (ToUnicode-independent) glyph locator: records, per text-showing operator, the
|
||||
* 0-based indexes of the non-blank glyphs whose box intersects a rect, plus each operator's total
|
||||
* glyph count. Blank glyphs are ignored so a space left behind by removal is not a "leak".
|
||||
*/
|
||||
final class TokenIndexCollector extends PDFTextStripper {
|
||||
private final List<Rectangle2D.Float> rects;
|
||||
final Map<Integer, Set<Integer>> dropGlyphsByOp = new HashMap<>();
|
||||
final Map<Integer, Integer> glyphCountByOp = new HashMap<>();
|
||||
private int showTextOpCounter = -1;
|
||||
private int glyphInOp = 0;
|
||||
|
||||
TokenIndexCollector(List<Rectangle2D.Float> rects) throws IOException {
|
||||
this.rects = rects;
|
||||
setSortByPosition(false);
|
||||
}
|
||||
|
||||
int totalTextOps() {
|
||||
return showTextOpCounter + 1;
|
||||
}
|
||||
|
||||
boolean anyGlyphInRect() {
|
||||
return !dropGlyphsByOp.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processTextPosition(TextPosition text) {
|
||||
int op = showTextOpCounter;
|
||||
int idx = glyphInOp++;
|
||||
String u = text.getUnicode();
|
||||
if (u != null && !u.isBlank()) {
|
||||
// PDFBox reports coordinates with top-left origin here.
|
||||
float x = text.getX();
|
||||
float y = text.getY() - text.getHeight();
|
||||
Rectangle2D.Float glyph =
|
||||
new Rectangle2D.Float(x, y, text.getWidth(), text.getHeight());
|
||||
for (Rectangle2D.Float rect : rects) {
|
||||
if (rect.intersects(glyph)) {
|
||||
dropGlyphsByOp.computeIfAbsent(op, k -> new HashSet<>()).add(idx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
super.processTextPosition(text);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processOperator(Operator operator, List<COSBase> operands) throws IOException {
|
||||
String name = operator.getName();
|
||||
boolean textOp = RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name);
|
||||
if (textOp) {
|
||||
showTextOpCounter++;
|
||||
glyphInOp = 0;
|
||||
}
|
||||
super.processOperator(operator, operands);
|
||||
if (textOp) {
|
||||
glyphCountByOp.put(showTextOpCounter, glyphInOp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -86,14 +86,21 @@ public class TextFinderUtils {
|
||||
return patterns;
|
||||
}
|
||||
|
||||
private static String applyWordBoundaries(String originalTerm, String patternString) {
|
||||
if (originalTerm.length() == 1 && Character.isDigit(originalTerm.charAt(0))) {
|
||||
/** Shared whole-word wrapper so the finder and RedactionPipeline never diverge (see F2). */
|
||||
public static String applyWordBoundaries(String originalTerm, String patternString) {
|
||||
// Lookarounds for single/non-word-edge terms (\b no-ops on chars like '&'); \b otherwise.
|
||||
boolean nonWordEdge =
|
||||
originalTerm.length() == 1
|
||||
|| !isWordChar(originalTerm.charAt(0))
|
||||
|| !isWordChar(originalTerm.charAt(originalTerm.length() - 1));
|
||||
if (nonWordEdge) {
|
||||
return "(?<![\\w])" + patternString + "(?![\\w])";
|
||||
} else if (originalTerm.length() == 1) {
|
||||
return "(?<![\\w])" + patternString + "(?![\\w])";
|
||||
} else {
|
||||
return "\\b" + patternString + "\\b";
|
||||
}
|
||||
return "\\b" + patternString + "\\b";
|
||||
}
|
||||
|
||||
private static boolean isWordChar(char c) {
|
||||
return Character.isLetterOrDigit(c) || c == '_';
|
||||
}
|
||||
|
||||
public static boolean hasProblematicFonts(PDPage page) {
|
||||
|
||||
+45
-5
@@ -19,6 +19,7 @@ import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -568,7 +569,15 @@ class ManualRedactionServiceTest {
|
||||
byPage.put(0, new ArrayList<>(Arrays.asList(text(0, 72, 690, 300, 710))));
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 1.0f, false, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
1.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertNotNull(result.getFile());
|
||||
@@ -590,7 +599,15 @@ class ManualRedactionServiceTest {
|
||||
Map<Integer, List<PDFText>> byPage = new HashMap<>();
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 0.0f, false, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getFile().exists());
|
||||
@@ -610,7 +627,15 @@ class ManualRedactionServiceTest {
|
||||
Map<Integer, List<PDFText>> byPage = new HashMap<>();
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 0.0f, null, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
null,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getFile().exists());
|
||||
@@ -630,7 +655,15 @@ class ManualRedactionServiceTest {
|
||||
byPage.put(0, new ArrayList<>(Arrays.asList(text(0, 72, 690, 300, 710))));
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#FF0000", 2.0f, false, true);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#FF0000",
|
||||
2.0f,
|
||||
false,
|
||||
true,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
try (PDDocument reloaded = Loader.loadPDF(result.getFile())) {
|
||||
@@ -658,7 +691,14 @@ class ManualRedactionServiceTest {
|
||||
IOException.class,
|
||||
() ->
|
||||
service.finalizeRedaction(
|
||||
doc, byPage, "#000000", 0.0f, false, false));
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList()));
|
||||
// The failing temp file is closed on the error path.
|
||||
verify(failing).close();
|
||||
} finally {
|
||||
|
||||
+16
-4
@@ -192,15 +192,27 @@ class RedactControllerTest {
|
||||
when(mockCOSStream.createOutputStream()).thenReturn(mockOutputStream);
|
||||
when(mockCOSStream.createOutputStream(any())).thenReturn(mockOutputStream);
|
||||
|
||||
// The pipeline saves then reloads the bytes to verify, so the mock must emit a real,
|
||||
// reloadable PDF (not a placeholder) or verification throws on reopen.
|
||||
byte[] realPdfBytes = createSimplePdfContent();
|
||||
lenient()
|
||||
.doAnswer(
|
||||
inv -> {
|
||||
File f = inv.getArgument(0);
|
||||
java.nio.file.Files.write(f.toPath(), "mock pdf".getBytes());
|
||||
java.nio.file.Files.write(f.toPath(), realPdfBytes);
|
||||
return null;
|
||||
})
|
||||
.when(mockDocument)
|
||||
.save(any(File.class));
|
||||
lenient()
|
||||
.doAnswer(
|
||||
inv -> {
|
||||
java.io.OutputStream os = inv.getArgument(0);
|
||||
os.write(realPdfBytes);
|
||||
return null;
|
||||
})
|
||||
.when(mockDocument)
|
||||
.save(any(java.io.OutputStream.class));
|
||||
doNothing().when(mockDocument).close();
|
||||
|
||||
// Build real service instances so tests exercise actual logic
|
||||
@@ -347,7 +359,7 @@ class RedactControllerTest {
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
|
||||
verify(mockDocument).save(any(File.class));
|
||||
verify(mockDocument).save(any(java.io.OutputStream.class));
|
||||
verify(mockDocument).close();
|
||||
}
|
||||
}
|
||||
@@ -753,7 +765,7 @@ class RedactControllerTest {
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response.getBody());
|
||||
assertTrue(drainBody(response).length > 0);
|
||||
verify(mockDocument, times(1)).save(any(File.class));
|
||||
verify(mockDocument, times(1)).save(any(java.io.OutputStream.class));
|
||||
verify(mockDocument, times(1)).close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
@@ -776,7 +788,7 @@ class RedactControllerTest {
|
||||
if (response != null) {
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
verify(mockDocument, times(1)).save(any(File.class));
|
||||
verify(mockDocument, times(1)).save(any(java.io.OutputStream.class));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.info("Manual redaction test completed with graceful handling: {}", e.getMessage());
|
||||
|
||||
+755
@@ -0,0 +1,755 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSInteger;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSStream;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDFormContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.common.PDStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFontDescriptor;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.MacRomanEncoding;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.WinAnsiEncoding;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
/** Security matrix for redaction across PDF shapes: standard and embedded */
|
||||
@DisplayName("Redaction PDF-variety security matrix")
|
||||
class RedactionPdfVarietyTest {
|
||||
|
||||
private static final String LIBERATION =
|
||||
"/org/apache/pdfbox/resources/ttf/LiberationSans-Regular.ttf";
|
||||
private static final float FONT_SIZE = 12f;
|
||||
private static final float LEFT_X = 72f;
|
||||
private static final float TOP_Y = PDRectangle.LETTER.getHeight() - 80f;
|
||||
|
||||
private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private TempFileManager tempFileManager;
|
||||
private RedactController controller;
|
||||
|
||||
private final List<File> createdTempFiles = new ArrayList<>();
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws IOException {
|
||||
pdfDocumentFactory = mock(CustomPDFDocumentFactory.class);
|
||||
tempFileManager = mock(TempFileManager.class);
|
||||
|
||||
lenient()
|
||||
.when(tempFileManager.createManagedTempFile(anyString()))
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
File f =
|
||||
Files.createTempFile(
|
||||
"redact-variety", inv.<String>getArgument(0))
|
||||
.toFile();
|
||||
createdTempFiles.add(f);
|
||||
TempFile tf = mock(TempFile.class);
|
||||
lenient().when(tf.getFile()).thenReturn(f);
|
||||
lenient().when(tf.getPath()).thenReturn(f.toPath());
|
||||
return tf;
|
||||
});
|
||||
|
||||
controller =
|
||||
new RedactController(
|
||||
pdfDocumentFactory,
|
||||
tempFileManager,
|
||||
new ManualRedactionService(tempFileManager),
|
||||
new TextRedactionService(),
|
||||
mock(RedactExecuteService.class));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
for (File f : createdTempFiles) {
|
||||
if (f != null && f.exists()) {
|
||||
f.delete();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Matrix cases (all through the real /auto-redact controller path)
|
||||
|
||||
@Test
|
||||
@DisplayName("standard Helvetica: target gone, neighbours survive")
|
||||
void standard14Helvetica() throws IOException {
|
||||
byte[] out = autoRedact(helveticaPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("standard-14 Times and Courier: target gone, neighbours survive")
|
||||
void standard14TimesAndCourier() throws IOException {
|
||||
for (Standard14Fonts.FontName fn :
|
||||
new Standard14Fonts.FontName[] {
|
||||
Standard14Fonts.FontName.TIMES_ROMAN, Standard14Fonts.FontName.COURIER
|
||||
}) {
|
||||
byte[] pdf = std14Pdf(fn, "alpha SECRET omega");
|
||||
byte[] out = autoRedact(pdf, "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).as("%s keeps neighbours", fn).contains("alpha");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple TrueType with MacRoman encoding: target gone, neighbours survive")
|
||||
void simpleTrueTypeMacRoman() throws IOException {
|
||||
byte[] out = autoRedact(macRomanTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded Type0 with /ToUnicode stripped: target still gone, neighbours survive")
|
||||
void type0WithoutToUnicode() throws IOException {
|
||||
byte[] out = autoRedact(type0NoToUnicodePdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("symbolic TrueType (no encoding, no ToUnicode): target not extractable")
|
||||
void symbolicTrueType() throws IOException {
|
||||
// PDFBox extraction is glyph-blind here, so removal is guaranteed by the native pass +
|
||||
// rasterise fallback rather than surgical decode; this pins that the pipeline handles a
|
||||
// symbolic font without error and leaves no extractable target.
|
||||
byte[] out = autoRedact(symbolicTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("subset-tagged font (ABCDEF+): target gone, neighbours survive")
|
||||
void subsetTaggedFont() throws IOException {
|
||||
byte[] out = autoRedact(subsetTaggedPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("real Type3 font (glyphs as content streams, no ToUnicode): target gone")
|
||||
void type3GlyphProcs() throws IOException {
|
||||
// crop_test.pdf uses DejaVuSans embedded as a subset Type3 font with no
|
||||
byte[] input;
|
||||
try (InputStream in = getClass().getResourceAsStream("/redaction/type3_dejavu.pdf")) {
|
||||
input = in.readAllBytes();
|
||||
}
|
||||
byte[] out = autoRedact(input, "EXAMPLE");
|
||||
assertGone(out, "EXAMPLE");
|
||||
assertThat(pdfText(out)).contains("CROP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded subset Type0 (CID) font: target gone, neighbours survive")
|
||||
void embeddedSubsetType0() throws IOException {
|
||||
byte[] out = autoRedact(type0Pdf(true, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded full Type0 (CID) font: target gone, neighbours survive")
|
||||
void embeddedFullType0() throws IOException {
|
||||
byte[] out = autoRedact(type0Pdf(false, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded Type1 (PFB) font: target gone, neighbours survive")
|
||||
void embeddedType1() throws IOException {
|
||||
byte[] out = autoRedact(type1Pdf("PUBLICA SECRETWORD KEEP"), "SECRETWORD");
|
||||
assertGone(out, "SECRETWORD");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded CJK Type0 (CIDFontType2) font: CJK target gone, neighbours survive")
|
||||
void embeddedCjkType0() throws IOException {
|
||||
byte[] out = autoRedact(cjkPdf("PUBLICA \u79D8\u5BC6 KEEP"), "\u79D8\u5BC6");
|
||||
assertGone(out, "\u79D8\u5BC6");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded CIDFontType0 (CFF) font: target gone, neighbours survive")
|
||||
void embeddedCidFontType0() throws IOException {
|
||||
byte[] out = autoRedact(cidFontType0Pdf("PUBLICA SECRETWORD KEEP"), "SECRETWORD");
|
||||
assertGone(out, "SECRETWORD");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple embedded TrueType (WinAnsi) font: target gone, neighbours survive")
|
||||
void simpleTrueTypeWinAnsi() throws IOException {
|
||||
byte[] out = autoRedact(simpleTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rotated pages (90/180/270): target gone on every rotation")
|
||||
void rotatedPages() throws IOException {
|
||||
for (int rotation : new int[] {90, 180, 270}) {
|
||||
byte[] out = autoRedact(rotatedPdf(rotation, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).as("rotation %d keeps neighbours", rotation).contains("alpha");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target split across TJ array operands is removed")
|
||||
void tjArraySplit() throws IOException {
|
||||
byte[] out = autoRedact(tjSplitPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("public");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target split across separate Tj operators is removed, other pages keep text")
|
||||
void crossOperatorSplit() throws IOException {
|
||||
byte[] out = autoRedact(crossOperatorPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
// Page 2 was never touched; whatever path handled page 1, page 2 text must
|
||||
assertThat(pdfText(out)).contains("PUBLIC PAGE TWO");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target inside a Form XObject is removed")
|
||||
void formXObjectText() throws IOException {
|
||||
byte[] out = autoRedact(formXObjectPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("CropBox smaller than MediaBox: target gone, neighbours survive")
|
||||
void cropBoxOffset() throws IOException {
|
||||
byte[] out = autoRedact(cropBoxPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("multi-page: target on middle page only, other pages keep their text")
|
||||
void multiPageMiddleTarget() throws IOException {
|
||||
byte[] out =
|
||||
autoRedact(
|
||||
multiPagePdf("PUBLIC ONE", "middle SECRET line", "PUBLIC THREE"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("PUBLIC ONE").contains("PUBLIC THREE");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("case variant: doc contains 'Secret', target 'SECRET' removes it")
|
||||
void caseVariantRemoved() throws IOException {
|
||||
byte[] out = autoRedact(helveticaPdf("alpha Secret omega"), "SECRET");
|
||||
assertGone(out, "Secret");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("whole-word with case variant: 'Cat' removed, substrings survive")
|
||||
void wholeWordCaseVariant() throws IOException {
|
||||
byte[] bytes = helveticaPdf("Cat classification scatter");
|
||||
factoryReturns(bytes);
|
||||
RedactPdfRequest request = baseRequest(bytes, "cat");
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
byte[] out = drainBody(controller.redactPdf(request));
|
||||
String text = pdfText(out);
|
||||
assertThat(text).contains("classification").contains("scatter");
|
||||
// The standalone word must be gone in any case variant.
|
||||
assertThat(text).doesNotContainPattern("(?i)\\bcat\\b");
|
||||
}
|
||||
|
||||
// Page-scoped rasterisation (pipeline-level, deterministic)
|
||||
|
||||
@Test
|
||||
@DisplayName("verification leak rasterises only the leaking page, others keep text")
|
||||
void leakRasterisesOnlyLeakingPage() throws IOException {
|
||||
// Cross-operator split defeats the per-operand literal rewriter
|
||||
byte[] input = crossOperatorPdf();
|
||||
byte[] out;
|
||||
try (PDDocument doc = Loader.loadPDF(input)) {
|
||||
Set<String> targets = Set.of("SECRET");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"SECRET"}, false, false);
|
||||
RedactionPipeline.redactLiteralTerms(doc, targets, patterns);
|
||||
out = RedactionPipeline.finalize(doc, targets, patterns);
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(out)) {
|
||||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
stripper.setStartPage(1);
|
||||
stripper.setEndPage(1);
|
||||
String page1 = stripper.getText(reopened);
|
||||
stripper.setStartPage(2);
|
||||
stripper.setEndPage(2);
|
||||
String page2 = stripper.getText(reopened);
|
||||
|
||||
assertThat(page1.toLowerCase()).doesNotContain("secret");
|
||||
assertThat(page2).contains("PUBLIC PAGE TWO");
|
||||
}
|
||||
}
|
||||
|
||||
// Drivers and assertions
|
||||
|
||||
private byte[] autoRedact(byte[] pdfBytes, String target) throws IOException {
|
||||
factoryReturns(pdfBytes);
|
||||
RedactPdfRequest request = baseRequest(pdfBytes, target);
|
||||
ResponseEntity<Resource> response = controller.redactPdf(request);
|
||||
assertThat(response.getStatusCode().value()).isEqualTo(200);
|
||||
return drainBody(response);
|
||||
}
|
||||
|
||||
private RedactPdfRequest baseRequest(byte[] pdfBytes, String target) {
|
||||
RedactPdfRequest request = new RedactPdfRequest();
|
||||
request.setFileInput(pdfFile(pdfBytes));
|
||||
request.setListOfText(target);
|
||||
request.setUseRegex(false);
|
||||
request.setWholeWordSearch(false);
|
||||
request.setRedactColor("#000000");
|
||||
request.setConvertPDFToImage(false);
|
||||
return request;
|
||||
}
|
||||
|
||||
private void assertGone(byte[] out, String target) throws IOException {
|
||||
assertThat(pdfText(out).toLowerCase()).doesNotContain(target.toLowerCase());
|
||||
}
|
||||
|
||||
private void factoryReturns(byte[] pdfBytes) throws IOException {
|
||||
lenient()
|
||||
.when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(pdfBytes));
|
||||
}
|
||||
|
||||
private MockMultipartFile pdfFile(byte[] bytes) {
|
||||
return new MockMultipartFile("fileInput", "doc.pdf", "application/pdf", bytes);
|
||||
}
|
||||
|
||||
private byte[] drainBody(ResponseEntity<Resource> response) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try (InputStream in = response.getBody().getInputStream()) {
|
||||
in.transferTo(baos);
|
||||
}
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
private String pdfText(byte[] pdfBytes) throws IOException {
|
||||
try (PDDocument doc = Loader.loadPDF(pdfBytes)) {
|
||||
return new PDFTextStripper().getText(doc);
|
||||
}
|
||||
}
|
||||
|
||||
// PDF builders
|
||||
|
||||
private static PDFont helvetica() {
|
||||
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
|
||||
}
|
||||
|
||||
private byte[] helveticaPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] std14Pdf(Standard14Fonts.FontName fontName, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, new PDType1Font(fontName), line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] macRomanTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, MacRomanEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
// GID order emitted by generators/gen_cff_cidfont.py: 0=.notdef 1=space then the letters.
|
||||
private static final String CFF_GLYPHS = "\0 ABCDEIKLOPRSTUW";
|
||||
|
||||
private static int cffGid(char c) {
|
||||
int i = CFF_GLYPHS.indexOf(c);
|
||||
return i < 0 ? 0 : i;
|
||||
}
|
||||
|
||||
/**
|
||||
* PDFBox 3.0.7 has no CFF embedder, so hand-assemble a Type0 -> CIDFontType0 (CFF in
|
||||
* /FontFile3, Identity-H, CID==GID, explicit /ToUnicode) from the generated CFF program.
|
||||
*/
|
||||
private byte[] cidFontType0Pdf(String line) throws IOException {
|
||||
byte[] cff;
|
||||
try (InputStream in = getClass().getResourceAsStream("/redaction/cidfont0_cff.bin")) {
|
||||
cff = in.readAllBytes();
|
||||
}
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
|
||||
PDStream fontFile = new PDStream(doc);
|
||||
try (var os = fontFile.createOutputStream()) {
|
||||
os.write(cff);
|
||||
}
|
||||
COSStream ff = fontFile.getCOSObject();
|
||||
ff.setItem(COSName.SUBTYPE, COSName.getPDFName("CIDFontType0C"));
|
||||
|
||||
COSDictionary fdesc = new COSDictionary();
|
||||
fdesc.setItem(COSName.TYPE, COSName.FONT_DESC);
|
||||
fdesc.setName(COSName.FONT_NAME, "MinimalCFFCID");
|
||||
fdesc.setInt(COSName.FLAGS, 4);
|
||||
COSArray bbox = new COSArray();
|
||||
for (int v : new int[] {0, -200, 600, 800}) {
|
||||
bbox.add(COSInteger.get(v));
|
||||
}
|
||||
fdesc.setItem(COSName.FONT_BBOX, bbox);
|
||||
fdesc.setInt(COSName.ITALIC_ANGLE, 0);
|
||||
fdesc.setInt(COSName.ASCENT, 800);
|
||||
fdesc.setInt(COSName.DESCENT, -200);
|
||||
fdesc.setInt(COSName.CAP_HEIGHT, 700);
|
||||
fdesc.setInt(COSName.STEM_V, 80);
|
||||
fdesc.setItem(COSName.getPDFName("FontFile3"), ff);
|
||||
|
||||
COSDictionary cidSystemInfo = new COSDictionary();
|
||||
cidSystemInfo.setString(COSName.REGISTRY, "Adobe");
|
||||
cidSystemInfo.setString(COSName.ORDERING, "Identity");
|
||||
cidSystemInfo.setInt(COSName.SUPPLEMENT, 0);
|
||||
|
||||
COSDictionary cidFont = new COSDictionary();
|
||||
cidFont.setItem(COSName.TYPE, COSName.FONT);
|
||||
cidFont.setItem(COSName.SUBTYPE, COSName.CID_FONT_TYPE0);
|
||||
cidFont.setName(COSName.BASE_FONT, "MinimalCFFCID");
|
||||
cidFont.setItem(COSName.CIDSYSTEMINFO, cidSystemInfo);
|
||||
cidFont.setItem(COSName.FONT_DESC, fdesc);
|
||||
cidFont.setInt(COSName.DW, 600);
|
||||
|
||||
COSStream toUni = new PDStream(doc).getCOSObject();
|
||||
try (var os = toUni.createOutputStream()) {
|
||||
os.write(cffToUnicode().getBytes(java.nio.charset.StandardCharsets.US_ASCII));
|
||||
}
|
||||
|
||||
COSDictionary type0 = new COSDictionary();
|
||||
type0.setItem(COSName.TYPE, COSName.FONT);
|
||||
type0.setItem(COSName.SUBTYPE, COSName.TYPE0);
|
||||
type0.setName(COSName.BASE_FONT, "MinimalCFFCID");
|
||||
type0.setItem(COSName.ENCODING, COSName.IDENTITY_H);
|
||||
COSArray descendants = new COSArray();
|
||||
descendants.add(cidFont);
|
||||
type0.setItem(COSName.DESCENDANT_FONTS, descendants);
|
||||
type0.setItem(COSName.TO_UNICODE, toUni);
|
||||
|
||||
PDFont font = org.apache.pdfbox.pdmodel.font.PDFontFactory.createFont(type0);
|
||||
PDResources res = new PDResources();
|
||||
COSName fName = res.add(font);
|
||||
page.setResources(res);
|
||||
|
||||
StringBuilder csb = new StringBuilder();
|
||||
csb.append("BT\n/").append(fName.getName()).append(" 24 Tf\n50 700 Td\n<");
|
||||
for (int i = 0; i < line.length(); i++) {
|
||||
csb.append(String.format("%04X", cffGid(line.charAt(i))));
|
||||
}
|
||||
csb.append("> Tj\nET\n");
|
||||
PDStream content = new PDStream(doc);
|
||||
try (var os = content.createOutputStream()) {
|
||||
os.write(csb.toString().getBytes(java.nio.charset.StandardCharsets.US_ASCII));
|
||||
}
|
||||
page.setContents(content);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private static String cffToUnicode() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n");
|
||||
sb.append("/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n");
|
||||
sb.append("/CMapName /Adobe-Identity-UCS def\n/CMapType 2 def\n");
|
||||
sb.append("1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n");
|
||||
StringBuilder body = new StringBuilder();
|
||||
int count = 0;
|
||||
for (int gid = 1; gid < CFF_GLYPHS.length(); gid++) {
|
||||
body.append(String.format("<%04X> <%04X>\n", gid, (int) CFF_GLYPHS.charAt(gid)));
|
||||
count++;
|
||||
}
|
||||
sb.append(count).append(" beginbfchar\n").append(body).append("endbfchar\n");
|
||||
sb.append("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private byte[] type1Pdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream pfb = getClass().getResourceAsStream("/redaction/type1_minimal.pfb")) {
|
||||
font = new PDType1Font(doc, pfb, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] cjkPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = getClass().getResourceAsStream("/redaction/cjk_minimal.ttf")) {
|
||||
font = PDType0Font.load(doc, ttf, true);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] symbolicTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDTrueTypeFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
// Strip /Encoding and flag the descriptor Symbolic so extraction must use the cmap.
|
||||
font.getCOSObject().removeItem(org.apache.pdfbox.cos.COSName.ENCODING);
|
||||
PDFontDescriptor fd = font.getFontDescriptor();
|
||||
fd.setFlags((fd.getFlags() | 4) & ~32);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] type0NoToUnicodePdf(String line) throws IOException {
|
||||
byte[] pdf;
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
// Full embed, not subset: the subsetter regenerates /ToUnicode at save, undoing the
|
||||
// strip.
|
||||
font = PDType0Font.load(doc, ttf, false);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
font.getCOSObject().removeItem(org.apache.pdfbox.cos.COSName.TO_UNICODE);
|
||||
pdf = save(doc);
|
||||
}
|
||||
// Guard: the strip must survive save so this fixture genuinely exercises the no-mapping
|
||||
// path.
|
||||
try (PDDocument reopened = Loader.loadPDF(pdf)) {
|
||||
PDResources res = reopened.getPage(0).getResources();
|
||||
for (org.apache.pdfbox.cos.COSName n : res.getFontNames()) {
|
||||
assertThat(
|
||||
res.getFont(n)
|
||||
.getCOSObject()
|
||||
.containsKey(org.apache.pdfbox.cos.COSName.TO_UNICODE))
|
||||
.as("no-ToUnicode fixture must not ship /ToUnicode")
|
||||
.isFalse();
|
||||
}
|
||||
}
|
||||
return pdf;
|
||||
}
|
||||
|
||||
private byte[] subsetTaggedPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDType0Font.load(doc, ttf, true);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
String tagged = "ABCDEF+" + font.getName();
|
||||
font.getCOSObject().setName(org.apache.pdfbox.cos.COSName.BASE_FONT, tagged);
|
||||
if (font.getFontDescriptor() != null) {
|
||||
font.getFontDescriptor()
|
||||
.getCOSObject()
|
||||
.setName(org.apache.pdfbox.cos.COSName.FONT_NAME, tagged);
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] type0Pdf(boolean subset, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDType0Font.load(doc, ttf, subset);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] simpleTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] rotatedPdf(int rotation, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, rotation, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] cropBoxPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, 0, new PDRectangle(40, 40, 500, 700));
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] multiPagePdf(String... pageLines) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
for (String line : pageLines) {
|
||||
addTextPage(doc, helvetica(), line, 0, null);
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
/** One page whose text is emitted as a TJ array: ["public ", "SEC", -20 */
|
||||
private byte[] tjSplitPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(helvetica(), FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
cs.showTextWithPositioning(
|
||||
new Object[] {"public ", "SEC", Float.valueOf(-20f), "RET", " end"});
|
||||
cs.endText();
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
/** Page 1 shows "SEC" and "RET" as separate adjacent Tj operators; page 2 */
|
||||
private byte[] crossOperatorPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
PDFont font = helvetica();
|
||||
float secWidth = font.getStringWidth("SEC") / 1000f * FONT_SIZE;
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
cs.showText("SEC");
|
||||
cs.endText();
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X + secWidth, TOP_Y);
|
||||
cs.showText("RET");
|
||||
cs.endText();
|
||||
}
|
||||
addTextPage(doc, helvetica(), "PUBLIC PAGE TWO", 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] formXObjectPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
|
||||
PDFormXObject form = new PDFormXObject(doc);
|
||||
form.setBBox(new PDRectangle(0, 0, 400, 60));
|
||||
form.setResources(new PDResources());
|
||||
try (PDFormContentStream fcs = new PDFormContentStream(form)) {
|
||||
fcs.beginText();
|
||||
fcs.setFont(helvetica(), FONT_SIZE);
|
||||
fcs.newLineAtOffset(10, 20);
|
||||
fcs.showText("xobj SECRET payload");
|
||||
fcs.endText();
|
||||
}
|
||||
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.transform(Matrix.getTranslateInstance(LEFT_X, TOP_Y - 60));
|
||||
cs.drawForm(form);
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private void addTextPage(
|
||||
PDDocument doc, PDFont font, String line, int rotation, PDRectangle cropBox)
|
||||
throws IOException {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
if (rotation != 0) {
|
||||
page.setRotation(rotation);
|
||||
}
|
||||
if (cropBox != null) {
|
||||
page.setCropBox(cropBox);
|
||||
}
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
if (rotation != 0) {
|
||||
// Real generators compensate the text matrix so text reads upright
|
||||
cs.setTextMatrix(
|
||||
Matrix.getRotateInstance(
|
||||
Math.toRadians(rotation),
|
||||
PDRectangle.LETTER.getWidth() / 2,
|
||||
PDRectangle.LETTER.getHeight() / 2));
|
||||
} else {
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
}
|
||||
cs.showText(line);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] save(PDDocument doc) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
doc.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
+6
-2
@@ -287,8 +287,12 @@ class TextRedactionServiceExtraTest {
|
||||
|
||||
service.createTokensWithoutTargetText(doc, page, Set.of("SECRET"), false, false);
|
||||
|
||||
// The deep traversal must have rewritten the inner form's content stream.
|
||||
assertThat(inner.getCOSObject().containsKey(COSName.CONTENTS)).isTrue();
|
||||
// The deep traversal must have rewritten the inner form's own stream in place (F3).
|
||||
String innerContent;
|
||||
try (var in = inner.getContents()) {
|
||||
innerContent = new String(in.readAllBytes(), StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
assertThat(innerContent).doesNotContain("SECRET");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+12
-8
@@ -226,19 +226,23 @@ class TextRedactionServiceMoreTest {
|
||||
}
|
||||
page.setContents(pageStream);
|
||||
|
||||
// Processing the page walks into the XObject graph; when a match is found inside
|
||||
// the
|
||||
// form, writeRedactedContentToXObject runs and sets a /Contents item on the form's
|
||||
// COS dictionary. Asserting that item appears proves the XObject redaction path
|
||||
// executed end-to-end without throwing.
|
||||
// Processing the page walks into the XObject graph and rewrites the form's own
|
||||
// stream body in place (F3), so the target is physically gone from the form.
|
||||
List<Object> tokens =
|
||||
service.createTokensWithoutTargetText(
|
||||
doc, page, Set.of("SECRET"), false, false);
|
||||
|
||||
assertThat(tokens).isNotNull();
|
||||
assertThat(form.getCOSObject().containsKey(COSName.CONTENTS))
|
||||
.as("form XObject redaction path should have written a new content item")
|
||||
.isTrue();
|
||||
String rewrittenForm;
|
||||
try (var in = form.getContents()) {
|
||||
rewrittenForm =
|
||||
new String(
|
||||
in.readAllBytes(),
|
||||
java.nio.charset.StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
assertThat(rewrittenForm)
|
||||
.as("form XObject content should have the target removed in place")
|
||||
.doesNotContain("SECRET");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Integration tests that exercise the real redaction pipeline against PDFs */
|
||||
class RedactionPipelineIntegrationTest {
|
||||
|
||||
private static byte[] loadFixture() throws Exception {
|
||||
try (InputStream in =
|
||||
RedactionPipelineIntegrationTest.class.getResourceAsStream(
|
||||
"/redaction/test_pdf_1.pdf")) {
|
||||
assertNotNull(in, "fixture resource must exist on classpath");
|
||||
return in.readAllBytes();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"finalize on rotated ReportLab PDF without a rewrite pass falls back to rasterisation so target disappears")
|
||||
void finaliseFallsBackToRasterisationWhenTargetWouldSurvive() throws Exception {
|
||||
byte[] fixtureBytes = loadFixture();
|
||||
byte[] outputBytes;
|
||||
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Test");
|
||||
// No content-stream rewriting performed.
|
||||
outputBytes = RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
|
||||
}
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("test"),
|
||||
"Rasterisation fallback must have removed target. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"finalize with zero rewrite + synthetic upright PDF triggers RedactionVerificationFailedException only when fallback disabled")
|
||||
void verificationExceptionWiringSanityCheck() throws Exception {
|
||||
// This test proves the verification hook itself still throws -
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
cs.newLineAtOffset(100, 700);
|
||||
cs.showText("Top Smith Classified");
|
||||
cs.endText();
|
||||
}
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Smith");
|
||||
// Even with the rasterisation fallback active, the output must not contain
|
||||
byte[] outBytes =
|
||||
RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
|
||||
try (PDDocument reopened = Loader.loadPDF(outBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("smith"),
|
||||
"Neither primary pass nor rasterisation removed target. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"auto-word redact on rotated PDF with target packed in single Tj removes the word from text stream")
|
||||
void autoWordRedactRemovesTargetOnRotatedSingleTjPdf() throws Exception {
|
||||
byte[] fixtureBytes = loadFixture();
|
||||
byte[] outputBytes;
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Test");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"Test"}, false, false);
|
||||
|
||||
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
|
||||
RedactionPipeline.redactLiteralTerms(doc, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(doc, literalTargets, patterns);
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
String extracted = stripper.getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("test"),
|
||||
"Target term must not be extractable after redact. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple upright Helvetica fixture still gets word redacted via the pipeline")
|
||||
void autoWordRedactRemovesTargetOnSyntheticPdf() throws Exception {
|
||||
byte[] outputBytes;
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Secret");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"Secret"}, false, false);
|
||||
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
cs.newLineAtOffset(100, 700);
|
||||
cs.showText("Top Secret Classified");
|
||||
cs.endText();
|
||||
}
|
||||
ByteArrayOutputStream tmp = new ByteArrayOutputStream();
|
||||
doc.save(tmp);
|
||||
// reload from bytes so we are processing an already-saved PDF similar
|
||||
try (PDDocument reloaded = Loader.loadPDF(tmp.toByteArray())) {
|
||||
RedactionPipeline.redactLiteralTerms(reloaded, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(reloaded, literalTargets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("secret"),
|
||||
"Target term must not be extractable after redact on synthetic PDF. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
+1019
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a minimal OpenType-CFF (.otf) font for a CIDFontType0 redaction fixture.
|
||||
|
||||
Why this exists
|
||||
---------------
|
||||
Real-world Adobe/InDesign PDFs embed CFF-based CID fonts (Type0 -> CIDFontType0,
|
||||
program in /FontFile3 /Subtype /CIDFontType0C). PDFBox 3.0.7 can *read* those but
|
||||
ships NO CFF embedder: every PDType0Font.load(...) overload routes CFF outlines
|
||||
through the TrueType path (which either rejects "TTF using CFF outlines" or, via the
|
||||
OTFParser+TrueTypeFont overload, emits a CIDFontType2 and then fails to subset CFF
|
||||
because there is no glyf table). So to build a genuine CIDFontType0 fixture you must:
|
||||
1. generate the CFF .otf with fontTools (this script), and
|
||||
2. assemble the Type0/CIDFontType0 dictionary by hand, attaching the raw CFF table
|
||||
as /FontFile3 /Subtype /CIDFontType0C (see the companion Java harness).
|
||||
|
||||
This script emits BOTH minimal_cff.otf and cff_raw.bin (the raw CFF table bytes,
|
||||
ready to drop into /FontFile3).
|
||||
|
||||
Glyph coverage: only the ASCII letters in "PUBLICA SECRETWORD KEEP"
|
||||
(P U B L I C A S E R T W O D K) plus .notdef and space. Outlines are trivial filled
|
||||
boxes; only the cmap / encoding correctness matters so the drawn text extracts.
|
||||
"""
|
||||
import sys
|
||||
from fontTools.fontBuilder import FontBuilder
|
||||
from fontTools.pens.t2CharStringPen import T2CharStringPen
|
||||
from fontTools.ttLib import TTFont
|
||||
|
||||
UPM = 1000
|
||||
LETTERS = sorted(set("PUBLICASECRETWORDKEEP")) # A B C D E I K L O P R S T U W
|
||||
|
||||
|
||||
def box_charstring(width=600, height=700):
|
||||
"""A simple filled rectangle as a Type2 (CFF) charstring."""
|
||||
pen = T2CharStringPen(width, None)
|
||||
pen.moveTo((50, 0))
|
||||
pen.lineTo((width - 50, 0))
|
||||
pen.lineTo((width - 50, height))
|
||||
pen.lineTo((50, height))
|
||||
pen.closePath()
|
||||
return pen.getCharString()
|
||||
|
||||
|
||||
def build_otf(out_otf):
|
||||
# GID order: .notdef (GID 0), space (GID 1), then the letters. The companion
|
||||
# harness relies on this exact order for its Identity-H CID==GID mapping.
|
||||
glyph_order = [".notdef", "space"] + list(LETTERS)
|
||||
|
||||
fb = FontBuilder(unitsPerEm=UPM, isTTF=False) # isTTF=False -> CFF/OTF
|
||||
fb.setupGlyphOrder(glyph_order)
|
||||
|
||||
# Unicode cmap: codepoint -> glyph name. PDFBox reads this to build /ToUnicode,
|
||||
# and it also lets any consumer resolve the glyphs by character.
|
||||
cmap = {0x20: "space"}
|
||||
for c in LETTERS:
|
||||
cmap[ord(c)] = c
|
||||
fb.setupCharacterMap(cmap)
|
||||
|
||||
charstrings = {
|
||||
".notdef": box_charstring(),
|
||||
"space": box_charstring(width=250, height=0), # empty advance-only glyph
|
||||
}
|
||||
for c in LETTERS:
|
||||
charstrings[c] = box_charstring()
|
||||
|
||||
fb.setupCFF(
|
||||
psName="MinimalCFFCID",
|
||||
charStringsDict=charstrings,
|
||||
fontInfo={
|
||||
"FullName": "Minimal CFF CID",
|
||||
"FamilyName": "MinimalCFFCID",
|
||||
"Weight": "Regular",
|
||||
},
|
||||
privateDict={},
|
||||
)
|
||||
|
||||
metrics = {".notdef": (600, 50), "space": (250, 0)}
|
||||
for c in LETTERS:
|
||||
metrics[c] = (600, 50)
|
||||
fb.setupHorizontalMetrics(metrics)
|
||||
fb.setupHorizontalHeader(ascent=800, descent=-200)
|
||||
|
||||
fb.setupNameTable(
|
||||
{
|
||||
"familyName": "MinimalCFFCID",
|
||||
"styleName": "Regular",
|
||||
"psName": "MinimalCFFCID",
|
||||
"fullName": "Minimal CFF CID",
|
||||
"version": "Version 1.000",
|
||||
"uniqueFontIdentifier": "MinimalCFFCID;1.000",
|
||||
}
|
||||
)
|
||||
fb.setupOS2(
|
||||
sTypoAscender=800,
|
||||
sTypoDescender=-200,
|
||||
usWinAscent=800,
|
||||
usWinDescent=200,
|
||||
sCapHeight=700,
|
||||
)
|
||||
fb.setupPost(keepGlyphNames=True)
|
||||
|
||||
fb.save(out_otf)
|
||||
print("wrote", out_otf, "(", len(glyph_order), "glyphs )")
|
||||
|
||||
|
||||
def dump_cff(out_otf, out_cff):
|
||||
"""Extract the raw CFF table bytes for use as PDF /FontFile3."""
|
||||
f = TTFont(out_otf)
|
||||
cff = f["CFF "].compile(f)
|
||||
with open(out_cff, "wb") as fh:
|
||||
fh.write(cff)
|
||||
print("wrote", out_cff, "(", len(cff), "bytes CFF )")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
otf = sys.argv[1] if len(sys.argv) > 1 else "minimal_cff.otf"
|
||||
cffbin = sys.argv[2] if len(sys.argv) > 2 else "cff_raw.bin"
|
||||
build_otf(otf)
|
||||
dump_cff(otf, cffbin)
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate a minimal TrueType font (.ttf) that covers exactly the characters
|
||||
needed for a CJK Type0 (Identity-H) redaction test:
|
||||
|
||||
- ASCII letters in "PUBLICA SECRETWORD KEEP": P U B L I C A S E R T W O D K + space
|
||||
- Two CJK ideographs used as the redaction target:
|
||||
U+79D8 (secret1) and U+5BC6 (secret2) -> "秘密"
|
||||
- .notdef
|
||||
|
||||
Glyph outlines are trivial boxes; the point of the test is that the cmap /
|
||||
encoding is correct so PDFBox can build a working /ToUnicode and the text is
|
||||
EXTRACTABLE. When PDFBox loads this via PDType0Font.load(doc, in, true) it
|
||||
becomes an Identity-H Type0 font with a CIDFontType2 descendant.
|
||||
|
||||
Usage: python gen_cjk_font.py <output.ttf>
|
||||
"""
|
||||
import sys
|
||||
from fontTools.fontBuilder import FontBuilder
|
||||
from fontTools.pens.ttGlyphPen import TTGlyphPen
|
||||
|
||||
UPM = 1000
|
||||
|
||||
|
||||
def box_glyph(width_units):
|
||||
"""A trivial filled box outline so the glyph has real contours."""
|
||||
pen = TTGlyphPen(None)
|
||||
x0, y0, x1, y1 = 80, 0, max(120, width_units - 80), 700
|
||||
pen.moveTo((x0, y0))
|
||||
pen.lineTo((x0, y1))
|
||||
pen.lineTo((x1, y1))
|
||||
pen.lineTo((x1, y0))
|
||||
pen.closePath()
|
||||
return pen.glyph()
|
||||
|
||||
|
||||
def empty_glyph():
|
||||
return TTGlyphPen(None).glyph()
|
||||
|
||||
|
||||
def main(out_path):
|
||||
# Characters we must cover. Order defines glyph order (after .notdef).
|
||||
ascii_chars = list("PUBLICA SECRETWORD KEEP")
|
||||
ascii_unique = []
|
||||
for ch in ascii_chars:
|
||||
if ch not in ascii_unique:
|
||||
ascii_unique.append(ch)
|
||||
|
||||
cjk = ["秘", "密"] # 秘 密
|
||||
|
||||
# glyph name -> codepoint mapping
|
||||
def gname(cp):
|
||||
if cp == 0x20:
|
||||
return "space"
|
||||
return "uni%04X" % cp
|
||||
|
||||
codepoints = []
|
||||
for ch in ascii_unique:
|
||||
codepoints.append(ord(ch))
|
||||
for ch in cjk:
|
||||
codepoints.append(ord(ch))
|
||||
|
||||
glyph_order = [".notdef"] + [gname(cp) for cp in codepoints]
|
||||
|
||||
fb = FontBuilder(UPM, isTTF=True)
|
||||
fb.setupGlyphOrder(glyph_order)
|
||||
|
||||
# cmap: map every codepoint to its glyph. FontBuilder builds format-4
|
||||
# (BMP) + format-12 as needed. Both CJK codepoints are BMP so format-4
|
||||
# covers them.
|
||||
cmap = {cp: gname(cp) for cp in codepoints}
|
||||
fb.setupCharacterMap(cmap)
|
||||
|
||||
# Build glyf table with box outlines; advance widths.
|
||||
glyphs = {".notdef": empty_glyph()}
|
||||
metrics = {".notdef": (600, 0)}
|
||||
for cp in codepoints:
|
||||
name = gname(cp)
|
||||
if cp == 0x20:
|
||||
glyphs[name] = empty_glyph()
|
||||
metrics[name] = (300, 0) # space advance, no outline
|
||||
elif cp > 0x2000: # CJK -> full-width
|
||||
glyphs[name] = box_glyph(900)
|
||||
metrics[name] = (1000, 80)
|
||||
else:
|
||||
glyphs[name] = box_glyph(560)
|
||||
metrics[name] = (600, 80)
|
||||
|
||||
fb.setupGlyf(glyphs)
|
||||
fb.setupHorizontalMetrics(metrics)
|
||||
fb.setupHorizontalHeader(ascent=800, descent=-200)
|
||||
|
||||
# Names + the rest of the required tables.
|
||||
fb.setupNameTable({
|
||||
"familyName": "CJKRedactTest",
|
||||
"styleName": "Regular",
|
||||
"psName": "CJKRedactTest-Regular",
|
||||
"fullName": "CJKRedactTest Regular",
|
||||
})
|
||||
fb.setupOS2(sTypoAscender=800, sTypoDescender=-200, usWinAscent=800,
|
||||
usWinDescent=200)
|
||||
fb.setupPost()
|
||||
|
||||
fb.save(out_path)
|
||||
print("Wrote", out_path)
|
||||
print("Glyph order:", glyph_order)
|
||||
print("cmap entries:", {("U+%04X" % k): v for k, v in cmap.items()})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: gen_cjk_font.py <output.ttf>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
main(sys.argv[1])
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a minimal real Type1 font as a classic PFB (0x80 0x01 segment format).
|
||||
|
||||
Pure-python, uses fontTools.t1Lib.T1Font.createData() + writePFB. The font contains
|
||||
.notdef plus the ASCII letters needed for "PUBLICA SECRETWORD KEEP", with a proper
|
||||
Encoding array + standard PostScript glyph names. Because the glyph names are the
|
||||
standard ones and the PDF font uses WinAnsiEncoding, PDFBox extracts text via the
|
||||
base encoding (code -> glyphname -> Unicode); no /ToUnicode CMap is required.
|
||||
|
||||
Usage: python gen_type1.py <out.pfb>
|
||||
"""
|
||||
import sys
|
||||
from fontTools import t1Lib
|
||||
from fontTools.misc import psCharStrings
|
||||
|
||||
|
||||
# Glyph name for each character we must render. PostScript standard glyph names.
|
||||
CHARS = {
|
||||
"P": "P", "U": "U", "B": "B", "L": "L", "I": "I", "C": "C", "A": "A",
|
||||
"S": "S", "E": "E", "R": "R", "T": "T", "W": "W", "O": "O", "D": "D",
|
||||
"K": "K", " ": "space",
|
||||
}
|
||||
|
||||
EM = 1000
|
||||
WIDTH = 600
|
||||
BOX_L, BOX_B, BOX_R, BOX_T = 50, 0, 550, 700
|
||||
|
||||
|
||||
def box_charstring(width):
|
||||
"""A trivial filled box outline. hsbw sets sidebearing+width; then a closed rect."""
|
||||
prog = [
|
||||
0, width, "hsbw", # sb=0, width
|
||||
BOX_L, BOX_B, "rmoveto", # move to bottom-left (relative from 0,0)
|
||||
(BOX_R - BOX_L), "hlineto", # right
|
||||
(BOX_T - BOX_B), "vlineto", # up
|
||||
-(BOX_R - BOX_L), "hlineto", # left
|
||||
"closepath",
|
||||
"endchar",
|
||||
]
|
||||
cs = psCharStrings.T1CharString(program=prog)
|
||||
return cs
|
||||
|
||||
|
||||
def space_charstring(width):
|
||||
prog = [0, width, "hsbw", "endchar"]
|
||||
return psCharStrings.T1CharString(program=prog)
|
||||
|
||||
|
||||
def build_font(font_name="MinType1Test"):
|
||||
# 256-slot encoding, default .notdef, filled at each char's codepoint.
|
||||
encoding = [".notdef"] * 256
|
||||
for ch, gname in CHARS.items():
|
||||
encoding[ord(ch)] = gname
|
||||
|
||||
charstrings = {".notdef": box_charstring(WIDTH)}
|
||||
for ch, gname in CHARS.items():
|
||||
if gname == "space":
|
||||
charstrings[gname] = space_charstring(WIDTH)
|
||||
else:
|
||||
charstrings[gname] = box_charstring(WIDTH)
|
||||
|
||||
# Subrs: the 5 standard Type1 subrs (division/hint machinery). Provide as
|
||||
# T1CharString objects so createData can compile+encrypt them.
|
||||
subrs = [psCharStrings.T1CharString(bytecode=s) for s in t1Lib.std_subrs]
|
||||
|
||||
# Order matters. fontTools emits a stray "put" after the Subrs array; fontbox's
|
||||
# Type1Parser breaks its Private loop on the first non-LITERAL after Subrs, so
|
||||
# every key AFTER Subrs is dropped. Keep Subrs LAST so nothing gets skipped.
|
||||
private = {
|
||||
"RD": t1Lib.RD_value, # ('string','currentfile','exch','readstring','pop')
|
||||
"ND": t1Lib.ND_values[0], # ('def',) codespell:ignore
|
||||
"NP": t1Lib.PD_values[0], # ('put',)
|
||||
"MinFeature": [16, 16],
|
||||
# PDFBox reads BlueValues[2] for CapHeight, so >=3 entries required.
|
||||
# Pairs: baseline (0 0), then cap-height zone (700 712).
|
||||
"BlueValues": [0, 0, BOX_T, BOX_T + 12],
|
||||
"StdHW": [60],
|
||||
"StdVW": [80],
|
||||
"lenIV": 4,
|
||||
"password": 5839,
|
||||
"Subrs": subrs,
|
||||
}
|
||||
|
||||
font_info = {
|
||||
"version": "001.001",
|
||||
"FullName": font_name,
|
||||
"FamilyName": font_name,
|
||||
"Weight": "Regular",
|
||||
"ItalicAngle": 0,
|
||||
"isFixedPitch": False,
|
||||
"UnderlinePosition": -100,
|
||||
"UnderlineThickness": 50,
|
||||
}
|
||||
|
||||
font = {
|
||||
"FontName": font_name,
|
||||
"FontInfo": font_info,
|
||||
"FontType": 1,
|
||||
"PaintType": 0,
|
||||
"FontMatrix": [0.001, 0, 0, 0.001, 0, 0],
|
||||
"FontBBox": [0, 0, WIDTH, BOX_T],
|
||||
"Encoding": encoding,
|
||||
"Private": private,
|
||||
"CharStrings": charstrings,
|
||||
}
|
||||
return font
|
||||
|
||||
|
||||
def main():
|
||||
out = sys.argv[1] if len(sys.argv) > 1 else "min.pfb"
|
||||
# Construct a T1Font without reading a file, then hand-build .font.
|
||||
t1 = t1Lib.T1Font.__new__(t1Lib.T1Font)
|
||||
t1.encoding = "ascii"
|
||||
t1.font = build_font()
|
||||
data = t1.createData() # full ASCII+eexec Type1 program (binary eexec inline)
|
||||
t1.data = data
|
||||
t1.saveAs(out, "PFB") # split into 0x80 0x01 / 0x80 0x02 / 0x80 0x03 segments
|
||||
# Report first bytes to confirm PFB marker.
|
||||
with open(out, "rb") as f:
|
||||
head = f.read(6)
|
||||
print("WROTE", out)
|
||||
print("PFB head bytes:", " ".join(f"{b:02x}" for b in head))
|
||||
assert head[0] == 0x80 and head[1] == 0x01, "not a classic PFB 0x80 0x01 header"
|
||||
print("OK: classic PFB 0x80 0x01 ASCII segment marker present")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,143 @@
|
||||
@security @redact
|
||||
Feature: PDF redaction physically removes text
|
||||
Target text must be gone from the extracted text layer, not just covered by a box.
|
||||
|
||||
Scenario: Auto-redact physically removes the target word
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "PUBLIC alpha SECRET99 omega PUBLIC"
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | SECRET99 |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response content type should be "application/pdf"
|
||||
And the response PDF should not contain the text "SECRET99"
|
||||
And the response PDF should contain the text "omega"
|
||||
|
||||
Scenario: Auto-redact leaves substrings when whole-word search is on
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "cat classification scatter"
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | cat |
|
||||
| wholeWordSearch | true |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should contain the text "classification"
|
||||
And the response PDF should contain the text "scatter"
|
||||
|
||||
Scenario: Auto-redact with a regex pattern removes matches
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "call 123-45-6789 today"
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | \d{3}-\d{2}-\d{4} |
|
||||
| useRegex | true |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "123-45-6789"
|
||||
And the response PDF should contain the text "today"
|
||||
|
||||
Scenario: Auto-redact convert-to-image drops the entire text layer
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "keep SECRET77 hidden"
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | SECRET77 |
|
||||
| convertPDFToImage | true |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "SECRET77"
|
||||
And the response PDF should not contain the text "keep"
|
||||
|
||||
Scenario: Auto-redact also scrubs the target from bookmark titles
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "body SECRET55 text"
|
||||
And the pdf has a bookmark titled "Chapter SECRET55 overview"
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | SECRET55 |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "SECRET55"
|
||||
And the response PDF bookmarks should not contain "SECRET55"
|
||||
|
||||
Scenario: Manual whole-page redaction wipes every word on the page
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "TOP SECRET material"
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| pageNumbers | 1 |
|
||||
When I send the API request to the endpoint "/api/v1/security/redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "SECRET"
|
||||
|
||||
Scenario: Auto-redact removes case variants of the target
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "alpha Secret99x omega"
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | SECRET99X |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "Secret99x"
|
||||
And the response PDF should contain the text "omega"
|
||||
|
||||
Scenario: Auto-redact only touches pages containing the target
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf contains 3 pages
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | Page 2 |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "Page 2"
|
||||
And the response PDF should contain the text "Page 1"
|
||||
And the response PDF should contain the text "Page 3"
|
||||
|
||||
Scenario: Auto-redact handles a Type3 font PDF (glyphs as content streams)
|
||||
Given I use an example file at "../crop_test.pdf" as parameter "fileInput"
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | EXAMPLE |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "EXAMPLE"
|
||||
And the response PDF should contain the text "CROP"
|
||||
|
||||
Scenario: Auto-redact removes the target from an embedded TrueType font PDF
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "keepA SECRET88 keepB" in an embedded font
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | SECRET88 |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response content type should be "application/pdf"
|
||||
And the response PDF should not contain the text "SECRET88"
|
||||
And the response PDF should contain the text "keepA"
|
||||
And the response PDF should contain the text "keepB"
|
||||
|
||||
Scenario: Auto-redact with a regex pattern on an embedded-font PDF
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "invoice 987-65-4321 total" in an embedded font
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | \d{3}-\d{2}-\d{4} |
|
||||
| useRegex | true |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "987-65-4321"
|
||||
And the response PDF should contain the text "invoice"
|
||||
|
||||
Scenario: Auto-redact convert-to-image on an embedded-font PDF drops the text layer
|
||||
Given I generate a PDF file as "fileInput"
|
||||
And the pdf pages all contain the text "keep SECRET66 hidden" in an embedded font
|
||||
And the request data includes
|
||||
| parameter | value |
|
||||
| listOfText | SECRET66 |
|
||||
| convertPDFToImage | true |
|
||||
When I send the API request to the endpoint "/api/v1/security/auto-redact"
|
||||
Then the response status code should be 200
|
||||
And the response PDF should not contain the text "SECRET66"
|
||||
And the response PDF should not contain the text "keep"
|
||||
@@ -17,6 +17,9 @@ from PIL import Image, ImageDraw
|
||||
|
||||
API_HEADERS = {"X-API-KEY": "123456789"}
|
||||
|
||||
# Backend under test; override STIRLING_BASE_URL to target another port.
|
||||
BASE_URL = os.environ.get("STIRLING_BASE_URL", "http://localhost:8080")
|
||||
|
||||
#########
|
||||
# GIVEN #
|
||||
#########
|
||||
@@ -277,6 +280,47 @@ def step_pdf_pages_contain_text(context, text):
|
||||
context.files[context.param_name] = open(context.file_name, "rb")
|
||||
|
||||
|
||||
# DejaVuSans is a repo asset; embedding it exercises redaction over a real embedded (Type0/CID,
|
||||
# non Standard-14) font whose text pypdf can still extract to verify removal end-to-end.
|
||||
_DEJAVU_TTF = os.path.normpath(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..", "..", "..", "..",
|
||||
"app", "core", "src", "main", "resources", "static", "fonts", "DejaVuSans.ttf",
|
||||
)
|
||||
)
|
||||
_embedded_font_registered = False
|
||||
|
||||
|
||||
def _ensure_embedded_font():
|
||||
global _embedded_font_registered
|
||||
if not _embedded_font_registered:
|
||||
from reportlab.pdfbase.ttfonts import TTFont
|
||||
from reportlab.pdfbase import pdfmetrics
|
||||
|
||||
pdfmetrics.registerFont(TTFont("DejaVuEmbedded", _DEJAVU_TTF))
|
||||
_embedded_font_registered = True
|
||||
|
||||
|
||||
@given('the pdf pages all contain the text "{text}" in an embedded font')
|
||||
def step_pdf_pages_contain_text_embedded(context, text):
|
||||
_ensure_embedded_font()
|
||||
buffer = io.BytesIO()
|
||||
c = canvas.Canvas(buffer, pagesize=letter)
|
||||
width, height = letter
|
||||
for _ in range(len(PdfReader(context.file_name).pages)):
|
||||
c.setFont("DejaVuEmbedded", 14)
|
||||
c.drawString(100, height - 100, text)
|
||||
c.showPage()
|
||||
c.save()
|
||||
|
||||
with open(context.file_name, "wb") as f:
|
||||
f.write(buffer.getvalue())
|
||||
|
||||
context.files[context.param_name].close()
|
||||
context.files[context.param_name] = open(context.file_name, "rb")
|
||||
|
||||
|
||||
@given('the pdf is encrypted with password "{password}"')
|
||||
def step_encrypt_pdf(context, password):
|
||||
writer = PdfWriter()
|
||||
@@ -581,7 +625,7 @@ def step_request_json_part(context, part_name, json_content):
|
||||
|
||||
@when('I send a GET request to "{endpoint}"')
|
||||
def step_send_get_request(context, endpoint):
|
||||
base_url = "http://localhost:8080"
|
||||
base_url = BASE_URL
|
||||
full_url = f"{base_url}{endpoint}"
|
||||
response = requests.get(full_url, headers=API_HEADERS, timeout=60)
|
||||
context.response = response
|
||||
@@ -589,7 +633,7 @@ def step_send_get_request(context, endpoint):
|
||||
|
||||
@when('I send a GET request to "{endpoint}" with parameters')
|
||||
def step_send_get_request_with_params(context, endpoint):
|
||||
base_url = "http://localhost:8080"
|
||||
base_url = BASE_URL
|
||||
params = {row["parameter"]: row["value"] for row in context.table}
|
||||
full_url = f"{base_url}{endpoint}"
|
||||
response = requests.get(full_url, params=params, headers=API_HEADERS, timeout=60)
|
||||
@@ -598,7 +642,7 @@ def step_send_get_request_with_params(context, endpoint):
|
||||
|
||||
@when('I send the API request to the endpoint "{endpoint}"')
|
||||
def step_send_api_request(context, endpoint):
|
||||
url = f"http://localhost:8080{endpoint}"
|
||||
url = f"{BASE_URL}{endpoint}"
|
||||
files = context.files if hasattr(context, "files") else {}
|
||||
|
||||
if not hasattr(context, "request_data") or context.request_data is None:
|
||||
@@ -803,3 +847,69 @@ def step_response_matches_regex(context, pattern):
|
||||
assert re.match(
|
||||
pattern, response_text
|
||||
), f"Response '{response_text}' does not match the expected pattern '{pattern}'"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Redaction: text-layer and catalog assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_response_pdf_text(context):
|
||||
reader = PdfReader(io.BytesIO(context.response.content))
|
||||
return "\n".join((page.extract_text() or "") for page in reader.pages)
|
||||
|
||||
|
||||
@then('the response PDF should contain the text "{text}"')
|
||||
def step_response_pdf_contains_text(context, text):
|
||||
extracted = _extract_response_pdf_text(context)
|
||||
assert text in extracted, (
|
||||
f"Expected redacted PDF to still contain '{text}', but it was missing. "
|
||||
f"Extracted text: {extracted!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('the response PDF should not contain the text "{text}"')
|
||||
def step_response_pdf_not_contains_text(context, text):
|
||||
extracted = _extract_response_pdf_text(context)
|
||||
assert text not in extracted, (
|
||||
f"Redacted PDF still contains '{text}' - redaction did not remove it. "
|
||||
f"Extracted text: {extracted!r}"
|
||||
)
|
||||
|
||||
|
||||
def _collect_outline_titles(outline, titles):
|
||||
for item in outline:
|
||||
if isinstance(item, list):
|
||||
_collect_outline_titles(item, titles)
|
||||
else:
|
||||
title = getattr(item, "title", None)
|
||||
if title:
|
||||
titles.append(title)
|
||||
|
||||
|
||||
@then('the response PDF bookmarks should not contain "{text}"')
|
||||
def step_response_pdf_bookmarks_not_contain(context, text):
|
||||
reader = PdfReader(io.BytesIO(context.response.content))
|
||||
titles = []
|
||||
try:
|
||||
_collect_outline_titles(reader.outline, titles)
|
||||
except Exception:
|
||||
titles = []
|
||||
joined = " ".join(titles)
|
||||
assert text not in joined, (
|
||||
f"Redacted PDF bookmark titles still contain '{text}': {titles!r}"
|
||||
)
|
||||
|
||||
|
||||
@given('the pdf has a bookmark titled "{title}"')
|
||||
def step_pdf_has_bookmark_titled(context, title):
|
||||
"""Add a single top-level outline entry with an explicit title."""
|
||||
reader = PdfReader(context.file_name)
|
||||
writer = PdfWriter()
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
writer.add_outline_item(title, 0)
|
||||
with open(context.file_name, "wb") as f:
|
||||
writer.write(f)
|
||||
context.files[context.param_name].close()
|
||||
context.files[context.param_name] = open(context.file_name, "rb")
|
||||
|
||||
Reference in New Issue
Block a user