Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b764320e06 | ||
|
|
3feb757821 | ||
|
|
faca7486c1 | ||
|
|
bc0ad7d0c3 | ||
|
|
dbe25402c1 | ||
|
|
5400c02b81 | ||
|
|
835863ebe9 | ||
|
|
d0b4438c47 | ||
|
|
7217e6ac15 | ||
|
|
e91d535c36 | ||
|
|
7eaa54d633 | ||
|
|
4e8dd18594 | ||
|
|
b88cbe6bfc | ||
|
|
18cc6dcdc8 |
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-desktop
|
||||
pkgver=2.14.2
|
||||
pkgver=2.14.1
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-server-bin
|
||||
pkgver=2.14.2
|
||||
pkgver=2.14.1
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
|
||||
arch=('any')
|
||||
|
||||
@@ -510,7 +510,6 @@ jobs:
|
||||
# cargo output unsigned, so checking it produces false negatives.
|
||||
- name: Verify Windows Code Signature
|
||||
if: ${{ matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master') }}
|
||||
timeout-minutes: 15
|
||||
shell: pwsh
|
||||
run: |
|
||||
$allSigned = $true
|
||||
@@ -532,26 +531,11 @@ jobs:
|
||||
|
||||
# Extract MSI and verify the inner exe (the file that actually gets installed).
|
||||
# This is the critical check - AV flags the installed exe at runtime.
|
||||
# Use lessmsi, not `msiexec /a`: msiexec serializes on the global
|
||||
# _MSIExecute mutex and hangs forever on hosted runners when another
|
||||
# installer is busy. lessmsi reads MSI tables directly - no mutex, no service.
|
||||
$msi = $msiFiles[0].FullName
|
||||
$extractDir = Join-Path $env:RUNNER_TEMP "msi-verify"
|
||||
if (Test-Path $extractDir) { Remove-Item $extractDir -Recurse -Force }
|
||||
New-Item -ItemType Directory -Force -Path $extractDir | Out-Null
|
||||
|
||||
choco install lessmsi -y --no-progress --limit-output | Out-Null
|
||||
|
||||
# Bound the extraction and kill on hang (defence in depth over timeout-minutes).
|
||||
$proc = Start-Process lessmsi -ArgumentList 'x', "`"$msi`"", "`"$extractDir\`"" -PassThru -NoNewWindow
|
||||
if (-not $proc.WaitForExit(120000)) {
|
||||
try { $proc.Kill() } catch {}
|
||||
Write-Host "[ERROR] MSI extraction timed out after 120s"
|
||||
$allSigned = $false
|
||||
} elseif ($proc.ExitCode -ne 0) {
|
||||
Write-Host "[ERROR] Failed to extract MSI for verification (exit code: $($proc.ExitCode))"
|
||||
$allSigned = $false
|
||||
} else {
|
||||
$proc = Start-Process msiexec.exe -ArgumentList '/a', $msi, '/qn', "TARGETDIR=$extractDir" -Wait -PassThru -NoNewWindow
|
||||
if ($proc.ExitCode -eq 0) {
|
||||
$innerExe = Get-ChildItem -Path $extractDir -Filter "stirling-pdf.exe" -Recurse -File | Select-Object -First 1
|
||||
if ($innerExe) {
|
||||
$sig = Get-AuthenticodeSignature -FilePath $innerExe.FullName
|
||||
@@ -564,6 +548,9 @@ jobs:
|
||||
Write-Host "[ERROR] Could not find stirling-pdf.exe inside MSI"
|
||||
$allSigned = $false
|
||||
}
|
||||
} else {
|
||||
Write-Host "[ERROR] Failed to extract MSI for verification (exit code: $($proc.ExitCode))"
|
||||
$allSigned = $false
|
||||
}
|
||||
|
||||
if (-not $allSigned) {
|
||||
@@ -813,11 +800,7 @@ jobs:
|
||||
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0
|
||||
with:
|
||||
tag_name: v${{ needs.determine-matrix.outputs.version }}
|
||||
# Don't regenerate/append notes on re-runs, and don't force this into the
|
||||
# "Latest" slot - leave the release body and latest marker as they are.
|
||||
generate_release_notes: false
|
||||
append_body: false
|
||||
make_latest: false
|
||||
generate_release_notes: true
|
||||
fail_on_unmatched_files: true
|
||||
# Installers + updater payloads + manifest. .sig contents are embedded
|
||||
# in latest.json so the .sig files themselves are not uploaded.
|
||||
|
||||
+152
-100
@@ -5,8 +5,12 @@ import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
@@ -22,6 +26,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
@@ -38,15 +43,18 @@ class ManualRedactionService {
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Area and page redaction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactAreas(List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
AreaRedactionResult redactAreas(
|
||||
List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Set<String> capturedStrings = new LinkedHashSet<>();
|
||||
Map<Integer, List<PDRectangle>> rectsByPage = new HashMap<>();
|
||||
Set<Integer> forceRasterPages = new LinkedHashSet<>();
|
||||
|
||||
if (redactionAreas == null || redactionAreas.isEmpty()) {
|
||||
return;
|
||||
return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings);
|
||||
}
|
||||
|
||||
Map<Integer, List<RedactionArea>> redactionsByPage = new HashMap<>();
|
||||
@@ -74,57 +82,57 @@ class ManualRedactionService {
|
||||
continue;
|
||||
}
|
||||
|
||||
PDPage page = allPages.get(pageNumber - 1);
|
||||
int pageIndex = pageNumber - 1;
|
||||
PDPage page = allPages.get(pageIndex);
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
for (RedactionArea redactionArea : areasForPage) {
|
||||
Color redactColor = decodeOrDefault(redactionArea.getColor());
|
||||
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
float x = redactionArea.getX().floatValue();
|
||||
float y = redactionArea.getY().floatValue();
|
||||
float width = redactionArea.getWidth().floatValue();
|
||||
float height = redactionArea.getHeight().floatValue();
|
||||
|
||||
float pdfY = page.getBBox().getHeight() - y - height;
|
||||
|
||||
contentStream.addRect(x, pdfY, width, height);
|
||||
contentStream.fill();
|
||||
}
|
||||
contentStream.restoreGraphicsState();
|
||||
// Group rects by their decoded colour so each area keeps its own overlay tint.
|
||||
Map<Color, List<PDRectangle>> byColor = new LinkedHashMap<>();
|
||||
List<PDRectangle> allRects = new ArrayList<>();
|
||||
for (RedactionArea area : areasForPage) {
|
||||
float x = area.getX().floatValue();
|
||||
float y = area.getY().floatValue();
|
||||
float width = area.getWidth().floatValue();
|
||||
float height = area.getHeight().floatValue();
|
||||
// Request coords are top-left origin; convert to PDF user space (bottom-left).
|
||||
float pdfY = pageHeight - y - height;
|
||||
PDRectangle rect = new PDRectangle(x, pdfY, width, height);
|
||||
allRects.add(rect);
|
||||
byColor.computeIfAbsent(decodeOrDefault(area.getColor()), k -> new ArrayList<>())
|
||||
.add(rect);
|
||||
}
|
||||
|
||||
for (Map.Entry<Color, List<PDRectangle>> colorEntry : byColor.entrySet()) {
|
||||
Map<Integer, List<PDRectangle>> singlePage = new HashMap<>();
|
||||
singlePage.put(pageIndex, colorEntry.getValue());
|
||||
RedactionPipeline.RedactionResult result =
|
||||
RedactionPipeline.redactAreas(document, singlePage, colorEntry.getKey());
|
||||
capturedStrings.addAll(result.getCapturedStrings());
|
||||
forceRasterPages.addAll(result.getForceRasterPages());
|
||||
}
|
||||
rectsByPage.put(pageIndex, allRects);
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Manual area redaction captured {} text run(s) across {} page(s)",
|
||||
capturedStrings.size(),
|
||||
rectsByPage.size());
|
||||
return new AreaRedactionResult(rectsByPage, forceRasterPages, capturedStrings);
|
||||
}
|
||||
|
||||
void redactPages(ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
List<Integer> redactPages(
|
||||
ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Color redactColor = decodeOrDefault(request.getPageRedactionColor());
|
||||
List<Integer> pageNumbers = getPageNumbers(request, allPages.getCount());
|
||||
List<Integer> pageIndexes = getPageNumbers(request, allPages.getCount());
|
||||
|
||||
for (Integer pageNumber : pageNumbers) {
|
||||
PDPage page = allPages.get(pageNumber);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
PDRectangle box = page.getBBox();
|
||||
contentStream.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
contentStream.fill();
|
||||
}
|
||||
}
|
||||
// Whole-page wipe: drop the content stream, resources and annotations, then fill.
|
||||
RedactionPipeline.redactWholePages(document, pageIndexes, redactColor);
|
||||
return new ArrayList<>(pageIndexes);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Overlay drawing
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactFoundText(
|
||||
PDDocument document,
|
||||
@@ -158,7 +166,10 @@ class ManualRedactionService {
|
||||
|
||||
try {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
PDRectangle pageBox = page.getBBox();
|
||||
// TextPosition coords are relative to the CropBox origin.
|
||||
PDRectangle crop = page.getCropBox();
|
||||
float cropX = crop.getLowerLeftX();
|
||||
float cropY = crop.getLowerLeftY();
|
||||
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
@@ -179,8 +190,8 @@ class ManualRedactionService {
|
||||
}
|
||||
|
||||
contentStream.addRect(
|
||||
boxX,
|
||||
pageBox.getHeight() - block.getY2() - padding,
|
||||
cropX + boxX,
|
||||
cropY + crop.getHeight() - block.getY2() - padding,
|
||||
boxWidth,
|
||||
block.getY2() - block.getY1() + 2 * padding);
|
||||
}
|
||||
@@ -192,8 +203,8 @@ class ManualRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent
|
||||
// users from hovering over redacted URLs and seeing the underlying destination.
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent users
|
||||
// from hovering over redacted URLs and seeing the underlying destination.
|
||||
try {
|
||||
float pageH = page.getBBox().getHeight();
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
@@ -247,14 +258,19 @@ class ManualRedactionService {
|
||||
continue;
|
||||
}
|
||||
PDPage page = pages.get(pageIdx);
|
||||
PDRectangle crop = page.getCropBox();
|
||||
float cropX = crop.getLowerLeftX();
|
||||
float cropY = crop.getLowerLeftY();
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(color);
|
||||
for (float[] box : entry.getValue()) {
|
||||
// Box coords are CropBox-relative; offset by the CropBox origin
|
||||
// (cropbox-overlay).
|
||||
float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4];
|
||||
cs.addRect(x1, y1, x2 - x1, y2 - y1);
|
||||
cs.addRect(cropX + x1, cropY + y1, x2 - x1, y2 - y1);
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
@@ -262,9 +278,7 @@ class ManualRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Page element extraction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns bounding boxes for every text line and image on {@code page} in PDF user-space
|
||||
@@ -288,9 +302,7 @@ class ManualRedactionService {
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Finalization
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
@@ -298,7 +310,37 @@ class ManualRedactionService {
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode)
|
||||
boolean isTextRemovalMode,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns)
|
||||
throws IOException {
|
||||
return finalizeRedaction(
|
||||
document,
|
||||
allFoundTextsByPage,
|
||||
colorString,
|
||||
customPadding,
|
||||
convertToImage,
|
||||
isTextRemovalMode,
|
||||
literalTargets,
|
||||
patterns,
|
||||
Collections.emptySet());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param geometricRasterPages 0-based pages carrying a range / image-box redaction, whose
|
||||
* covered content (text under an overlay, or an image) is not text-removable and so must be
|
||||
* rasterised to guarantee removal.
|
||||
*/
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> allFoundTextsByPage,
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> geometricRasterPages)
|
||||
throws IOException {
|
||||
|
||||
List<PDFText> allFoundTexts = new ArrayList<>();
|
||||
@@ -309,74 +351,84 @@ class ManualRedactionService {
|
||||
if (!allFoundTexts.isEmpty()) {
|
||||
Color redactColor = decodeOrDefault(colorString);
|
||||
redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode);
|
||||
cleanDocumentMetadata(document);
|
||||
}
|
||||
|
||||
byte[] outputBytes;
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
cleanDocumentMetadata(convertedPdf);
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
convertedPdf.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized (image mode): {} pages ➜ {} KB",
|
||||
convertedPdf.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
// Convert-to-image physically removes all text, so verification is a plain save.
|
||||
outputBytes =
|
||||
RedactionPipeline.finalize(
|
||||
convertedPdf, Collections.emptySet(), Collections.emptyList());
|
||||
}
|
||||
} else {
|
||||
// Strip matched glyphs, then scrub/verify/rasterise via finalize.
|
||||
RedactionPipeline.redactLiteralTerms(document, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(document, literalTargets, patterns);
|
||||
// Geometric (range/image-box) redactions can't be text-removed; rasterise their pages.
|
||||
outputBytes =
|
||||
RedactionPipeline.rasteriseSpecificPages(
|
||||
outputBytes, geometricRasterPages, literalTargets, patterns);
|
||||
}
|
||||
|
||||
return writeBytes(outputBytes, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
/** Finalize a manual redaction; forced + verified-leaking pages are rasterised. */
|
||||
TempFile finalizeManual(
|
||||
PDDocument document, AreaRedactionResult areaResult, Boolean convertToImage)
|
||||
throws IOException {
|
||||
|
||||
byte[] outputBytes;
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
outputBytes =
|
||||
RedactionPipeline.finalize(
|
||||
convertedPdf, Collections.emptySet(), Collections.emptyList());
|
||||
}
|
||||
} else {
|
||||
outputBytes =
|
||||
RedactionPipeline.finalizeAreas(
|
||||
document,
|
||||
areaResult.rectsByPage,
|
||||
areaResult.forceRasterPages,
|
||||
areaResult.capturedStrings);
|
||||
}
|
||||
|
||||
return writeBytes(outputBytes, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
private TempFile writeBytes(byte[] outputBytes, int pageCount) throws IOException {
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
document.save(tempOut.getFile());
|
||||
java.nio.file.Files.write(tempOut.getFile().toPath(), outputBytes);
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized: {} pages ➜ {} KB",
|
||||
document.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
log.info("Redaction finalized: {} pages -> {} KB", pageCount, outputBytes.length / 1024);
|
||||
return tempOut;
|
||||
}
|
||||
|
||||
private void cleanDocumentMetadata(PDDocument document) {
|
||||
try {
|
||||
var documentInfo = document.getDocumentInformation();
|
||||
if (documentInfo != null) {
|
||||
documentInfo.setAuthor(null);
|
||||
documentInfo.setSubject(null);
|
||||
documentInfo.setKeywords(null);
|
||||
documentInfo.setModificationDate(java.util.Calendar.getInstance());
|
||||
log.debug("Cleaned document metadata for security");
|
||||
}
|
||||
// Utilities
|
||||
|
||||
if (document.getDocumentCatalog() != null) {
|
||||
try {
|
||||
document.getDocumentCatalog().setMetadata(null);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not clear XMP metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
/** Redaction rectangles, pages to force-rasterise, and the text captured under the rects. */
|
||||
static final class AreaRedactionResult {
|
||||
final Map<Integer, List<PDRectangle>> rectsByPage;
|
||||
final Set<Integer> forceRasterPages;
|
||||
final Set<String> capturedStrings;
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to clean document metadata: {}", e.getMessage());
|
||||
AreaRedactionResult(
|
||||
Map<Integer, List<PDRectangle>> rectsByPage,
|
||||
Set<Integer> forceRasterPages,
|
||||
Set<String> capturedStrings) {
|
||||
this.rectsByPage = rectsByPage;
|
||||
this.forceRasterPages = forceRasterPages;
|
||||
this.capturedStrings = capturedStrings;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Utilities
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
static Color decodeOrDefault(String hex) {
|
||||
if (hex == null) {
|
||||
return Color.BLACK;
|
||||
|
||||
+54
-30
@@ -1,9 +1,15 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
@@ -29,13 +35,13 @@ import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.SecurityApi;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
@@ -93,33 +99,24 @@ public class RedactController {
|
||||
throws IOException {
|
||||
|
||||
MultipartFile file = request.getFileInput();
|
||||
String filename =
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(file.getOriginalFilename())))
|
||||
+ "_redacted.pdf";
|
||||
|
||||
try (PDDocument document = pdfDocumentFactory.load(file)) {
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
|
||||
// Whole-page wipes drop content; areas drop glyphs + overlay, verified later.
|
||||
manualRedactionService.redactPages(request, document, allPages);
|
||||
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
|
||||
ManualRedactionService.AreaRedactionResult areaResult =
|
||||
manualRedactionService.redactAreas(request.getRedactions(), document, allPages);
|
||||
|
||||
if (Boolean.TRUE.equals(request.getConvertPDFToImage())) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
convertedPdf,
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(
|
||||
file.getOriginalFilename())))
|
||||
+ "_redacted.pdf",
|
||||
tempFileManager);
|
||||
}
|
||||
}
|
||||
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
document,
|
||||
removeFileExtension(
|
||||
Objects.requireNonNull(
|
||||
Filenames.toSimpleFileName(file.getOriginalFilename())))
|
||||
+ "_redacted.pdf",
|
||||
tempFileManager);
|
||||
TempFile out =
|
||||
manualRedactionService.finalizeManual(
|
||||
document, areaResult, request.getConvertPDFToImage());
|
||||
return WebResponseUtils.pdfFileToWebResponse(out, filename);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,9 +185,32 @@ public class RedactController {
|
||||
request.getFileInput().getOriginalFilename())))
|
||||
+ "_redacted.pdf";
|
||||
|
||||
Set<String> literalTargets =
|
||||
Arrays.stream(listOfText)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
List<Pattern> compiledPatterns =
|
||||
RedactionPipeline.buildPatterns(listOfText, useRegex, wholeWordSearchBool);
|
||||
// Bare literals match substrings; regex/whole-word rely on compiled patterns.
|
||||
Set<String> verificationTargets =
|
||||
(useRegex || wholeWordSearchBool) ? Collections.emptySet() : literalTargets;
|
||||
|
||||
if (allFoundTextsByPage.isEmpty()) {
|
||||
log.info("No text found matching redaction patterns");
|
||||
return WebResponseUtils.pdfDocToWebResponse(document, filename, tempFileManager);
|
||||
// No page hit, but the target may live in a bookmark/annotation/form/JS carrier, so
|
||||
// still run the scrub + verify path rather than just wiping metadata.
|
||||
log.info("No page text matched; scrubbing catalog carriers and verifying");
|
||||
TempFile finalized =
|
||||
manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
Collections.emptyMap(),
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
true,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
}
|
||||
|
||||
boolean fallbackToBoxOnlyMode;
|
||||
@@ -211,7 +231,8 @@ public class RedactController {
|
||||
|
||||
if (fallbackToBoxOnlyMode) {
|
||||
log.warn(
|
||||
"Font compatibility issues detected. Using box-only redaction mode for better reliability.");
|
||||
"Font compatibility issue in placeholder pass; the true-removal pass and "
|
||||
+ "verification still guarantee the target is gone.");
|
||||
|
||||
fallbackDocument = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
@@ -226,7 +247,9 @@ public class RedactController {
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
false);
|
||||
false,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
}
|
||||
@@ -238,7 +261,9 @@ public class RedactController {
|
||||
request.getRedactColor(),
|
||||
request.getCustomPadding(),
|
||||
request.getConvertPDFToImage(),
|
||||
true);
|
||||
true,
|
||||
verificationTargets,
|
||||
compiledPatterns);
|
||||
|
||||
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
|
||||
|
||||
@@ -247,11 +272,10 @@ public class RedactController {
|
||||
throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e);
|
||||
|
||||
} finally {
|
||||
// Both are distinct PDDocument handles.
|
||||
if (document != null) {
|
||||
try {
|
||||
if (fallbackDocument == null) {
|
||||
document.close();
|
||||
}
|
||||
document.close();
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to close main document: {}", e.getMessage());
|
||||
}
|
||||
|
||||
+55
-29
@@ -7,8 +7,11 @@ import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
@@ -30,6 +33,7 @@ import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyl
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
@@ -90,8 +94,8 @@ class RedactExecuteService {
|
||||
try {
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF
|
||||
// stripper only once across the entire execute() call rather than once per target.
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF stripper only
|
||||
// once across the entire execute() call rather than once per target.
|
||||
Map<Integer, List<PDFText>> foundTexts =
|
||||
hasTextOps ? collectTextMatches(document, request) : new HashMap<>();
|
||||
|
||||
@@ -101,13 +105,13 @@ class RedactExecuteService {
|
||||
totalMatches,
|
||||
foundTexts.size());
|
||||
|
||||
// Text removal (content-stream rewriting) — skipped in overlay-only mode.
|
||||
// Text removal (content-stream rewriting) - skipped in overlay-only mode.
|
||||
boolean needsOverlayOnly = overlayOnly;
|
||||
if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) {
|
||||
needsOverlayOnly = applyTextRemoval(document, request);
|
||||
} else if (overlayOnly) {
|
||||
log.info(
|
||||
"[redact/execute] overlay-only mode requested — skipping content-stream rewriting");
|
||||
"[redact/execute] overlay-only mode requested - skipping content-stream rewriting");
|
||||
}
|
||||
|
||||
// Reload fresh document on fallback so we overlay onto clean content.
|
||||
@@ -128,25 +132,46 @@ class RedactExecuteService {
|
||||
applyPageWipe(document, wipePages, style);
|
||||
}
|
||||
|
||||
// Range + image-box redactions are geometric overlays.
|
||||
Set<Integer> geometricRasterPages = new HashSet<>();
|
||||
for (TextRange range : ranges) {
|
||||
applyRangeRedaction(document, range, style, layoutCache);
|
||||
geometricRasterPages.addAll(
|
||||
applyRangeRedaction(document, range, style, layoutCache));
|
||||
}
|
||||
|
||||
for (ImageBox box : imageBoxes) {
|
||||
applyImageBoxRedaction(document, box, style);
|
||||
geometricRasterPages.addAll(applyImageBoxRedaction(document, box, style));
|
||||
}
|
||||
|
||||
if (request.getRedactImagePages() != null) {
|
||||
applyAllImagesRedaction(document, request.getRedactImagePages(), style);
|
||||
}
|
||||
|
||||
// Overlay-only skips removal/verify; font-fallback overlays still pass targets.
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
for (String value : textValues) {
|
||||
String trimmed = value == null ? "" : value.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
literalTargets.add(trimmed);
|
||||
}
|
||||
}
|
||||
List<Pattern> verificationPatterns =
|
||||
RedactionPipeline.buildPatterns(
|
||||
regexPatterns.toArray(new String[0]), true, false);
|
||||
Set<String> finalizeTargets = overlayOnly ? Collections.emptySet() : literalTargets;
|
||||
List<Pattern> finalizePatterns =
|
||||
overlayOnly ? Collections.emptyList() : verificationPatterns;
|
||||
|
||||
return manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
foundTexts,
|
||||
style.getColor(),
|
||||
style.getPadding(),
|
||||
convertToImage,
|
||||
!needsOverlayOnly);
|
||||
!needsOverlayOnly,
|
||||
finalizeTargets,
|
||||
finalizePatterns,
|
||||
geometricRasterPages);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Execute redaction failed: {}", e.getMessage(), e);
|
||||
@@ -162,9 +187,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Single-pass text scan (one stripper pass per execute() call)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit
|
||||
@@ -197,9 +220,7 @@ class RedactExecuteService {
|
||||
return found;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Text removal (content-stream rewriting)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the
|
||||
@@ -233,7 +254,7 @@ class RedactExecuteService {
|
||||
|
||||
if (fallback) {
|
||||
log.warn(
|
||||
"[redact/execute] font compatibility issue — falling back to overlay-only");
|
||||
"[redact/execute] font compatibility issue - falling back to overlay-only");
|
||||
} else {
|
||||
log.info("[redact/execute] content-stream text removal applied successfully");
|
||||
}
|
||||
@@ -246,9 +267,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Per-operation dispatch methods
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void applyPageWipe(PDDocument document, List<Integer> pageNumbers, RedactStyle style)
|
||||
throws IOException {
|
||||
@@ -304,7 +323,10 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
private void applyRangeRedaction(
|
||||
/**
|
||||
* @return 0-based pages covered by the range overlay; those pages must be rasterised.
|
||||
*/
|
||||
private Set<Integer> applyRangeRedaction(
|
||||
PDDocument document,
|
||||
TextRange range,
|
||||
RedactStyle style,
|
||||
@@ -322,6 +344,11 @@ class RedactExecuteService {
|
||||
style.getPadding(),
|
||||
ManualRedactionService.decodeOrDefault(style.getColor()),
|
||||
false);
|
||||
Set<Integer> pages = new HashSet<>();
|
||||
for (PDFText block : blocks) {
|
||||
pages.add(block.getPageIndex());
|
||||
}
|
||||
return pages;
|
||||
} else {
|
||||
log.warn(
|
||||
"[redact/execute] range not found: start='{}' end='{}'",
|
||||
@@ -331,10 +358,14 @@ class RedactExecuteService {
|
||||
} catch (Exception e) {
|
||||
log.warn("[redact/execute] range redaction failed: {}", e.getMessage());
|
||||
}
|
||||
return Collections.emptySet();
|
||||
}
|
||||
|
||||
private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style)
|
||||
throws IOException {
|
||||
/**
|
||||
* @return 0-based page covered by the image-box overlay; that page must be rasterised.
|
||||
*/
|
||||
private Set<Integer> applyImageBoxRedaction(
|
||||
PDDocument document, ImageBox box, RedactStyle style) throws IOException {
|
||||
List<float[]> boxes =
|
||||
List.of(
|
||||
new float[] {
|
||||
@@ -343,6 +374,7 @@ class RedactExecuteService {
|
||||
log.info("[redact/execute] image box overlay on page {}", box.pageIndex());
|
||||
Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
manualRedactionService.redactImageBoxes(document, boxes, boxColor);
|
||||
return Set.of(box.pageIndex());
|
||||
}
|
||||
|
||||
private void applyAllImagesRedaction(
|
||||
@@ -386,9 +418,7 @@ class RedactExecuteService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range collection helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text
|
||||
@@ -458,7 +488,7 @@ class RedactExecuteService {
|
||||
}
|
||||
if (end == null) {
|
||||
log.warn(
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping",
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) - skipping",
|
||||
start.page + 1,
|
||||
start.col,
|
||||
start.y);
|
||||
@@ -479,7 +509,7 @@ class RedactExecuteService {
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range '{}'→'{}': {} total blocks",
|
||||
"[redact/execute] range '{}'->'{}': {} total blocks",
|
||||
startStr,
|
||||
openEnded ? "<end of document>" : endStr,
|
||||
blocks.size());
|
||||
@@ -511,7 +541,6 @@ class RedactExecuteService {
|
||||
float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2();
|
||||
|
||||
// Line-box cache: populated lazily per page, reused across range iterations.
|
||||
// Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException.
|
||||
Map<Integer, List<float[]>> lineBoxCache = new HashMap<>();
|
||||
|
||||
for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) {
|
||||
@@ -677,7 +706,7 @@ class RedactExecuteService {
|
||||
if (i > 0) sb.append(", ");
|
||||
sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y));
|
||||
}
|
||||
if (anchors.size() > max) sb.append(", …");
|
||||
if (anchors.size() > max) sb.append(", ...");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@@ -703,9 +732,8 @@ class RedactExecuteService {
|
||||
candidates.add(new Candidate(tolerant, true));
|
||||
}
|
||||
|
||||
// If the anchor spans multiple lines (model provided entire paragraph instead of a short
|
||||
// phrase), try just the first non-empty line — it's usually sufficient to locate the
|
||||
// position and avoids mismatches from mid-paragraph text extraction artifacts.
|
||||
// Long multi-line anchors mismatch on extraction artifacts, so try just the first non-empty
|
||||
// line.
|
||||
if (trimmed.contains("\n")) {
|
||||
String firstLine =
|
||||
Arrays.stream(trimmed.split("\n"))
|
||||
@@ -733,7 +761,7 @@ class RedactExecuteService {
|
||||
if (!m.isEmpty()) {
|
||||
if (!c.pattern.equals(trimmed)) {
|
||||
log.info(
|
||||
"[redact/execute] range boundary matched via fallback: '{}' → '{}'",
|
||||
"[redact/execute] range boundary matched via fallback: '{}' -> '{}'",
|
||||
trimmed,
|
||||
c.pattern);
|
||||
}
|
||||
@@ -745,9 +773,7 @@ class RedactExecuteService {
|
||||
|
||||
private record Candidate(String pattern, boolean useRegex) {}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Static helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation
|
||||
|
||||
+5
-20
@@ -46,12 +46,11 @@ class TextRedactionService {
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
private static final float PRECISION_THRESHOLD = 1e-3f;
|
||||
private static final int FONT_SCALE_FACTOR = 1000;
|
||||
private static final Set<String> TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\"");
|
||||
private static final Set<String> TEXT_SHOWING_OPERATORS =
|
||||
stirling.software.SPDF.pdf.redaction.RedactionPipeline.TEXT_SHOWING_OPERATORS;
|
||||
private static final COSString EMPTY_COS_STRING = new COSString("");
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Public API
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
Map<Integer, List<PDFText>> findTextToRedact(
|
||||
PDDocument document, String[] listOfText, boolean useRegex, boolean wholeWordSearch) {
|
||||
@@ -135,9 +134,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Content stream manipulation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
List<Object> createTokensWithoutTargetText(
|
||||
PDDocument document,
|
||||
@@ -271,9 +268,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Placeholder creation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
String createPlaceholderWithFont(String originalWord, PDFont font) {
|
||||
if (originalWord == null || originalWord.isEmpty()) {
|
||||
@@ -410,9 +405,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Width calculation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private float safeGetStringWidth(PDFont font, String text) {
|
||||
if (font == null || text == null || text.isEmpty()) {
|
||||
@@ -601,9 +594,7 @@ class TextRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Token and segment operations
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void processPageXObjects(
|
||||
PDDocument document,
|
||||
@@ -706,16 +697,12 @@ class TextRedactionService {
|
||||
private void writeRedactedContentToXObject(
|
||||
PDDocument document, PDFormXObject formXObject, List<Object> redactedTokens)
|
||||
throws IOException {
|
||||
|
||||
PDStream newStream = new PDStream(document);
|
||||
|
||||
try (var out = newStream.createOutputStream()) {
|
||||
// A form XObject's content IS its own stream body; overwrite it in place.
|
||||
PDStream formStream = new PDStream(formXObject.getCOSObject());
|
||||
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
ContentStreamWriter writer = new ContentStreamWriter(out);
|
||||
writer.writeTokens(redactedTokens);
|
||||
}
|
||||
|
||||
formXObject.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
formXObject.getCOSObject().setItem(COSName.CONTENTS, newStream.getCOSObject());
|
||||
}
|
||||
|
||||
private List<TextSegment> extractTextSegments(PDPage page, List<Object> tokens) {
|
||||
@@ -1168,9 +1155,7 @@ class TextRedactionService {
|
||||
};
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Inner data classes
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@Data
|
||||
private static class GraphicsState {
|
||||
|
||||
@@ -0,0 +1,755 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSObject;
|
||||
import org.apache.pdfbox.cos.COSStream;
|
||||
import org.apache.pdfbox.cos.COSString;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentNameDictionary;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDMetadata;
|
||||
import org.apache.pdfbox.pdmodel.common.PDNameTreeNode;
|
||||
import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureTreeRoot;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocumentOutline;
|
||||
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/** Removes/rewrites every catalog carrier that could leak redacted text. */
|
||||
@Slf4j
|
||||
public final class CatalogScrubber {
|
||||
|
||||
// A literal shorter than this won't delete a whole carrier (JS/XFA/action/embedded file).
|
||||
private static final int MIN_CARRIER_DROP_LITERAL = 3;
|
||||
|
||||
private CatalogScrubber() {}
|
||||
|
||||
/** Scrub all catalog-level carriers of the given literal/regex targets. */
|
||||
public static void scrub(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (document == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
PDDocumentCatalog catalog = document.getDocumentCatalog();
|
||||
if (catalog == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
scrubOutline(catalog.getDocumentOutline(), literalTargets, patterns);
|
||||
scrubAcroForm(catalog.getAcroForm(), literalTargets, patterns);
|
||||
scrubAnnotations(document, literalTargets, patterns);
|
||||
scrubStructTree(catalog.getStructureTreeRoot(), literalTargets, patterns);
|
||||
scrubNames(catalog.getNames(), literalTargets, patterns);
|
||||
scrubCatalogActions(catalog, literalTargets, patterns);
|
||||
scrubDocumentInfo(document, literalTargets, patterns);
|
||||
}
|
||||
|
||||
// Catalog actions: OpenAction, AA, and any JavaScript / URI payloads
|
||||
|
||||
private static void scrubCatalogActions(
|
||||
PDDocumentCatalog catalog, Set<String> targets, List<Pattern> patterns) {
|
||||
COSDictionary root = catalog.getCOSObject();
|
||||
if (root == null) {
|
||||
return;
|
||||
}
|
||||
// OpenAction may be either an action dict (with /URI or /JS) or an explicit
|
||||
scrubActionIfMatching(root, COSName.getPDFName("OpenAction"), targets, patterns);
|
||||
scrubActionIfMatching(root, COSName.getPDFName("AA"), targets, patterns);
|
||||
}
|
||||
|
||||
/** Drop the action at key if any target appears in its URI/JS payload. */
|
||||
private static void scrubActionIfMatching(
|
||||
COSDictionary parent, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
if (parent == null || key == null) {
|
||||
return;
|
||||
}
|
||||
COSBase value = parent.getDictionaryObject(key);
|
||||
if (value == null) {
|
||||
return;
|
||||
}
|
||||
if (containsTarget(value, carrierDropLiterals(targets), patterns, new HashSet<>())) {
|
||||
log.debug("Removing catalog {} due to target match", key.getName());
|
||||
parent.removeItem(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Literals specific enough to justify DELETING an entire carrier (JS / XFA / action / embedded
|
||||
* file). Sub-threshold literals (a single digit, a 2-char run) would nuke unrelated carriers,
|
||||
* so they are excluded from whole-carrier drops - they are still removed in-string by
|
||||
* stripMatches where the carrier is a plain string.
|
||||
*/
|
||||
private static Set<String> carrierDropLiterals(Set<String> targets) {
|
||||
if (targets == null) {
|
||||
return java.util.Collections.emptySet();
|
||||
}
|
||||
Set<String> specific = new LinkedHashSet<>();
|
||||
for (String t : targets) {
|
||||
if (t != null && t.trim().length() >= MIN_CARRIER_DROP_LITERAL) {
|
||||
specific.add(t);
|
||||
}
|
||||
}
|
||||
return specific;
|
||||
}
|
||||
|
||||
private static boolean containsTarget(
|
||||
COSBase base, Set<String> targets, List<Pattern> patterns, Set<COSBase> seen) {
|
||||
if (base == null) {
|
||||
return false;
|
||||
}
|
||||
COSBase resolved = base instanceof COSObject obj ? obj.getObject() : base;
|
||||
if (resolved == null || !seen.add(resolved)) {
|
||||
return false;
|
||||
}
|
||||
if (resolved instanceof COSString cs) {
|
||||
return matches(cs.getString(), targets, patterns);
|
||||
}
|
||||
if (resolved instanceof COSStream stream) {
|
||||
// Streams in XFA / OpenAction contexts are text (XML, JavaScript).
|
||||
try (java.io.InputStream is = stream.createInputStream()) {
|
||||
byte[] buf = new byte[2 * 1024 * 1024];
|
||||
int total = 0;
|
||||
int n;
|
||||
while ((n = is.read(buf, total, buf.length - total)) > 0) {
|
||||
total += n;
|
||||
if (total >= buf.length) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
String text = new String(buf, 0, total, java.nio.charset.StandardCharsets.UTF_8);
|
||||
if (matches(text, targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
// Fail closed: content past the 2 MiB cap is unproven, so treat as a match.
|
||||
return total >= buf.length && is.read() >= 0;
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scan stream for targets: {}", e.getMessage());
|
||||
// Fail closed: if we cannot read it we cannot prove it is clean, so treat
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (resolved instanceof COSDictionary dict) {
|
||||
for (COSName k : new HashSet<>(dict.keySet())) {
|
||||
if (containsTarget(dict.getItem(k), targets, patterns, seen)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (resolved instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
if (containsTarget(array.getObject(i), targets, patterns, seen)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the redaction target from /Info string entries and drop the XMP packet only if it
|
||||
* carries a target. Non-matching document properties (Title, Author, dates) are left intact -
|
||||
* redaction removes the target, it does not blanket-wipe metadata.
|
||||
*/
|
||||
private static void scrubDocumentInfo(
|
||||
PDDocument document, Set<String> targets, List<Pattern> patterns) {
|
||||
PDDocumentInformation info = document.getDocumentInformation();
|
||||
if (info != null && info.getCOSObject() != null) {
|
||||
COSDictionary infoDict = info.getCOSObject();
|
||||
for (COSName key : new HashSet<>(infoDict.keySet())) {
|
||||
if (infoDict.getDictionaryObject(key) instanceof COSString cs) {
|
||||
String stripped = stripMatches(cs.getString(), targets, patterns);
|
||||
if (!stripped.equals(cs.getString())) {
|
||||
if (stripped.isEmpty()) {
|
||||
infoDict.removeItem(key);
|
||||
} else {
|
||||
infoDict.setString(key, stripped);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PDDocumentCatalog catalog = document.getDocumentCatalog();
|
||||
if (catalog != null && catalog.getMetadata() != null) {
|
||||
scrubXmp(document, catalog, targets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* XMP is RDF/XML that usually mirrors /Info. Strip only the target occurrences so non-matching
|
||||
* properties (dates, rights, custom schema) survive; fall back to dropping the whole packet if
|
||||
* the edit can't be proven to have removed the target (e.g. entity-encoded) or anything throws.
|
||||
*/
|
||||
private static void scrubXmp(
|
||||
PDDocument document,
|
||||
PDDocumentCatalog catalog,
|
||||
Set<String> targets,
|
||||
List<Pattern> patterns) {
|
||||
String xmp;
|
||||
try {
|
||||
xmp = new String(catalog.getMetadata().toByteArray(), StandardCharsets.UTF_8);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not read XMP metadata: {}", e.getMessage());
|
||||
return;
|
||||
}
|
||||
if (!matches(xmp, targets, patterns)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String stripped = stripMatches(xmp, targets, patterns);
|
||||
// Only keep the edited packet if the target is provably gone from it.
|
||||
if (!matches(stripped, targets, patterns)) {
|
||||
catalog.setMetadata(
|
||||
new PDMetadata(
|
||||
document,
|
||||
new ByteArrayInputStream(
|
||||
stripped.getBytes(StandardCharsets.UTF_8))));
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Surgical XMP scrub failed; dropping packet: {}", e.getMessage());
|
||||
}
|
||||
catalog.setMetadata(null);
|
||||
}
|
||||
|
||||
// Outline
|
||||
|
||||
private static void scrubOutline(
|
||||
PDDocumentOutline outline, Set<String> targets, List<Pattern> patterns) {
|
||||
if (outline == null) {
|
||||
return;
|
||||
}
|
||||
scrubOutlineItems(outline.children(), targets, patterns);
|
||||
}
|
||||
|
||||
private static void scrubOutlineItems(
|
||||
Iterable<PDOutlineItem> items, Set<String> targets, List<Pattern> patterns) {
|
||||
if (items == null) {
|
||||
return;
|
||||
}
|
||||
for (PDOutlineItem item : items) {
|
||||
try {
|
||||
String title = item.getTitle();
|
||||
if (title != null) {
|
||||
String stripped = stripMatches(title, targets, patterns);
|
||||
if (!stripped.equals(title)) {
|
||||
item.setTitle(stripped);
|
||||
}
|
||||
}
|
||||
// Bookmark actions: /A is an action dict which may carry a /URI or /JS
|
||||
COSDictionary itemDict = item.getCOSObject();
|
||||
if (itemDict != null) {
|
||||
scrubActionIfMatching(itemDict, COSName.A, targets, patterns);
|
||||
scrubActionIfMatching(itemDict, COSName.getPDFName("AA"), targets, patterns);
|
||||
}
|
||||
scrubOutlineItems(item.children(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub outline item: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AcroForm
|
||||
|
||||
private static void scrubAcroForm(
|
||||
PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
|
||||
if (form == null) {
|
||||
return;
|
||||
}
|
||||
// XFA forms: scrubbed separately because the XFA XML packet carries
|
||||
scrubXfa(form, targets, patterns);
|
||||
|
||||
try {
|
||||
for (PDField field : form.getFieldTree()) {
|
||||
scrubField(field, targets, patterns);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to walk AcroForm field tree: {}", e.getMessage());
|
||||
}
|
||||
|
||||
// Force viewers to regenerate appearance streams from the (scrubbed) /V
|
||||
try {
|
||||
form.setNeedAppearances(true);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to set /NeedAppearances on AcroForm: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubXfa(PDAcroForm form, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
COSBase xfaBase = form.getCOSObject().getDictionaryObject(COSName.XFA);
|
||||
if (xfaBase == null) {
|
||||
return;
|
||||
}
|
||||
boolean hit =
|
||||
containsTarget(
|
||||
xfaBase, carrierDropLiterals(targets), patterns, new HashSet<>());
|
||||
if (hit) {
|
||||
// Simplest safe move: strip the XFA entry entirely.
|
||||
log.warn(
|
||||
"Removing XFA form packet from AcroForm - XFA XML contained a redaction "
|
||||
+ "target and has been dropped so viewers render AcroForm widgets "
|
||||
+ "instead.");
|
||||
form.getCOSObject().removeItem(COSName.XFA);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub XFA: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubField(PDField field, Set<String> targets, List<Pattern> patterns) {
|
||||
if (field == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
COSDictionary dict = field.getCOSObject();
|
||||
// Only touch fields whose own values actually contain a target: clearing /AP
|
||||
// document-wide blanks unrelated fields in viewers that ignore /NeedAppearances.
|
||||
boolean matched =
|
||||
dictValueMatches(dict, COSName.V, targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.DV, targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.getPDFName("RV"), targets, patterns)
|
||||
|| dictValueMatches(dict, COSName.getPDFName("TU"), targets, patterns)
|
||||
|| buttonCaptionMatches(dict, targets, patterns)
|
||||
|| fieldValueMatches(field, targets, patterns);
|
||||
if (!matched) {
|
||||
return;
|
||||
}
|
||||
scrubDictStrings(dict, COSName.V, targets, patterns);
|
||||
scrubDictStrings(dict, COSName.DV, targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("RV"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("TU"), targets, patterns);
|
||||
// Keep field appearance streams in sync with value where possible.
|
||||
try {
|
||||
if (field.getValueAsString() != null) {
|
||||
String stripped = stripMatches(field.getValueAsString(), targets, patterns);
|
||||
if (!stripped.equals(field.getValueAsString())) {
|
||||
field.setValue(stripped);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to rewrite field value via setValue: {}", e.getMessage());
|
||||
}
|
||||
if (!isButtonField(dict)) {
|
||||
clearWidgetAppearances(dict);
|
||||
} else if (scrubButtonCaption(dict, targets, patterns)) {
|
||||
// A button caption (/MK /CA,/RC,/AC) that carried the target renders via /AP that
|
||||
// /NeedAppearances can't rebuild, so drop the stale /AP too.
|
||||
clearWidgetAppearances(dict);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub field: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static final String[] MK_CAPTION_KEYS = {"CA", "RC", "AC"};
|
||||
|
||||
/** True if a button widget's /MK caption (down/rollover/alternate) contains a target. */
|
||||
private static boolean buttonCaptionMatches(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
for (COSDictionary mk : mkDicts(dict)) {
|
||||
for (String k : MK_CAPTION_KEYS) {
|
||||
if (dictValueMatches(mk, COSName.getPDFName(k), targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Strip target substrings from a button widget's /MK captions; true if any matched. */
|
||||
private static boolean scrubButtonCaption(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
boolean hit = false;
|
||||
for (COSDictionary mk : mkDicts(dict)) {
|
||||
for (String k : MK_CAPTION_KEYS) {
|
||||
COSName key = COSName.getPDFName(k);
|
||||
if (dictValueMatches(mk, key, targets, patterns)) {
|
||||
hit = true;
|
||||
scrubDictStrings(mk, key, targets, patterns);
|
||||
}
|
||||
}
|
||||
}
|
||||
return hit;
|
||||
}
|
||||
|
||||
/** The /MK appearance-characteristics dicts on a field and any widget kids. */
|
||||
private static List<COSDictionary> mkDicts(COSDictionary dict) {
|
||||
List<COSDictionary> out = new java.util.ArrayList<>();
|
||||
if (dict.getDictionaryObject(COSName.getPDFName("MK")) instanceof COSDictionary mk) {
|
||||
out.add(mk);
|
||||
}
|
||||
if (dict.getDictionaryObject(COSName.KIDS) instanceof COSArray kids) {
|
||||
for (int i = 0; i < kids.size(); i++) {
|
||||
if (kids.getObject(i) instanceof COSDictionary kid
|
||||
&& kid.getDictionaryObject(COSName.getPDFName("MK"))
|
||||
instanceof COSDictionary mk) {
|
||||
out.add(mk);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static boolean isButtonField(COSDictionary dict) {
|
||||
COSBase ftBase = dict.getDictionaryObject(COSName.FT);
|
||||
COSName ft = ftBase instanceof COSName n ? n : null;
|
||||
if (ft == null) {
|
||||
COSBase parent = dict.getDictionaryObject(COSName.PARENT);
|
||||
if (parent instanceof COSDictionary p
|
||||
&& p.getDictionaryObject(COSName.FT) instanceof COSName pn) {
|
||||
ft = pn;
|
||||
}
|
||||
}
|
||||
return COSName.getPDFName("Btn").equals(ft);
|
||||
}
|
||||
|
||||
private static boolean dictValueMatches(
|
||||
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
COSBase value = dict.getDictionaryObject(key);
|
||||
if (value instanceof COSString cs) {
|
||||
return matches(cs.getString(), targets, patterns);
|
||||
}
|
||||
if (value instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
if (array.getObject(i) instanceof COSString element
|
||||
&& matches(element.getString(), targets, patterns)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static boolean fieldValueMatches(
|
||||
PDField field, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
String value = field.getValueAsString();
|
||||
return value != null && matches(value, targets, patterns);
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static void clearWidgetAppearances(COSDictionary fieldDict) {
|
||||
if (fieldDict == null) {
|
||||
return;
|
||||
}
|
||||
// The field itself may be a widget (single-widget field) and/or have Kids.
|
||||
fieldDict.removeItem(COSName.AP);
|
||||
COSBase kids = fieldDict.getDictionaryObject(COSName.KIDS);
|
||||
if (kids instanceof COSArray arr) {
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase kidBase = arr.getObject(i);
|
||||
if (kidBase instanceof COSDictionary kidDict) {
|
||||
kidDict.removeItem(COSName.AP);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Annotations
|
||||
|
||||
private static void scrubAnnotations(
|
||||
PDDocument document, Set<String> targets, List<Pattern> patterns) {
|
||||
try {
|
||||
for (PDPage page : document.getPages()) {
|
||||
// Page-level additional actions (/AA - open/close JS) can carry the target too.
|
||||
scrubActionIfMatching(
|
||||
page.getCOSObject(), COSName.getPDFName("AA"), targets, patterns);
|
||||
List<PDAnnotation> annotations;
|
||||
try {
|
||||
annotations = page.getAnnotations();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to load annotations for page: {}", e.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (annotations == null) {
|
||||
continue;
|
||||
}
|
||||
for (PDAnnotation annotation : annotations) {
|
||||
scrubAnnotation(annotation, targets, patterns);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Annotation scrub walk failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubAnnotation(
|
||||
PDAnnotation annotation, Set<String> targets, List<Pattern> patterns) {
|
||||
if (annotation == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String contents = annotation.getContents();
|
||||
if (contents != null) {
|
||||
String stripped = stripMatches(contents, targets, patterns);
|
||||
if (!stripped.equals(contents)) {
|
||||
annotation.setContents(stripped);
|
||||
}
|
||||
}
|
||||
COSDictionary dict = annotation.getCOSObject();
|
||||
scrubDictStrings(dict, COSName.getPDFName("RC"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Subj"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("NM"), targets, patterns);
|
||||
// Link/widget annotation actions can encode the target in a /URI or JS body.
|
||||
scrubActionIfMatching(dict, COSName.A, targets, patterns);
|
||||
scrubActionIfMatching(dict, COSName.getPDFName("AA"), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub annotation: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Structure tree
|
||||
|
||||
private static void scrubStructTree(
|
||||
PDStructureTreeRoot root, Set<String> targets, List<Pattern> patterns) {
|
||||
if (root == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
scrubStructDict(root.getCOSObject(), targets, patterns, new HashSet<>());
|
||||
} catch (Exception e) {
|
||||
log.debug("Structure tree scrub failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void scrubStructDict(
|
||||
COSBase base, Set<String> targets, List<Pattern> patterns, Set<COSBase> seen) {
|
||||
if (base == null) {
|
||||
return;
|
||||
}
|
||||
COSBase resolved = base instanceof COSObject obj ? obj.getObject() : base;
|
||||
if (resolved == null || !seen.add(resolved)) {
|
||||
return;
|
||||
}
|
||||
if (resolved instanceof COSDictionary dict) {
|
||||
// Do not walk into content streams - those are handled by content-stream
|
||||
if (resolved instanceof COSStream) {
|
||||
return;
|
||||
}
|
||||
scrubDictStrings(dict, COSName.getPDFName("ActualText"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Alt"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("E"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("T"), targets, patterns);
|
||||
scrubDictStrings(dict, COSName.getPDFName("Lang"), targets, patterns);
|
||||
for (COSName key : new HashSet<>(dict.keySet())) {
|
||||
COSBase value = dict.getItem(key);
|
||||
if (value instanceof COSDictionary
|
||||
|| value instanceof COSArray
|
||||
|| value instanceof COSObject) {
|
||||
scrubStructDict(value, targets, patterns, seen);
|
||||
}
|
||||
}
|
||||
} else if (resolved instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
scrubStructDict(array.getObject(i), targets, patterns, seen);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Names tree (JavaScript + embedded files)
|
||||
|
||||
private static void scrubNames(
|
||||
PDDocumentNameDictionary names, Set<String> targets, List<Pattern> patterns) {
|
||||
if (names == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
dropMatchingNames(names.getJavaScript(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub JavaScript names: {}", e.getMessage());
|
||||
}
|
||||
try {
|
||||
dropMatchingNames(names.getEmbeddedFiles(), targets, patterns);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to scrub embedded-file names: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void dropMatchingNames(
|
||||
PDNameTreeNode<?> node, Set<String> targets, List<Pattern> patterns) {
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
COSDictionary dict = node.getCOSObject();
|
||||
if (dict == null) {
|
||||
return;
|
||||
}
|
||||
scrubNameTreeDict(dict, targets, patterns);
|
||||
}
|
||||
|
||||
private static void scrubNameTreeDict(
|
||||
COSDictionary dict, Set<String> targets, List<Pattern> patterns) {
|
||||
if (dict == null) {
|
||||
return;
|
||||
}
|
||||
// Guarded casts: a malformed /Names or /Kids must skip that carrier, not throw a swallowed
|
||||
// ClassCastException.
|
||||
COSArray namesArray =
|
||||
dict.getDictionaryObject(COSName.NAMES) instanceof COSArray a ? a : null;
|
||||
if (namesArray != null) {
|
||||
Set<String> dropLiterals = carrierDropLiterals(targets);
|
||||
for (int i = namesArray.size() - 2; i >= 0; i -= 2) {
|
||||
COSBase keyBase = namesArray.getObject(i);
|
||||
String key = keyBase instanceof COSString s ? s.getString() : null;
|
||||
// Drop the pair when the KEY or the VALUE (JS /JS stream, embedded-file bytes)
|
||||
// contains a target - not just the key.
|
||||
boolean keyHit = key != null && matches(key, dropLiterals, patterns);
|
||||
boolean valueHit =
|
||||
i + 1 < namesArray.size()
|
||||
&& containsTarget(
|
||||
namesArray.getObject(i + 1),
|
||||
dropLiterals,
|
||||
patterns,
|
||||
new HashSet<>());
|
||||
if (keyHit || valueHit) {
|
||||
namesArray.remove(i + 1);
|
||||
namesArray.remove(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
COSArray kids = dict.getDictionaryObject(COSName.KIDS) instanceof COSArray a ? a : null;
|
||||
if (kids != null) {
|
||||
for (int i = 0; i < kids.size(); i++) {
|
||||
COSBase kid = kids.getObject(i);
|
||||
if (kid instanceof COSDictionary kidDict) {
|
||||
scrubNameTreeDict(kidDict, targets, patterns);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helpers
|
||||
|
||||
private static void scrubDictStrings(
|
||||
COSDictionary dict, COSName key, Set<String> targets, List<Pattern> patterns) {
|
||||
if (dict == null || key == null) {
|
||||
return;
|
||||
}
|
||||
COSBase value = dict.getDictionaryObject(key);
|
||||
if (value instanceof COSString cosString) {
|
||||
String stripped = stripMatches(cosString.getString(), targets, patterns);
|
||||
if (!stripped.equals(cosString.getString())) {
|
||||
dict.setString(key, stripped);
|
||||
}
|
||||
} else if (value instanceof COSArray array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
COSBase element = array.getObject(i);
|
||||
if (element instanceof COSString elementString) {
|
||||
String stripped = stripMatches(elementString.getString(), targets, patterns);
|
||||
if (!stripped.equals(elementString.getString())) {
|
||||
array.set(i, new COSString(stripped));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static String stripMatches(String source, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (source == null || source.isEmpty()) {
|
||||
return source;
|
||||
}
|
||||
String result = source;
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// Case-insensitive literal removal. Verification is case-insensitive
|
||||
result = caseInsensitiveReplaceAll(result, target);
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
// Force case-insensitive matching for catalog carriers regardless
|
||||
Pattern ci = withCaseInsensitive(pattern);
|
||||
result = ci.matcher(DeadlineCharSequence.of(result)).replaceAll("");
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
// Fail closed: a throwing regex means we cannot prove the carrier clean, so
|
||||
// drop the whole string rather than leaving it intact.
|
||||
log.warn("Pattern replace failed for {}; dropping carrier text", pattern);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static boolean matches(String source, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (source == null || source.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
String lower = source.toLowerCase(Locale.ROOT);
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target != null
|
||||
&& !target.isEmpty()
|
||||
&& lower.contains(target.toLowerCase(Locale.ROOT))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
if (withCaseInsensitive(pattern)
|
||||
.matcher(DeadlineCharSequence.of(source))
|
||||
.find()) {
|
||||
return true;
|
||||
}
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
// Fail closed: a throwing regex counts as a match so the carrier is scrubbed.
|
||||
log.warn("Pattern match failed for {}; treating carrier as a match", pattern);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String caseInsensitiveReplaceAll(String source, String target) {
|
||||
if (target.isEmpty()) {
|
||||
return source;
|
||||
}
|
||||
Pattern literal =
|
||||
Pattern.compile(
|
||||
Pattern.quote(target), Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
|
||||
return literal.matcher(source).replaceAll("");
|
||||
}
|
||||
|
||||
private static Pattern withCaseInsensitive(Pattern pattern) {
|
||||
if ((pattern.flags() & Pattern.CASE_INSENSITIVE) != 0) {
|
||||
return pattern;
|
||||
}
|
||||
try {
|
||||
return Pattern.compile(
|
||||
pattern.pattern(),
|
||||
pattern.flags() | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
|
||||
} catch (Exception e) {
|
||||
return pattern;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
/**
|
||||
* Wraps a match subject so a runaway (catastrophic-backtracking) regex aborts instead of hanging
|
||||
* the request thread: every {@link #charAt(int)} the matcher performs checks a wall-clock deadline
|
||||
* and throws once it is exceeded. The thrown {@link RegexTimeoutException} is a {@link
|
||||
* RuntimeException}, so the redaction match sites' existing fail-closed catches handle it.
|
||||
*/
|
||||
final class DeadlineCharSequence implements CharSequence {
|
||||
|
||||
/** Per-match wall-clock budget. A legitimate whole-document match finishes far inside this. */
|
||||
static final long DEFAULT_BUDGET_MILLIS = 2_000L;
|
||||
|
||||
static final class RegexTimeoutException extends RuntimeException {
|
||||
RegexTimeoutException(long budgetMillis) {
|
||||
super("Regex evaluation exceeded " + budgetMillis + " ms (possible ReDoS)");
|
||||
}
|
||||
}
|
||||
|
||||
private final CharSequence inner;
|
||||
private final long deadlineNanos;
|
||||
private final long budgetMillis;
|
||||
|
||||
private DeadlineCharSequence(CharSequence inner, long budgetMillis) {
|
||||
this.inner = inner;
|
||||
this.budgetMillis = budgetMillis;
|
||||
this.deadlineNanos = System.nanoTime() + budgetMillis * 1_000_000L;
|
||||
}
|
||||
|
||||
/** Wrap {@code text} with the default budget; null becomes an empty sequence. */
|
||||
static DeadlineCharSequence of(String text) {
|
||||
return new DeadlineCharSequence(text == null ? "" : text, DEFAULT_BUDGET_MILLIS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public char charAt(int index) {
|
||||
if (System.nanoTime() > deadlineNanos) {
|
||||
throw new RegexTimeoutException(budgetMillis);
|
||||
}
|
||||
return inner.charAt(index);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int length() {
|
||||
return inner.length();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharSequence subSequence(int start, int end) {
|
||||
return inner.subSequence(start, end);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return inner.toString();
|
||||
}
|
||||
}
|
||||
+748
@@ -0,0 +1,748 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSFloat;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSString;
|
||||
import org.apache.pdfbox.pdfparser.PDFStreamParser;
|
||||
import org.apache.pdfbox.pdfwriter.ContentStreamWriter;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.common.PDStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.text.PDFTextStripperByArea;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
|
||||
/**
|
||||
* Content-stream editing: physically removes glyphs from page and form-XObject streams, both by
|
||||
* literal/regex match (auto-word) and by rect intersection (manual area), plus the area-capture and
|
||||
* overlay/annotation helpers those paths rely on.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RedactionContentEditor {
|
||||
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
|
||||
private RedactionContentEditor() {}
|
||||
|
||||
/** True only for upright pages whose CropBox origin is (0,0), where the rect flip is exact. */
|
||||
static boolean isSurgicallySafe(PDPage page) {
|
||||
if (page.getRotation() != 0) {
|
||||
return false;
|
||||
}
|
||||
PDRectangle crop = page.getCropBox();
|
||||
return crop.getLowerLeftX() == 0f && crop.getLowerLeftY() == 0f;
|
||||
}
|
||||
|
||||
/** Drop annotations whose rectangle overlaps any redaction rect. */
|
||||
static void removeOverlappingAnnotations(PDPage page, List<PDRectangle> rects) {
|
||||
try {
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
for (var ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
boolean overlaps = false;
|
||||
if (ar != null) {
|
||||
Rectangle2D.Float a =
|
||||
new Rectangle2D.Float(
|
||||
ar.getLowerLeftX(),
|
||||
ar.getLowerLeftY(),
|
||||
ar.getWidth(),
|
||||
ar.getHeight());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (a.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
overlaps = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!overlaps) {
|
||||
kept.add(ann);
|
||||
}
|
||||
}
|
||||
page.setAnnotations(kept);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not strip overlapping annotations: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
static List<Pattern> effectivePatterns(Set<String> literalTargets, List<Pattern> patterns) {
|
||||
List<Pattern> result = new ArrayList<>();
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// Case-insensitive to match TextFinderUtils; removal must not miss finder hits.
|
||||
result.add(
|
||||
Pattern.compile(
|
||||
Pattern.quote(target),
|
||||
Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE));
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
result.addAll(patterns);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Per-page content-stream rewrite (literal/regex based)
|
||||
|
||||
static void rewritePageContent(PDDocument document, PDPage page, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
PDResources resources = page.getResources();
|
||||
if (resources == null) {
|
||||
return;
|
||||
}
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(page));
|
||||
boolean modified = rewriteTokens(tokens, resources, patterns);
|
||||
if (modified) {
|
||||
writePageTokens(document, page, tokens);
|
||||
}
|
||||
// Recurse into form XObjects referenced by this page (shared visited set + depth cap).
|
||||
rewriteFormXObjects(resources, resources, patterns, new HashSet<>(), 0);
|
||||
}
|
||||
|
||||
private static void rewriteFormXObjects(
|
||||
PDResources resources,
|
||||
PDResources parentResources,
|
||||
List<Pattern> patterns,
|
||||
Set<COSBase> visited,
|
||||
int depth)
|
||||
throws IOException {
|
||||
if (depth > MAX_XOBJECT_DEPTH) {
|
||||
log.warn("XObject nesting exceeded {}; stopping recursion", MAX_XOBJECT_DEPTH);
|
||||
return;
|
||||
}
|
||||
for (COSName name : resources.getXObjectNames()) {
|
||||
try {
|
||||
var xobj = resources.getXObject(name);
|
||||
if (!(xobj instanceof PDFormXObject form)) {
|
||||
continue;
|
||||
}
|
||||
if (!visited.add(form.getCOSObject())) {
|
||||
continue;
|
||||
}
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(form));
|
||||
// Forms may inherit fonts from the parent when they carry no own /Resources.
|
||||
PDResources formResources =
|
||||
form.getResources() != null ? form.getResources() : parentResources;
|
||||
boolean modified = rewriteTokens(tokens, formResources, patterns);
|
||||
if (modified) {
|
||||
// A form XObject's content IS its own stream body; overwrite it in place.
|
||||
PDStream formStream = new PDStream(form.getCOSObject());
|
||||
try (var out = formStream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
new ContentStreamWriter(out).writeTokens(tokens);
|
||||
}
|
||||
}
|
||||
rewriteFormXObjects(formResources, formResources, patterns, visited, depth + 1);
|
||||
} catch (IOException e) {
|
||||
log.debug("Failed to rewrite XObject {}: {}", name.getName(), e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Object> parseTokens(PDFStreamParser parser) throws IOException {
|
||||
List<Object> tokens = new ArrayList<>();
|
||||
Object t;
|
||||
while ((t = parser.parseNextToken()) != null) {
|
||||
tokens.add(t);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/** Walk tokens keeping a tiny text state (current font). */
|
||||
private static boolean rewriteTokens(
|
||||
List<Object> tokens, PDResources resources, List<Pattern> patterns) {
|
||||
boolean modified = false;
|
||||
PDFont currentFont = null;
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
Object tok = tokens.get(i);
|
||||
if (!(tok instanceof Operator op)) {
|
||||
continue;
|
||||
}
|
||||
String name = op.getName();
|
||||
if ("Tf".equals(name) && i >= 2) {
|
||||
Object fontNameTok = tokens.get(i - 2);
|
||||
if (fontNameTok instanceof COSName fontName) {
|
||||
try {
|
||||
currentFont = resources.getFont(fontName);
|
||||
} catch (IOException ex) {
|
||||
log.debug(
|
||||
"Could not resolve font {}: {}",
|
||||
fontName.getName(),
|
||||
ex.getMessage());
|
||||
currentFont = null;
|
||||
}
|
||||
}
|
||||
} else if (RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name) && i >= 1) {
|
||||
int operandIdx = i - 1;
|
||||
Object operand = tokens.get(operandIdx);
|
||||
if (operand instanceof COSString cosString) {
|
||||
COSString replacement = rewriteCosString(cosString, currentFont, patterns);
|
||||
if (replacement != null) {
|
||||
tokens.set(operandIdx, replacement);
|
||||
modified = true;
|
||||
}
|
||||
} else if (operand instanceof COSArray arr) {
|
||||
COSArray newArr = rewriteCosArray(arr, currentFont, patterns);
|
||||
if (newArr != null) {
|
||||
tokens.set(operandIdx, newArr);
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return modified;
|
||||
}
|
||||
|
||||
private static COSString rewriteCosString(
|
||||
COSString cosString, PDFont font, List<Pattern> patterns) {
|
||||
if (font == null) {
|
||||
// Without a font we cannot decode safely.
|
||||
return rewriteRawLatin(cosString, patterns);
|
||||
}
|
||||
DecodeResult decoded = decodeCosString(cosString, font);
|
||||
if (decoded == null) {
|
||||
return rewriteRawLatin(cosString, patterns);
|
||||
}
|
||||
boolean[] drop = findDroppedCharsMask(decoded.text, patterns);
|
||||
if (drop == null) {
|
||||
return null;
|
||||
}
|
||||
return buildFilteredCosString(decoded, drop, font);
|
||||
}
|
||||
|
||||
private static COSArray rewriteCosArray(COSArray arr, PDFont font, List<Pattern> patterns) {
|
||||
// Build a concatenated decode across all COSString elements so that matches
|
||||
List<DecodeResult> parts = new ArrayList<>();
|
||||
StringBuilder concat = new StringBuilder();
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase elem = arr.get(i);
|
||||
if (elem instanceof COSString cs) {
|
||||
DecodeResult decoded = font != null ? decodeCosString(cs, font) : null;
|
||||
if (decoded == null) {
|
||||
decoded = decodeAsLatin(cs);
|
||||
}
|
||||
parts.add(decoded);
|
||||
concat.append(decoded.text);
|
||||
} else {
|
||||
parts.add(null);
|
||||
}
|
||||
}
|
||||
boolean[] fullDrop = findDroppedCharsMask(concat.toString(), patterns);
|
||||
if (fullDrop == null) {
|
||||
return null;
|
||||
}
|
||||
COSArray out = new COSArray();
|
||||
int cursor = 0;
|
||||
for (int i = 0; i < arr.size(); i++) {
|
||||
COSBase elem = arr.get(i);
|
||||
if (elem instanceof COSString cs) {
|
||||
DecodeResult decoded = parts.get(i);
|
||||
int partLen = decoded.text.length();
|
||||
boolean[] partDrop = new boolean[partLen];
|
||||
System.arraycopy(fullDrop, cursor, partDrop, 0, partLen);
|
||||
cursor += partLen;
|
||||
COSString rebuilt =
|
||||
buildFilteredCosStringRaw(decoded, partDrop, font, cs.getBytes());
|
||||
// A zero-length COSString is valid; PDFBox emits it as () producing no
|
||||
out.add(rebuilt);
|
||||
} else {
|
||||
out.add(elem);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** For cases where we have no font - treat the bytes as latin-1 characters. */
|
||||
private static COSString rewriteRawLatin(COSString cosString, List<Pattern> patterns) {
|
||||
DecodeResult decoded = decodeAsLatin(cosString);
|
||||
boolean[] drop = findDroppedCharsMask(decoded.text, patterns);
|
||||
if (drop == null) {
|
||||
return null;
|
||||
}
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
for (int i = 0; i < decoded.text.length(); i++) {
|
||||
if (drop[i]) continue;
|
||||
out.write(decoded.text.charAt(i) & 0xFF);
|
||||
}
|
||||
return new COSString(out.toByteArray());
|
||||
}
|
||||
|
||||
private static DecodeResult decodeAsLatin(COSString cosString) {
|
||||
byte[] bytes = cosString.getBytes();
|
||||
StringBuilder sb = new StringBuilder(bytes.length);
|
||||
int[] codeStart = new int[bytes.length];
|
||||
int[] codeLen = new int[bytes.length];
|
||||
for (int i = 0; i < bytes.length; i++) {
|
||||
sb.append((char) (bytes[i] & 0xFF));
|
||||
codeStart[i] = i;
|
||||
codeLen[i] = 1;
|
||||
}
|
||||
return new DecodeResult(sb.toString(), bytes, codeStart, codeLen);
|
||||
}
|
||||
|
||||
/** Decode a COSString to Unicode via the font; null if decoding fails. */
|
||||
private static DecodeResult decodeCosString(COSString cosString, PDFont font) {
|
||||
byte[] bytes = cosString.getBytes();
|
||||
StringBuilder text = new StringBuilder();
|
||||
List<Integer> starts = new ArrayList<>();
|
||||
List<Integer> lens = new ArrayList<>();
|
||||
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes)) {
|
||||
int pos = 0;
|
||||
while (in.available() > 0) {
|
||||
int before = in.available();
|
||||
int code;
|
||||
try {
|
||||
code = font.readCode(in);
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
log.debug(
|
||||
"Font {} failed to decode byte sequence: {}",
|
||||
font.getName(),
|
||||
ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
int consumed = before - in.available();
|
||||
String unicode;
|
||||
try {
|
||||
unicode = font.toUnicode(code);
|
||||
} catch (Exception ex) {
|
||||
unicode = null;
|
||||
}
|
||||
if (unicode == null) {
|
||||
// If the font has no ToUnicode mapping we cannot match reliably - return
|
||||
return null;
|
||||
}
|
||||
text.append(unicode);
|
||||
// Associate every Unicode character produced with the same code byte range
|
||||
for (int c = 0; c < unicode.length(); c++) {
|
||||
starts.add(pos);
|
||||
lens.add(consumed);
|
||||
}
|
||||
if (consumed == 0) {
|
||||
// Defensive: avoid infinite loop on malformed fonts.
|
||||
break;
|
||||
}
|
||||
pos += consumed;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
}
|
||||
int[] startArr = starts.stream().mapToInt(Integer::intValue).toArray();
|
||||
int[] lenArr = lens.stream().mapToInt(Integer::intValue).toArray();
|
||||
return new DecodeResult(text.toString(), bytes, startArr, lenArr);
|
||||
}
|
||||
|
||||
/** Null if no pattern matches; else a per-char drop mask over the text. */
|
||||
private static boolean[] findDroppedCharsMask(String text, List<Pattern> patterns) {
|
||||
boolean any = false;
|
||||
boolean[] mask = new boolean[text.length()];
|
||||
for (Pattern pattern : patterns) {
|
||||
Matcher m;
|
||||
try {
|
||||
m = pattern.matcher(DeadlineCharSequence.of(text));
|
||||
} catch (Exception ex) {
|
||||
continue;
|
||||
}
|
||||
while (m.find()) {
|
||||
int s = m.start();
|
||||
int e = m.end();
|
||||
if (e <= s) continue;
|
||||
for (int i = s; i < e; i++) {
|
||||
mask[i] = true;
|
||||
}
|
||||
any = true;
|
||||
}
|
||||
}
|
||||
return any ? mask : null;
|
||||
}
|
||||
|
||||
private static COSString buildFilteredCosString(
|
||||
DecodeResult decoded, boolean[] drop, PDFont font) {
|
||||
return buildFilteredCosStringRaw(decoded, drop, font, decoded.bytes);
|
||||
}
|
||||
|
||||
private static COSString buildFilteredCosStringRaw(
|
||||
DecodeResult decoded, boolean[] drop, PDFont font, byte[] originalBytes) {
|
||||
// Collect code byte-ranges to drop.
|
||||
Set<Integer> dropStarts = new HashSet<>();
|
||||
for (int i = 0; i < drop.length; i++) {
|
||||
if (drop[i]) {
|
||||
dropStarts.add(decoded.codeStarts[i]);
|
||||
}
|
||||
}
|
||||
// Map byte-start -> code length once (O(n)); findCodeLenAt was an O(n^2) linear scan.
|
||||
Map<Integer, Integer> lenByStart = new HashMap<>();
|
||||
for (int j = 0; j < decoded.codeStarts.length; j++) {
|
||||
lenByStart.putIfAbsent(decoded.codeStarts[j], decoded.codeLens[j]);
|
||||
}
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
int i = 0;
|
||||
while (i < originalBytes.length) {
|
||||
int start = i;
|
||||
int len = lenByStart.getOrDefault(start, -1);
|
||||
if (len <= 0) {
|
||||
// Unknown - keep the byte verbatim.
|
||||
out.write(originalBytes[i] & 0xFF);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (dropStarts.contains(start)) {
|
||||
// Replace dropped code with encoded space if possible.
|
||||
byte[] spaceBytes = tryEncodeSpace(font);
|
||||
if (spaceBytes != null) {
|
||||
out.write(spaceBytes, 0, spaceBytes.length);
|
||||
}
|
||||
} else {
|
||||
out.write(originalBytes, start, len);
|
||||
}
|
||||
i += len;
|
||||
}
|
||||
return new COSString(out.toByteArray());
|
||||
}
|
||||
|
||||
private static byte[] tryEncodeSpace(PDFont font) {
|
||||
if (font == null) {
|
||||
return new byte[] {0x20};
|
||||
}
|
||||
try {
|
||||
return font.encode(" ");
|
||||
} catch (Exception e) {
|
||||
// Some fonts cannot encode a space at all (e.g. Type3 throws
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class DecodeResult {
|
||||
final String text;
|
||||
final byte[] bytes;
|
||||
final int[] codeStarts;
|
||||
final int[] codeLens;
|
||||
|
||||
DecodeResult(String text, byte[] bytes, int[] codeStarts, int[] codeLens) {
|
||||
this.text = text;
|
||||
this.bytes = bytes;
|
||||
this.codeStarts = codeStarts;
|
||||
this.codeLens = codeLens;
|
||||
}
|
||||
}
|
||||
|
||||
// Area capture
|
||||
|
||||
static List<String> captureTextInRects(PDPage page, List<PDRectangle> rects)
|
||||
throws IOException {
|
||||
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
|
||||
stripper.setSortByPosition(true);
|
||||
for (int i = 0; i < rects.size(); i++) {
|
||||
PDRectangle rect = rects.get(i);
|
||||
// PDFTextStripperByArea uses a Java2D rectangle in the same coordinate
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
Rectangle2D.Float region =
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight());
|
||||
stripper.addRegion("r" + i, region);
|
||||
}
|
||||
try {
|
||||
stripper.extractRegions(page);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to extract text in rects: {}", e.getMessage());
|
||||
return Collections.emptyList();
|
||||
}
|
||||
Set<String> captured = new LinkedHashSet<>();
|
||||
for (int i = 0; i < rects.size(); i++) {
|
||||
String text = stripper.getTextForRegion("r" + i);
|
||||
if (text == null) {
|
||||
continue;
|
||||
}
|
||||
for (String token : text.split("\\s+")) {
|
||||
String trimmed = token.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
captured.add(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ArrayList<>(captured);
|
||||
}
|
||||
|
||||
// Content-stream rewriting (rect-driven glyph removal)
|
||||
|
||||
/**
|
||||
* Blanks show-text operands whose glyphs fall in a rect. Returns false (surgical removal is
|
||||
* unreliable, caller must rasterise) when text-bearing form XObjects skew the operator ordinals
|
||||
* or an image sits under a rect - leaving the page stream untouched so the raster is correct.
|
||||
*/
|
||||
static boolean removeTokensIntersectingRects(
|
||||
PDDocument document, PDPage page, List<PDRectangle> rects) throws IOException {
|
||||
|
||||
List<Rectangle2D.Float> areaRects = new ArrayList<>();
|
||||
for (PDRectangle rect : rects) {
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
areaRects.add(
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight()));
|
||||
}
|
||||
|
||||
int pageIndex = document.getPages().indexOf(page);
|
||||
TokenIndexCollector collector = new TokenIndexCollector(areaRects);
|
||||
collector.setStartPage(pageIndex + 1);
|
||||
collector.setEndPage(pageIndex + 1);
|
||||
collector.getText(document);
|
||||
int collectorTextOps = collector.totalTextOps();
|
||||
|
||||
List<Object> tokens = parseTokens(new PDFStreamParser(page));
|
||||
int pageTextOps = 0;
|
||||
for (Object t : tokens) {
|
||||
if (t instanceof Operator op
|
||||
&& RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(op.getName())) {
|
||||
pageTextOps++;
|
||||
}
|
||||
}
|
||||
// Collector ordinals span form XObjects; images under a rect can't be surgically removed.
|
||||
if (collectorTextOps != pageTextOps || imageIntersectsAnyRect(page, pageIndex, rects)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Drop ONLY the in-rect glyphs of each operand (not the whole run), tracking the active
|
||||
// font so codes decode. Bail to rasterise if the font can't decode or glyph!=code counts.
|
||||
PDResources resources = page.getResources();
|
||||
PDFont currentFont = null;
|
||||
int opCounter = -1;
|
||||
boolean modified = false;
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
Object tok = tokens.get(i);
|
||||
if (!(tok instanceof Operator op)) {
|
||||
continue;
|
||||
}
|
||||
String name = op.getName();
|
||||
if ("Tf".equals(name) && i >= 2 && tokens.get(i - 2) instanceof COSName fn) {
|
||||
try {
|
||||
currentFont = resources != null ? resources.getFont(fn) : null;
|
||||
} catch (IOException e) {
|
||||
currentFont = null;
|
||||
}
|
||||
} else if (RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name)) {
|
||||
opCounter++;
|
||||
Set<Integer> dropGlyphs = collector.dropGlyphsByOp.get(opCounter);
|
||||
if (dropGlyphs == null || dropGlyphs.isEmpty() || i < 1) {
|
||||
continue;
|
||||
}
|
||||
// ' and " also move to the next line; converting them to TJ would lose that -
|
||||
// raster.
|
||||
if ("'".equals(name) || "\"".equals(name)) {
|
||||
return false;
|
||||
}
|
||||
int expected = collector.glyphCountByOp.getOrDefault(opCounter, -1);
|
||||
COSArray rebuilt =
|
||||
rebuildAdvancePreserving(
|
||||
tokens.get(i - 1), currentFont, dropGlyphs, expected);
|
||||
if (rebuilt == null) {
|
||||
return false; // undecodable / glyph-count mismatch - rasterise instead
|
||||
}
|
||||
// Emit as TJ so the removed glyphs' advance is preserved (no reflow).
|
||||
tokens.set(i - 1, rebuilt);
|
||||
tokens.set(i, Operator.getOperator("TJ"));
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
writePageTokens(document, page, tokens);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Distinct code byte-starts in order (one per glyph/code) from a per-char DecodeResult. */
|
||||
private static int[] distinctCodeStarts(DecodeResult d) {
|
||||
List<Integer> starts = new ArrayList<>();
|
||||
int last = -1;
|
||||
for (int s : d.codeStarts) {
|
||||
if (s != last) {
|
||||
starts.add(s);
|
||||
last = s;
|
||||
}
|
||||
}
|
||||
return starts.stream().mapToInt(Integer::intValue).toArray();
|
||||
}
|
||||
|
||||
private static int codeAt(PDFont font, byte[] bytes, int start, int len) throws IOException {
|
||||
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes, start, len)) {
|
||||
return font.readCode(in);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild a Tj/TJ operand as a TJ array that keeps every glyph EXCEPT the given 0-based
|
||||
* indexes, replacing each dropped run with a numeric adjustment equal to its advance so the
|
||||
* surviving text does not reflow into (or out of) the box. Returns null (caller rasterises) if
|
||||
* anything can't be decoded or the glyph count doesn't line up with the position pass.
|
||||
*/
|
||||
private static COSArray rebuildAdvancePreserving(
|
||||
Object operand, PDFont font, Set<Integer> dropGlyphs, int expectedGlyphs) {
|
||||
if (font == null) {
|
||||
return null;
|
||||
}
|
||||
List<COSBase> elements = new ArrayList<>();
|
||||
if (operand instanceof COSString cs) {
|
||||
elements.add(cs);
|
||||
} else if (operand instanceof COSArray arr) {
|
||||
for (COSBase b : arr) {
|
||||
elements.add(b);
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
COSArray out = new COSArray();
|
||||
ByteArrayOutputStream seg = new ByteArrayOutputStream();
|
||||
float pendingAdvance = 0f; // accumulated width of dropped codes, in 1/1000 text units
|
||||
int globalCode = 0;
|
||||
for (COSBase el : elements) {
|
||||
if (!(el instanceof COSString s)) {
|
||||
// Existing numeric adjustment: flush whichever run is pending, then preserve it.
|
||||
if (seg.size() > 0) {
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
seg.reset();
|
||||
} else if (pendingAdvance != 0f) {
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
pendingAdvance = 0f;
|
||||
}
|
||||
out.add(el);
|
||||
continue;
|
||||
}
|
||||
DecodeResult d = decodeCosString(s, font);
|
||||
if (d == null) {
|
||||
return null;
|
||||
}
|
||||
byte[] b = s.getBytes();
|
||||
int[] starts = distinctCodeStarts(d);
|
||||
Map<Integer, Integer> lenByStart = new HashMap<>();
|
||||
for (int j = 0; j < d.codeStarts.length; j++) {
|
||||
lenByStart.putIfAbsent(d.codeStarts[j], d.codeLens[j]);
|
||||
}
|
||||
for (int k = 0; k < starts.length; k++) {
|
||||
int bs = starts[k];
|
||||
int len = lenByStart.getOrDefault(bs, -1);
|
||||
if (len <= 0) {
|
||||
return null;
|
||||
}
|
||||
if (dropGlyphs.contains(globalCode)) {
|
||||
if (seg.size() > 0) { // close the kept run before accumulating advance
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
seg.reset();
|
||||
}
|
||||
try {
|
||||
pendingAdvance += font.getWidth(codeAt(font, b, bs, len));
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
if (pendingAdvance != 0f) { // emit the removed run's advance, then keep glyphs
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
pendingAdvance = 0f;
|
||||
}
|
||||
seg.write(b, bs, len);
|
||||
}
|
||||
globalCode++;
|
||||
}
|
||||
}
|
||||
if (pendingAdvance != 0f) {
|
||||
out.add(new COSFloat(-pendingAdvance));
|
||||
}
|
||||
if (seg.size() > 0) {
|
||||
out.add(new COSString(seg.toByteArray()));
|
||||
}
|
||||
if (expectedGlyphs >= 0 && globalCode != expectedGlyphs) {
|
||||
return null;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any image on the page overlaps a redaction rect (would survive under the overlay).
|
||||
*/
|
||||
private static boolean imageIntersectsAnyRect(
|
||||
PDPage page, int pageIndex, List<PDRectangle> rects) {
|
||||
try {
|
||||
PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
locator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) {
|
||||
Rectangle2D.Float img =
|
||||
new Rectangle2D.Float(
|
||||
ib.x1(), ib.y1(), ib.x2() - ib.x1(), ib.y2() - ib.y1());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (img.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
// Cannot prove no image sits under a box - treat as unsafe (force raster).
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static void writePageTokens(PDDocument document, PDPage page, List<Object> tokens)
|
||||
throws IOException {
|
||||
PDStream stream = new PDStream(document);
|
||||
try (var out = stream.createOutputStream(COSName.FLATE_DECODE)) {
|
||||
ContentStreamWriter writer = new ContentStreamWriter(out);
|
||||
writer.writeTokens(tokens);
|
||||
}
|
||||
page.setContents(stream);
|
||||
}
|
||||
|
||||
static void drawOverlay(
|
||||
PDDocument document, PDPage page, List<PDRectangle> rects, Color overlayColor)
|
||||
throws IOException {
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(overlayColor);
|
||||
for (PDRectangle rect : rects) {
|
||||
cs.addRect(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight());
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.utils.text.TextFinderUtils;
|
||||
import stirling.software.common.util.RegexPatternUtils;
|
||||
|
||||
/**
|
||||
* Public facade for the manual-area, whole-page and auto-word redaction paths. Orchestrates the
|
||||
* collaborators that do the work: {@link RedactionContentEditor} (content-stream glyph removal),
|
||||
* {@link RedactionVerifier} (fail-closed independent verification) and {@link RedactionRasteriser}
|
||||
* (last-resort rasterisation), with {@link CatalogScrubber} for out-of-page carriers.
|
||||
*/
|
||||
@Slf4j
|
||||
public final class RedactionPipeline {
|
||||
|
||||
public static final Set<String> TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\"");
|
||||
|
||||
private RedactionPipeline() {}
|
||||
|
||||
/** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */
|
||||
static void setJpdfiumAvailableForTest(boolean available) {
|
||||
RedactionVerifier.setJpdfiumAvailableForTest(available);
|
||||
}
|
||||
|
||||
/** Redact page-local rects (PDF user-space), dropping intersecting glyphs + overlay. */
|
||||
public static RedactionResult redactAreas(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDRectangle>> rectsByPageIndex,
|
||||
Color overlayColor)
|
||||
throws IOException {
|
||||
|
||||
Set<String> capturedStrings = new LinkedHashSet<>();
|
||||
Set<Integer> forceRasterPages = new LinkedHashSet<>();
|
||||
|
||||
for (Map.Entry<Integer, List<PDRectangle>> entry : rectsByPageIndex.entrySet()) {
|
||||
int pageIndex = entry.getKey();
|
||||
List<PDRectangle> rects = entry.getValue();
|
||||
if (pageIndex < 0 || pageIndex >= document.getNumberOfPages() || rects.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
capturedStrings.addAll(RedactionContentEditor.captureTextInRects(page, rects));
|
||||
|
||||
// Rotation / non-zero CropBox origin break the coordinate flip, and text-bearing forms
|
||||
// or in-rect images defeat page-stream ordinal removal.
|
||||
boolean surgical =
|
||||
RedactionContentEditor.isSurgicallySafe(page)
|
||||
&& RedactionContentEditor.removeTokensIntersectingRects(
|
||||
document, page, rects);
|
||||
if (!surgical) {
|
||||
forceRasterPages.add(pageIndex);
|
||||
}
|
||||
RedactionContentEditor.removeOverlappingAnnotations(page, rects);
|
||||
RedactionContentEditor.drawOverlay(document, page, rects, overlayColor);
|
||||
}
|
||||
|
||||
return new RedactionResult(capturedStrings, forceRasterPages);
|
||||
}
|
||||
|
||||
/** Wipe whole pages: drop all content/resources, fill with a rectangle. */
|
||||
public static void redactWholePages(
|
||||
PDDocument document, List<Integer> pageIndexes, Color overlayColor) throws IOException {
|
||||
for (Integer pageIndex : pageIndexes) {
|
||||
if (pageIndex == null || pageIndex < 0 || pageIndex >= document.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PDRectangle media = page.getMediaBox();
|
||||
|
||||
// Drop existing content streams and page resources outright.
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
page.getCOSObject().removeItem(COSName.ANNOTS);
|
||||
// /Thumb is a rendered image of the original page - drop it too.
|
||||
page.getCOSObject().removeItem(COSName.getPDFName("Thumb"));
|
||||
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.OVERWRITE, true, true)) {
|
||||
cs.setNonStrokingColor(overlayColor);
|
||||
cs.addRect(
|
||||
media.getLowerLeftX(),
|
||||
media.getLowerLeftY(),
|
||||
media.getWidth(),
|
||||
media.getHeight());
|
||||
cs.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Physically remove every literal/regex match from all page content streams. */
|
||||
public static void redactLiteralTerms(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
List<Pattern> effectivePatterns =
|
||||
RedactionContentEditor.effectivePatterns(literalTargets, patterns);
|
||||
if (effectivePatterns.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int pageIndex = 0;
|
||||
for (PDPage page : document.getPages()) {
|
||||
try {
|
||||
RedactionContentEditor.rewritePageContent(document, page, effectivePatterns);
|
||||
} catch (IOException | RuntimeException e) {
|
||||
// Never let one page's font quirk (e.g. Type3 encode, damaged program)
|
||||
log.warn(
|
||||
"Content-stream rewrite failed on page {} ({}); leaving it for the "
|
||||
+ "verification/rasterisation pass.",
|
||||
pageIndex + 1,
|
||||
e.toString());
|
||||
}
|
||||
pageIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalize with document-wide verification: scrub, wipe metadata, save, verify. */
|
||||
public static byte[] finalize(
|
||||
PDDocument document, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
return finalize(document, literalTargets, patterns, null);
|
||||
}
|
||||
|
||||
/** Finalize with verification scoped to a specific set of page indexes. */
|
||||
public static byte[] finalize(
|
||||
PDDocument document,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> affectedPages)
|
||||
throws IOException {
|
||||
|
||||
CatalogScrubber.scrub(document, literalTargets, patterns);
|
||||
RedactionVerifier.warnAboutEmbeddedFontGlyphs(document);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
try {
|
||||
RedactionVerifier.verify(bytes, literalTargets, patterns, affectedPages);
|
||||
return bytes;
|
||||
} catch (RedactionVerificationFailedException primaryFailure) {
|
||||
// Rewriter could not guarantee removal; rasterise as a last resort.
|
||||
log.warn(
|
||||
"Primary redaction verification failed ({}); falling back to page-scoped "
|
||||
+ "rasterisation to guarantee removal.",
|
||||
primaryFailure.getMessage());
|
||||
// With no page scope (auto-word), rasterise only the pages that still leak.
|
||||
Set<Integer> pagesToRaster =
|
||||
(affectedPages == null || affectedPages.isEmpty())
|
||||
? RedactionVerifier.findLeakingPages(bytes, literalTargets, patterns)
|
||||
: new HashSet<>(affectedPages);
|
||||
try (PDDocument rasterised = RedactionRasteriser.rasterisePages(bytes, pagesToRaster)) {
|
||||
CatalogScrubber.scrub(rasterised, literalTargets, patterns);
|
||||
ByteArrayOutputStream rasterOut = new ByteArrayOutputStream();
|
||||
rasterised.save(rasterOut);
|
||||
byte[] rasterBytes = rasterOut.toByteArray();
|
||||
RedactionVerifier.verify(rasterBytes, literalTargets, patterns, affectedPages);
|
||||
return rasterBytes;
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Rasterisation fallback failed after primary redaction leak", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalize a manual area redaction; rasterises forced + verified-leaking pages. */
|
||||
public static byte[] finalizeAreas(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDRectangle>> rectsByPage,
|
||||
Set<Integer> forceRasterPages,
|
||||
Set<String> capturedTargets)
|
||||
throws IOException {
|
||||
|
||||
// Text captured under the rects may also live in a bookmark / form value / annotation / JS
|
||||
// carrier.
|
||||
Set<String> carrierTargets =
|
||||
capturedTargets == null ? Collections.emptySet() : capturedTargets;
|
||||
CatalogScrubber.scrub(document, carrierTargets, Collections.emptyList());
|
||||
RedactionVerifier.warnAboutEmbeddedFontGlyphs(document);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
Set<Integer> toRaster =
|
||||
new HashSet<>(forceRasterPages == null ? Set.of() : forceRasterPages);
|
||||
toRaster.addAll(RedactionVerifier.findLeakingRectPages(bytes, rectsByPage));
|
||||
if (toRaster.isEmpty()) {
|
||||
return bytes;
|
||||
}
|
||||
log.warn("Manual redaction rasterising page(s) {} to guarantee removal", toRaster);
|
||||
try (PDDocument rasterised = RedactionRasteriser.rasterisePages(bytes, toRaster)) {
|
||||
ByteArrayOutputStream rasterOut = new ByteArrayOutputStream();
|
||||
rasterised.save(rasterOut);
|
||||
return rasterOut.toByteArray();
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Rasterisation fallback failed after manual redaction leak", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rasterise the given pages of already-finalized bytes to guarantee removal of geometric (range
|
||||
* / image-box) redactions whose covered content - text under an overlay, or an image - is not
|
||||
* removed by content-stream text redaction. Scrubs carriers and re-saves.
|
||||
*/
|
||||
public static byte[] rasteriseSpecificPages(
|
||||
byte[] bytes, Set<Integer> pages, Set<String> literalTargets, List<Pattern> patterns)
|
||||
throws IOException {
|
||||
if (pages == null || pages.isEmpty()) {
|
||||
return bytes;
|
||||
}
|
||||
log.warn("Rasterising page(s) {} to guarantee geometric redaction removal", pages);
|
||||
try (PDDocument rasterised =
|
||||
RedactionRasteriser.rasterisePages(bytes, new HashSet<>(pages))) {
|
||||
CatalogScrubber.scrub(rasterised, literalTargets, patterns);
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
rasterised.save(out);
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
/** Build case-insensitive regex patterns from user input. */
|
||||
public static List<Pattern> buildPatterns(
|
||||
String[] rawEntries, boolean useRegex, boolean wholeWordSearch) {
|
||||
List<Pattern> patterns = new ArrayList<>();
|
||||
if (rawEntries == null) {
|
||||
return patterns;
|
||||
}
|
||||
for (String raw : rawEntries) {
|
||||
if (raw == null) {
|
||||
continue;
|
||||
}
|
||||
String trimmed = raw.trim();
|
||||
if (trimmed.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String core = useRegex ? trimmed : Pattern.quote(trimmed);
|
||||
if (wholeWordSearch) {
|
||||
// Shared with the finder so removal + verification use identical boundaries.
|
||||
core = TextFinderUtils.applyWordBoundaries(trimmed, core);
|
||||
}
|
||||
// Compile via the shared cache (same path as the finder) for consistent
|
||||
// handling of user-supplied regex; ReDoS is bounded by DeadlineCharSequence.
|
||||
patterns.add(RegexPatternUtils.getInstance().createSearchPattern(core, true));
|
||||
} catch (PatternSyntaxException e) {
|
||||
log.debug("Skipping invalid regex '{}': {}", trimmed, e.getMessage());
|
||||
}
|
||||
}
|
||||
return patterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any font is not provably reliable for PDFBox glyph extraction (delegates to {@link
|
||||
* RedactionVerifier}); exposed here for the pipeline's callers and tests.
|
||||
*/
|
||||
static boolean documentHasUnreliableFont(PDDocument document) {
|
||||
return RedactionVerifier.documentHasUnreliableFont(document);
|
||||
}
|
||||
|
||||
/** Captured strings plus pages that must be rasterised (surgical removal was unreliable). */
|
||||
public static final class RedactionResult {
|
||||
private final Set<String> capturedStrings;
|
||||
private final Set<Integer> forceRasterPages;
|
||||
|
||||
public RedactionResult(Set<String> capturedStrings, Set<Integer> forceRasterPages) {
|
||||
this.capturedStrings = capturedStrings == null ? Set.of() : capturedStrings;
|
||||
this.forceRasterPages = forceRasterPages == null ? Set.of() : forceRasterPages;
|
||||
}
|
||||
|
||||
public Set<String> getCapturedStrings() {
|
||||
return capturedStrings;
|
||||
}
|
||||
|
||||
public Set<Integer> getForceRasterPages() {
|
||||
return forceRasterPages;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.rendering.ImageType;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/** Rasterisation fallback: renders whole pages to an image so no text/vector content survives. */
|
||||
@Slf4j
|
||||
final class RedactionRasteriser {
|
||||
|
||||
private RedactionRasteriser() {}
|
||||
|
||||
/** Rasterise the listed pages (all when null) at 150 DPI, replacing their content. */
|
||||
static PDDocument rasterisePages(byte[] sourceBytes, Set<Integer> pagesToRaster)
|
||||
throws IOException {
|
||||
// Load the document directly and mutate in place: rewriting
|
||||
PDDocument source = Loader.loadPDF(sourceBytes);
|
||||
try {
|
||||
PDFRenderer renderer = new PDFRenderer(source);
|
||||
int pageCount = source.getNumberOfPages();
|
||||
for (int i = 0; i < pageCount; i++) {
|
||||
if (pagesToRaster != null && !pagesToRaster.contains(i)) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = source.getPage(i);
|
||||
// PDFRenderer renders the CropBox region, so draw the raster over the CropBox (not
|
||||
// the MediaBox) or a CropBox != MediaBox page is stretched/offset.
|
||||
PDRectangle crop = page.getCropBox();
|
||||
|
||||
BufferedImage img = renderer.renderImageWithDPI(i, 150, ImageType.RGB);
|
||||
ByteArrayOutputStream imgOut = new ByteArrayOutputStream();
|
||||
ImageIO.write(img, "png", imgOut);
|
||||
PDImageXObject imageXObject =
|
||||
PDImageXObject.createFromByteArray(
|
||||
source, imgOut.toByteArray(), "redacted-page-" + i);
|
||||
|
||||
// Drop all prior content / resources / annotations / thumbnail; the raster is the
|
||||
// page.
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
page.getCOSObject().removeItem(COSName.ANNOTS);
|
||||
page.getCOSObject().removeItem(COSName.getPDFName("Thumb"));
|
||||
// Rotation is already baked into the rendered image, so reset it to zero.
|
||||
page.setRotation(0);
|
||||
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
source,
|
||||
page,
|
||||
PDPageContentStream.AppendMode.OVERWRITE,
|
||||
false,
|
||||
true)) {
|
||||
// The rendered image already has the rotation baked in visually
|
||||
cs.drawImage(
|
||||
imageXObject,
|
||||
crop.getLowerLeftX(),
|
||||
crop.getLowerLeftY(),
|
||||
crop.getWidth(),
|
||||
crop.getHeight());
|
||||
}
|
||||
}
|
||||
return source;
|
||||
} catch (IOException | RuntimeException e) {
|
||||
source.close();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
/** Thrown when post-redaction verification still finds a target in the saved PDF. */
|
||||
public class RedactionVerificationFailedException extends RuntimeException {
|
||||
|
||||
public RedactionVerificationFailedException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public RedactionVerificationFailedException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,495 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.text.PdfTextExtractor;
|
||||
|
||||
/**
|
||||
* Independent, fail-closed verification that redacted text is truly gone: an /ActualText-blind
|
||||
* PDFBox pass plus an additive native (PDFium) pass for fonts PDFBox cannot reliably extract, with
|
||||
* per-page leak localisation for the rasterisation fallback.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RedactionVerifier {
|
||||
|
||||
private static final int MAX_XOBJECT_DEPTH = 10;
|
||||
|
||||
// Latched false when the native PDFium binding can't load, so the host falls back to the PDFBox
|
||||
// pass.
|
||||
private static volatile boolean jpdfiumAvailable = true;
|
||||
|
||||
// Skip the additive native pass above this size to bound off-heap copy + native runtime on
|
||||
// adversarial inputs; the PDFBox glyph-blind pass still verifies.
|
||||
private static final long MAX_JPDFIUM_VERIFY_BYTES = 100L * 1024 * 1024;
|
||||
|
||||
private RedactionVerifier() {}
|
||||
|
||||
/** Test hook to simulate the native binding being unavailable (drives the fail-closed path). */
|
||||
static void setJpdfiumAvailableForTest(boolean available) {
|
||||
jpdfiumAvailable = available;
|
||||
}
|
||||
|
||||
/** Warns when the document still carries embedded Type0/TrueType font */
|
||||
static void warnAboutEmbeddedFontGlyphs(PDDocument document) {
|
||||
boolean anyEmbedded = false;
|
||||
Set<PDFont> visited = new HashSet<>();
|
||||
for (PDPage page : document.getPages()) {
|
||||
PDResources resources = page.getResources();
|
||||
if (resources == null) {
|
||||
continue;
|
||||
}
|
||||
for (COSName name : resources.getFontNames()) {
|
||||
PDFont font;
|
||||
try {
|
||||
font = resources.getFont(name);
|
||||
} catch (IOException ioe) {
|
||||
continue;
|
||||
}
|
||||
if (font == null || !visited.add(font)) {
|
||||
continue;
|
||||
}
|
||||
if (font instanceof PDType0Font || font instanceof PDTrueTypeFont) {
|
||||
anyEmbedded = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (anyEmbedded) {
|
||||
log.warn(
|
||||
"Redacted document contains embedded Type0/TrueType fonts; glyph outlines for "
|
||||
+ "redacted characters may remain in the font program. Text is not "
|
||||
+ "extractable via content-stream reading, but raw font inspection can "
|
||||
+ "still recover glyph shapes. Use the convert-to-image fallback for "
|
||||
+ "maximum assurance.");
|
||||
}
|
||||
}
|
||||
|
||||
static void verify(
|
||||
byte[] bytes,
|
||||
Set<String> literalTargets,
|
||||
List<Pattern> patterns,
|
||||
Set<Integer> affectedPages) {
|
||||
if ((literalTargets == null || literalTargets.isEmpty())
|
||||
&& (patterns == null || patterns.isEmpty())) {
|
||||
return;
|
||||
}
|
||||
Set<Integer> pageSet =
|
||||
(affectedPages == null || affectedPages.isEmpty())
|
||||
? null
|
||||
: new TreeSet<>(affectedPages);
|
||||
// PDFBox pass, blind to /ActualText so a benign override can't mask real glyphs.
|
||||
boolean needNativePass;
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
assertNoTarget(extractText(reopened, pageSet), literalTargets, patterns);
|
||||
needNativePass = documentHasUnreliableFont(reopened);
|
||||
} catch (IOException e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Failed to reopen redacted PDF for verification", e);
|
||||
}
|
||||
// Additive producer-independent pass: native PDFium sees glyphs PDFBox may miss (fonts with
|
||||
// no ToUnicode).
|
||||
if (needNativePass) {
|
||||
String nativeText = extractTextJPDFium(bytes, pageSet);
|
||||
if (nativeText == null) {
|
||||
// Required independent pass could not run (native unavailable or doc over the size
|
||||
// guard); fail closed.
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Independent native verification could not run for a document whose fonts "
|
||||
+ "PDFBox cannot reliably extract; cannot confirm removal");
|
||||
}
|
||||
assertNoTarget(nativeText, literalTargets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any font (on any page or nested form XObject) is not provably reliable for PDFBox
|
||||
* glyph extraction - i.e. it is neither a built-in Standard-14 font nor carries a /ToUnicode
|
||||
* map. These are exactly the fonts (CID/Type3/symbolic without ToUnicode) where the PDFBox pass
|
||||
* can go blind, so the independent native pass earns its cost. Biased to {@code true} on any
|
||||
* inspection failure so the native pass runs whenever reliability is uncertain.
|
||||
*/
|
||||
static boolean documentHasUnreliableFont(PDDocument document) {
|
||||
try {
|
||||
Set<COSBase> visited = new HashSet<>();
|
||||
for (PDPage page : document.getPages()) {
|
||||
if (resourcesHaveUnreliableFont(page.getResources(), visited, 0)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} catch (RuntimeException e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean resourcesHaveUnreliableFont(
|
||||
PDResources res, Set<COSBase> visited, int depth) {
|
||||
if (res == null) {
|
||||
return false;
|
||||
}
|
||||
if (depth > MAX_XOBJECT_DEPTH) {
|
||||
return true; // too deeply nested to fully verify - run the native pass to be safe
|
||||
}
|
||||
if (!visited.add(res.getCOSObject())) {
|
||||
return false; // already inspected this resource dictionary
|
||||
}
|
||||
for (COSName name : res.getFontNames()) {
|
||||
PDFont font;
|
||||
try {
|
||||
font = res.getFont(name);
|
||||
} catch (Exception e) {
|
||||
return true; // font won't load for inspection - assume unreliable
|
||||
}
|
||||
// Reliable only for built-in Standard-14 fonts or a /ToUnicode map we can trust.
|
||||
if (font != null
|
||||
&& !font.isStandard14()
|
||||
&& (!font.getCOSObject().containsKey(COSName.TO_UNICODE)
|
||||
|| isSubsetFont(font))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for (COSName name : res.getXObjectNames()) {
|
||||
try {
|
||||
if (res.getXObject(name) instanceof PDFormXObject form
|
||||
&& resourcesHaveUnreliableFont(form.getResources(), visited, depth + 1)) {
|
||||
return true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return true; // can't inspect the XObject - assume unreliable
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* A subset-embedded font carries a 6-uppercase-letter '+' BaseFont tag (e.g. {@code ABCDEF+}).
|
||||
* Its /ToUnicode is custom-built and cannot be trusted for the reliability gate: a partial map
|
||||
* silently loses text, and a crafted map can defeat text-based redaction entirely (only
|
||||
* convert-to-image fully mitigates that adversarial case).
|
||||
*/
|
||||
private static boolean isSubsetFont(PDFont font) {
|
||||
String name = font.getName();
|
||||
if (name == null || name.length() < 8 || name.charAt(6) != '+') {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < 6; i++) {
|
||||
char c = name.charAt(i);
|
||||
if (c < 'A' || c > 'Z') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Fail-closed match check with whitespace-normalised literals and X2 regex semantics. */
|
||||
private static void assertNoTarget(
|
||||
String extracted, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
if (extracted == null) {
|
||||
return;
|
||||
}
|
||||
String normalised = extracted.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
// De-hyphenated view catches a target split by a soft hyphen (U+00AD) or a line-break
|
||||
// hyphen ("-" + space).
|
||||
String dehyphenated = normalised.replace("\u00ad", "").replaceAll("-\\s+", "");
|
||||
if (literalTargets != null) {
|
||||
for (String target : literalTargets) {
|
||||
if (target == null || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
String needle = target.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
if (normalised.contains(needle) || dehyphenated.contains(needle)) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Redacted text still extractable: '" + target + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (patterns != null) {
|
||||
for (Pattern pattern : patterns) {
|
||||
try {
|
||||
if (pattern.matcher(DeadlineCharSequence.of(extracted)).find()) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Redacted pattern still extractable: " + pattern.pattern());
|
||||
}
|
||||
} catch (RedactionVerificationFailedException rvf) {
|
||||
throw rvf;
|
||||
} catch (RuntimeException | StackOverflowError e) {
|
||||
throw new RedactionVerificationFailedException(
|
||||
"Verification regex failed ("
|
||||
+ pattern.pattern()
|
||||
+ "): "
|
||||
+ e.getMessage(),
|
||||
e instanceof Exception ? (Exception) e : new Exception(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Extract text from pageIndexes (0-based) using an /ActualText-blind stripper. */
|
||||
private static String extractText(PDDocument document, Set<Integer> pageIndexes)
|
||||
throws IOException {
|
||||
PDFTextStripper stripper = new GlyphOnlyTextStripper();
|
||||
if (pageIndexes == null) {
|
||||
return stripper.getText(document);
|
||||
}
|
||||
StringBuilder out = new StringBuilder();
|
||||
for (Integer pageIdx : pageIndexes) {
|
||||
if (pageIdx == null || pageIdx < 0 || pageIdx >= document.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
stripper.setStartPage(pageIdx + 1);
|
||||
stripper.setEndPage(pageIdx + 1);
|
||||
String pageText = stripper.getText(document);
|
||||
if (pageText != null) {
|
||||
out.append(pageText);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Independent native (PDFium) extraction; null if the binding is unavailable (additive only).
|
||||
*/
|
||||
private static String extractTextJPDFium(byte[] bytes, Set<Integer> pageIndexes) {
|
||||
if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) {
|
||||
return null;
|
||||
}
|
||||
try (PdfDocument doc = PdfDocument.open(bytes)) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int n = doc.pageCount();
|
||||
if (pageIndexes == null) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
sb.append(jpdfiumPlainText(doc, i)).append('\n');
|
||||
}
|
||||
} else {
|
||||
for (Integer p : pageIndexes) {
|
||||
if (p != null && p >= 0 && p < n) {
|
||||
sb.append(jpdfiumPlainText(doc, p)).append('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (RuntimeException | Error e) {
|
||||
onJpdfiumFailure(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String jpdfiumPlainText(PdfDocument doc, int i) {
|
||||
try {
|
||||
return PdfTextExtractor.extractPage(doc, i).plainText();
|
||||
} catch (RuntimeException | Error e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** One native open, all pages' plain text; null if the binding is unavailable. */
|
||||
private static List<String> extractPagesJPDFium(byte[] bytes) {
|
||||
if (!jpdfiumAvailable || bytes.length > MAX_JPDFIUM_VERIFY_BYTES) {
|
||||
return null;
|
||||
}
|
||||
try (PdfDocument doc = PdfDocument.open(bytes)) {
|
||||
List<String> pages = new ArrayList<>();
|
||||
int n = doc.pageCount();
|
||||
for (int i = 0; i < n; i++) {
|
||||
pages.add(jpdfiumPlainText(doc, i));
|
||||
}
|
||||
return pages;
|
||||
} catch (RuntimeException | Error e) {
|
||||
onJpdfiumFailure(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A native-binding load error latches the pass off process-wide (warn once) so a host without
|
||||
* the native stops retrying; a per-document error only skips this one document (debug).
|
||||
*/
|
||||
private static void onJpdfiumFailure(Throwable e) {
|
||||
boolean nativeUnavailable =
|
||||
e instanceof UnsatisfiedLinkError
|
||||
|| e instanceof NoClassDefFoundError
|
||||
|| e instanceof ExceptionInInitializerError
|
||||
|| e.getClass().getSimpleName().contains("NativeLoad");
|
||||
if (nativeUnavailable) {
|
||||
jpdfiumAvailable = false;
|
||||
log.warn(
|
||||
"JPDFium native unavailable; redaction verification will use the PDFBox pass "
|
||||
+ "only: {}",
|
||||
e.toString());
|
||||
} else {
|
||||
log.debug("JPDFium verification skipped for this document: {}", e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
/** Pages whose surviving text (PDFBox glyph-blind OR native PDFium) still matches a target. */
|
||||
static Set<Integer> findLeakingPages(
|
||||
byte[] bytes, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
List<String> jpdfiumPages = extractPagesJPDFium(bytes);
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
// Native pass required but unavailable: PDFBox can't localise the leak, so rasterise
|
||||
// every page.
|
||||
if (jpdfiumPages == null && documentHasUnreliableFont(reopened)) {
|
||||
log.warn(
|
||||
"Independent native pass unavailable on an unreliable-font document; "
|
||||
+ "rasterising all pages to guarantee removal.");
|
||||
return null;
|
||||
}
|
||||
Set<Integer> leaking = new TreeSet<>();
|
||||
GlyphOnlyTextStripper stripper = new GlyphOnlyTextStripper();
|
||||
for (int i = 0; i < reopened.getNumberOfPages(); i++) {
|
||||
stripper.setStartPage(i + 1);
|
||||
stripper.setEndPage(i + 1);
|
||||
String pdfboxText = stripper.getText(reopened);
|
||||
String jpdfiumText =
|
||||
jpdfiumPages != null && i < jpdfiumPages.size() ? jpdfiumPages.get(i) : "";
|
||||
String combined = (pdfboxText == null ? "" : pdfboxText) + "\n" + jpdfiumText;
|
||||
if (pageLeaks(combined, literalTargets, patterns)) {
|
||||
leaking.add(i);
|
||||
}
|
||||
}
|
||||
if (leaking.isEmpty()) {
|
||||
log.warn(
|
||||
"Verification failed but no single page leaks in isolation; rasterising "
|
||||
+ "all pages to be safe.");
|
||||
return null;
|
||||
}
|
||||
log.info("Leak detection: rasterising only page(s) {}", leaking);
|
||||
return leaking;
|
||||
} catch (Exception e) {
|
||||
log.warn("Per-page leak detection failed ({}); rasterising all pages.", e.toString());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean pageLeaks(
|
||||
String pageText, Set<String> literalTargets, List<Pattern> patterns) {
|
||||
try {
|
||||
assertNoTarget(pageText, literalTargets, patterns);
|
||||
return false;
|
||||
} catch (RedactionVerificationFailedException e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/** Redacted pages that still show text inside a rect or carry an overlapping annotation. */
|
||||
static Set<Integer> findLeakingRectPages(
|
||||
byte[] bytes, Map<Integer, List<PDRectangle>> rectsByPage) {
|
||||
Set<Integer> leaking = new HashSet<>();
|
||||
if (rectsByPage == null || rectsByPage.isEmpty()) {
|
||||
return leaking;
|
||||
}
|
||||
try (PDDocument reopened = Loader.loadPDF(bytes)) {
|
||||
for (Map.Entry<Integer, List<PDRectangle>> entry : rectsByPage.entrySet()) {
|
||||
int pageIndex = entry.getKey();
|
||||
if (pageIndex < 0 || pageIndex >= reopened.getNumberOfPages()) {
|
||||
continue;
|
||||
}
|
||||
PDPage page = reopened.getPage(pageIndex);
|
||||
if (glyphStillInRects(reopened, page, pageIndex, entry.getValue())
|
||||
|| annotationOverlapsRect(page, entry.getValue())) {
|
||||
leaking.add(pageIndex);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Cannot verify - rasterise every redacted page to be safe.
|
||||
return new HashSet<>(rectsByPage.keySet());
|
||||
}
|
||||
return leaking;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if any non-blank glyph is still painted inside a rect. Position-based (not ToUnicode),
|
||||
* so it catches residual CID/Type3/no-ToUnicode glyphs the text stripper would miss; fails
|
||||
* closed.
|
||||
*/
|
||||
private static boolean glyphStillInRects(
|
||||
PDDocument doc, PDPage page, int pageIndex, List<PDRectangle> rects) {
|
||||
try {
|
||||
List<Rectangle2D.Float> areaRects = new ArrayList<>();
|
||||
for (PDRectangle rect : rects) {
|
||||
float pdfY = page.getBBox().getHeight() - rect.getUpperRightY();
|
||||
areaRects.add(
|
||||
new Rectangle2D.Float(
|
||||
rect.getLowerLeftX(), pdfY, rect.getWidth(), rect.getHeight()));
|
||||
}
|
||||
TokenIndexCollector collector = new TokenIndexCollector(areaRects);
|
||||
collector.setStartPage(pageIndex + 1);
|
||||
collector.setEndPage(pageIndex + 1);
|
||||
collector.getText(doc);
|
||||
return collector.anyGlyphInRect();
|
||||
} catch (Exception e) {
|
||||
return true; // cannot prove the rect is clean - rasterise to be safe
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean annotationOverlapsRect(PDPage page, List<PDRectangle> rects) {
|
||||
try {
|
||||
for (var ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
if (ar == null) {
|
||||
continue;
|
||||
}
|
||||
Rectangle2D.Float a =
|
||||
new Rectangle2D.Float(
|
||||
ar.getLowerLeftX(),
|
||||
ar.getLowerLeftY(),
|
||||
ar.getWidth(),
|
||||
ar.getHeight());
|
||||
for (PDRectangle rect : rects) {
|
||||
if (a.intersects(
|
||||
rect.getLowerLeftX(),
|
||||
rect.getLowerLeftY(),
|
||||
rect.getWidth(),
|
||||
rect.getHeight())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** PDFTextStripper that strips /ActualText so verification sees the real glyph stream. */
|
||||
private static final class GlyphOnlyTextStripper extends PDFTextStripper {
|
||||
private static final COSName ACTUAL_TEXT = COSName.getPDFName("ActualText");
|
||||
|
||||
GlyphOnlyTextStripper() throws IOException {}
|
||||
|
||||
@Override
|
||||
public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {
|
||||
COSDictionary safe = properties;
|
||||
if (properties != null && properties.containsKey(ACTUAL_TEXT)) {
|
||||
safe = new COSDictionary(properties);
|
||||
safe.removeItem(ACTUAL_TEXT);
|
||||
}
|
||||
super.beginMarkedContentSequence(tag, safe);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* Position-based (ToUnicode-independent) glyph locator: records, per text-showing operator, the
|
||||
* 0-based indexes of the non-blank glyphs whose box intersects a rect, plus each operator's total
|
||||
* glyph count. Blank glyphs are ignored so a space left behind by removal is not a "leak".
|
||||
*/
|
||||
final class TokenIndexCollector extends PDFTextStripper {
|
||||
private final List<Rectangle2D.Float> rects;
|
||||
final Map<Integer, Set<Integer>> dropGlyphsByOp = new HashMap<>();
|
||||
final Map<Integer, Integer> glyphCountByOp = new HashMap<>();
|
||||
private int showTextOpCounter = -1;
|
||||
private int glyphInOp = 0;
|
||||
|
||||
TokenIndexCollector(List<Rectangle2D.Float> rects) throws IOException {
|
||||
this.rects = rects;
|
||||
setSortByPosition(false);
|
||||
}
|
||||
|
||||
int totalTextOps() {
|
||||
return showTextOpCounter + 1;
|
||||
}
|
||||
|
||||
boolean anyGlyphInRect() {
|
||||
return !dropGlyphsByOp.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processTextPosition(TextPosition text) {
|
||||
int op = showTextOpCounter;
|
||||
int idx = glyphInOp++;
|
||||
String u = text.getUnicode();
|
||||
if (u != null && !u.isBlank()) {
|
||||
// PDFBox reports coordinates with top-left origin here.
|
||||
float x = text.getX();
|
||||
float y = text.getY() - text.getHeight();
|
||||
Rectangle2D.Float glyph =
|
||||
new Rectangle2D.Float(x, y, text.getWidth(), text.getHeight());
|
||||
for (Rectangle2D.Float rect : rects) {
|
||||
if (rect.intersects(glyph)) {
|
||||
dropGlyphsByOp.computeIfAbsent(op, k -> new HashSet<>()).add(idx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
super.processTextPosition(text);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processOperator(Operator operator, List<COSBase> operands) throws IOException {
|
||||
String name = operator.getName();
|
||||
boolean textOp = RedactionPipeline.TEXT_SHOWING_OPERATORS.contains(name);
|
||||
if (textOp) {
|
||||
showTextOpCounter++;
|
||||
glyphInOp = 0;
|
||||
}
|
||||
super.processOperator(operator, operands);
|
||||
if (textOp) {
|
||||
glyphCountByOp.put(showTextOpCounter, glyphInOp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -86,14 +86,21 @@ public class TextFinderUtils {
|
||||
return patterns;
|
||||
}
|
||||
|
||||
private static String applyWordBoundaries(String originalTerm, String patternString) {
|
||||
if (originalTerm.length() == 1 && Character.isDigit(originalTerm.charAt(0))) {
|
||||
/** Shared whole-word wrapper so the finder and RedactionPipeline never diverge (see F2). */
|
||||
public static String applyWordBoundaries(String originalTerm, String patternString) {
|
||||
// Lookarounds for single/non-word-edge terms (\b no-ops on chars like '&'); \b otherwise.
|
||||
boolean nonWordEdge =
|
||||
originalTerm.length() == 1
|
||||
|| !isWordChar(originalTerm.charAt(0))
|
||||
|| !isWordChar(originalTerm.charAt(originalTerm.length() - 1));
|
||||
if (nonWordEdge) {
|
||||
return "(?<![\\w])" + patternString + "(?![\\w])";
|
||||
} else if (originalTerm.length() == 1) {
|
||||
return "(?<![\\w])" + patternString + "(?![\\w])";
|
||||
} else {
|
||||
return "\\b" + patternString + "\\b";
|
||||
}
|
||||
return "\\b" + patternString + "\\b";
|
||||
}
|
||||
|
||||
private static boolean isWordChar(char c) {
|
||||
return Character.isLetterOrDigit(c) || c == '_';
|
||||
}
|
||||
|
||||
public static boolean hasProblematicFonts(PDPage page) {
|
||||
|
||||
+45
-5
@@ -19,6 +19,7 @@ import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -568,7 +569,15 @@ class ManualRedactionServiceTest {
|
||||
byPage.put(0, new ArrayList<>(Arrays.asList(text(0, 72, 690, 300, 710))));
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 1.0f, false, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
1.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertNotNull(result.getFile());
|
||||
@@ -590,7 +599,15 @@ class ManualRedactionServiceTest {
|
||||
Map<Integer, List<PDFText>> byPage = new HashMap<>();
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 0.0f, false, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getFile().exists());
|
||||
@@ -610,7 +627,15 @@ class ManualRedactionServiceTest {
|
||||
Map<Integer, List<PDFText>> byPage = new HashMap<>();
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#000000", 0.0f, null, false);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
null,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getFile().exists());
|
||||
@@ -630,7 +655,15 @@ class ManualRedactionServiceTest {
|
||||
byPage.put(0, new ArrayList<>(Arrays.asList(text(0, 72, 690, 300, 710))));
|
||||
|
||||
TempFile result =
|
||||
service.finalizeRedaction(doc, byPage, "#FF0000", 2.0f, false, true);
|
||||
service.finalizeRedaction(
|
||||
doc,
|
||||
byPage,
|
||||
"#FF0000",
|
||||
2.0f,
|
||||
false,
|
||||
true,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList());
|
||||
|
||||
assertNotNull(result);
|
||||
try (PDDocument reloaded = Loader.loadPDF(result.getFile())) {
|
||||
@@ -658,7 +691,14 @@ class ManualRedactionServiceTest {
|
||||
IOException.class,
|
||||
() ->
|
||||
service.finalizeRedaction(
|
||||
doc, byPage, "#000000", 0.0f, false, false));
|
||||
doc,
|
||||
byPage,
|
||||
"#000000",
|
||||
0.0f,
|
||||
false,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Collections.emptyList()));
|
||||
// The failing temp file is closed on the error path.
|
||||
verify(failing).close();
|
||||
} finally {
|
||||
|
||||
+16
-4
@@ -192,15 +192,27 @@ class RedactControllerTest {
|
||||
when(mockCOSStream.createOutputStream()).thenReturn(mockOutputStream);
|
||||
when(mockCOSStream.createOutputStream(any())).thenReturn(mockOutputStream);
|
||||
|
||||
// The pipeline saves then reloads the bytes to verify, so the mock must emit a real,
|
||||
// reloadable PDF (not a placeholder) or verification throws on reopen.
|
||||
byte[] realPdfBytes = createSimplePdfContent();
|
||||
lenient()
|
||||
.doAnswer(
|
||||
inv -> {
|
||||
File f = inv.getArgument(0);
|
||||
java.nio.file.Files.write(f.toPath(), "mock pdf".getBytes());
|
||||
java.nio.file.Files.write(f.toPath(), realPdfBytes);
|
||||
return null;
|
||||
})
|
||||
.when(mockDocument)
|
||||
.save(any(File.class));
|
||||
lenient()
|
||||
.doAnswer(
|
||||
inv -> {
|
||||
java.io.OutputStream os = inv.getArgument(0);
|
||||
os.write(realPdfBytes);
|
||||
return null;
|
||||
})
|
||||
.when(mockDocument)
|
||||
.save(any(java.io.OutputStream.class));
|
||||
doNothing().when(mockDocument).close();
|
||||
|
||||
// Build real service instances so tests exercise actual logic
|
||||
@@ -347,7 +359,7 @@ class RedactControllerTest {
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
|
||||
verify(mockDocument).save(any(File.class));
|
||||
verify(mockDocument).save(any(java.io.OutputStream.class));
|
||||
verify(mockDocument).close();
|
||||
}
|
||||
}
|
||||
@@ -753,7 +765,7 @@ class RedactControllerTest {
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response.getBody());
|
||||
assertTrue(drainBody(response).length > 0);
|
||||
verify(mockDocument, times(1)).save(any(File.class));
|
||||
verify(mockDocument, times(1)).save(any(java.io.OutputStream.class));
|
||||
verify(mockDocument, times(1)).close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
@@ -776,7 +788,7 @@ class RedactControllerTest {
|
||||
if (response != null) {
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
verify(mockDocument, times(1)).save(any(File.class));
|
||||
verify(mockDocument, times(1)).save(any(java.io.OutputStream.class));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.info("Manual redaction test completed with graceful handling: {}", e.getMessage());
|
||||
|
||||
+755
@@ -0,0 +1,755 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSArray;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSInteger;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSStream;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDFormContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.common.PDStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFontDescriptor;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.MacRomanEncoding;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.WinAnsiEncoding;
|
||||
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.redaction.RedactionPipeline;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
/** Security matrix for redaction across PDF shapes: standard and embedded */
|
||||
@DisplayName("Redaction PDF-variety security matrix")
|
||||
class RedactionPdfVarietyTest {
|
||||
|
||||
private static final String LIBERATION =
|
||||
"/org/apache/pdfbox/resources/ttf/LiberationSans-Regular.ttf";
|
||||
private static final float FONT_SIZE = 12f;
|
||||
private static final float LEFT_X = 72f;
|
||||
private static final float TOP_Y = PDRectangle.LETTER.getHeight() - 80f;
|
||||
|
||||
private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private TempFileManager tempFileManager;
|
||||
private RedactController controller;
|
||||
|
||||
private final List<File> createdTempFiles = new ArrayList<>();
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws IOException {
|
||||
pdfDocumentFactory = mock(CustomPDFDocumentFactory.class);
|
||||
tempFileManager = mock(TempFileManager.class);
|
||||
|
||||
lenient()
|
||||
.when(tempFileManager.createManagedTempFile(anyString()))
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
File f =
|
||||
Files.createTempFile(
|
||||
"redact-variety", inv.<String>getArgument(0))
|
||||
.toFile();
|
||||
createdTempFiles.add(f);
|
||||
TempFile tf = mock(TempFile.class);
|
||||
lenient().when(tf.getFile()).thenReturn(f);
|
||||
lenient().when(tf.getPath()).thenReturn(f.toPath());
|
||||
return tf;
|
||||
});
|
||||
|
||||
controller =
|
||||
new RedactController(
|
||||
pdfDocumentFactory,
|
||||
tempFileManager,
|
||||
new ManualRedactionService(tempFileManager),
|
||||
new TextRedactionService(),
|
||||
mock(RedactExecuteService.class));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
for (File f : createdTempFiles) {
|
||||
if (f != null && f.exists()) {
|
||||
f.delete();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Matrix cases (all through the real /auto-redact controller path)
|
||||
|
||||
@Test
|
||||
@DisplayName("standard Helvetica: target gone, neighbours survive")
|
||||
void standard14Helvetica() throws IOException {
|
||||
byte[] out = autoRedact(helveticaPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("standard-14 Times and Courier: target gone, neighbours survive")
|
||||
void standard14TimesAndCourier() throws IOException {
|
||||
for (Standard14Fonts.FontName fn :
|
||||
new Standard14Fonts.FontName[] {
|
||||
Standard14Fonts.FontName.TIMES_ROMAN, Standard14Fonts.FontName.COURIER
|
||||
}) {
|
||||
byte[] pdf = std14Pdf(fn, "alpha SECRET omega");
|
||||
byte[] out = autoRedact(pdf, "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).as("%s keeps neighbours", fn).contains("alpha");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple TrueType with MacRoman encoding: target gone, neighbours survive")
|
||||
void simpleTrueTypeMacRoman() throws IOException {
|
||||
byte[] out = autoRedact(macRomanTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded Type0 with /ToUnicode stripped: target still gone, neighbours survive")
|
||||
void type0WithoutToUnicode() throws IOException {
|
||||
byte[] out = autoRedact(type0NoToUnicodePdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("symbolic TrueType (no encoding, no ToUnicode): target not extractable")
|
||||
void symbolicTrueType() throws IOException {
|
||||
// PDFBox extraction is glyph-blind here, so removal is guaranteed by the native pass +
|
||||
// rasterise fallback rather than surgical decode; this pins that the pipeline handles a
|
||||
// symbolic font without error and leaves no extractable target.
|
||||
byte[] out = autoRedact(symbolicTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("subset-tagged font (ABCDEF+): target gone, neighbours survive")
|
||||
void subsetTaggedFont() throws IOException {
|
||||
byte[] out = autoRedact(subsetTaggedPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("real Type3 font (glyphs as content streams, no ToUnicode): target gone")
|
||||
void type3GlyphProcs() throws IOException {
|
||||
// crop_test.pdf uses DejaVuSans embedded as a subset Type3 font with no
|
||||
byte[] input;
|
||||
try (InputStream in = getClass().getResourceAsStream("/redaction/type3_dejavu.pdf")) {
|
||||
input = in.readAllBytes();
|
||||
}
|
||||
byte[] out = autoRedact(input, "EXAMPLE");
|
||||
assertGone(out, "EXAMPLE");
|
||||
assertThat(pdfText(out)).contains("CROP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded subset Type0 (CID) font: target gone, neighbours survive")
|
||||
void embeddedSubsetType0() throws IOException {
|
||||
byte[] out = autoRedact(type0Pdf(true, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded full Type0 (CID) font: target gone, neighbours survive")
|
||||
void embeddedFullType0() throws IOException {
|
||||
byte[] out = autoRedact(type0Pdf(false, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded Type1 (PFB) font: target gone, neighbours survive")
|
||||
void embeddedType1() throws IOException {
|
||||
byte[] out = autoRedact(type1Pdf("PUBLICA SECRETWORD KEEP"), "SECRETWORD");
|
||||
assertGone(out, "SECRETWORD");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded CJK Type0 (CIDFontType2) font: CJK target gone, neighbours survive")
|
||||
void embeddedCjkType0() throws IOException {
|
||||
byte[] out = autoRedact(cjkPdf("PUBLICA \u79D8\u5BC6 KEEP"), "\u79D8\u5BC6");
|
||||
assertGone(out, "\u79D8\u5BC6");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("embedded CIDFontType0 (CFF) font: target gone, neighbours survive")
|
||||
void embeddedCidFontType0() throws IOException {
|
||||
byte[] out = autoRedact(cidFontType0Pdf("PUBLICA SECRETWORD KEEP"), "SECRETWORD");
|
||||
assertGone(out, "SECRETWORD");
|
||||
assertThat(pdfText(out)).contains("PUBLICA").contains("KEEP");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple embedded TrueType (WinAnsi) font: target gone, neighbours survive")
|
||||
void simpleTrueTypeWinAnsi() throws IOException {
|
||||
byte[] out = autoRedact(simpleTtfPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rotated pages (90/180/270): target gone on every rotation")
|
||||
void rotatedPages() throws IOException {
|
||||
for (int rotation : new int[] {90, 180, 270}) {
|
||||
byte[] out = autoRedact(rotatedPdf(rotation, "alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).as("rotation %d keeps neighbours", rotation).contains("alpha");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target split across TJ array operands is removed")
|
||||
void tjArraySplit() throws IOException {
|
||||
byte[] out = autoRedact(tjSplitPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("public");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target split across separate Tj operators is removed, other pages keep text")
|
||||
void crossOperatorSplit() throws IOException {
|
||||
byte[] out = autoRedact(crossOperatorPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
// Page 2 was never touched; whatever path handled page 1, page 2 text must
|
||||
assertThat(pdfText(out)).contains("PUBLIC PAGE TWO");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("target inside a Form XObject is removed")
|
||||
void formXObjectText() throws IOException {
|
||||
byte[] out = autoRedact(formXObjectPdf(), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("CropBox smaller than MediaBox: target gone, neighbours survive")
|
||||
void cropBoxOffset() throws IOException {
|
||||
byte[] out = autoRedact(cropBoxPdf("alpha SECRET omega"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("alpha");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("multi-page: target on middle page only, other pages keep their text")
|
||||
void multiPageMiddleTarget() throws IOException {
|
||||
byte[] out =
|
||||
autoRedact(
|
||||
multiPagePdf("PUBLIC ONE", "middle SECRET line", "PUBLIC THREE"), "SECRET");
|
||||
assertGone(out, "SECRET");
|
||||
assertThat(pdfText(out)).contains("PUBLIC ONE").contains("PUBLIC THREE");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("case variant: doc contains 'Secret', target 'SECRET' removes it")
|
||||
void caseVariantRemoved() throws IOException {
|
||||
byte[] out = autoRedact(helveticaPdf("alpha Secret omega"), "SECRET");
|
||||
assertGone(out, "Secret");
|
||||
assertThat(pdfText(out)).contains("alpha").contains("omega");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("whole-word with case variant: 'Cat' removed, substrings survive")
|
||||
void wholeWordCaseVariant() throws IOException {
|
||||
byte[] bytes = helveticaPdf("Cat classification scatter");
|
||||
factoryReturns(bytes);
|
||||
RedactPdfRequest request = baseRequest(bytes, "cat");
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
byte[] out = drainBody(controller.redactPdf(request));
|
||||
String text = pdfText(out);
|
||||
assertThat(text).contains("classification").contains("scatter");
|
||||
// The standalone word must be gone in any case variant.
|
||||
assertThat(text).doesNotContainPattern("(?i)\\bcat\\b");
|
||||
}
|
||||
|
||||
// Page-scoped rasterisation (pipeline-level, deterministic)
|
||||
|
||||
@Test
|
||||
@DisplayName("verification leak rasterises only the leaking page, others keep text")
|
||||
void leakRasterisesOnlyLeakingPage() throws IOException {
|
||||
// Cross-operator split defeats the per-operand literal rewriter
|
||||
byte[] input = crossOperatorPdf();
|
||||
byte[] out;
|
||||
try (PDDocument doc = Loader.loadPDF(input)) {
|
||||
Set<String> targets = Set.of("SECRET");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"SECRET"}, false, false);
|
||||
RedactionPipeline.redactLiteralTerms(doc, targets, patterns);
|
||||
out = RedactionPipeline.finalize(doc, targets, patterns);
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(out)) {
|
||||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
stripper.setStartPage(1);
|
||||
stripper.setEndPage(1);
|
||||
String page1 = stripper.getText(reopened);
|
||||
stripper.setStartPage(2);
|
||||
stripper.setEndPage(2);
|
||||
String page2 = stripper.getText(reopened);
|
||||
|
||||
assertThat(page1.toLowerCase()).doesNotContain("secret");
|
||||
assertThat(page2).contains("PUBLIC PAGE TWO");
|
||||
}
|
||||
}
|
||||
|
||||
// Drivers and assertions
|
||||
|
||||
private byte[] autoRedact(byte[] pdfBytes, String target) throws IOException {
|
||||
factoryReturns(pdfBytes);
|
||||
RedactPdfRequest request = baseRequest(pdfBytes, target);
|
||||
ResponseEntity<Resource> response = controller.redactPdf(request);
|
||||
assertThat(response.getStatusCode().value()).isEqualTo(200);
|
||||
return drainBody(response);
|
||||
}
|
||||
|
||||
private RedactPdfRequest baseRequest(byte[] pdfBytes, String target) {
|
||||
RedactPdfRequest request = new RedactPdfRequest();
|
||||
request.setFileInput(pdfFile(pdfBytes));
|
||||
request.setListOfText(target);
|
||||
request.setUseRegex(false);
|
||||
request.setWholeWordSearch(false);
|
||||
request.setRedactColor("#000000");
|
||||
request.setConvertPDFToImage(false);
|
||||
return request;
|
||||
}
|
||||
|
||||
private void assertGone(byte[] out, String target) throws IOException {
|
||||
assertThat(pdfText(out).toLowerCase()).doesNotContain(target.toLowerCase());
|
||||
}
|
||||
|
||||
private void factoryReturns(byte[] pdfBytes) throws IOException {
|
||||
lenient()
|
||||
.when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(pdfBytes));
|
||||
}
|
||||
|
||||
private MockMultipartFile pdfFile(byte[] bytes) {
|
||||
return new MockMultipartFile("fileInput", "doc.pdf", "application/pdf", bytes);
|
||||
}
|
||||
|
||||
private byte[] drainBody(ResponseEntity<Resource> response) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try (InputStream in = response.getBody().getInputStream()) {
|
||||
in.transferTo(baos);
|
||||
}
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
private String pdfText(byte[] pdfBytes) throws IOException {
|
||||
try (PDDocument doc = Loader.loadPDF(pdfBytes)) {
|
||||
return new PDFTextStripper().getText(doc);
|
||||
}
|
||||
}
|
||||
|
||||
// PDF builders
|
||||
|
||||
private static PDFont helvetica() {
|
||||
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
|
||||
}
|
||||
|
||||
private byte[] helveticaPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] std14Pdf(Standard14Fonts.FontName fontName, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, new PDType1Font(fontName), line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] macRomanTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, MacRomanEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
// GID order emitted by generators/gen_cff_cidfont.py: 0=.notdef 1=space then the letters.
|
||||
private static final String CFF_GLYPHS = "\0 ABCDEIKLOPRSTUW";
|
||||
|
||||
private static int cffGid(char c) {
|
||||
int i = CFF_GLYPHS.indexOf(c);
|
||||
return i < 0 ? 0 : i;
|
||||
}
|
||||
|
||||
/**
|
||||
* PDFBox 3.0.7 has no CFF embedder, so hand-assemble a Type0 -> CIDFontType0 (CFF in
|
||||
* /FontFile3, Identity-H, CID==GID, explicit /ToUnicode) from the generated CFF program.
|
||||
*/
|
||||
private byte[] cidFontType0Pdf(String line) throws IOException {
|
||||
byte[] cff;
|
||||
try (InputStream in = getClass().getResourceAsStream("/redaction/cidfont0_cff.bin")) {
|
||||
cff = in.readAllBytes();
|
||||
}
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
|
||||
PDStream fontFile = new PDStream(doc);
|
||||
try (var os = fontFile.createOutputStream()) {
|
||||
os.write(cff);
|
||||
}
|
||||
COSStream ff = fontFile.getCOSObject();
|
||||
ff.setItem(COSName.SUBTYPE, COSName.getPDFName("CIDFontType0C"));
|
||||
|
||||
COSDictionary fdesc = new COSDictionary();
|
||||
fdesc.setItem(COSName.TYPE, COSName.FONT_DESC);
|
||||
fdesc.setName(COSName.FONT_NAME, "MinimalCFFCID");
|
||||
fdesc.setInt(COSName.FLAGS, 4);
|
||||
COSArray bbox = new COSArray();
|
||||
for (int v : new int[] {0, -200, 600, 800}) {
|
||||
bbox.add(COSInteger.get(v));
|
||||
}
|
||||
fdesc.setItem(COSName.FONT_BBOX, bbox);
|
||||
fdesc.setInt(COSName.ITALIC_ANGLE, 0);
|
||||
fdesc.setInt(COSName.ASCENT, 800);
|
||||
fdesc.setInt(COSName.DESCENT, -200);
|
||||
fdesc.setInt(COSName.CAP_HEIGHT, 700);
|
||||
fdesc.setInt(COSName.STEM_V, 80);
|
||||
fdesc.setItem(COSName.getPDFName("FontFile3"), ff);
|
||||
|
||||
COSDictionary cidSystemInfo = new COSDictionary();
|
||||
cidSystemInfo.setString(COSName.REGISTRY, "Adobe");
|
||||
cidSystemInfo.setString(COSName.ORDERING, "Identity");
|
||||
cidSystemInfo.setInt(COSName.SUPPLEMENT, 0);
|
||||
|
||||
COSDictionary cidFont = new COSDictionary();
|
||||
cidFont.setItem(COSName.TYPE, COSName.FONT);
|
||||
cidFont.setItem(COSName.SUBTYPE, COSName.CID_FONT_TYPE0);
|
||||
cidFont.setName(COSName.BASE_FONT, "MinimalCFFCID");
|
||||
cidFont.setItem(COSName.CIDSYSTEMINFO, cidSystemInfo);
|
||||
cidFont.setItem(COSName.FONT_DESC, fdesc);
|
||||
cidFont.setInt(COSName.DW, 600);
|
||||
|
||||
COSStream toUni = new PDStream(doc).getCOSObject();
|
||||
try (var os = toUni.createOutputStream()) {
|
||||
os.write(cffToUnicode().getBytes(java.nio.charset.StandardCharsets.US_ASCII));
|
||||
}
|
||||
|
||||
COSDictionary type0 = new COSDictionary();
|
||||
type0.setItem(COSName.TYPE, COSName.FONT);
|
||||
type0.setItem(COSName.SUBTYPE, COSName.TYPE0);
|
||||
type0.setName(COSName.BASE_FONT, "MinimalCFFCID");
|
||||
type0.setItem(COSName.ENCODING, COSName.IDENTITY_H);
|
||||
COSArray descendants = new COSArray();
|
||||
descendants.add(cidFont);
|
||||
type0.setItem(COSName.DESCENDANT_FONTS, descendants);
|
||||
type0.setItem(COSName.TO_UNICODE, toUni);
|
||||
|
||||
PDFont font = org.apache.pdfbox.pdmodel.font.PDFontFactory.createFont(type0);
|
||||
PDResources res = new PDResources();
|
||||
COSName fName = res.add(font);
|
||||
page.setResources(res);
|
||||
|
||||
StringBuilder csb = new StringBuilder();
|
||||
csb.append("BT\n/").append(fName.getName()).append(" 24 Tf\n50 700 Td\n<");
|
||||
for (int i = 0; i < line.length(); i++) {
|
||||
csb.append(String.format("%04X", cffGid(line.charAt(i))));
|
||||
}
|
||||
csb.append("> Tj\nET\n");
|
||||
PDStream content = new PDStream(doc);
|
||||
try (var os = content.createOutputStream()) {
|
||||
os.write(csb.toString().getBytes(java.nio.charset.StandardCharsets.US_ASCII));
|
||||
}
|
||||
page.setContents(content);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private static String cffToUnicode() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n");
|
||||
sb.append("/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n");
|
||||
sb.append("/CMapName /Adobe-Identity-UCS def\n/CMapType 2 def\n");
|
||||
sb.append("1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n");
|
||||
StringBuilder body = new StringBuilder();
|
||||
int count = 0;
|
||||
for (int gid = 1; gid < CFF_GLYPHS.length(); gid++) {
|
||||
body.append(String.format("<%04X> <%04X>\n", gid, (int) CFF_GLYPHS.charAt(gid)));
|
||||
count++;
|
||||
}
|
||||
sb.append(count).append(" beginbfchar\n").append(body).append("endbfchar\n");
|
||||
sb.append("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private byte[] type1Pdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream pfb = getClass().getResourceAsStream("/redaction/type1_minimal.pfb")) {
|
||||
font = new PDType1Font(doc, pfb, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] cjkPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = getClass().getResourceAsStream("/redaction/cjk_minimal.ttf")) {
|
||||
font = PDType0Font.load(doc, ttf, true);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] symbolicTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDTrueTypeFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
// Strip /Encoding and flag the descriptor Symbolic so extraction must use the cmap.
|
||||
font.getCOSObject().removeItem(org.apache.pdfbox.cos.COSName.ENCODING);
|
||||
PDFontDescriptor fd = font.getFontDescriptor();
|
||||
fd.setFlags((fd.getFlags() | 4) & ~32);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] type0NoToUnicodePdf(String line) throws IOException {
|
||||
byte[] pdf;
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
// Full embed, not subset: the subsetter regenerates /ToUnicode at save, undoing the
|
||||
// strip.
|
||||
font = PDType0Font.load(doc, ttf, false);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
font.getCOSObject().removeItem(org.apache.pdfbox.cos.COSName.TO_UNICODE);
|
||||
pdf = save(doc);
|
||||
}
|
||||
// Guard: the strip must survive save so this fixture genuinely exercises the no-mapping
|
||||
// path.
|
||||
try (PDDocument reopened = Loader.loadPDF(pdf)) {
|
||||
PDResources res = reopened.getPage(0).getResources();
|
||||
for (org.apache.pdfbox.cos.COSName n : res.getFontNames()) {
|
||||
assertThat(
|
||||
res.getFont(n)
|
||||
.getCOSObject()
|
||||
.containsKey(org.apache.pdfbox.cos.COSName.TO_UNICODE))
|
||||
.as("no-ToUnicode fixture must not ship /ToUnicode")
|
||||
.isFalse();
|
||||
}
|
||||
}
|
||||
return pdf;
|
||||
}
|
||||
|
||||
private byte[] subsetTaggedPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDType0Font.load(doc, ttf, true);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
String tagged = "ABCDEF+" + font.getName();
|
||||
font.getCOSObject().setName(org.apache.pdfbox.cos.COSName.BASE_FONT, tagged);
|
||||
if (font.getFontDescriptor() != null) {
|
||||
font.getFontDescriptor()
|
||||
.getCOSObject()
|
||||
.setName(org.apache.pdfbox.cos.COSName.FONT_NAME, tagged);
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] type0Pdf(boolean subset, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDType0Font.load(doc, ttf, subset);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] simpleTtfPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDFont font;
|
||||
try (InputStream ttf = PDDocument.class.getResourceAsStream(LIBERATION)) {
|
||||
font = PDTrueTypeFont.load(doc, ttf, WinAnsiEncoding.INSTANCE);
|
||||
}
|
||||
addTextPage(doc, font, line, 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] rotatedPdf(int rotation, String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, rotation, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] cropBoxPdf(String line) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
addTextPage(doc, helvetica(), line, 0, new PDRectangle(40, 40, 500, 700));
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] multiPagePdf(String... pageLines) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
for (String line : pageLines) {
|
||||
addTextPage(doc, helvetica(), line, 0, null);
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
/** One page whose text is emitted as a TJ array: ["public ", "SEC", -20 */
|
||||
private byte[] tjSplitPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(helvetica(), FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
cs.showTextWithPositioning(
|
||||
new Object[] {"public ", "SEC", Float.valueOf(-20f), "RET", " end"});
|
||||
cs.endText();
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
/** Page 1 shows "SEC" and "RET" as separate adjacent Tj operators; page 2 */
|
||||
private byte[] crossOperatorPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
PDFont font = helvetica();
|
||||
float secWidth = font.getStringWidth("SEC") / 1000f * FONT_SIZE;
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
cs.showText("SEC");
|
||||
cs.endText();
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
cs.newLineAtOffset(LEFT_X + secWidth, TOP_Y);
|
||||
cs.showText("RET");
|
||||
cs.endText();
|
||||
}
|
||||
addTextPage(doc, helvetica(), "PUBLIC PAGE TWO", 0, null);
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] formXObjectPdf() throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
|
||||
PDFormXObject form = new PDFormXObject(doc);
|
||||
form.setBBox(new PDRectangle(0, 0, 400, 60));
|
||||
form.setResources(new PDResources());
|
||||
try (PDFormContentStream fcs = new PDFormContentStream(form)) {
|
||||
fcs.beginText();
|
||||
fcs.setFont(helvetica(), FONT_SIZE);
|
||||
fcs.newLineAtOffset(10, 20);
|
||||
fcs.showText("xobj SECRET payload");
|
||||
fcs.endText();
|
||||
}
|
||||
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.transform(Matrix.getTranslateInstance(LEFT_X, TOP_Y - 60));
|
||||
cs.drawForm(form);
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private void addTextPage(
|
||||
PDDocument doc, PDFont font, String line, int rotation, PDRectangle cropBox)
|
||||
throws IOException {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
if (rotation != 0) {
|
||||
page.setRotation(rotation);
|
||||
}
|
||||
if (cropBox != null) {
|
||||
page.setCropBox(cropBox);
|
||||
}
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(font, FONT_SIZE);
|
||||
if (rotation != 0) {
|
||||
// Real generators compensate the text matrix so text reads upright
|
||||
cs.setTextMatrix(
|
||||
Matrix.getRotateInstance(
|
||||
Math.toRadians(rotation),
|
||||
PDRectangle.LETTER.getWidth() / 2,
|
||||
PDRectangle.LETTER.getHeight() / 2));
|
||||
} else {
|
||||
cs.newLineAtOffset(LEFT_X, TOP_Y);
|
||||
}
|
||||
cs.showText(line);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] save(PDDocument doc) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
doc.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
+6
-2
@@ -287,8 +287,12 @@ class TextRedactionServiceExtraTest {
|
||||
|
||||
service.createTokensWithoutTargetText(doc, page, Set.of("SECRET"), false, false);
|
||||
|
||||
// The deep traversal must have rewritten the inner form's content stream.
|
||||
assertThat(inner.getCOSObject().containsKey(COSName.CONTENTS)).isTrue();
|
||||
// The deep traversal must have rewritten the inner form's own stream in place (F3).
|
||||
String innerContent;
|
||||
try (var in = inner.getContents()) {
|
||||
innerContent = new String(in.readAllBytes(), StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
assertThat(innerContent).doesNotContain("SECRET");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+12
-8
@@ -226,19 +226,23 @@ class TextRedactionServiceMoreTest {
|
||||
}
|
||||
page.setContents(pageStream);
|
||||
|
||||
// Processing the page walks into the XObject graph; when a match is found inside
|
||||
// the
|
||||
// form, writeRedactedContentToXObject runs and sets a /Contents item on the form's
|
||||
// COS dictionary. Asserting that item appears proves the XObject redaction path
|
||||
// executed end-to-end without throwing.
|
||||
// Processing the page walks into the XObject graph and rewrites the form's own
|
||||
// stream body in place (F3), so the target is physically gone from the form.
|
||||
List<Object> tokens =
|
||||
service.createTokensWithoutTargetText(
|
||||
doc, page, Set.of("SECRET"), false, false);
|
||||
|
||||
assertThat(tokens).isNotNull();
|
||||
assertThat(form.getCOSObject().containsKey(COSName.CONTENTS))
|
||||
.as("form XObject redaction path should have written a new content item")
|
||||
.isTrue();
|
||||
String rewrittenForm;
|
||||
try (var in = form.getContents()) {
|
||||
rewrittenForm =
|
||||
new String(
|
||||
in.readAllBytes(),
|
||||
java.nio.charset.StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
assertThat(rewrittenForm)
|
||||
.as("form XObject content should have the target removed in place")
|
||||
.doesNotContain("SECRET");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package stirling.software.SPDF.pdf.redaction;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Integration tests that exercise the real redaction pipeline against PDFs */
|
||||
class RedactionPipelineIntegrationTest {
|
||||
|
||||
private static byte[] loadFixture() throws Exception {
|
||||
try (InputStream in =
|
||||
RedactionPipelineIntegrationTest.class.getResourceAsStream(
|
||||
"/redaction/test_pdf_1.pdf")) {
|
||||
assertNotNull(in, "fixture resource must exist on classpath");
|
||||
return in.readAllBytes();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"finalize on rotated ReportLab PDF without a rewrite pass falls back to rasterisation so target disappears")
|
||||
void finaliseFallsBackToRasterisationWhenTargetWouldSurvive() throws Exception {
|
||||
byte[] fixtureBytes = loadFixture();
|
||||
byte[] outputBytes;
|
||||
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Test");
|
||||
// No content-stream rewriting performed.
|
||||
outputBytes = RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
|
||||
}
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("test"),
|
||||
"Rasterisation fallback must have removed target. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"finalize with zero rewrite + synthetic upright PDF triggers RedactionVerificationFailedException only when fallback disabled")
|
||||
void verificationExceptionWiringSanityCheck() throws Exception {
|
||||
// This test proves the verification hook itself still throws -
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
cs.newLineAtOffset(100, 700);
|
||||
cs.showText("Top Smith Classified");
|
||||
cs.endText();
|
||||
}
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Smith");
|
||||
// Even with the rasterisation fallback active, the output must not contain
|
||||
byte[] outBytes =
|
||||
RedactionPipeline.finalize(doc, literalTargets, Collections.emptyList());
|
||||
try (PDDocument reopened = Loader.loadPDF(outBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("smith"),
|
||||
"Neither primary pass nor rasterisation removed target. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"auto-word redact on rotated PDF with target packed in single Tj removes the word from text stream")
|
||||
void autoWordRedactRemovesTargetOnRotatedSingleTjPdf() throws Exception {
|
||||
byte[] fixtureBytes = loadFixture();
|
||||
byte[] outputBytes;
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Test");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"Test"}, false, false);
|
||||
|
||||
try (PDDocument doc = Loader.loadPDF(fixtureBytes)) {
|
||||
RedactionPipeline.redactLiteralTerms(doc, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(doc, literalTargets, patterns);
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
String extracted = stripper.getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("test"),
|
||||
"Target term must not be extractable after redact. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("simple upright Helvetica fixture still gets word redacted via the pipeline")
|
||||
void autoWordRedactRemovesTargetOnSyntheticPdf() throws Exception {
|
||||
byte[] outputBytes;
|
||||
Set<String> literalTargets = new LinkedHashSet<>();
|
||||
literalTargets.add("Secret");
|
||||
List<Pattern> patterns =
|
||||
RedactionPipeline.buildPatterns(new String[] {"Secret"}, false, false);
|
||||
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.beginText();
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
cs.newLineAtOffset(100, 700);
|
||||
cs.showText("Top Secret Classified");
|
||||
cs.endText();
|
||||
}
|
||||
ByteArrayOutputStream tmp = new ByteArrayOutputStream();
|
||||
doc.save(tmp);
|
||||
// reload from bytes so we are processing an already-saved PDF similar
|
||||
try (PDDocument reloaded = Loader.loadPDF(tmp.toByteArray())) {
|
||||
RedactionPipeline.redactLiteralTerms(reloaded, literalTargets, patterns);
|
||||
outputBytes = RedactionPipeline.finalize(reloaded, literalTargets, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
try (PDDocument reopened = Loader.loadPDF(outputBytes)) {
|
||||
String extracted = new PDFTextStripper().getText(reopened);
|
||||
String lower = extracted == null ? "" : extracted.toLowerCase(Locale.ROOT);
|
||||
assertFalse(
|
||||
lower.contains("secret"),
|
||||
"Target term must not be extractable after redact on synthetic PDF. Extracted: '"
|
||||
+ extracted
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
+1019
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a minimal OpenType-CFF (.otf) font for a CIDFontType0 redaction fixture.
|
||||
|
||||
Why this exists
|
||||
---------------
|
||||
Real-world Adobe/InDesign PDFs embed CFF-based CID fonts (Type0 -> CIDFontType0,
|
||||
program in /FontFile3 /Subtype /CIDFontType0C). PDFBox 3.0.7 can *read* those but
|
||||
ships NO CFF embedder: every PDType0Font.load(...) overload routes CFF outlines
|
||||
through the TrueType path (which either rejects "TTF using CFF outlines" or, via the
|
||||
OTFParser+TrueTypeFont overload, emits a CIDFontType2 and then fails to subset CFF
|
||||
because there is no glyf table). So to build a genuine CIDFontType0 fixture you must:
|
||||
1. generate the CFF .otf with fontTools (this script), and
|
||||
2. assemble the Type0/CIDFontType0 dictionary by hand, attaching the raw CFF table
|
||||
as /FontFile3 /Subtype /CIDFontType0C (see the companion Java harness).
|
||||
|
||||
This script emits BOTH minimal_cff.otf and cff_raw.bin (the raw CFF table bytes,
|
||||
ready to drop into /FontFile3).
|
||||
|
||||
Glyph coverage: only the ASCII letters in "PUBLICA SECRETWORD KEEP"
|
||||
(P U B L I C A S E R T W O D K) plus .notdef and space. Outlines are trivial filled
|
||||
boxes; only the cmap / encoding correctness matters so the drawn text extracts.
|
||||
"""
|
||||
import sys
|
||||
from fontTools.fontBuilder import FontBuilder
|
||||
from fontTools.pens.t2CharStringPen import T2CharStringPen
|
||||
from fontTools.ttLib import TTFont
|
||||
|
||||
UPM = 1000
|
||||
LETTERS = sorted(set("PUBLICASECRETWORDKEEP")) # A B C D E I K L O P R S T U W
|
||||
|
||||
|
||||
def box_charstring(width=600, height=700):
|
||||
"""A simple filled rectangle as a Type2 (CFF) charstring."""
|
||||
pen = T2CharStringPen(width, None)
|
||||
pen.moveTo((50, 0))
|
||||
pen.lineTo((width - 50, 0))
|
||||
pen.lineTo((width - 50, height))
|
||||
pen.lineTo((50, height))
|
||||
pen.closePath()
|
||||
return pen.getCharString()
|
||||
|
||||
|
||||
def build_otf(out_otf):
|
||||
# GID order: .notdef (GID 0), space (GID 1), then the letters. The companion
|
||||
# harness relies on this exact order for its Identity-H CID==GID mapping.
|
||||
glyph_order = [".notdef", "space"] + list(LETTERS)
|
||||
|
||||
fb = FontBuilder(unitsPerEm=UPM, isTTF=False) # isTTF=False -> CFF/OTF
|
||||
fb.setupGlyphOrder(glyph_order)
|
||||
|
||||
# Unicode cmap: codepoint -> glyph name. PDFBox reads this to build /ToUnicode,
|
||||
# and it also lets any consumer resolve the glyphs by character.
|
||||
cmap = {0x20: "space"}
|
||||
for c in LETTERS:
|
||||
cmap[ord(c)] = c
|
||||
fb.setupCharacterMap(cmap)
|
||||
|
||||
charstrings = {
|
||||
".notdef": box_charstring(),
|
||||
"space": box_charstring(width=250, height=0), # empty advance-only glyph
|
||||
}
|
||||
for c in LETTERS:
|
||||
charstrings[c] = box_charstring()
|
||||
|
||||
fb.setupCFF(
|
||||
psName="MinimalCFFCID",
|
||||
charStringsDict=charstrings,
|
||||
fontInfo={
|
||||
"FullName": "Minimal CFF CID",
|
||||
"FamilyName": "MinimalCFFCID",
|
||||
"Weight": "Regular",
|
||||
},
|
||||
privateDict={},
|
||||
)
|
||||
|
||||
metrics = {".notdef": (600, 50), "space": (250, 0)}
|
||||
for c in LETTERS:
|
||||
metrics[c] = (600, 50)
|
||||
fb.setupHorizontalMetrics(metrics)
|
||||
fb.setupHorizontalHeader(ascent=800, descent=-200)
|
||||
|
||||
fb.setupNameTable(
|
||||
{
|
||||
"familyName": "MinimalCFFCID",
|
||||
"styleName": "Regular",
|
||||
"psName": "MinimalCFFCID",
|
||||
"fullName": "Minimal CFF CID",
|
||||
"version": "Version 1.000",
|
||||
"uniqueFontIdentifier": "MinimalCFFCID;1.000",
|
||||
}
|
||||
)
|
||||
fb.setupOS2(
|
||||
sTypoAscender=800,
|
||||
sTypoDescender=-200,
|
||||
usWinAscent=800,
|
||||
usWinDescent=200,
|
||||
sCapHeight=700,
|
||||
)
|
||||
fb.setupPost(keepGlyphNames=True)
|
||||
|
||||
fb.save(out_otf)
|
||||
print("wrote", out_otf, "(", len(glyph_order), "glyphs )")
|
||||
|
||||
|
||||
def dump_cff(out_otf, out_cff):
|
||||
"""Extract the raw CFF table bytes for use as PDF /FontFile3."""
|
||||
f = TTFont(out_otf)
|
||||
cff = f["CFF "].compile(f)
|
||||
with open(out_cff, "wb") as fh:
|
||||
fh.write(cff)
|
||||
print("wrote", out_cff, "(", len(cff), "bytes CFF )")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
otf = sys.argv[1] if len(sys.argv) > 1 else "minimal_cff.otf"
|
||||
cffbin = sys.argv[2] if len(sys.argv) > 2 else "cff_raw.bin"
|
||||
build_otf(otf)
|
||||
dump_cff(otf, cffbin)
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate a minimal TrueType font (.ttf) that covers exactly the characters
|
||||
needed for a CJK Type0 (Identity-H) redaction test:
|
||||
|
||||
- ASCII letters in "PUBLICA SECRETWORD KEEP": P U B L I C A S E R T W O D K + space
|
||||
- Two CJK ideographs used as the redaction target:
|
||||
U+79D8 (secret1) and U+5BC6 (secret2) -> "秘密"
|
||||
- .notdef
|
||||
|
||||
Glyph outlines are trivial boxes; the point of the test is that the cmap /
|
||||
encoding is correct so PDFBox can build a working /ToUnicode and the text is
|
||||
EXTRACTABLE. When PDFBox loads this via PDType0Font.load(doc, in, true) it
|
||||
becomes an Identity-H Type0 font with a CIDFontType2 descendant.
|
||||
|
||||
Usage: python gen_cjk_font.py <output.ttf>
|
||||
"""
|
||||
import sys
|
||||
from fontTools.fontBuilder import FontBuilder
|
||||
from fontTools.pens.ttGlyphPen import TTGlyphPen
|
||||
|
||||
UPM = 1000
|
||||
|
||||
|
||||
def box_glyph(width_units):
|
||||
"""A trivial filled box outline so the glyph has real contours."""
|
||||
pen = TTGlyphPen(None)
|
||||
x0, y0, x1, y1 = 80, 0, max(120, width_units - 80), 700
|
||||
pen.moveTo((x0, y0))
|
||||
pen.lineTo((x0, y1))
|
||||
pen.lineTo((x1, y1))
|
||||
pen.lineTo((x1, y0))
|
||||
pen.closePath()
|
||||
return pen.glyph()
|
||||
|
||||
|
||||
def empty_glyph():
|
||||
return TTGlyphPen(None).glyph()
|
||||
|
||||
|
||||
def main(out_path):
|
||||
# Characters we must cover. Order defines glyph order (after .notdef).
|
||||
ascii_chars = list("PUBLICA SECRETWORD KEEP")
|
||||
ascii_unique = []
|
||||
for ch in ascii_chars:
|
||||
if ch not in ascii_unique:
|
||||
ascii_unique.append(ch)
|
||||
|
||||
cjk = ["秘", "密"] # 秘 密
|
||||
|
||||
# glyph name -> codepoint mapping
|
||||
def gname(cp):
|
||||
if cp == 0x20:
|
||||
return "space"
|
||||
return "uni%04X" % cp
|
||||
|
||||
codepoints = []
|
||||
for ch in ascii_unique:
|
||||
codepoints.append(ord(ch))
|
||||
for ch in cjk:
|
||||
codepoints.append(ord(ch))
|
||||
|
||||
glyph_order = [".notdef"] + [gname(cp) for cp in codepoints]
|
||||
|
||||
fb = FontBuilder(UPM, isTTF=True)
|
||||
fb.setupGlyphOrder(glyph_order)
|
||||
|
||||
# cmap: map every codepoint to its glyph. FontBuilder builds format-4
|
||||
# (BMP) + format-12 as needed. Both CJK codepoints are BMP so format-4
|
||||
# covers them.
|
||||
cmap = {cp: gname(cp) for cp in codepoints}
|
||||
fb.setupCharacterMap(cmap)
|
||||
|
||||
# Build glyf table with box outlines; advance widths.
|
||||
glyphs = {".notdef": empty_glyph()}
|
||||
metrics = {".notdef": (600, 0)}
|
||||
for cp in codepoints:
|
||||
name = gname(cp)
|
||||
if cp == 0x20:
|
||||
glyphs[name] = empty_glyph()
|
||||
metrics[name] = (300, 0) # space advance, no outline
|
||||
elif cp > 0x2000: # CJK -> full-width
|
||||
glyphs[name] = box_glyph(900)
|
||||
metrics[name] = (1000, 80)
|
||||
else:
|
||||
glyphs[name] = box_glyph(560)
|
||||
metrics[name] = (600, 80)
|
||||
|
||||
fb.setupGlyf(glyphs)
|
||||
fb.setupHorizontalMetrics(metrics)
|
||||
fb.setupHorizontalHeader(ascent=800, descent=-200)
|
||||
|
||||
# Names + the rest of the required tables.
|
||||
fb.setupNameTable({
|
||||
"familyName": "CJKRedactTest",
|
||||
"styleName": "Regular",
|
||||
"psName": "CJKRedactTest-Regular",
|
||||
"fullName": "CJKRedactTest Regular",
|
||||
})
|
||||
fb.setupOS2(sTypoAscender=800, sTypoDescender=-200, usWinAscent=800,
|
||||
usWinDescent=200)
|
||||
fb.setupPost()
|
||||
|
||||
fb.save(out_path)
|
||||
print("Wrote", out_path)
|
||||
print("Glyph order:", glyph_order)
|
||||
print("cmap entries:", {("U+%04X" % k): v for k, v in cmap.items()})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: gen_cjk_font.py <output.ttf>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
main(sys.argv[1])
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a minimal real Type1 font as a classic PFB (0x80 0x01 segment format).
|
||||
|
||||
Pure-python, uses fontTools.t1Lib.T1Font.createData() + writePFB. The font contains
|
||||
.notdef plus the ASCII letters needed for "PUBLICA SECRETWORD KEEP", with a proper
|
||||
Encoding array + standard PostScript glyph names. Because the glyph names are the
|
||||
standard ones and the PDF font uses WinAnsiEncoding, PDFBox extracts text via the
|
||||
base encoding (code -> glyphname -> Unicode); no /ToUnicode CMap is required.
|
||||
|
||||
Usage: python gen_type1.py <out.pfb>
|
||||
"""
|
||||
import sys
|
||||
from fontTools import t1Lib
|
||||
from fontTools.misc import psCharStrings
|
||||
|
||||
|
||||
# Glyph name for each character we must render. PostScript standard glyph names.
|
||||
CHARS = {
|
||||
"P": "P", "U": "U", "B": "B", "L": "L", "I": "I", "C": "C", "A": "A",
|
||||
"S": "S", "E": "E", "R": "R", "T": "T", "W": "W", "O": "O", "D": "D",
|
||||
"K": "K", " ": "space",
|
||||
}
|
||||
|
||||
EM = 1000
|
||||
WIDTH = 600
|
||||
BOX_L, BOX_B, BOX_R, BOX_T = 50, 0, 550, 700
|
||||
|
||||
|
||||
def box_charstring(width):
|
||||
"""A trivial filled box outline. hsbw sets sidebearing+width; then a closed rect."""
|
||||
prog = [
|
||||
0, width, "hsbw", # sb=0, width
|
||||
BOX_L, BOX_B, "rmoveto", # move to bottom-left (relative from 0,0)
|
||||
(BOX_R - BOX_L), "hlineto", # right
|
||||
(BOX_T - BOX_B), "vlineto", # up
|
||||
-(BOX_R - BOX_L), "hlineto", # left
|
||||
"closepath",
|
||||
"endchar",
|
||||
]
|
||||
cs = psCharStrings.T1CharString(program=prog)
|
||||
return cs
|
||||
|
||||
|
||||
def space_charstring(width):
|
||||
prog = [0, width, "hsbw", "endchar"]
|
||||
return psCharStrings.T1CharString(program=prog)
|
||||
|
||||
|
||||
def build_font(font_name="MinType1Test"):
|
||||
# 256-slot encoding, default .notdef, filled at each char's codepoint.
|
||||
encoding = [".notdef"] * 256
|
||||
for ch, gname in CHARS.items():
|
||||
encoding[ord(ch)] = gname
|
||||
|
||||
charstrings = {".notdef": box_charstring(WIDTH)}
|
||||
for ch, gname in CHARS.items():
|
||||
if gname == "space":
|
||||
charstrings[gname] = space_charstring(WIDTH)
|
||||
else:
|
||||
charstrings[gname] = box_charstring(WIDTH)
|
||||
|
||||
# Subrs: the 5 standard Type1 subrs (division/hint machinery). Provide as
|
||||
# T1CharString objects so createData can compile+encrypt them.
|
||||
subrs = [psCharStrings.T1CharString(bytecode=s) for s in t1Lib.std_subrs]
|
||||
|
||||
# Order matters. fontTools emits a stray "put" after the Subrs array; fontbox's
|
||||
# Type1Parser breaks its Private loop on the first non-LITERAL after Subrs, so
|
||||
# every key AFTER Subrs is dropped. Keep Subrs LAST so nothing gets skipped.
|
||||
private = {
|
||||
"RD": t1Lib.RD_value, # ('string','currentfile','exch','readstring','pop')
|
||||
"ND": t1Lib.ND_values[0], # ('def',) codespell:ignore
|
||||
"NP": t1Lib.PD_values[0], # ('put',)
|
||||
"MinFeature": [16, 16],
|
||||
# PDFBox reads BlueValues[2] for CapHeight, so >=3 entries required.
|
||||
# Pairs: baseline (0 0), then cap-height zone (700 712).
|
||||
"BlueValues": [0, 0, BOX_T, BOX_T + 12],
|
||||
"StdHW": [60],
|
||||
"StdVW": [80],
|
||||
"lenIV": 4,
|
||||
"password": 5839,
|
||||
"Subrs": subrs,
|
||||
}
|
||||
|
||||
font_info = {
|
||||
"version": "001.001",
|
||||
"FullName": font_name,
|
||||
"FamilyName": font_name,
|
||||
"Weight": "Regular",
|
||||
"ItalicAngle": 0,
|
||||
"isFixedPitch": False,
|
||||
"UnderlinePosition": -100,
|
||||
"UnderlineThickness": 50,
|
||||
}
|
||||
|
||||
font = {
|
||||
"FontName": font_name,
|
||||
"FontInfo": font_info,
|
||||
"FontType": 1,
|
||||
"PaintType": 0,
|
||||
"FontMatrix": [0.001, 0, 0, 0.001, 0, 0],
|
||||
"FontBBox": [0, 0, WIDTH, BOX_T],
|
||||
"Encoding": encoding,
|
||||
"Private": private,
|
||||
"CharStrings": charstrings,
|
||||
}
|
||||
return font
|
||||
|
||||
|
||||
def main():
|
||||
out = sys.argv[1] if len(sys.argv) > 1 else "min.pfb"
|
||||
# Construct a T1Font without reading a file, then hand-build .font.
|
||||
t1 = t1Lib.T1Font.__new__(t1Lib.T1Font)
|
||||
t1.encoding = "ascii"
|
||||
t1.font = build_font()
|
||||
data = t1.createData() # full ASCII+eexec Type1 program (binary eexec inline)
|
||||
t1.data = data
|
||||
t1.saveAs(out, "PFB") # split into 0x80 0x01 / 0x80 0x02 / 0x80 0x03 segments
|
||||
# Report first bytes to confirm PFB marker.
|
||||
with open(out, "rb") as f:
|
||||
head = f.read(6)
|
||||
print("WROTE", out)
|
||||
print("PFB head bytes:", " ".join(f"{b:02x}" for b in head))
|
||||
assert head[0] == 0x80 and head[1] == 0x01, "not a classic PFB 0x80 0x01 header"
|
||||
print("OK: classic PFB 0x80 0x01 ASCII segment marker present")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
+4
-40
@@ -43,10 +43,6 @@ public class IntegrationConfigService {
|
||||
private final OwnershipService ownership;
|
||||
private final SecretMasker secretMasker;
|
||||
private final ResourceGrantRepository grantRepository;
|
||||
// Bean-discovered extension points: features that understand a type contribute its config
|
||||
// schema and report what still references a config, without this module depending on them.
|
||||
private final List<IntegrationConfigValidator> validators;
|
||||
private final List<IntegrationConfigUsageCheck> usageChecks;
|
||||
|
||||
// ---- commands ----
|
||||
|
||||
@@ -70,21 +66,13 @@ public class IntegrationConfigService {
|
||||
? DefaultAccessPolicy.EXPLICIT_ONLY
|
||||
: request.defaultAccess());
|
||||
|
||||
// TEAM scope may omit the team id: default to the caller's own team so clients (the
|
||||
// portal) need not know it. assignOwnership still enforces admin-or-leader of that team.
|
||||
Long ownerTeamId = request.ownerTeamId();
|
||||
if (ownerTeamId == null && scope == OwnerScope.TEAM && currentUser.getTeam() != null) {
|
||||
ownerTeamId = currentUser.getTeam().getId();
|
||||
}
|
||||
ownership.assignOwnership(
|
||||
cfg,
|
||||
scope,
|
||||
ownerTeamId,
|
||||
request.ownerTeamId(),
|
||||
currentUser,
|
||||
() -> lockedServerExists(cfg.getIntegrationType()));
|
||||
Map<String, Object> config = secretMasker.sanitize(request.config());
|
||||
validateConfig(cfg.getIntegrationType(), config);
|
||||
cfg.setConfig(writeJson(config));
|
||||
cfg.setConfig(writeJson(secretMasker.sanitize(request.config())));
|
||||
return repository.save(cfg);
|
||||
}
|
||||
|
||||
@@ -113,10 +101,8 @@ public class IntegrationConfigService {
|
||||
cfg.setDefaultAccess(request.defaultAccess());
|
||||
}
|
||||
if (request.config() != null) {
|
||||
Map<String, Object> merged =
|
||||
secretMasker.merge(readJson(cfg.getConfig()), request.config());
|
||||
validateConfig(cfg.getIntegrationType(), merged);
|
||||
cfg.setConfig(writeJson(merged));
|
||||
cfg.setConfig(
|
||||
writeJson(secretMasker.merge(readJson(cfg.getConfig()), request.config())));
|
||||
}
|
||||
return repository.save(cfg);
|
||||
}
|
||||
@@ -127,15 +113,6 @@ public class IntegrationConfigService {
|
||||
if (!ownership.canManage(TYPE, cfg, currentUser)) {
|
||||
throw forbidden("You cannot manage this integration");
|
||||
}
|
||||
// Refuse to pull a connection out from under whatever still references it.
|
||||
List<String> usages =
|
||||
usageChecks.stream()
|
||||
.flatMap(check -> check.usagesOf(cfg.getId()).stream())
|
||||
.toList();
|
||||
if (!usages.isEmpty()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.CONFLICT, "Integration is in use by: " + String.join(", ", usages));
|
||||
}
|
||||
// Drop grants sharing this config so they do not dangle as dead rows.
|
||||
grantRepository.deleteByResourceTypeAndResourceId(TYPE, String.valueOf(cfg.getId()));
|
||||
repository.delete(cfg);
|
||||
@@ -211,19 +188,6 @@ public class IntegrationConfigService {
|
||||
|
||||
// ---- integration-specific glue ----
|
||||
|
||||
/** Runs every registered validator for the type; unknown types save free-form. */
|
||||
private void validateConfig(IntegrationType type, Map<String, Object> config) {
|
||||
for (IntegrationConfigValidator validator : validators) {
|
||||
if (validator.type() == type) {
|
||||
try {
|
||||
validator.validate(config == null ? Map.of() : config);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A non-admin can't create a personal config of a type an admin has locked at server scope. */
|
||||
private boolean lockedServerExists(IntegrationType type) {
|
||||
return repository.findByScope(OwnerScope.SERVER).stream()
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package stirling.software.proprietary.integration.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Reports what still references an integration config, so deletion can be refused instead of
|
||||
* pulling a connection out from under a live consumer. Implementations are beans discovered by
|
||||
* {@link IntegrationConfigService} (e.g. the policy subsystem reporting sources and pipelines that
|
||||
* reference a connection).
|
||||
*/
|
||||
public interface IntegrationConfigUsageCheck {
|
||||
|
||||
/** Human-readable labels of everything still using the config; empty when unreferenced. */
|
||||
List<String> usagesOf(long configId);
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package stirling.software.proprietary.integration.service;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
|
||||
/**
|
||||
* Validates one integration type's config map at save time. Implementations are beans discovered by
|
||||
* {@link IntegrationConfigService}, so the feature that understands a type (e.g. the policy S3
|
||||
* backend) owns its schema without the integration module depending on it. Types with no registered
|
||||
* validator save free-form.
|
||||
*/
|
||||
public interface IntegrationConfigValidator {
|
||||
|
||||
/** The type this validator understands. */
|
||||
IntegrationType type();
|
||||
|
||||
/**
|
||||
* Validates the config as it will be stored (secrets already sanitized/merged, so values are
|
||||
* real, never the redaction mask). Throws {@link IllegalArgumentException} on bad config.
|
||||
*/
|
||||
void validate(Map<String, Object> config);
|
||||
}
|
||||
-20
@@ -120,7 +120,6 @@ public class PolicyController {
|
||||
throws IOException {
|
||||
stampPolicyAudit(definition);
|
||||
requireRunnable(definition);
|
||||
validateAdHocOutput(definition);
|
||||
PolicyInputs inputs = toInputs(files);
|
||||
PolicyRunHandle handle =
|
||||
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP);
|
||||
@@ -141,7 +140,6 @@ public class PolicyController {
|
||||
throws IOException {
|
||||
stampPolicyAudit(definition);
|
||||
requireRunnable(definition);
|
||||
validateAdHocOutput(definition);
|
||||
PolicyInputs inputs = toInputs(files);
|
||||
|
||||
SseEmitter emitter =
|
||||
@@ -532,24 +530,6 @@ public class PolicyController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Authorization-check an ad-hoc run's output while the caller's principal is present (this
|
||||
* request thread). The worker thread that later delivers carries no security context, so an S3
|
||||
* output's connection-access check would be skipped there; without this gate a caller could
|
||||
* reference another tenant's connection by id and write to it (confused deputy). Stored
|
||||
* policies are covered by save-time {@link PolicyValidator#validate} instead.
|
||||
*/
|
||||
private void validateAdHocOutput(PipelineDefinition definition) {
|
||||
if (definition.output() == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
policyValidator.validateOutput(definition.output());
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ad-hoc runs (AI / one-off pipelines) are still editor activity, so their supplied documents
|
||||
* feed the same virtual editor source as stored editor policies, counted against the caller's
|
||||
|
||||
+1
-13
@@ -52,19 +52,7 @@ public class PolicyValidator {
|
||||
InputSpec spec = source.toInputSpec();
|
||||
inputSourceFor(spec).validate(spec);
|
||||
}
|
||||
validateOutput(policy.output());
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an output spec against its sink. Must be called on a request thread (caller's
|
||||
* principal present) so an S3 output's connection is authorization-checked against the caller -
|
||||
* ad-hoc runs are never persisted and so never hit {@link #validate(Policy)}, and the worker
|
||||
* thread that later delivers has no principal, so this is their only access gate.
|
||||
*
|
||||
* @throws IllegalArgumentException if the type is unknown or the config is invalid/inaccessible
|
||||
*/
|
||||
public void validateOutput(OutputSpec output) {
|
||||
outputSinkFor(output).validate(output);
|
||||
outputSinkFor(policy.output()).validate(policy.output());
|
||||
}
|
||||
|
||||
private PolicyTrigger triggerFor(TriggerConfig config) {
|
||||
|
||||
+13
-14
@@ -18,7 +18,6 @@ import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.s3.S3Config;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
|
||||
import stirling.software.proprietary.policy.s3.S3Identities;
|
||||
|
||||
import software.amazon.awssdk.core.exception.SdkException;
|
||||
@@ -37,14 +36,15 @@ import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
* Reads input files from an Amazon S3 (or S3-compatible) bucket; each listed object is its own unit
|
||||
* of work, claimed through the {@link ResolveContext} ledger and tracked in place. Identity and
|
||||
* version gate come from {@link S3Identities}, so the steady-state sweep never downloads content.
|
||||
* Options: "connectionId" references the stored S3 connection (an {@code IntegrationConfig} owning
|
||||
* bucket, region, endpoint, and credentials - resolved by {@link S3ConnectionResolver}); "prefix"
|
||||
* (only keys starting with it are read) and "mode" are per-source, where mode is "consume"
|
||||
* (default: a processed object is deleted once every policy that claimed it has settled
|
||||
* successfully and it is still the version that ran; failures stay in place and are not retried
|
||||
* until they change) or "snapshot" (stateless, every run sees the full set). Keys ending in "/"
|
||||
* (folder placeholders) and keys with a dot-prefixed path segment are never picked up, mirroring
|
||||
* the folder source's hidden-file rule.
|
||||
* Options (see {@link S3Config}): "bucket" (required), "region" (default us-east-1), "prefix" (only
|
||||
* keys starting with it are read), "endpoint" (S3-compatible stores such as MinIO; path-style
|
||||
* addressing is used automatically), "accessKeyId" and "secretAccessKey" (required; requests are
|
||||
* never signed with the server's own AWS identity), and "mode" which is "consume" (default: a
|
||||
* processed object is deleted once every policy that claimed it has settled successfully and it is
|
||||
* still the version that ran; failures stay in place and are not retried until they change) or
|
||||
* "snapshot" (stateless, every run sees the full set). Keys ending in "/" (folder placeholders) and
|
||||
* keys with a dot-prefixed path segment are never picked up, mirroring the folder source's
|
||||
* hidden-file rule.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -55,7 +55,6 @@ public class S3InputSource implements InputSource {
|
||||
private static final String TYPE = "s3";
|
||||
|
||||
private final S3ConnectionPool connectionPool;
|
||||
private final S3ConnectionResolver connectionResolver;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
@@ -68,12 +67,12 @@ public class S3InputSource implements InputSource {
|
||||
}
|
||||
|
||||
/**
|
||||
* Fails fast at save time: an unknown/disabled/unusable connection, bad config shape, a private
|
||||
* endpoint without the operator opt-in, or a bucket the connection cannot list.
|
||||
* Fails fast at save time: bad config shape, a private endpoint without the operator opt-in, or
|
||||
* a bucket the supplied credentials cannot list.
|
||||
*/
|
||||
@Override
|
||||
public void validate(InputSpec spec) {
|
||||
S3Config config = connectionResolver.resolve(spec.options());
|
||||
S3Config config = S3Config.from(spec.options());
|
||||
try {
|
||||
connectionPool.clientFor(config).listObjectsV2(listRequest(config).maxKeys(1).build());
|
||||
} catch (SdkException e) {
|
||||
@@ -90,7 +89,7 @@ public class S3InputSource implements InputSource {
|
||||
|
||||
@Override
|
||||
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
|
||||
S3Config config = connectionResolver.resolve(spec.options());
|
||||
S3Config config = S3Config.from(spec.options());
|
||||
S3Client client = connectionPool.clientFor(config);
|
||||
// A listing failure propagates so the sweep reads it as "could not list" (which vetoes
|
||||
// presence cleanup), never as "verifiably no objects".
|
||||
|
||||
+5
-7
@@ -27,7 +27,6 @@ import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3Config;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionResolver;
|
||||
import stirling.software.proprietary.policy.s3.S3Identities;
|
||||
|
||||
import software.amazon.awssdk.core.exception.SdkException;
|
||||
@@ -60,7 +59,6 @@ public class S3OutputSink implements PolicyOutputSink {
|
||||
private static final String TYPE = "s3";
|
||||
|
||||
private final S3ConnectionPool connectionPool;
|
||||
private final S3ConnectionResolver connectionResolver;
|
||||
private final ProcessedLedger processedLedger;
|
||||
|
||||
@Override
|
||||
@@ -74,19 +72,19 @@ public class S3OutputSink implements PolicyOutputSink {
|
||||
}
|
||||
|
||||
/**
|
||||
* Connection resolution (including the saving user's right to use it) and endpoint guard only -
|
||||
* no network probe, since write-only credentials (s3:PutObject without s3:ListBucket) are a
|
||||
* legitimate setup for an output bucket and a listing probe would wrongly reject them.
|
||||
* Config shape and endpoint guard only - no network probe, since write-only credentials
|
||||
* (s3:PutObject without s3:ListBucket) are a legitimate setup for an output bucket and a
|
||||
* listing probe would wrongly reject them.
|
||||
*/
|
||||
@Override
|
||||
public void validate(OutputSpec spec) {
|
||||
connectionPool.clientFor(connectionResolver.resolve(spec.options()));
|
||||
connectionPool.clientFor(S3Config.from(spec.options()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResultFile> deliver(
|
||||
OutputDelivery delivery, List<Resource> outputs, OutputSpec spec) throws IOException {
|
||||
S3Config config = connectionResolver.resolve(spec.options());
|
||||
S3Config config = S3Config.from(spec.options());
|
||||
S3Client client = connectionPool.clientFor(config);
|
||||
|
||||
List<ResultFile> results = new ArrayList<>();
|
||||
|
||||
-212
@@ -1,212 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.access.model.DefaultAccessPolicy;
|
||||
import stirling.software.proprietary.access.model.OwnerScope;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* One-time, idempotent extraction of legacy embedded S3 credentials into stored connections:
|
||||
* sources and policy outputs written before connections shipped carry bucket/credentials in their
|
||||
* own options; this rewrites each to reference a (deduplicated) S3 {@link IntegrationConfig} and
|
||||
* keeps only per-use options (prefix, mode). MUST be programmatic - the option JSON is encrypted at
|
||||
* the application layer, so no SQL migration can read it.
|
||||
*
|
||||
* <p>Idempotent by construction: rewritten rows no longer embed credentials, so re-runs find
|
||||
* nothing to do. Connections are deduplicated against both this run's extractions and existing S3
|
||||
* connections; a concurrent multi-node boot can at worst create a redundant connection row, never
|
||||
* corrupt a source. Ownership follows the owning row: team-scoped when the source/policy has a
|
||||
* team, server-scoped otherwise (single-operator self-hosted).
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class EmbeddedS3CredentialMigration {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
private static final List<String> CONNECTION_OPTIONS =
|
||||
List.of("bucket", "region", "endpoint", "accessKeyId", "secretAccessKey");
|
||||
// Field separator for the dedup key: a unit-separator control char that cannot appear in a
|
||||
// bucket/region/endpoint/credential, so distinct field sets can never collide.
|
||||
private static final char DELIMITER = '\u001f';
|
||||
|
||||
private final SourceStore sourceStore;
|
||||
private final PolicyStore policyStore;
|
||||
private final IntegrationConfigRepository connections;
|
||||
private final TeamRepository teamRepository;
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
@Transactional
|
||||
public void migrate() {
|
||||
Map<String, IntegrationConfig> byCredentialKey = indexExistingConnections();
|
||||
int migrated = 0;
|
||||
for (Source source : sourceStore.all()) {
|
||||
if (!"s3".equals(source.type()) || !embedsCredentials(source.options())) {
|
||||
continue;
|
||||
}
|
||||
IntegrationConfig connection =
|
||||
connectionFor(source.options(), source.teamId(), byCredentialKey);
|
||||
sourceStore.save(withOptions(source, referencing(connection, source.options(), true)));
|
||||
migrated++;
|
||||
}
|
||||
for (Policy policy : policyStore.all()) {
|
||||
OutputSpec output = policy.output();
|
||||
if (!"s3".equals(output.type()) || !embedsCredentials(output.options())) {
|
||||
continue;
|
||||
}
|
||||
IntegrationConfig connection =
|
||||
connectionFor(output.options(), policy.teamId(), byCredentialKey);
|
||||
policyStore.save(
|
||||
withOutput(
|
||||
policy,
|
||||
new OutputSpec(
|
||||
output.type(),
|
||||
referencing(connection, output.options(), false))));
|
||||
migrated++;
|
||||
}
|
||||
if (migrated > 0) {
|
||||
log.info("Extracted embedded S3 credentials from {} row(s) into connections", migrated);
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean embedsCredentials(Map<String, Object> options) {
|
||||
return options.get("accessKeyId") != null;
|
||||
}
|
||||
|
||||
/** Reuses an existing connection with identical coordinates+credentials, else creates one. */
|
||||
private IntegrationConfig connectionFor(
|
||||
Map<String, Object> options, Long teamId, Map<String, IntegrationConfig> byKey) {
|
||||
String key = credentialKey(options);
|
||||
IntegrationConfig existing = byKey.get(key);
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
IntegrationConfig connection = new IntegrationConfig();
|
||||
connection.setIntegrationType(IntegrationType.S3);
|
||||
connection.setName(connectionName(options, byKey));
|
||||
connection.setEnabled(true);
|
||||
connection.setLocked(false);
|
||||
connection.setDefaultAccess(DefaultAccessPolicy.EXPLICIT_ONLY);
|
||||
Team team = teamId == null ? null : teamRepository.findById(teamId).orElse(null);
|
||||
if (team != null) {
|
||||
connection.setScope(OwnerScope.TEAM);
|
||||
connection.setOwnerTeam(team);
|
||||
} else {
|
||||
// No team (teamless self-hosted, or a source whose team was since deleted): server
|
||||
// scope, i.e. admin-owned. An orphaned-team source's non-admin editor would then need
|
||||
// an admin to re-share the connection - acceptable for the narrow orphaned case.
|
||||
connection.setScope(OwnerScope.SERVER);
|
||||
}
|
||||
Map<String, Object> config = new LinkedHashMap<>();
|
||||
for (String option : CONNECTION_OPTIONS) {
|
||||
Object value = options.get(option);
|
||||
if (value != null && !value.toString().isBlank()) {
|
||||
config.put(option, value);
|
||||
}
|
||||
}
|
||||
connection.setConfig(OBJECT_MAPPER.writeValueAsString(config));
|
||||
IntegrationConfig saved = connections.save(connection);
|
||||
byKey.put(key, saved);
|
||||
return saved;
|
||||
}
|
||||
|
||||
/** The rewritten options: the connection reference plus per-use settings only. */
|
||||
private static Map<String, Object> referencing(
|
||||
IntegrationConfig connection, Map<String, Object> legacy, boolean keepMode) {
|
||||
Map<String, Object> options = new LinkedHashMap<>();
|
||||
options.put(S3ConnectionResolver.CONNECTION_ID_OPTION, connection.getId());
|
||||
Object prefix = legacy.get("prefix");
|
||||
if (prefix != null && !prefix.toString().isBlank()) {
|
||||
options.put("prefix", prefix);
|
||||
}
|
||||
Object mode = legacy.get("mode");
|
||||
if (keepMode && mode != null && !mode.toString().isBlank()) {
|
||||
options.put("mode", mode);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
private Map<String, IntegrationConfig> indexExistingConnections() {
|
||||
Map<String, IntegrationConfig> byKey = new LinkedHashMap<>();
|
||||
for (IntegrationConfig connection : connections.findAll()) {
|
||||
if (connection.getIntegrationType() != IntegrationType.S3) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
Map<String, Object> config =
|
||||
OBJECT_MAPPER.readValue(connection.getConfig(), Map.class);
|
||||
byKey.putIfAbsent(credentialKey(config), connection);
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Skipping unreadable S3 connection {} while indexing: {}",
|
||||
connection.getId(),
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
return byKey;
|
||||
}
|
||||
|
||||
private static String credentialKey(Map<String, Object> options) {
|
||||
StringBuilder key = new StringBuilder();
|
||||
for (String option : CONNECTION_OPTIONS) {
|
||||
Object value = options.get(option);
|
||||
key.append(value == null ? "" : value.toString().trim()).append(DELIMITER);
|
||||
}
|
||||
return key.toString();
|
||||
}
|
||||
|
||||
private static String connectionName(
|
||||
Map<String, Object> options, Map<String, IntegrationConfig> byKey) {
|
||||
String base = "S3: " + options.getOrDefault("bucket", "bucket");
|
||||
long sameName = byKey.values().stream().filter(c -> c.getName().startsWith(base)).count();
|
||||
return sameName == 0 ? base : base + " (" + (sameName + 1) + ")";
|
||||
}
|
||||
|
||||
private static Source withOptions(Source source, Map<String, Object> options) {
|
||||
return new Source(
|
||||
source.id(),
|
||||
source.name(),
|
||||
source.type(),
|
||||
options,
|
||||
source.enabled(),
|
||||
source.owner(),
|
||||
source.teamId());
|
||||
}
|
||||
|
||||
private static Policy withOutput(Policy policy, OutputSpec output) {
|
||||
return new Policy(
|
||||
policy.id(),
|
||||
policy.name(),
|
||||
policy.owner(),
|
||||
policy.enabled(),
|
||||
policy.trigger(),
|
||||
policy.sourceIds(),
|
||||
policy.steps(),
|
||||
output,
|
||||
policy.teamId());
|
||||
}
|
||||
}
|
||||
-56
@@ -1,56 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.proprietary.integration.service.IntegrationConfigUsageCheck;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
|
||||
/**
|
||||
* Reports the policy sources and pipeline outputs referencing an S3 connection, so the connection
|
||||
* cannot be deleted out from under them (mirrors {@code SourceController}'s referenced-source
|
||||
* delete guard). Scans in memory - fine at admin-dashboard scale, always consistent with the live
|
||||
* stores.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class PolicyS3ConnectionUsageCheck implements IntegrationConfigUsageCheck {
|
||||
|
||||
private final SourceStore sourceStore;
|
||||
private final PolicyStore policyStore;
|
||||
|
||||
@Override
|
||||
public List<String> usagesOf(long configId) {
|
||||
List<String> usages = new ArrayList<>();
|
||||
for (Source source : sourceStore.all()) {
|
||||
if (references(source.options(), configId)) {
|
||||
usages.add("source '" + source.name() + "'");
|
||||
}
|
||||
}
|
||||
for (Policy policy : policyStore.all()) {
|
||||
if (references(policy.output().options(), configId)) {
|
||||
usages.add("pipeline '" + policy.name() + "'");
|
||||
}
|
||||
}
|
||||
return usages;
|
||||
}
|
||||
|
||||
private static boolean references(Map<String, Object> options, long configId) {
|
||||
try {
|
||||
Long reference = S3ConnectionResolver.connectionId(options);
|
||||
return reference != null && reference == configId;
|
||||
} catch (IllegalArgumentException unparseable) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,12 +5,10 @@ import java.net.URISyntaxException;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* The fully resolved connection settings the S3 input source and output sink run with - normally
|
||||
* produced by {@link S3ConnectionResolver} merging a stored connection (bucket, region, endpoint,
|
||||
* credentials) with per-use options (prefix, mode), or parsed directly from legacy options that
|
||||
* still embed credentials. Credentials are required: there is deliberately no fallback to the
|
||||
* server's own AWS credential chain, so user-supplied config can never borrow the host's identity.
|
||||
* {@code snapshot} is input-only and ignored by the sink.
|
||||
* Connection settings shared by the S3 input source and output sink, parsed from a spec's options
|
||||
* map. Credentials are required: there is deliberately no fallback to the server's own AWS
|
||||
* credential chain, so user-supplied config can never borrow the host's identity. {@code snapshot}
|
||||
* is input-only and ignored by the sink.
|
||||
*/
|
||||
public record S3Config(
|
||||
String bucket,
|
||||
|
||||
-151
@@ -1,151 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.security.core.userdetails.UserDetails;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.access.model.ResourceType;
|
||||
import stirling.software.proprietary.access.service.OwnershipService;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Turns a source's or output's options into a full {@link S3Config} by dereferencing its {@code
|
||||
* connectionId} to a stored S3 {@link IntegrationConfig} (the connection owns bucket, region,
|
||||
* endpoint, and credentials; the options own per-use settings such as prefix and mode). Options
|
||||
* with no {@code connectionId} fall back to legacy embedded credentials, so rows written before
|
||||
* connections shipped keep working until {@link EmbeddedS3CredentialMigration} rewrites them.
|
||||
*
|
||||
* <p>When an authenticated caller is present (save-time validation), they must be allowed to use
|
||||
* the connection. Background sweeps and deliveries run with no caller and skip that check: the
|
||||
* referencing source or policy was access-checked when it was saved.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Transactional(readOnly = true)
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class S3ConnectionResolver {
|
||||
|
||||
static final String CONNECTION_ID_OPTION = "connectionId";
|
||||
private static final String PREFIX_OPTION = "prefix";
|
||||
private static final String MODE_OPTION = "mode";
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private final IntegrationConfigRepository connections;
|
||||
private final OwnershipService ownership;
|
||||
private final UserService userService;
|
||||
|
||||
public S3Config resolve(Map<String, Object> options) {
|
||||
Long connectionId = connectionId(options);
|
||||
if (connectionId == null) {
|
||||
// Legacy embedded credentials, pending migration.
|
||||
return S3Config.from(options);
|
||||
}
|
||||
IntegrationConfig connection =
|
||||
connections
|
||||
.findById(connectionId)
|
||||
.filter(cfg -> cfg.getIntegrationType() == IntegrationType.S3)
|
||||
.filter(this::usableByCurrentUser)
|
||||
// Existence and access collapse into one error: a caller must not be able
|
||||
// to tell "no such connection" from "someone else's connection" and
|
||||
// enumerate ids. The id/name are never echoed.
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalArgumentException(
|
||||
"unknown or inaccessible s3 connection"));
|
||||
if (!connection.isEnabled()) {
|
||||
throw new IllegalArgumentException("s3 connection is disabled");
|
||||
}
|
||||
Map<String, Object> merged = new LinkedHashMap<>(connectionConfig(connection));
|
||||
copyPerUseOption(options, merged, PREFIX_OPTION);
|
||||
copyPerUseOption(options, merged, MODE_OPTION);
|
||||
return S3Config.from(merged);
|
||||
}
|
||||
|
||||
/** The {@code connectionId} option as a long, or null when the options are legacy-embedded. */
|
||||
static Long connectionId(Map<String, Object> options) {
|
||||
Object reference = options.get(CONNECTION_ID_OPTION);
|
||||
if (reference == null || (reference instanceof String s && s.isBlank())) {
|
||||
return null;
|
||||
}
|
||||
if (reference instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
try {
|
||||
return Long.valueOf(reference.toString().trim());
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException(
|
||||
"s3 'connectionId' is not a valid connection reference: " + reference);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the current caller may use this connection. With no principal - a background sweep or
|
||||
* delivery on a worker thread that carries no {@code SecurityContext} - access is treated as
|
||||
* already established: stored policies are validated with the caller present at save time, and
|
||||
* ad-hoc runs are validated on the request thread before dispatch (see {@code
|
||||
* PolicyValidator#validateOutput}). A missing principal must therefore never be the ONLY thing
|
||||
* standing between a caller and a connection, or the check becomes a confused deputy.
|
||||
*/
|
||||
private boolean usableByCurrentUser(IntegrationConfig connection) {
|
||||
User user = currentUser();
|
||||
return user == null || ownership.canUse(ResourceType.INTEGRATION_CONFIG, connection, user);
|
||||
}
|
||||
|
||||
// Mirrors ResourceAccessSecurity's principal resolution; null when unauthenticated.
|
||||
private User currentUser() {
|
||||
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
|
||||
if (auth == null || !auth.isAuthenticated()) {
|
||||
return null;
|
||||
}
|
||||
Object principal = auth.getPrincipal();
|
||||
if (principal instanceof User user) {
|
||||
return user;
|
||||
}
|
||||
if (principal instanceof UserDetails userDetails) {
|
||||
return userService.findByUsername(userDetails.getUsername()).orElse(null);
|
||||
}
|
||||
if (principal instanceof String username && !"anonymousUser".equals(username)) {
|
||||
return userService.findByUsername(username).orElse(null);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Map<String, Object> connectionConfig(IntegrationConfig connection) {
|
||||
String json = connection.getConfig();
|
||||
if (json == null || json.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
return OBJECT_MAPPER.readValue(
|
||||
json, new TypeReference<LinkedHashMap<String, Object>>() {});
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException(
|
||||
"s3 connection '" + connection.getName() + "' has unreadable config", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static void copyPerUseOption(
|
||||
Map<String, Object> options, Map<String, Object> merged, String key) {
|
||||
Object value = options.get(key);
|
||||
if (value != null && !value.toString().isBlank()) {
|
||||
merged.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
-51
@@ -1,51 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.cluster.s3.S3Clients;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.service.IntegrationConfigValidator;
|
||||
|
||||
/**
|
||||
* The S3 connection schema, enforced when an S3 {@link IntegrationType} config is saved: bucket and
|
||||
* credentials required, endpoint an http(s) URL that must not reach private addresses without the
|
||||
* operator opt-in - the same rules {@link S3ConnectionPool} enforces before signing, moved to save
|
||||
* time so a bad connection fails in the form rather than in a sweep.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnBooleanProperty(name = "policies.enabled")
|
||||
public class S3IntegrationValidator implements IntegrationConfigValidator {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@Override
|
||||
public IntegrationType type() {
|
||||
return IntegrationType.S3;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> config) {
|
||||
S3Config parsed = S3Config.from(config);
|
||||
if (parsed.endpoint() == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
S3Clients.validateEndpointHost(
|
||||
URI.create(parsed.endpoint()),
|
||||
applicationProperties.getPolicies().isAllowPrivateS3Endpoints(),
|
||||
"S3 connection endpoint",
|
||||
"set policies.allowPrivateS3Endpoints=true to opt in (e.g. for a local"
|
||||
+ " MinIO).");
|
||||
} catch (IllegalStateException e) {
|
||||
throw new IllegalArgumentException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -103,6 +103,7 @@ public class User implements UserDetails, Serializable {
|
||||
|
||||
@ElementCollection
|
||||
@MapKeyColumn(name = "setting_key")
|
||||
@Lob
|
||||
@Column(name = "setting_value", columnDefinition = "text")
|
||||
@CollectionTable(name = "user_settings", joinColumns = @JoinColumn(name = "user_id"))
|
||||
@JsonIgnore
|
||||
|
||||
+2
-53
@@ -13,9 +13,9 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpStatus;
|
||||
@@ -52,58 +52,7 @@ class IntegrationConfigServiceTest {
|
||||
@Mock
|
||||
private stirling.software.proprietary.access.repository.ResourceGrantRepository grantRepository;
|
||||
|
||||
@Mock private IntegrationConfigValidator validator;
|
||||
@Mock private IntegrationConfigUsageCheck usageCheck;
|
||||
|
||||
private IntegrationConfigService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service =
|
||||
new IntegrationConfigService(
|
||||
repository,
|
||||
ownership,
|
||||
secretMasker,
|
||||
grantRepository,
|
||||
List.of(validator),
|
||||
List.of(usageCheck));
|
||||
}
|
||||
|
||||
@Test
|
||||
void createRejectsAConfigItsTypeValidatorRefuses() {
|
||||
when(secretMasker.sanitize(any())).thenReturn(Map.of());
|
||||
when(validator.type()).thenReturn(IntegrationType.API);
|
||||
org.mockito.Mockito.doThrow(new IllegalArgumentException("api config needs a 'url'"))
|
||||
.when(validator)
|
||||
.validate(any());
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
service.create(
|
||||
request(IntegrationType.API, OwnerScope.USER, null),
|
||||
user(7)))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.BAD_REQUEST));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteRefusedWhileAnythingStillReferencesTheConfig() {
|
||||
IntegrationConfig cfg = config(9L);
|
||||
when(repository.findById(9L)).thenReturn(Optional.of(cfg));
|
||||
when(ownership.canManage(any(), eq(cfg), any())).thenReturn(true);
|
||||
when(usageCheck.usagesOf(9L)).thenReturn(List.of("source 'Claims intake'"));
|
||||
|
||||
assertThatThrownBy(() -> service.delete(9L, user(7)))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.CONFLICT));
|
||||
verify(repository, org.mockito.Mockito.never()).delete(any(IntegrationConfig.class));
|
||||
}
|
||||
@InjectMocks private IntegrationConfigService service;
|
||||
|
||||
@Test
|
||||
void createDelegatesOwnershipAndSanitizesConfig() {
|
||||
|
||||
-24
@@ -4,7 +4,6 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -195,29 +194,6 @@ class PolicyControllerTest {
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.BAD_REQUEST));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects an ad-hoc output the caller cannot use, on the request thread")
|
||||
void rejectsUnauthorizedAdHocOutput() {
|
||||
// The confused-deputy guard: an S3 output referencing a connection the caller may not
|
||||
// use is validated here (principal present) and refused before any worker dispatch.
|
||||
PipelineDefinition definition =
|
||||
new PipelineDefinition(
|
||||
"pipe",
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", null)),
|
||||
new OutputSpec("s3", Map.of("connectionId", 999)));
|
||||
doThrow(new IllegalArgumentException("unknown or inaccessible s3 connection"))
|
||||
.when(policyValidator)
|
||||
.validateOutput(any());
|
||||
|
||||
assertThatThrownBy(() -> controller.run(definition, new PolicyRunFiles()))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((ResponseStatusException) e).getStatusCode())
|
||||
.isEqualTo(HttpStatus.BAD_REQUEST));
|
||||
verify(policyRunner, never()).runAdHoc(any(), any(), any());
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
|
||||
-22
@@ -82,28 +82,6 @@ class PolicyValidatorTest {
|
||||
assertTrue(ex.getMessage().contains("schedule"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateOutputDelegatesToTheSink() {
|
||||
when(outputSink.supports(any())).thenReturn(true);
|
||||
OutputSpec output = new OutputSpec("s3", Map.of("connectionId", 1));
|
||||
|
||||
validator.validateOutput(output);
|
||||
|
||||
verify(outputSink).validate(output);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateOutputSurfacesAnInaccessibleConnection() {
|
||||
when(outputSink.supports(any())).thenReturn(true);
|
||||
doThrow(new IllegalArgumentException("unknown or inaccessible s3 connection"))
|
||||
.when(outputSink)
|
||||
.validate(any());
|
||||
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> validator.validateOutput(new OutputSpec("s3", Map.of("connectionId", 1))));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsAnUnknownTriggerType() {
|
||||
when(trigger.type()).thenReturn("schedule");
|
||||
|
||||
+2
-7
@@ -23,7 +23,6 @@ import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3TestConnections;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
@@ -83,9 +82,7 @@ class S3InputSourceMinioTest {
|
||||
// The MinIO endpoint resolves to loopback, so the operator opt-in must be on.
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getPolicies().setAllowPrivateS3Endpoints(true);
|
||||
source =
|
||||
new S3InputSource(
|
||||
new S3ConnectionPool(properties), S3TestConnections.legacyResolver());
|
||||
source = new S3InputSource(new S3ConnectionPool(properties));
|
||||
ledger = new InProcessProcessedLedger();
|
||||
ctx = new RecordingContext();
|
||||
}
|
||||
@@ -164,9 +161,7 @@ class S3InputSourceMinioTest {
|
||||
@Test
|
||||
void aPrivateEndpointIsRejectedWithoutTheOperatorOptIn() {
|
||||
S3InputSource guarded =
|
||||
new S3InputSource(
|
||||
new S3ConnectionPool(new ApplicationProperties()),
|
||||
S3TestConnections.legacyResolver());
|
||||
new S3InputSource(new S3ConnectionPool(new ApplicationProperties()));
|
||||
|
||||
assertThatThrownBy(() -> guarded.validate(spec(Map.of())))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
|
||||
+1
-3
@@ -29,7 +29,6 @@ import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3TestConnections;
|
||||
|
||||
import software.amazon.awssdk.core.ResponseInputStream;
|
||||
import software.amazon.awssdk.core.exception.SdkClientException;
|
||||
@@ -65,8 +64,7 @@ class S3InputSourceTest {
|
||||
void setUp() {
|
||||
source =
|
||||
new S3InputSource(
|
||||
new S3ConnectionPool(new ApplicationProperties(), config -> s3Client),
|
||||
S3TestConnections.legacyResolver());
|
||||
new S3ConnectionPool(new ApplicationProperties(), config -> s3Client));
|
||||
ledger = new InProcessProcessedLedger();
|
||||
ctx = new RecordingContext();
|
||||
}
|
||||
|
||||
+2
-3
@@ -28,7 +28,6 @@ import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3TestConnections;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
@@ -91,8 +90,8 @@ class S3OutputSinkMinioTest {
|
||||
properties.getPolicies().setAllowPrivateS3Endpoints(true);
|
||||
S3ConnectionPool pool = new S3ConnectionPool(properties);
|
||||
ledger = new InProcessProcessedLedger();
|
||||
sink = new S3OutputSink(pool, S3TestConnections.legacyResolver(), ledger);
|
||||
source = new S3InputSource(pool, S3TestConnections.legacyResolver());
|
||||
sink = new S3OutputSink(pool, ledger);
|
||||
source = new S3InputSource(pool);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
-2
@@ -32,7 +32,6 @@ import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedFileStatus;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.s3.S3ConnectionPool;
|
||||
import stirling.software.proprietary.policy.s3.S3TestConnections;
|
||||
|
||||
import software.amazon.awssdk.awscore.exception.AwsServiceException;
|
||||
import software.amazon.awssdk.core.exception.SdkClientException;
|
||||
@@ -67,7 +66,6 @@ class S3OutputSinkTest {
|
||||
sink =
|
||||
new S3OutputSink(
|
||||
new S3ConnectionPool(new ApplicationProperties(), config -> s3Client),
|
||||
S3TestConnections.legacyResolver(),
|
||||
ledger);
|
||||
}
|
||||
|
||||
|
||||
-212
@@ -1,212 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.proprietary.access.model.OwnerScope;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.store.InProcessPolicyStore;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
|
||||
/**
|
||||
* Tests for {@link EmbeddedS3CredentialMigration}: legacy embedded credentials become deduplicated
|
||||
* team-scoped connections, rewritten rows keep only per-use options, and re-runs are no-ops.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class EmbeddedS3CredentialMigrationTest {
|
||||
|
||||
@Mock private IntegrationConfigRepository connections;
|
||||
@Mock private TeamRepository teamRepository;
|
||||
|
||||
private final InProcessSourceStore sourceStore = new InProcessSourceStore();
|
||||
private final InProcessPolicyStore policyStore = new InProcessPolicyStore();
|
||||
private EmbeddedS3CredentialMigration migration;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
migration =
|
||||
new EmbeddedS3CredentialMigration(
|
||||
sourceStore, policyStore, connections, teamRepository);
|
||||
AtomicLong ids = new AtomicLong(100);
|
||||
// Lenient: the nothing-to-migrate cases never create a connection.
|
||||
lenient().when(connections.findAll()).thenReturn(List.of());
|
||||
lenient()
|
||||
.when(connections.save(any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
IntegrationConfig saved = invocation.getArgument(0);
|
||||
if (saved.getId() == null) {
|
||||
saved.setId(ids.incrementAndGet());
|
||||
}
|
||||
return saved;
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void extractsSharedCredentialsIntoOneTeamScopedConnection() {
|
||||
Team team = new Team();
|
||||
team.setId(7L);
|
||||
when(teamRepository.findById(7L)).thenReturn(Optional.of(team));
|
||||
Source source =
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
null,
|
||||
"Claims intake",
|
||||
"s3",
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"prefix", "incoming/",
|
||||
"mode", "snapshot",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh"),
|
||||
true,
|
||||
"alice",
|
||||
7L));
|
||||
Policy policy =
|
||||
policyStore.save(
|
||||
new Policy(
|
||||
null,
|
||||
"Rotate",
|
||||
"alice",
|
||||
true,
|
||||
null,
|
||||
List.of(source.id()),
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
|
||||
new OutputSpec(
|
||||
"s3",
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"prefix", "processed/",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh")),
|
||||
7L));
|
||||
|
||||
migration.migrate();
|
||||
|
||||
// Same bucket + credentials on both rows: exactly one connection extracted.
|
||||
verify(connections, times(1)).save(any());
|
||||
Map<String, Object> sourceOptions = sourceStore.get(source.id()).orElseThrow().options();
|
||||
assertEquals(101L, sourceOptions.get("connectionId"));
|
||||
assertEquals("incoming/", sourceOptions.get("prefix"));
|
||||
assertEquals("snapshot", sourceOptions.get("mode"));
|
||||
assertNull(sourceOptions.get("accessKeyId"));
|
||||
assertNull(sourceOptions.get("secretAccessKey"));
|
||||
assertNull(sourceOptions.get("bucket"));
|
||||
|
||||
Map<String, Object> outputOptions =
|
||||
policyStore.get(policy.id()).orElseThrow().output().options();
|
||||
assertEquals(101L, outputOptions.get("connectionId"));
|
||||
assertEquals("processed/", outputOptions.get("prefix"));
|
||||
assertNull(outputOptions.get("secretAccessKey"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void connectionOwnershipFollowsTheSourceTeam() {
|
||||
Team team = new Team();
|
||||
team.setId(7L);
|
||||
when(teamRepository.findById(7L)).thenReturn(Optional.of(team));
|
||||
sourceStore.save(s3Source("teamed", 7L));
|
||||
|
||||
migration.migrate();
|
||||
|
||||
verify(connections)
|
||||
.save(
|
||||
org.mockito.ArgumentMatchers.argThat(
|
||||
connection ->
|
||||
connection.getScope() == OwnerScope.TEAM
|
||||
&& connection.getOwnerTeam() == team));
|
||||
}
|
||||
|
||||
@Test
|
||||
void teamlessRowsBecomeServerScopedConnections() {
|
||||
sourceStore.save(s3Source("solo", null));
|
||||
|
||||
migration.migrate();
|
||||
|
||||
verify(connections)
|
||||
.save(
|
||||
org.mockito.ArgumentMatchers.argThat(
|
||||
connection -> connection.getScope() == OwnerScope.SERVER));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSecondRunFindsNothingToDo() {
|
||||
sourceStore.save(s3Source("once", null));
|
||||
|
||||
migration.migrate();
|
||||
migration.migrate();
|
||||
|
||||
// One connection from the first run; the rewritten source no longer embeds credentials.
|
||||
verify(connections, times(1)).save(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonS3AndAlreadyMigratedRowsAreUntouched() {
|
||||
Source folder =
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
null,
|
||||
"Folder",
|
||||
"folder",
|
||||
Map.of("directory", "/in"),
|
||||
true,
|
||||
"alice",
|
||||
null));
|
||||
Source migrated =
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
null,
|
||||
"Done already",
|
||||
"s3",
|
||||
Map.of("connectionId", 55L, "prefix", "in/"),
|
||||
true,
|
||||
"alice",
|
||||
null));
|
||||
|
||||
migration.migrate();
|
||||
|
||||
verify(connections, times(0)).save(any());
|
||||
assertEquals(
|
||||
Map.of("directory", "/in"), sourceStore.get(folder.id()).orElseThrow().options());
|
||||
assertEquals(
|
||||
Map.of("connectionId", 55L, "prefix", "in/"),
|
||||
sourceStore.get(migrated.id()).orElseThrow().options());
|
||||
}
|
||||
|
||||
private static Source s3Source(String name, Long teamId) {
|
||||
return new Source(
|
||||
null,
|
||||
name,
|
||||
"s3",
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh"),
|
||||
true,
|
||||
"alice",
|
||||
teamId);
|
||||
}
|
||||
}
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.store.InProcessPolicyStore;
|
||||
|
||||
/** Tests for {@link PolicyS3ConnectionUsageCheck}'s reference scan across sources and outputs. */
|
||||
class PolicyS3ConnectionUsageCheckTest {
|
||||
|
||||
private final InProcessSourceStore sourceStore = new InProcessSourceStore();
|
||||
private final InProcessPolicyStore policyStore = new InProcessPolicyStore();
|
||||
private final PolicyS3ConnectionUsageCheck check =
|
||||
new PolicyS3ConnectionUsageCheck(sourceStore, policyStore);
|
||||
|
||||
@Test
|
||||
void reportsSourcesAndOutputsReferencingTheConnection() {
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
null,
|
||||
"Claims intake",
|
||||
"s3",
|
||||
Map.of("connectionId", 5L, "prefix", "in/"),
|
||||
true,
|
||||
"alice",
|
||||
null));
|
||||
policyStore.save(
|
||||
new Policy(
|
||||
null,
|
||||
"Rotate",
|
||||
"alice",
|
||||
true,
|
||||
null,
|
||||
List.of(),
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
|
||||
new OutputSpec("s3", Map.of("connectionId", "5")),
|
||||
null));
|
||||
|
||||
assertThat(check.usagesOf(5))
|
||||
.containsExactlyInAnyOrder("source 'Claims intake'", "pipeline 'Rotate'");
|
||||
assertThat(check.usagesOf(6)).isEmpty();
|
||||
}
|
||||
}
|
||||
-149
@@ -1,149 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
|
||||
import stirling.software.proprietary.access.service.OwnershipService;
|
||||
import stirling.software.proprietary.integration.model.IntegrationConfig;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
/**
|
||||
* Tests for {@link S3ConnectionResolver}: connection dereferencing with per-use overrides, the
|
||||
* legacy embedded fallback, and the save-time access check that background sweeps skip.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class S3ConnectionResolverTest {
|
||||
|
||||
@Mock private IntegrationConfigRepository connections;
|
||||
@Mock private OwnershipService ownership;
|
||||
@Mock private UserService userService;
|
||||
|
||||
@AfterEach
|
||||
void clearSecurityContext() {
|
||||
SecurityContextHolder.clearContext();
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvesAConnectionAndMergesPerUseOptions() {
|
||||
when(connections.findById(9L)).thenReturn(Optional.of(s3Connection(9L, true)));
|
||||
|
||||
S3Config config =
|
||||
resolver()
|
||||
.resolve(
|
||||
Map.of(
|
||||
"connectionId", 9L,
|
||||
"prefix", "incoming/",
|
||||
"mode", "snapshot"));
|
||||
|
||||
assertEquals("inbox", config.bucket());
|
||||
assertEquals("AKIAEXAMPLE", config.accessKeyId());
|
||||
assertEquals("incoming/", config.prefix());
|
||||
assertTrue(config.snapshot());
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsAStringConnectionReference() {
|
||||
when(connections.findById(9L)).thenReturn(Optional.of(s3Connection(9L, true)));
|
||||
|
||||
assertEquals("inbox", resolver().resolve(Map.of("connectionId", "9")).bucket());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fallsBackToLegacyEmbeddedCredentials() {
|
||||
S3Config config =
|
||||
resolver()
|
||||
.resolve(
|
||||
Map.of(
|
||||
"bucket", "legacy",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh"));
|
||||
|
||||
assertEquals("legacy", config.bucket());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsUnknownDisabledOrWrongTypeConnections() {
|
||||
when(connections.findById(1L)).thenReturn(Optional.empty());
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> resolver().resolve(Map.of("connectionId", 1L)));
|
||||
|
||||
when(connections.findById(2L)).thenReturn(Optional.of(s3Connection(2L, false)));
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> resolver().resolve(Map.of("connectionId", 2L)));
|
||||
|
||||
IntegrationConfig mcp = s3Connection(3L, true);
|
||||
mcp.setIntegrationType(IntegrationType.MCP);
|
||||
when(connections.findById(3L)).thenReturn(Optional.of(mcp));
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> resolver().resolve(Map.of("connectionId", 3L)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void anAuthenticatedSaverMustBeAllowedToUseTheConnection() {
|
||||
when(connections.findById(9L)).thenReturn(Optional.of(s3Connection(9L, true)));
|
||||
User saver = new User();
|
||||
saver.setUsername("alice");
|
||||
SecurityContextHolder.getContext()
|
||||
.setAuthentication(
|
||||
new UsernamePasswordAuthenticationToken(saver, null, java.util.List.of()));
|
||||
when(ownership.canUse(any(), any(IntegrationConfig.class), eq(saver))).thenReturn(false);
|
||||
|
||||
// Denied reads the same as unknown and never echoes the connection name, so ids can't be
|
||||
// enumerated by probing.
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> resolver().resolve(Map.of("connectionId", 9L)));
|
||||
try {
|
||||
resolver().resolve(Map.of("connectionId", 9L));
|
||||
} catch (IllegalArgumentException e) {
|
||||
org.junit.jupiter.api.Assertions.assertFalse(
|
||||
e.getMessage().contains("Claims bucket"),
|
||||
"access-denied error must not leak the connection name");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void backgroundSweepsWithNoUserSkipTheAccessCheck() {
|
||||
when(connections.findById(9L)).thenReturn(Optional.of(s3Connection(9L, true)));
|
||||
|
||||
// No authentication in the context: resolution succeeds without consulting ownership.
|
||||
assertEquals("inbox", resolver().resolve(Map.of("connectionId", 9L)).bucket());
|
||||
}
|
||||
|
||||
private S3ConnectionResolver resolver() {
|
||||
return new S3ConnectionResolver(connections, ownership, userService);
|
||||
}
|
||||
|
||||
private static IntegrationConfig s3Connection(long id, boolean enabled) {
|
||||
IntegrationConfig connection = new IntegrationConfig();
|
||||
connection.setId(id);
|
||||
connection.setIntegrationType(IntegrationType.S3);
|
||||
connection.setName("Claims bucket");
|
||||
connection.setEnabled(enabled);
|
||||
connection.setConfig(
|
||||
"{\"bucket\":\"inbox\",\"accessKeyId\":\"AKIAEXAMPLE\","
|
||||
+ "\"secretAccessKey\":\"shh\"}");
|
||||
return connection;
|
||||
}
|
||||
}
|
||||
-71
@@ -1,71 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.integration.model.IntegrationType;
|
||||
|
||||
/**
|
||||
* Tests for {@link S3IntegrationValidator}: the S3 connection schema fails at save time - missing
|
||||
* credentials, bad endpoints, and private endpoints without the operator opt-in.
|
||||
*/
|
||||
class S3IntegrationValidatorTest {
|
||||
|
||||
@Test
|
||||
void acceptsACompleteConnection() {
|
||||
assertThatCode(
|
||||
() ->
|
||||
validator(false)
|
||||
.validate(
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh")))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsMissingCredentialsOrBucket() {
|
||||
assertThatThrownBy(() -> validator(false).validate(Map.of("bucket", "inbox")))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
validator(false)
|
||||
.validate(
|
||||
Map.of(
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh")))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsAPrivateEndpointWithoutTheOperatorOptIn() {
|
||||
Map<String, Object> config =
|
||||
Map.of(
|
||||
"bucket", "inbox",
|
||||
"accessKeyId", "AKIAEXAMPLE",
|
||||
"secretAccessKey", "shh",
|
||||
"endpoint", "http://localhost:9000");
|
||||
|
||||
assertThatThrownBy(() -> validator(false).validate(config))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("allowPrivateS3Endpoints");
|
||||
assertThatCode(() -> validator(true).validate(config)).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void itOnlyClaimsTheS3Type() {
|
||||
org.junit.jupiter.api.Assertions.assertEquals(IntegrationType.S3, validator(false).type());
|
||||
}
|
||||
|
||||
private static S3IntegrationValidator validator(boolean allowPrivateEndpoints) {
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getPolicies().setAllowPrivateS3Endpoints(allowPrivateEndpoints);
|
||||
return new S3IntegrationValidator(properties);
|
||||
}
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package stirling.software.proprietary.policy.s3;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import stirling.software.proprietary.access.service.OwnershipService;
|
||||
import stirling.software.proprietary.integration.repository.IntegrationConfigRepository;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
/** Test fixtures for S3 connection plumbing shared across the policy S3 tests. */
|
||||
public final class S3TestConnections {
|
||||
|
||||
private S3TestConnections() {}
|
||||
|
||||
/**
|
||||
* A resolver for tests whose options embed credentials directly (the legacy pass-through path),
|
||||
* so its collaborators are never touched.
|
||||
*/
|
||||
public static S3ConnectionResolver legacyResolver() {
|
||||
return new S3ConnectionResolver(
|
||||
mock(IntegrationConfigRepository.class),
|
||||
mock(OwnershipService.class),
|
||||
mock(UserService.class));
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -91,7 +91,7 @@ springBoot {
|
||||
|
||||
allprojects {
|
||||
group = 'stirling.software'
|
||||
version = '2.14.2'
|
||||
version = '2.14.1'
|
||||
|
||||
configurations.configureEach {
|
||||
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
|
||||
|
||||
@@ -43,30 +43,14 @@ function parseEnTranslation(): Record<string, unknown> {
|
||||
}
|
||||
}
|
||||
|
||||
const enTranslationResources = parseEnTranslation();
|
||||
if (!i18next.isInitialized) {
|
||||
// initImmediate: false → initialise synchronously from the inline resources
|
||||
// (there's no async backend here), so i18next is ready before the first story
|
||||
// renders. Without it the first render can beat init and stick on raw keys.
|
||||
void i18next.use(initReactI18next).init({
|
||||
lng: "en",
|
||||
fallbackLng: "en",
|
||||
resources: { en: { translation: enTranslationResources } },
|
||||
resources: { en: { translation: parseEnTranslation() } },
|
||||
interpolation: { escapeValue: false },
|
||||
react: { useSuspense: false },
|
||||
initImmediate: false,
|
||||
});
|
||||
} else {
|
||||
// Something initialised i18next first (e.g. the app's async TOML backend):
|
||||
// inject the shipped English copy synchronously so t() never renders raw keys.
|
||||
i18next.addResourceBundle(
|
||||
"en",
|
||||
"translation",
|
||||
enTranslationResources,
|
||||
true,
|
||||
true,
|
||||
);
|
||||
if (i18next.language !== "en") void i18next.changeLanguage("en");
|
||||
}
|
||||
|
||||
// Start MSW once. Storybook runs in a browser so this uses the service worker.
|
||||
|
||||
@@ -6240,10 +6240,6 @@ muted = "muted"
|
||||
nameLabel = "Name"
|
||||
namePlaceholder = "e.g. Compliance escalation"
|
||||
|
||||
[portal.agentBuilder.status]
|
||||
draft = "Draft"
|
||||
published = "Published"
|
||||
|
||||
[portal.agentBuilder.tabs]
|
||||
evals = "Evals"
|
||||
scenarios = "Scenarios"
|
||||
@@ -6467,7 +6463,7 @@ install = "Install"
|
||||
usage = "Usage"
|
||||
|
||||
[portal.catalogue.detail.locked]
|
||||
description = "{{name}} is included from the {{plan}}. Upgrade to embed it."
|
||||
description = "{{name}} is included from the {{tier}} plan. Upgrade to embed it."
|
||||
title = "Not available on your plan"
|
||||
|
||||
[portal.catalogue.detail.preview]
|
||||
@@ -6534,35 +6530,6 @@ title = "No components available"
|
||||
description = "GA components are available on Pay-as-you-go; a few Beta components are enterprise-only. Locked cards show an upgrade nudge."
|
||||
title = "Some components need a paid plan"
|
||||
|
||||
[portal.connections]
|
||||
createTitle = "New S3 connection"
|
||||
delete = "Delete"
|
||||
edit = "Edit"
|
||||
editTitle = "Edit S3 connection"
|
||||
subtitle = "Reusable S3 credentials that sources and pipeline outputs connect to."
|
||||
|
||||
[portal.connections.actions]
|
||||
new = "New connection"
|
||||
|
||||
[portal.connections.empty]
|
||||
description = "Add an S3 connection to reuse the same bucket and credentials across sources and pipeline outputs."
|
||||
title = "No connections yet"
|
||||
|
||||
[portal.connections.picker]
|
||||
cancel = "Cancel"
|
||||
createNew = "New connection..."
|
||||
placeholder = "Select a connection"
|
||||
save = "Save connection"
|
||||
|
||||
[portal.connections.s3.fields]
|
||||
name = "Connection name"
|
||||
namePlaceholder = "e.g. Claims bucket"
|
||||
|
||||
[portal.connections.table]
|
||||
bucket = "Bucket"
|
||||
name = "Name"
|
||||
region = "Region"
|
||||
|
||||
[portal.docs.authentication]
|
||||
codeCaption = "every request"
|
||||
eyebrow = "GETTING STARTED"
|
||||
@@ -7374,6 +7341,10 @@ operations_one = "Operation ({{count}})"
|
||||
operations_other = "Operations ({{count}})"
|
||||
output = "Output"
|
||||
removeStep = "Remove operation"
|
||||
s3Configure = "Configure"
|
||||
s3Done = "Done"
|
||||
s3ModalTitle = "Amazon S3 output"
|
||||
s3NotConfigured = "Not configured"
|
||||
s3PrefixHelp = "Outputs are uploaded under this key prefix."
|
||||
save = "Save changes"
|
||||
scheduleEvery = "Run every"
|
||||
@@ -7444,7 +7415,6 @@ manual = "Manual"
|
||||
schedule = "Scheduled"
|
||||
|
||||
[portal.policies]
|
||||
defaultName = "{{category}} Policy"
|
||||
subtitle = "Standing automations that enforce a tool pipeline on every document. Each policy fires on upload or export, runs its tool chain, and saves the enforced version alongside the original."
|
||||
title = "Policies"
|
||||
|
||||
@@ -7708,48 +7678,6 @@ soon = "Soon"
|
||||
managePlan = "Manage plan"
|
||||
volumeSuffix = "PDFs processed · last 30 days"
|
||||
|
||||
[portal.processorFlow]
|
||||
footnote = "Counts are over the last 24 hours. Flow speed is illustrative."
|
||||
liveBadge = "Live"
|
||||
stats = "{{connected}} connected · {{processed}} PDFs processed"
|
||||
title = "PDF Processor"
|
||||
|
||||
[portal.processorFlow.lens]
|
||||
ariaLabel = "View mode"
|
||||
flow = "Flow"
|
||||
sankey = "Sankey"
|
||||
|
||||
[portal.processorFlow.outcomes]
|
||||
count = "{{n}} · 24h"
|
||||
failed = "Failed"
|
||||
heading = "Outcomes"
|
||||
success = "Delivered"
|
||||
|
||||
[portal.processorFlow.policies]
|
||||
activeCount = "{{n}} active"
|
||||
count = "{{n}} · 24h"
|
||||
heading = "Policies"
|
||||
setUp = "Set up"
|
||||
soon = "Soon"
|
||||
|
||||
[portal.processorFlow.sankey]
|
||||
waist = "Policies · {{n}} active"
|
||||
|
||||
[portal.processorFlow.sankey.empty]
|
||||
description = "Connect a source and switch on a policy to watch documents flow through the processor."
|
||||
title = "No flow yet"
|
||||
|
||||
[portal.processorFlow.sources]
|
||||
comingSoonTag = "Connect"
|
||||
editor = "Stirling PDF Editor"
|
||||
heading = "Sources"
|
||||
perDay = "{{n}} / 24h"
|
||||
|
||||
[portal.processorFlow.sources.comingSoon]
|
||||
apiMcp = "API · MCP"
|
||||
cloud = "Drive · Box · S3"
|
||||
email = "Email intake"
|
||||
|
||||
[portal.procurement]
|
||||
subtitle = "Get your team evaluated, contracted, and onboarded. Every document in one place."
|
||||
title = "Procurement"
|
||||
@@ -8058,29 +7986,34 @@ primaryNav = "Primary navigation"
|
||||
switchApp = "Switch app"
|
||||
|
||||
[portal.sources]
|
||||
subtitle = "Reusable input connections that feed documents into Stirling. Configure a source once, then reference it from any number of pipelines."
|
||||
subtitle = "Reusable input connections that feed documents into Stirling. Configure a connection once, then reference it from any number of policies. Click a row for its config and which policies use it."
|
||||
title = "Sources"
|
||||
|
||||
[portal.sources.actions]
|
||||
agentBuilder = "Agent Builder"
|
||||
connectSource = "Connect source"
|
||||
|
||||
[portal.sources.builder]
|
||||
back = "Back to sources"
|
||||
cancel = "Cancel"
|
||||
create = "Create source"
|
||||
createTitle = "Connect a source"
|
||||
delete = "Delete"
|
||||
editTitle = "Edit source"
|
||||
enabled = "Enabled"
|
||||
save = "Save changes"
|
||||
|
||||
[portal.sources.delete]
|
||||
body = "Delete \"{{name}}\"? This can't be undone. Policies that reference it would need to be updated."
|
||||
cancel = "Cancel"
|
||||
confirm = "Delete"
|
||||
title = "Delete source?"
|
||||
|
||||
[portal.sources.detail]
|
||||
closeAriaLabel = "Close detail"
|
||||
delete = "Delete source"
|
||||
docs24h = "Last 24h"
|
||||
docs30d = "Last 30 days"
|
||||
docsTotal = "Total seen"
|
||||
docsTrend = "Documents over the last 30 days"
|
||||
documents = "Documents"
|
||||
edit = "Edit"
|
||||
notReferenced = "Not referenced by any policy, so it's safe to delete."
|
||||
pause = "Pause"
|
||||
resume = "Resume"
|
||||
subtitle = "{{type}} · {{status}}"
|
||||
usedBy = "Used by"
|
||||
|
||||
[portal.sources.empty]
|
||||
description = "Connect a folder (and, soon, cloud storage) so your policies have somewhere to pull documents from."
|
||||
title = "No sources connected yet"
|
||||
@@ -8096,15 +8029,10 @@ disabled = "Disabled"
|
||||
unused = "Unused"
|
||||
|
||||
[portal.sources.table]
|
||||
documents = "Documents"
|
||||
source = "Source"
|
||||
status = "Status"
|
||||
usedBy = "Policies"
|
||||
|
||||
[portal.sources.tabs]
|
||||
connections = "Connections"
|
||||
sources = "Sources"
|
||||
|
||||
[portal.sources.types.editor]
|
||||
description = "Documents your team has processed in the editor, across policy and AI runs."
|
||||
label = "Editor"
|
||||
@@ -8152,10 +8080,6 @@ label = "Access key ID"
|
||||
label = "Bucket"
|
||||
placeholder = "my-company-inbox"
|
||||
|
||||
[portal.sources.types.s3.fields.connection]
|
||||
helperText = "The stored connection holding the bucket and credentials. Reused by every source and pipeline output that references it."
|
||||
label = "Connection"
|
||||
|
||||
[portal.sources.types.s3.fields.endpoint]
|
||||
helperText = "Leave blank for Amazon S3. Set to use an S3-compatible service such as MinIO."
|
||||
label = "Custom endpoint"
|
||||
@@ -8185,14 +8109,21 @@ label = "Secret access key"
|
||||
label = "Source"
|
||||
|
||||
[portal.sources.wizard]
|
||||
back = "Back"
|
||||
cancel = "Cancel"
|
||||
continue = "Continue"
|
||||
editTitle = "Edit source"
|
||||
name = "Name"
|
||||
namePlaceholder = "e.g. Claims intake"
|
||||
save = "Save changes"
|
||||
subtitle = "Step {{current}} of {{total}} · {{label}}"
|
||||
title = "Connect a source"
|
||||
type = "Type"
|
||||
|
||||
[portal.tier]
|
||||
enterprise = "Enterprise plan"
|
||||
free = "Editor plan"
|
||||
pro = "Processor plan"
|
||||
[portal.sources.wizard.steps]
|
||||
chooseType = "Choose type"
|
||||
configure = "Configure"
|
||||
review = "Review & connect"
|
||||
|
||||
[portal.usage]
|
||||
managePayment = "Manage Payment"
|
||||
@@ -10208,19 +10139,6 @@ resetPw = "Reset password"
|
||||
suspend = "Suspend"
|
||||
unlock = "Unlock account"
|
||||
|
||||
[users.activity]
|
||||
daysAgo = "{{count}}d ago"
|
||||
hoursAgo = "{{count}}h ago"
|
||||
justNow = "Just now"
|
||||
minutesAgo = "{{count}}m ago"
|
||||
monthsAgo_one = "{{count}} month ago"
|
||||
monthsAgo_other = "{{count}} months ago"
|
||||
never = "Never"
|
||||
weeksAgo_one = "{{count}} week ago"
|
||||
weeksAgo_other = "{{count}} weeks ago"
|
||||
yearsAgo_one = "{{count}} year ago"
|
||||
yearsAgo_other = "{{count}} years ago"
|
||||
|
||||
[users.cap]
|
||||
addProcessor = "+ Processor"
|
||||
approver = "Approves policy"
|
||||
@@ -10291,9 +10209,7 @@ by = "Invited by {{who}}"
|
||||
cancel = "Cancel"
|
||||
count = "{{count}} pending"
|
||||
desc = "Invited people who haven't joined yet. They hold a seat until they accept."
|
||||
expiresInDays_one = "Expires in {{count}} day"
|
||||
expiresInDays_other = "Expires in {{count}} days"
|
||||
expiresToday = "Expires today"
|
||||
expires = "Expires"
|
||||
title = "Pending invitations"
|
||||
|
||||
[users.loadError]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
|
||||
"productName": "Stirling PDF",
|
||||
"mainBinaryName": "Stirling-PDF",
|
||||
"version": "2.14.2",
|
||||
"version": "2.14.1",
|
||||
"identifier": "stirling.pdf.dev",
|
||||
"build": {
|
||||
"frontendDist": "../dist",
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
import {
|
||||
deserializeToolStep,
|
||||
getExecutableTools,
|
||||
serializeStepFromEndpoint,
|
||||
serializeToolStep,
|
||||
stepRequiresUpload,
|
||||
type WorkingToolStep,
|
||||
@@ -198,6 +199,41 @@ describe("serialize/deserialize round-trip", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeStepFromEndpoint", () => {
|
||||
test("maps a wizard step's UI params to the backend contract, filling defaults", () => {
|
||||
// The shape the policy setup wizard holds: an endpoint plus UI-shaped params
|
||||
// (redact's `wordsToRedact`), with several fields left to their defaults.
|
||||
const api = serializeStepFromEndpoint(
|
||||
"/api/v1/security/auto-redact",
|
||||
{ mode: "automatic", useRegex: true, wordsToRedact: ["ssn", "card"] },
|
||||
dynamicRegistry,
|
||||
);
|
||||
|
||||
expect(api.operation).toBe("/api/v1/security/auto-redact");
|
||||
// wordsToRedact -> listOfText (the field the backend actually reads), and the
|
||||
// frontend-only `mode` is dropped.
|
||||
expect(api.parameters).toMatchObject({ listOfText: "ssn\ncard" });
|
||||
expect(api.parameters).not.toHaveProperty("wordsToRedact");
|
||||
expect(api.parameters).not.toHaveProperty("mode");
|
||||
// Fields the wizard never set still get their defaults so the body is complete.
|
||||
expect(api.parameters).toHaveProperty("wholeWordSearch");
|
||||
expect(api.parameters).toHaveProperty("customPadding");
|
||||
});
|
||||
|
||||
test("passes an unmapped endpoint's params through unchanged", () => {
|
||||
expect(
|
||||
serializeStepFromEndpoint(
|
||||
"/api/v1/unknown/thing",
|
||||
{ keep: true },
|
||||
dynamicRegistry,
|
||||
),
|
||||
).toEqual({
|
||||
operation: "/api/v1/unknown/thing",
|
||||
parameters: { keep: true },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("stepRequiresUpload", () => {
|
||||
const step = (params: Record<string, unknown>): WorkingToolStep => ({
|
||||
toolId: "compress" as ToolId,
|
||||
|
||||
@@ -197,6 +197,31 @@ export function serializeToolStep(
|
||||
return { operation, parameters };
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a step held as an endpoint path plus frontend-shaped params - the form the policy setup
|
||||
* wizard keeps, where params match the tool's UI shape (e.g. redact's `wordsToRedact`) rather than
|
||||
* the backend contract - into the backend step contract, mapping params through the tool's
|
||||
* `toApiParams` (merged over its defaults, so fields the wizard never set still get their defaults).
|
||||
* The endpoint maps to a tool by path, so this works for dynamic-endpoint tools whose config
|
||||
* endpoint is a function. Endpoints that map to no known tool pass through unchanged.
|
||||
*/
|
||||
export function serializeStepFromEndpoint(
|
||||
operation: string,
|
||||
params: ErasedToolParams,
|
||||
registry: Partial<ToolRegistry>,
|
||||
): ToolApiStep {
|
||||
const match = findToolByEndpoint({ operation, parameters: params }, registry);
|
||||
const config = match?.[1].operationConfig;
|
||||
if (!config) return { operation, parameters: params };
|
||||
const merged = { ...(config.defaultParameters ?? {}), ...params };
|
||||
return {
|
||||
operation: resolveEndpoint(config, merged) ?? operation,
|
||||
parameters: config.toApiParams
|
||||
? (config.toApiParams(merged) as Record<string, unknown>)
|
||||
: {},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the registry tool for a stored step's endpoint: exact match for static endpoints, else
|
||||
* membership in a dynamic tool's declared `endpoints` set (replaying its function can't recover a
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { describeToolOperation } from "@app/hooks/tools/shared/toolOperationDescriptor";
|
||||
|
||||
interface Params {
|
||||
a: number;
|
||||
}
|
||||
|
||||
// A minimal config that type-checks against the flatten endpoint's model.
|
||||
const CONFIG = {
|
||||
endpoint: "/api/v1/misc/flatten" as const,
|
||||
defaultParameters: { a: 1 } satisfies Params,
|
||||
toApiParams: (p: Params) => ({ renderDpi: p.a }),
|
||||
fromApiParams: (api: { renderDpi?: number }) => ({ a: api.renderDpi ?? 0 }),
|
||||
};
|
||||
|
||||
describe("describeToolOperation", () => {
|
||||
test("wraps the config's mappers and endpoint into a descriptor", () => {
|
||||
const d = describeToolOperation("/api/v1/misc/flatten", CONFIG);
|
||||
expect(d.endpoint).toBe("/api/v1/misc/flatten");
|
||||
expect(d.toApi({ a: 200 })).toEqual({ renderDpi: 200 });
|
||||
});
|
||||
|
||||
test("fromApi merges the mapped values over the defaults", () => {
|
||||
const d = describeToolOperation("/api/v1/misc/flatten", CONFIG);
|
||||
expect(d.fromApi({ renderDpi: 72 })).toEqual({ a: 72 });
|
||||
});
|
||||
|
||||
test("throws when the config lacks a mapper", () => {
|
||||
expect(() =>
|
||||
describeToolOperation("/api/v1/misc/flatten", {
|
||||
endpoint: "/api/v1/misc/flatten" as const,
|
||||
defaultParameters: { a: 1 },
|
||||
toApiParams: (p: Params) => ({ renderDpi: p.a }),
|
||||
}),
|
||||
).toThrow(/mappers/);
|
||||
});
|
||||
});
|
||||
@@ -1,59 +0,0 @@
|
||||
/**
|
||||
* Typed wrapper over a tool's `toApiParams`/`fromApiParams` mappers, binding one endpoint to safe
|
||||
* frontend<->backend parameter conversion.
|
||||
*/
|
||||
|
||||
import type { ToolApiParams, ToolEndpoint } from "@app/types/toolApiTypes";
|
||||
|
||||
export interface ToolOperationDescriptor<E extends ToolEndpoint, TParams> {
|
||||
readonly endpoint: E;
|
||||
readonly defaultParameters: TParams;
|
||||
toApi(params: TParams): ToolApiParams[E];
|
||||
/** Backend model -> full frontend params (defaults merged under the mapped values). */
|
||||
fromApi(api: ToolApiParams[E]): TParams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural subset of a tool's config. `CE` is the config's declared endpoint type, inferred from
|
||||
* the `endpoint` field: the literal for static tools, or the whole `ToolEndpoint` union for
|
||||
* dynamic-endpoint tools (whose endpoint is a function typed against the union).
|
||||
*/
|
||||
export interface BidirectionalToolConfig<TParams, CE extends ToolEndpoint> {
|
||||
endpoint: CE | null | ((params: TParams) => CE | null);
|
||||
defaultParameters?: TParams;
|
||||
toApiParams?(params: TParams): ToolApiParams[CE];
|
||||
fromApiParams?(api: ToolApiParams[CE]): Partial<TParams>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pin a config to `endpoint` (passed explicitly, since dynamic-endpoint tools declare `endpoint` as
|
||||
* a function). `E extends CE` rejects pairing a static tool's config with the wrong endpoint, while
|
||||
* allowing a dynamic tool whose `CE` is the full union. Throws when mappers or defaults are missing.
|
||||
*/
|
||||
export function describeToolOperation<
|
||||
E extends CE,
|
||||
CE extends ToolEndpoint,
|
||||
TParams,
|
||||
>(
|
||||
endpoint: E,
|
||||
config: BidirectionalToolConfig<TParams, CE>,
|
||||
): ToolOperationDescriptor<E, TParams> {
|
||||
const { toApiParams, fromApiParams, defaultParameters } = config;
|
||||
if (!toApiParams || !fromApiParams || defaultParameters === undefined) {
|
||||
throw new Error(
|
||||
`describeToolOperation: "${endpoint}" is missing mappers or defaults`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
endpoint,
|
||||
defaultParameters,
|
||||
// A dynamic tool's mapper is typed against the union; narrow to this endpoint (sound - the
|
||||
// runtime mapper produces this endpoint's model).
|
||||
toApi: (params) => toApiParams(params) as ToolApiParams[E],
|
||||
fromApi: (api) =>
|
||||
({
|
||||
...defaultParameters,
|
||||
...fromApiParams(api as ToolApiParams[CE]),
|
||||
}) as TParams,
|
||||
};
|
||||
}
|
||||
@@ -38,7 +38,7 @@ const FREE_LICENSE_INFO: LicenseInfo = {
|
||||
|
||||
const BASE_NO_LOGIN_CONFIG: AppConfig = {
|
||||
enableAnalytics: true,
|
||||
appVersion: "2.14.2",
|
||||
appVersion: "2.14.1",
|
||||
serverCertificateEnabled: false,
|
||||
enableAlphaFunctionality: false,
|
||||
serverPort: 8080,
|
||||
|
||||
@@ -270,29 +270,8 @@
|
||||
--grad-banner: linear-gradient(135deg, #0f172a 0%, #111827 50%, #1a1535 100%);
|
||||
}
|
||||
|
||||
/* Code palette — theme-aware: a light GitHub-style box in light mode, the dark
|
||||
terminal palette in dark mode. (CodeBlock renders plain text, so only the
|
||||
base colours are load-bearing; the syntax slots are kept for future Shiki. */
|
||||
:root,
|
||||
[data-theme="light"] {
|
||||
--code-bg: #f6f8fa;
|
||||
--code-bg-alt: #eef1f4;
|
||||
--code-bg-header: #eaeef2;
|
||||
--code-text: #1f2328;
|
||||
--code-dim: #656d76;
|
||||
--code-muted: #8c959f;
|
||||
--code-keyword: #cf222e;
|
||||
--code-string: #0a3069;
|
||||
--code-number: #0550ae;
|
||||
--code-fn: #8250df;
|
||||
--code-type: #953800;
|
||||
--code-property: #0550ae;
|
||||
--code-comment: #6e7781;
|
||||
--code-border: #d0d7de;
|
||||
/* Window-chrome traffic-light dots in the code-block header. */
|
||||
--code-dot: #d0d7de;
|
||||
}
|
||||
[data-theme="dark"] {
|
||||
/* Always-dark code palette. Not theme-switched. */
|
||||
:root {
|
||||
--code-bg: #0f172a;
|
||||
--code-bg-alt: #1e293b;
|
||||
--code-bg-header: #1a2332;
|
||||
@@ -307,16 +286,12 @@
|
||||
--code-property: #93c5fd;
|
||||
--code-comment: #475569;
|
||||
--code-border: #1e293b;
|
||||
/* Window-chrome traffic-light dots in the code-block header. */
|
||||
--code-dot: #475569;
|
||||
}
|
||||
|
||||
/* Radii / typography / motion / spacing / z-index — theme-stable */
|
||||
:root {
|
||||
/* Home hero strip navy. Theme-stable by design — the hero keeps this deep
|
||||
navy in both light and dark (it's a branded surface, like the assistant
|
||||
header), so it's defined once here rather than in the light/dark blocks. */
|
||||
--color-hero-navy: #16213e;
|
||||
|
||||
--radius-xs: 0.1875rem;
|
||||
--radius-sm: 0.25rem;
|
||||
--radius-md: 0.375rem;
|
||||
|
||||
@@ -61,14 +61,11 @@
|
||||
background: var(--button-bg, transparent) !important;
|
||||
}
|
||||
|
||||
/* Disabled buttons in dark read as a muted surface (not a dimmed accent that
|
||||
still looks clickable, and not an invisible transparent pill). Covers every
|
||||
variant so a disabled primary and a disabled secondary look alike. */
|
||||
[data-theme="dark"] .sui-btn.mantine-Button-root:disabled:not([data-loading]),
|
||||
[data-theme="dark"]
|
||||
.sui-btn.mantine-Button-root[data-disabled]:not([data-loading]) {
|
||||
background: var(--color-bg-muted);
|
||||
color: var(--color-text-5);
|
||||
border-color: transparent;
|
||||
opacity: 1;
|
||||
.sui-btn--primary.mantine-Button-root:disabled:not([data-loading]),
|
||||
[data-theme="dark"]
|
||||
.sui-btn--primary.mantine-Button-root[data-disabled]:not([data-loading]) {
|
||||
background: var(--button-bg);
|
||||
color: var(--button-color);
|
||||
opacity: 0.55;
|
||||
}
|
||||
|
||||
@@ -51,8 +51,7 @@
|
||||
transition: background var(--motion-fast);
|
||||
}
|
||||
.sui-code__copy:hover {
|
||||
/* Subtle tint that reads on both the light and dark code surfaces. */
|
||||
background: color-mix(in srgb, var(--code-text) 8%, transparent);
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
}
|
||||
.sui-code__pre {
|
||||
margin: 0;
|
||||
|
||||
@@ -19,12 +19,6 @@ export interface TableProps<T> {
|
||||
rowKey: (row: T) => string;
|
||||
/** Makes rows interactive (hover + click + keyboard). */
|
||||
onRowClick?: (row: T) => void;
|
||||
/**
|
||||
* Per-row gate for interactivity, checked only when {@link onRowClick} is set. A row for which
|
||||
* this returns false is inert: no click/keyboard, and not announced as a button. Defaults to
|
||||
* all rows interactive.
|
||||
*/
|
||||
isRowInteractive?: (row: T) => boolean;
|
||||
/** Rendered in place of the body when there are no rows. */
|
||||
empty?: ReactNode;
|
||||
className?: string;
|
||||
@@ -41,7 +35,6 @@ export function Table<T>({
|
||||
rows,
|
||||
rowKey,
|
||||
onRowClick,
|
||||
isRowInteractive,
|
||||
empty,
|
||||
className,
|
||||
}: TableProps<T>) {
|
||||
@@ -73,42 +66,38 @@ export function Table<T>({
|
||||
</td>
|
||||
</tr>
|
||||
) : (
|
||||
rows.map((row) => {
|
||||
const rowInteractive =
|
||||
interactive && (isRowInteractive?.(row) ?? true);
|
||||
return (
|
||||
<tr
|
||||
key={rowKey(row)}
|
||||
className={
|
||||
rowInteractive
|
||||
? "sui-table__row sui-table__row--interactive"
|
||||
: "sui-table__row"
|
||||
}
|
||||
onClick={rowInteractive ? () => onRowClick?.(row) : undefined}
|
||||
tabIndex={rowInteractive ? 0 : undefined}
|
||||
role={rowInteractive ? "button" : undefined}
|
||||
onKeyDown={
|
||||
rowInteractive
|
||||
? (e) => {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
onRowClick?.(row);
|
||||
}
|
||||
rows.map((row) => (
|
||||
<tr
|
||||
key={rowKey(row)}
|
||||
className={
|
||||
interactive
|
||||
? "sui-table__row sui-table__row--interactive"
|
||||
: "sui-table__row"
|
||||
}
|
||||
onClick={onRowClick ? () => onRowClick(row) : undefined}
|
||||
tabIndex={interactive ? 0 : undefined}
|
||||
role={interactive ? "button" : undefined}
|
||||
onKeyDown={
|
||||
interactive
|
||||
? (e) => {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
onRowClick?.(row);
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{columns.map((c) => (
|
||||
<td
|
||||
key={c.key}
|
||||
className={`sui-table__td sui-table__td--${c.align ?? "left"}`}
|
||||
>
|
||||
{c.render(row)}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
);
|
||||
})
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{columns.map((c) => (
|
||||
<td
|
||||
key={c.key}
|
||||
className={`sui-table__td sui-table__td--${c.align ?? "left"}`}
|
||||
>
|
||||
{c.render(row)}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
))
|
||||
)}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Documents } from "@portal/views/Documents";
|
||||
import { Pipelines } from "@portal/views/Pipelines";
|
||||
import { PipelineBuilder } from "@portal/views/PipelineBuilder";
|
||||
import { Sources } from "@portal/views/Sources";
|
||||
import { SourceBuilder } from "@portal/views/SourceBuilder";
|
||||
import { AgentBuilder } from "@portal/views/AgentBuilder";
|
||||
import { Policies } from "@portal/views/Policies";
|
||||
import { Components } from "@portal/views/Components";
|
||||
@@ -37,14 +36,6 @@ export function ViewRouter() {
|
||||
element={<PipelineBuilder />}
|
||||
/>
|
||||
<Route path={rel(VIEW_PATHS.sources)} element={<Sources />} />
|
||||
<Route
|
||||
path={`${rel(VIEW_PATHS.sources)}/new`}
|
||||
element={<SourceBuilder />}
|
||||
/>
|
||||
<Route
|
||||
path={`${rel(VIEW_PATHS.sources)}/:id`}
|
||||
element={<SourceBuilder />}
|
||||
/>
|
||||
<Route
|
||||
path={rel(VIEW_PATHS["agent-builder"])}
|
||||
element={<AgentBuilder />}
|
||||
|
||||
@@ -99,11 +99,6 @@ export const AGENT_STATUS_TONE: Record<AgentStatus, "success" | "neutral"> = {
|
||||
draft: "neutral",
|
||||
};
|
||||
|
||||
export const AGENT_STATUS_LABEL: Record<AgentStatus, string> = {
|
||||
published: "portal.agentBuilder.status.published",
|
||||
draft: "portal.agentBuilder.status.draft",
|
||||
};
|
||||
|
||||
/**
|
||||
* Catalogue of tools an agent can be granted or denied. Surfaced as the chip
|
||||
* palette in restricted mode so the deny list is picked from a known set
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
/**
|
||||
* Integrations service layer: stored connections (S3 today; MCP/API later) that
|
||||
* policy sources and pipeline outputs reference by id instead of embedding
|
||||
* credentials. Secrets are write-only - reads return them masked, and sending
|
||||
* the mask back on update keeps the stored value.
|
||||
*/
|
||||
import { apiClient } from "@portal/api/http";
|
||||
|
||||
export type IntegrationType = "S3" | "MCP" | "API";
|
||||
export type OwnerScope = "USER" | "TEAM" | "SERVER";
|
||||
|
||||
/** Mirrors the backend IntegrationConfigResponse; `config` values are masked. */
|
||||
export interface IntegrationConfig {
|
||||
id: number;
|
||||
integrationType: IntegrationType;
|
||||
name: string;
|
||||
scope: OwnerScope;
|
||||
ownerUserId: number | null;
|
||||
ownerTeamId: number | null;
|
||||
enabled: boolean;
|
||||
locked: boolean;
|
||||
defaultAccess: string;
|
||||
config: Record<string, unknown>;
|
||||
canManage: boolean;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/** Create/update body; omitted fields keep their stored values on update. */
|
||||
export interface IntegrationConfigRequest {
|
||||
integrationType?: IntegrationType;
|
||||
name?: string;
|
||||
scope?: OwnerScope;
|
||||
ownerTeamId?: number | null;
|
||||
enabled?: boolean;
|
||||
config?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export async function fetchIntegrations(): Promise<IntegrationConfig[]> {
|
||||
return apiClient.local.json<IntegrationConfig[]>("/api/v1/integrations");
|
||||
}
|
||||
|
||||
/** The S3 connections the caller may use, for source/output pickers. */
|
||||
export async function fetchS3Connections(): Promise<IntegrationConfig[]> {
|
||||
return (await fetchIntegrations()).filter(
|
||||
(integration) => integration.integrationType === "S3",
|
||||
);
|
||||
}
|
||||
|
||||
export async function createIntegration(
|
||||
body: IntegrationConfigRequest,
|
||||
): Promise<IntegrationConfig> {
|
||||
return apiClient.local.json<IntegrationConfig>("/api/v1/integrations", {
|
||||
method: "POST",
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
export async function updateIntegration(
|
||||
id: number,
|
||||
body: IntegrationConfigRequest,
|
||||
): Promise<IntegrationConfig> {
|
||||
return apiClient.local.json<IntegrationConfig>(
|
||||
`/api/v1/integrations/${encodeURIComponent(id)}`,
|
||||
{ method: "PUT", body },
|
||||
);
|
||||
}
|
||||
|
||||
export async function deleteIntegration(id: number): Promise<void> {
|
||||
await apiClient.local.json<void>(
|
||||
`/api/v1/integrations/${encodeURIComponent(id)}`,
|
||||
{ method: "DELETE" },
|
||||
);
|
||||
}
|
||||
@@ -9,12 +9,9 @@
|
||||
* approach the editor uses for its own catalogue view.
|
||||
*/
|
||||
|
||||
import type { TFunction } from "i18next";
|
||||
import { apiClient } from "@portal/api/http";
|
||||
import { fromWirePolicy, toWirePolicy } from "@app/policies/codec";
|
||||
import { runsToActivity, runsToStats } from "@app/policies/runs";
|
||||
import { policyStep, type PolicyToolStep } from "@app/policies/operations";
|
||||
import type { ToolEndpoint } from "@app/types/toolApiTypes";
|
||||
import type {
|
||||
PolicyDecodedState,
|
||||
PolicyRunView,
|
||||
@@ -68,7 +65,7 @@ export interface PolicyConfigDef {
|
||||
rules: string[];
|
||||
scopeLabel: string;
|
||||
fields: PolicyField[];
|
||||
defaultOperations: PolicyToolStep[];
|
||||
defaultOperations: WirePipelineStep[];
|
||||
}
|
||||
|
||||
export interface PolicyState {
|
||||
@@ -130,11 +127,20 @@ export interface CatalogueEntry {
|
||||
}
|
||||
|
||||
/* ──────────────────────────────────────────────────────────────────────── */
|
||||
/* Endpoint display labels */
|
||||
/* Tool → endpoint registry */
|
||||
/* ──────────────────────────────────────────────────────────────────────── */
|
||||
|
||||
/** i18n keys keyed by {@link ToolEndpoint}; labels stored steps in the detail view. */
|
||||
export const ENDPOINT_LABELS: Partial<Record<ToolEndpoint, string>> = {
|
||||
export const TOOL_ENDPOINTS: Record<string, string> = {
|
||||
redact: "/api/v1/security/auto-redact",
|
||||
sanitize: "/api/v1/security/sanitize-pdf",
|
||||
watermark: "/api/v1/security/add-watermark",
|
||||
ocr: "/api/v1/misc/ocr-pdf",
|
||||
flatten: "/api/v1/misc/flatten",
|
||||
compress: "/api/v1/misc/compress-pdf",
|
||||
};
|
||||
|
||||
/** Values are i18n keys — render with t(). */
|
||||
export const ENDPOINT_LABELS: Record<string, string> = {
|
||||
"/api/v1/security/auto-redact": "portal.policies.endpoints.autoRedact",
|
||||
"/api/v1/security/sanitize-pdf": "portal.policies.endpoints.sanitizePdf",
|
||||
"/api/v1/security/add-watermark": "portal.policies.endpoints.addWatermark",
|
||||
@@ -147,8 +153,7 @@ export function humanizeEndpoint(
|
||||
path: string,
|
||||
t: (key: string) => string,
|
||||
): string {
|
||||
const label = ENDPOINT_LABELS[path as ToolEndpoint];
|
||||
if (label) return t(label);
|
||||
if (ENDPOINT_LABELS[path]) return t(ENDPOINT_LABELS[path]);
|
||||
const last = path.split("/").filter(Boolean).pop() ?? path;
|
||||
return last
|
||||
.replace(/-/g, " ")
|
||||
@@ -224,7 +229,10 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
|
||||
"portal.policies.config.ingestion.rules.3",
|
||||
],
|
||||
scopeLabel: "portal.policies.config.scopeAll",
|
||||
defaultOperations: [policyStep("ocr"), policyStep("flatten")],
|
||||
defaultOperations: [
|
||||
{ operation: TOOL_ENDPOINTS.ocr, parameters: {} },
|
||||
{ operation: TOOL_ENDPOINTS.flatten, parameters: {} },
|
||||
],
|
||||
fields: [
|
||||
{
|
||||
label: "portal.policies.config.ingestion.fields.minConfidence",
|
||||
@@ -251,16 +259,32 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
|
||||
],
|
||||
scopeLabel: "portal.policies.config.scopeAll",
|
||||
defaultOperations: [
|
||||
// Flatten to image so redactions can't be lifted off.
|
||||
policyStep("redact", {
|
||||
useRegex: true,
|
||||
convertPDFToImage: true,
|
||||
wordsToRedact: DEFAULT_PII_PATTERNS,
|
||||
}),
|
||||
// JavaScript removal only; the tool enables removeEmbeddedFiles by default, so turn it off.
|
||||
policyStep("sanitize", { removeEmbeddedFiles: false }),
|
||||
// Bake in via image so it can't be stripped.
|
||||
policyStep("watermark", { convertPDFToImage: true }),
|
||||
{
|
||||
operation: TOOL_ENDPOINTS.redact,
|
||||
parameters: {
|
||||
mode: "automatic",
|
||||
useRegex: true,
|
||||
convertPDFToImage: true,
|
||||
wordsToRedact: DEFAULT_PII_PATTERNS,
|
||||
},
|
||||
},
|
||||
{
|
||||
operation: TOOL_ENDPOINTS.sanitize,
|
||||
parameters: {
|
||||
removeJavaScript: true,
|
||||
removeEmbeddedFiles: false,
|
||||
removeMetadata: false,
|
||||
removeLinks: false,
|
||||
removeFonts: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
operation: TOOL_ENDPOINTS.watermark,
|
||||
// convertPDFToImage bakes the watermark in so it can't be stripped
|
||||
parameters: {
|
||||
convertPDFToImage: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
fields: [],
|
||||
},
|
||||
@@ -272,7 +296,10 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
|
||||
"portal.policies.config.compliance.rules.2",
|
||||
],
|
||||
scopeLabel: "portal.policies.config.scopeAll",
|
||||
defaultOperations: [policyStep("sanitize"), policyStep("flatten")],
|
||||
defaultOperations: [
|
||||
{ operation: TOOL_ENDPOINTS.sanitize, parameters: {} },
|
||||
{ operation: TOOL_ENDPOINTS.flatten, parameters: {} },
|
||||
],
|
||||
fields: [
|
||||
{
|
||||
label: "portal.policies.config.compliance.fields.frameworks",
|
||||
@@ -315,7 +342,7 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
|
||||
"portal.policies.config.routing.rules.2",
|
||||
],
|
||||
scopeLabel: "portal.policies.config.scopeAll",
|
||||
defaultOperations: [policyStep("compress")],
|
||||
defaultOperations: [{ operation: TOOL_ENDPOINTS.compress, parameters: {} }],
|
||||
fields: [
|
||||
{
|
||||
label: "portal.policies.config.routing.fields.destination",
|
||||
@@ -346,7 +373,7 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
|
||||
"portal.policies.config.retention.rules.2",
|
||||
],
|
||||
scopeLabel: "portal.policies.config.scopeAll",
|
||||
defaultOperations: [policyStep("compress")],
|
||||
defaultOperations: [{ operation: TOOL_ENDPOINTS.compress, parameters: {} }],
|
||||
fields: [
|
||||
{
|
||||
label: "portal.policies.config.retention.fields.keepFor",
|
||||
@@ -529,30 +556,17 @@ const DEFAULT_RETRY_DELAY = 5;
|
||||
// POST /api/v1/policies endpoint. The real backend ignores unknown fields.
|
||||
type CatalogueWireBody = WirePolicy & { categoryId: string };
|
||||
|
||||
/**
|
||||
* The persisted policy name derived from its category, e.g. "Security Policy".
|
||||
* `category.label` is an i18n key, so translate it before building the name;
|
||||
* otherwise the raw key is persisted and surfaces in the UI (e.g. the Sources
|
||||
* "Used by" pill).
|
||||
*/
|
||||
function policyDisplayName(entry: CatalogueEntry, t: TFunction): string {
|
||||
return t("portal.policies.defaultName", {
|
||||
category: t(entry.category.label),
|
||||
});
|
||||
}
|
||||
|
||||
/** Build a wire policy from a setup wizard result. */
|
||||
export function buildWireFromSetup(
|
||||
entry: CatalogueEntry,
|
||||
result: PolicySetupResult,
|
||||
t: TFunction,
|
||||
enabled = true,
|
||||
): CatalogueWireBody {
|
||||
return {
|
||||
categoryId: entry.category.id,
|
||||
...toWirePolicy({
|
||||
id: entry.policy?.state.backendId ?? "",
|
||||
name: policyDisplayName(entry, t),
|
||||
name: `${entry.category.label} Policy`,
|
||||
enabled,
|
||||
categoryId: entry.category.id,
|
||||
sources: result.sources,
|
||||
@@ -575,14 +589,13 @@ export function buildWireFromState(
|
||||
entry: CatalogueEntry,
|
||||
policy: DecoratedPolicy,
|
||||
enabled: boolean,
|
||||
t: TFunction,
|
||||
): CatalogueWireBody {
|
||||
const s = policy.state;
|
||||
return {
|
||||
categoryId: entry.category.id,
|
||||
...toWirePolicy({
|
||||
id: s.backendId ?? "",
|
||||
name: policyDisplayName(entry, t),
|
||||
name: `${entry.category.label} Policy`,
|
||||
enabled,
|
||||
categoryId: entry.category.id,
|
||||
sources: s.sources,
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
/**
|
||||
* Processor-flow assembler for the home visualiser.
|
||||
*
|
||||
* Fans in the three real portal surfaces — sources (`/api/v1/sources`), policies
|
||||
* (`/api/v1/policies`) and their runs (`/api/v1/policies/runs`) — and derives the
|
||||
* left→middle→right shape the {@link ProcessorFlow} component renders:
|
||||
*
|
||||
* sources → policies → outcomes
|
||||
*
|
||||
* Everything here is real backend data. Per-run source attribution does not
|
||||
* exist (a `PolicyRunView` carries `policyId` but no source id), so the flow
|
||||
* animation is illustrative; the node counts are not — each source's `docs24h`,
|
||||
* each policy's trailing-24h run count, and the success/failure split all come
|
||||
* straight from the API.
|
||||
*/
|
||||
|
||||
import { apiClient } from "@portal/api/http";
|
||||
import { fetchSources } from "@portal/api/sources";
|
||||
import { POLICY_CATEGORIES } from "@portal/api/policies";
|
||||
import { fromWirePolicy } from "@app/policies/codec";
|
||||
import type { PolicyRunView, WirePolicy } from "@app/policies/types";
|
||||
|
||||
/** A source that actually feeds the processor today (editor, folder, S3, …). */
|
||||
export interface FlowSource {
|
||||
id: string;
|
||||
/** Display name (already resolved; editor rows get a friendly label). */
|
||||
name: string;
|
||||
type: string;
|
||||
/** Documents this source fed into runs over the trailing 24h. */
|
||||
docs24h: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* A connector type shown in the sources column but not yet a real source type —
|
||||
* a "coming soon" affordance only. `labelKey` is an i18n key.
|
||||
*/
|
||||
export interface FlowComingSoonSource {
|
||||
key: string;
|
||||
labelKey: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Row display state, mirroring the Policies page:
|
||||
* - `active` — configured + enabled; shows its live 24h run count
|
||||
* - `off` — available but not set up; offers a "Set up" CTA
|
||||
* - `locked` — a coming-soon category that doesn't exist yet
|
||||
*/
|
||||
export type FlowPolicyState = "active" | "off" | "locked";
|
||||
|
||||
/**
|
||||
* One row in the middle policies column — the full policy catalogue, in the
|
||||
* same order the Policies page shows, including the coming-soon categories.
|
||||
*/
|
||||
export interface FlowPolicy {
|
||||
/** Category id (also the lane key for the flow animation). */
|
||||
key: string;
|
||||
/** i18n key for the category label. */
|
||||
labelKey: string;
|
||||
/** Material Symbols icon name (from the catalogue). */
|
||||
icon: string;
|
||||
state: FlowPolicyState;
|
||||
configured: boolean;
|
||||
runs24h: number;
|
||||
}
|
||||
|
||||
export type FlowOutcomeKey = "success" | "failed";
|
||||
|
||||
/** A terminal audit outcome node on the right, counted over the trailing 24h. */
|
||||
export interface FlowOutcome {
|
||||
key: FlowOutcomeKey;
|
||||
labelKey: string;
|
||||
count24h: number;
|
||||
}
|
||||
|
||||
export interface ProcessorFlow {
|
||||
sources: FlowSource[];
|
||||
comingSoonSources: FlowComingSoonSource[];
|
||||
policies: FlowPolicy[];
|
||||
outcomes: FlowOutcome[];
|
||||
}
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/** Connector types the sources column advertises but can't create yet. */
|
||||
const COMING_SOON_SOURCES: FlowComingSoonSource[] = [
|
||||
{ key: "apiMcp", labelKey: "portal.processorFlow.sources.comingSoon.apiMcp" },
|
||||
{
|
||||
key: "cloud",
|
||||
labelKey: "portal.processorFlow.sources.comingSoon.cloud",
|
||||
},
|
||||
{
|
||||
key: "email",
|
||||
labelKey: "portal.processorFlow.sources.comingSoon.email",
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* The full policy catalogue, in the Policies-page order, including the
|
||||
* coming-soon categories (rendered as locked). `active` rows carry their
|
||||
* trailing-24h run count.
|
||||
*/
|
||||
function buildPolicies(
|
||||
wirePolicies: WirePolicy[],
|
||||
runs: PolicyRunView[],
|
||||
): FlowPolicy[] {
|
||||
const cutoff = Date.now() - DAY_MS;
|
||||
const decoded = wirePolicies.map(fromWirePolicy);
|
||||
|
||||
return POLICY_CATEGORIES.map((cat) => {
|
||||
const dp = decoded.find((p) => p.categoryId === cat.id);
|
||||
const configured = Boolean(dp?.enabled);
|
||||
const state: FlowPolicyState = configured
|
||||
? "active"
|
||||
: cat.comingSoon
|
||||
? "locked"
|
||||
: "off";
|
||||
const runs24h = dp
|
||||
? runs.filter((r) => r.policyId === dp.id && r.createdAt >= cutoff).length
|
||||
: 0;
|
||||
return {
|
||||
key: cat.id,
|
||||
labelKey: cat.label,
|
||||
icon: cat.icon,
|
||||
state,
|
||||
configured,
|
||||
runs24h,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Terminal audit outcomes over the trailing 24h — success vs failure. */
|
||||
function buildOutcomes(runs: PolicyRunView[]): FlowOutcome[] {
|
||||
const cutoff = Date.now() - DAY_MS;
|
||||
const recent = runs.filter((r) => r.createdAt >= cutoff);
|
||||
const success = recent.filter((r) => r.status === "COMPLETED").length;
|
||||
const failed = recent.filter(
|
||||
(r) => r.status === "FAILED" || r.status === "CANCELLED",
|
||||
).length;
|
||||
return [
|
||||
{
|
||||
key: "success",
|
||||
labelKey: "portal.processorFlow.outcomes.success",
|
||||
count24h: success,
|
||||
},
|
||||
{
|
||||
key: "failed",
|
||||
labelKey: "portal.processorFlow.outcomes.failed",
|
||||
count24h: failed,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/** Assemble the full flow model from the three live portal surfaces. */
|
||||
export async function fetchProcessorFlow(): Promise<ProcessorFlow> {
|
||||
const [sourcesResp, wirePolicies, runs] = await Promise.all([
|
||||
fetchSources(),
|
||||
apiClient.local.json<WirePolicy[]>("/api/v1/policies"),
|
||||
apiClient.local
|
||||
.json<PolicyRunView[]>("/api/v1/policies/runs")
|
||||
.catch(() => [] as PolicyRunView[]),
|
||||
]);
|
||||
|
||||
const sources: FlowSource[] = sourcesResp.sources.map((s) => ({
|
||||
id: s.id,
|
||||
name: s.name,
|
||||
type: s.type,
|
||||
docs24h: s.docs24h,
|
||||
}));
|
||||
|
||||
return {
|
||||
sources,
|
||||
comingSoonSources: COMING_SOON_SOURCES,
|
||||
policies: buildPolicies(wirePolicies, runs),
|
||||
outcomes: buildOutcomes(runs),
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,3 @@
|
||||
// The bare i18next singleton (the same instance @app/i18n initializes at
|
||||
// startup), imported directly so this data module doesn't pull i18n's
|
||||
// init side effects into unit tests that mock react-i18next.
|
||||
import i18n from "i18next";
|
||||
import { apiClient } from "@portal/api/http";
|
||||
import type { Tier } from "@portal/contexts/TierContext";
|
||||
|
||||
@@ -273,23 +269,22 @@ function roleIdFor(u: AdminUserSummaryDto): RoleId {
|
||||
|
||||
/** A member's last-seen time as plain language; "Never" when no session is tracked. */
|
||||
function relativeTime(value: number | string | undefined): string {
|
||||
if (value === undefined || value === null)
|
||||
return i18n.t("users.activity.never");
|
||||
if (value === undefined || value === null) return "Never";
|
||||
const ts = typeof value === "string" ? Date.parse(value) : value;
|
||||
if (!Number.isFinite(ts) || ts <= 0) return i18n.t("users.activity.never");
|
||||
if (!Number.isFinite(ts) || ts <= 0) return "Never";
|
||||
const mins = Math.max(0, Math.round((Date.now() - ts) / 60000));
|
||||
if (mins < 1) return i18n.t("users.activity.justNow");
|
||||
if (mins < 60) return i18n.t("users.activity.minutesAgo", { count: mins });
|
||||
if (mins < 1) return "Just now";
|
||||
if (mins < 60) return `${mins}m ago`;
|
||||
const hours = Math.round(mins / 60);
|
||||
if (hours < 24) return i18n.t("users.activity.hoursAgo", { count: hours });
|
||||
if (hours < 24) return `${hours}h ago`;
|
||||
const days = Math.round(hours / 24);
|
||||
if (days < 7) return i18n.t("users.activity.daysAgo", { count: days });
|
||||
if (days < 7) return `${days}d ago`;
|
||||
const weeks = Math.round(days / 7);
|
||||
if (weeks < 5) return i18n.t("users.activity.weeksAgo", { count: weeks });
|
||||
if (weeks < 5) return weeks === 1 ? "1 week ago" : `${weeks} weeks ago`;
|
||||
const months = Math.round(days / 30);
|
||||
if (months < 12) return i18n.t("users.activity.monthsAgo", { count: months });
|
||||
if (months < 12) return months <= 1 ? "1 month ago" : `${months} months ago`;
|
||||
const years = Math.round(days / 365);
|
||||
return i18n.t("users.activity.yearsAgo", { count: years });
|
||||
return years <= 1 ? "1 year ago" : `${years} years ago`;
|
||||
}
|
||||
|
||||
/** 0 / huge sentinel license values mean "no seat limit". */
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
align-items: center;
|
||||
gap: 1.25rem;
|
||||
padding: 1rem 1.25rem;
|
||||
background: var(--color-hero-navy);
|
||||
background: #16213e;
|
||||
}
|
||||
|
||||
.portal-editor-hero__logo {
|
||||
@@ -123,7 +123,7 @@
|
||||
.portal-editor-hero__action .portal-editor-hero__cta.sui-btn {
|
||||
background: #ffffff;
|
||||
border-color: #ffffff;
|
||||
color: var(--color-hero-navy);
|
||||
color: #16213e;
|
||||
}
|
||||
.portal-editor-hero__action .portal-editor-hero__cta.sui-btn:hover {
|
||||
background: rgba(255, 255, 255, 0.88);
|
||||
|
||||
@@ -26,7 +26,7 @@ export function ProcessingStatusStrip() {
|
||||
style={{ background: TIER_INFO[tier].dotColor }}
|
||||
aria-hidden
|
||||
/>
|
||||
{t(TIER_INFO[tier].labelKey)}
|
||||
{TIER_INFO[tier].label}
|
||||
</span>
|
||||
<span className="portal-statusstrip__sep" aria-hidden>
|
||||
·
|
||||
|
||||
@@ -1,394 +0,0 @@
|
||||
/* Processor-flow visualiser — sources → policies → outcomes. The flow is an
|
||||
SVG overlay measured from the HTML cards: bézier wires underneath, and a
|
||||
rAF-driven particle layer on top (see ProcessorFlow.tsx). */
|
||||
|
||||
.portal-pf {
|
||||
--pf-accent: var(--color-green);
|
||||
}
|
||||
|
||||
/* ── Header ──────────────────────────────────────────────────────────────── */
|
||||
.portal-pf__head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.portal-pf__head-text {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 0.5rem;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.portal-pf__head-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.625rem;
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.portal-pf__live {
|
||||
align-self: center;
|
||||
width: 0.5rem;
|
||||
height: 0.5rem;
|
||||
border-radius: 50%;
|
||||
background: var(--color-border);
|
||||
}
|
||||
|
||||
.portal-pf__live--on {
|
||||
background: var(--pf-accent);
|
||||
box-shadow: 0 0 0 0 color-mix(in srgb, var(--pf-accent) 55%, transparent);
|
||||
animation: pf-pulse 2.4s ease-out infinite;
|
||||
}
|
||||
|
||||
.portal-pf__title {
|
||||
margin: 0;
|
||||
font-size: 0.9375rem;
|
||||
font-weight: 600;
|
||||
color: var(--color-text-1);
|
||||
}
|
||||
|
||||
.portal-pf__connected {
|
||||
font-size: 0.75rem;
|
||||
color: var(--color-text-4);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
/* ── Stage: relative box the SVG overlays are measured against ───────────── */
|
||||
.portal-pf__stage {
|
||||
position: relative;
|
||||
padding: 0.25rem 0 0.5rem;
|
||||
}
|
||||
|
||||
.portal-pf__wires,
|
||||
.portal-pf__particles {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
pointer-events: none;
|
||||
overflow: visible;
|
||||
}
|
||||
|
||||
.portal-pf__wires {
|
||||
z-index: 0;
|
||||
}
|
||||
|
||||
.portal-pf__particles {
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
.portal-pf__wire-path {
|
||||
fill: none;
|
||||
stroke: var(--color-border-light);
|
||||
stroke-width: 1.25;
|
||||
}
|
||||
|
||||
/* ── Columns ─────────────────────────────────────────────────────────────── */
|
||||
.portal-pf__cols {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: stretch;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.portal-pf__col {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
gap: 0.5rem;
|
||||
flex: none;
|
||||
width: 15rem;
|
||||
}
|
||||
|
||||
.portal-pf__col-head,
|
||||
.portal-pf__policies-head {
|
||||
font-size: 0.6875rem;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.04em;
|
||||
text-transform: uppercase;
|
||||
color: var(--color-text-5);
|
||||
margin-bottom: 0.125rem;
|
||||
}
|
||||
|
||||
/* ── Nodes (source + outcome cards) ─────────────────────────────────────────── */
|
||||
.portal-pf__node {
|
||||
height: auto;
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
background: var(--color-bg-subtle);
|
||||
border: 1px solid var(--color-border-light);
|
||||
border-radius: var(--radius-md);
|
||||
transition:
|
||||
background var(--motion-fast),
|
||||
border-color var(--motion-fast);
|
||||
}
|
||||
|
||||
.portal-pf__node .mantine-Button-label {
|
||||
flex: 1 1 auto;
|
||||
justify-content: flex-start;
|
||||
overflow: visible;
|
||||
}
|
||||
|
||||
.portal-pf__node:hover {
|
||||
background: var(--color-bg-hover);
|
||||
border-color: var(--color-border);
|
||||
}
|
||||
|
||||
.portal-pf__node:focus-visible {
|
||||
outline: 2px solid var(--color-blue);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
.portal-pf__node--soon {
|
||||
background: transparent;
|
||||
border-style: dashed;
|
||||
opacity: 0.75;
|
||||
}
|
||||
|
||||
.portal-pf__node-icon {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex: none;
|
||||
width: 1.75rem;
|
||||
height: 1.75rem;
|
||||
border-radius: var(--radius-md);
|
||||
font-size: 0.9375rem;
|
||||
background: var(--color-bg-muted);
|
||||
color: var(--color-text-3);
|
||||
}
|
||||
|
||||
.portal-pf__node--soon .portal-pf__node-icon {
|
||||
color: var(--color-text-5);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.portal-pf__node--success .portal-pf__node-icon {
|
||||
background: var(--color-green-light);
|
||||
color: var(--color-green-dark);
|
||||
}
|
||||
|
||||
.portal-pf__node--failed .portal-pf__node-icon {
|
||||
background: var(--color-red-light);
|
||||
color: var(--color-red-dark);
|
||||
}
|
||||
|
||||
.portal-pf__node-text {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.portal-pf__node-text strong {
|
||||
font-size: 0.8125rem;
|
||||
font-weight: 600;
|
||||
color: var(--color-text-1);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.portal-pf__node-text span {
|
||||
font-size: 0.6875rem;
|
||||
color: var(--color-text-4);
|
||||
}
|
||||
|
||||
/* ── Policies card (core) ───────────────────────────────────────────────────── */
|
||||
.portal-pf__policies {
|
||||
align-self: center;
|
||||
flex: none;
|
||||
width: 18rem;
|
||||
padding: 0.75rem;
|
||||
background: var(--color-bg-subtle);
|
||||
border: 1px solid var(--color-border-light);
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.portal-pf__policies-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0 0.25rem 0.5rem;
|
||||
}
|
||||
|
||||
.portal-pf__policies-active {
|
||||
text-transform: none;
|
||||
letter-spacing: 0;
|
||||
color: var(--color-green-dark);
|
||||
}
|
||||
|
||||
.portal-pf__policy {
|
||||
padding: 0.5rem 0.25rem;
|
||||
}
|
||||
|
||||
.portal-pf__policy + .portal-pf__policy {
|
||||
border-top: 1px solid var(--color-border-light);
|
||||
}
|
||||
|
||||
.portal-pf__policy-line {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.portal-pf__policy-icon {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex: none;
|
||||
width: 1.5rem;
|
||||
height: 1.5rem;
|
||||
border-radius: 6px;
|
||||
font-size: 0.875rem;
|
||||
color: var(--color-text-4);
|
||||
transition:
|
||||
color 0.3s ease,
|
||||
background-color 0.3s ease,
|
||||
box-shadow 0.3s ease;
|
||||
}
|
||||
|
||||
.portal-pf__policy--active .portal-pf__policy-icon {
|
||||
color: var(--pf-accent);
|
||||
}
|
||||
|
||||
.portal-pf__policy--locked .portal-pf__policy-icon {
|
||||
color: var(--color-text-5);
|
||||
}
|
||||
|
||||
/* Leading-LED blink: added for 150ms as a particle threads this row's lane,
|
||||
then eased back out by the transition above. */
|
||||
.portal-pf__policy-icon.is-pulse {
|
||||
color: var(--color-green-dark);
|
||||
background-color: var(--color-green-light);
|
||||
box-shadow: 0 0 8px 1px
|
||||
color-mix(in srgb, var(--color-green) 55%, transparent);
|
||||
}
|
||||
|
||||
.portal-pf__policy-label {
|
||||
flex: 1 1 auto;
|
||||
font-size: 0.8125rem;
|
||||
font-weight: 600;
|
||||
color: var(--color-text-2);
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.portal-pf__policy--active .portal-pf__policy-label {
|
||||
color: var(--color-text-1);
|
||||
}
|
||||
|
||||
.portal-pf__policy--locked .portal-pf__policy-label {
|
||||
color: var(--color-text-4);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.portal-pf__policy-count {
|
||||
font-size: 0.75rem;
|
||||
color: var(--color-text-4);
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.portal-pf__policy-soon {
|
||||
flex: none;
|
||||
font-size: 0.6875rem;
|
||||
font-weight: 600;
|
||||
color: var(--color-text-5);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
|
||||
/* Vertical padding via the `py` prop; height auto so it grows with it. */
|
||||
.portal-pf__setup {
|
||||
flex: none;
|
||||
height: auto;
|
||||
}
|
||||
|
||||
/* ── Footnote / loading ─────────────────────────────────────────────────────── */
|
||||
.portal-pf__foot {
|
||||
margin: 0.875rem 0 0;
|
||||
font-size: 0.6875rem;
|
||||
color: var(--color-text-5);
|
||||
}
|
||||
|
||||
.portal-pf__loading {
|
||||
padding: 0.5rem 0;
|
||||
}
|
||||
|
||||
/* ── Sankey lens ────────────────────────────────────────────────────────────── */
|
||||
.portal-pf__sankey {
|
||||
max-width: 46rem;
|
||||
margin: 0.5rem auto 0;
|
||||
}
|
||||
|
||||
.portal-pf__sankey svg {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: auto;
|
||||
overflow: visible;
|
||||
}
|
||||
|
||||
.portal-pf__sankey-label {
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
fill: var(--color-text-2);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.portal-pf__sankey-caption {
|
||||
font-size: 10.5px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.05em;
|
||||
text-transform: uppercase;
|
||||
fill: var(--color-text-5);
|
||||
}
|
||||
|
||||
.portal-pf__sankey-empty {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 11rem;
|
||||
}
|
||||
|
||||
@keyframes pf-pulse {
|
||||
0% {
|
||||
box-shadow: 0 0 0 0 color-mix(in srgb, var(--pf-accent) 55%, transparent);
|
||||
}
|
||||
70% {
|
||||
box-shadow: 0 0 0 0.4rem transparent;
|
||||
}
|
||||
100% {
|
||||
box-shadow: 0 0 0 0 transparent;
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Responsive: stack the columns; the measured-geometry flow overlay only
|
||||
makes sense on the wide 3-column layout, so drop it below the breakpoint. ── */
|
||||
@media (max-width: 60rem) {
|
||||
.portal-pf__wires,
|
||||
.portal-pf__particles {
|
||||
display: none;
|
||||
}
|
||||
.portal-pf__cols {
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
.portal-pf__col,
|
||||
.portal-pf__policies {
|
||||
width: 100%;
|
||||
align-self: stretch;
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.portal-pf__live--on {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
import type { Meta, StoryObj } from "@storybook/react-vite";
|
||||
import { http, HttpResponse } from "msw";
|
||||
import { ProcessorFlow } from "@portal/components/ProcessorFlow";
|
||||
|
||||
/**
|
||||
* The home processor visualiser. Data is served by the global portal MSW
|
||||
* handlers (seeded sources + one active Security policy + its runs), so the
|
||||
* middle column shows Security "active" and Classification with a "Set up" CTA,
|
||||
* and the outcomes reflect the seeded 24h success/failure split.
|
||||
*
|
||||
* NB: the flow particles animate via requestAnimationFrame, which browsers
|
||||
* pause while the tab/preview is hidden — open the story in a focused tab to
|
||||
* see the dots move.
|
||||
*/
|
||||
const meta: Meta<typeof ProcessorFlow> = {
|
||||
title: "Portal/Components/ProcessorFlow",
|
||||
component: ProcessorFlow,
|
||||
parameters: { layout: "padded" },
|
||||
};
|
||||
export default meta;
|
||||
|
||||
type Story = StoryObj<typeof ProcessorFlow>;
|
||||
|
||||
/** Live machine: Security configured + real throughput → the flow runs. */
|
||||
export const Default: Story = {};
|
||||
|
||||
/**
|
||||
* Nothing set up and no activity — the empty state from the design. In
|
||||
* production the flow stays still here; the DEV_KEEP_FLOWING dev flag forces it
|
||||
* on with synthetic rates so the animation is visible while iterating.
|
||||
*/
|
||||
export const IdleEmpty: Story = {
|
||||
parameters: {
|
||||
msw: {
|
||||
handlers: [
|
||||
http.get("/api/v1/sources", () =>
|
||||
HttpResponse.json({
|
||||
kpis: [],
|
||||
sources: [
|
||||
{
|
||||
id: "editor",
|
||||
name: "Editor",
|
||||
type: "editor",
|
||||
status: "active",
|
||||
referenceCount: 0,
|
||||
referencingPolicies: [],
|
||||
config: [],
|
||||
docsTotal: 0,
|
||||
docs24h: 0,
|
||||
docs30d: 0,
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
http.get("/api/v1/policies", () => HttpResponse.json([])),
|
||||
http.get("/api/v1/policies/runs", () => HttpResponse.json([])),
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,184 +0,0 @@
|
||||
import { useState } from "react";
|
||||
import { useNavigate } from "react-router-dom";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Card, SegmentedControl, Skeleton, StatusBadge } from "@app/ui";
|
||||
import {
|
||||
useView,
|
||||
VIEW_PATHS,
|
||||
toPortalPath,
|
||||
} from "@portal/contexts/ViewContext";
|
||||
import { useAsync } from "@portal/hooks/useAsync";
|
||||
import {
|
||||
fetchProcessorFlow,
|
||||
type ProcessorFlow as ProcessorFlowModel,
|
||||
} from "@portal/api/processorFlow";
|
||||
import {
|
||||
DEV_KEEP_FLOWING,
|
||||
DEV_SYNTH_RATE,
|
||||
type Lens,
|
||||
} from "@portal/components/processor-flow/flowTypes";
|
||||
import { useFlowGeometry } from "@portal/components/processor-flow/useFlowGeometry";
|
||||
import { useFlowParticles } from "@portal/components/processor-flow/useFlowParticles";
|
||||
import { FlowSources } from "@portal/components/processor-flow/FlowSources";
|
||||
import { FlowPolicies } from "@portal/components/processor-flow/FlowPolicies";
|
||||
import { FlowOutcomes } from "@portal/components/processor-flow/FlowOutcomes";
|
||||
import { FlowSankey } from "@portal/components/processor-flow/FlowSankey";
|
||||
import "@portal/components/ProcessorFlow.css";
|
||||
|
||||
/**
|
||||
* Animated processor visualiser for the home surface: connected sources on the
|
||||
* left flow through the standing policies in the middle to their audit outcomes
|
||||
* on the right. Two lenses — a live particle flow and a Sankey summary.
|
||||
*
|
||||
* This module wires the data + gating together; the moving parts live under
|
||||
* `processor-flow/`: geometry ({@link useFlowGeometry}), the rAF particle loop
|
||||
* ({@link useFlowParticles}), the three columns, and the Sankey. The flow only
|
||||
* runs when something is set up AND there's activity; an idle machine stays
|
||||
* still (unless {@link DEV_KEEP_FLOWING} forces it while iterating).
|
||||
*/
|
||||
export function ProcessorFlow() {
|
||||
const { t } = useTranslation();
|
||||
const { setActiveView } = useView();
|
||||
const navigate = useNavigate();
|
||||
const { data, loading } = useAsync<ProcessorFlowModel>(
|
||||
() => fetchProcessorFlow(),
|
||||
[],
|
||||
);
|
||||
|
||||
const [lens, setLens] = useState<Lens>("flow");
|
||||
const isLoading = loading && data === null;
|
||||
|
||||
/** Deep-link to the Policies page and auto-open that policy's setup wizard. */
|
||||
const openPolicySetup = (key: string) =>
|
||||
navigate(
|
||||
`${toPortalPath(VIEW_PATHS.policies)}?setup=${encodeURIComponent(key)}`,
|
||||
);
|
||||
|
||||
/** Deep-link to Infrastructure with the audit-log tab open. */
|
||||
const openAuditLog = () =>
|
||||
navigate(`${toPortalPath(VIEW_PATHS.infrastructure)}?tab=audit`);
|
||||
|
||||
const sources = data?.sources ?? [];
|
||||
const policies = data?.policies ?? [];
|
||||
const outcomes = data?.outcomes ?? [];
|
||||
const comingSoonSources = data?.comingSoonSources ?? [];
|
||||
|
||||
// ── Flow gating: run only when something is set up AND there's activity.
|
||||
const totalRate = sources.reduce((sum, s) => sum + s.docs24h, 0);
|
||||
const hasConfigured = policies.some((p) => p.configured);
|
||||
const liveFlow = hasConfigured && totalRate > 0;
|
||||
// When forcing for dev with no live flow, synthesise rates + thread every row.
|
||||
const devForced = DEV_KEEP_FLOWING && !liveFlow;
|
||||
const animate = liveFlow || devForced;
|
||||
|
||||
// Particles only thread configured (active) policies; while dev-forcing with
|
||||
// no live flow, thread the available (non-locked) rows so the demo has lanes.
|
||||
const laneKeys = policies
|
||||
.filter((p) => (devForced ? p.state !== "locked" : p.state === "active"))
|
||||
.map((p) => p.key);
|
||||
|
||||
const activeCount = policies.filter((p) => p.state === "active").length;
|
||||
const pdfsProcessed = outcomes.reduce((sum, o) => sum + o.count24h, 0);
|
||||
const statsLabel = t("portal.processorFlow.stats", {
|
||||
connected: sources.length,
|
||||
processed: pdfsProcessed.toLocaleString(),
|
||||
});
|
||||
|
||||
// Per-source rates + outcome weights feeding the particle loop.
|
||||
const rates = sources.map((s) => (devForced ? DEV_SYNTH_RATE : s.docs24h));
|
||||
const weights = (() => {
|
||||
const raw = outcomes.map((o) => o.count24h);
|
||||
const sum = raw.reduce((a, b) => a + b, 0);
|
||||
if (sum > 0) return raw.map((v) => v / sum);
|
||||
// No real outcomes yet (dev flow): success-heavy default.
|
||||
return outcomes.map((o) => (o.key === "failed" ? 0.15 : 0.85));
|
||||
})();
|
||||
const outcomeKeys = outcomes.map((o) => o.key);
|
||||
|
||||
const { wrapRef, srcRefs, outRefs, coreRef, laneRefs, geoRef, wires } =
|
||||
useFlowGeometry();
|
||||
const pGroupRef = useFlowParticles({
|
||||
geoRef,
|
||||
animate,
|
||||
lens,
|
||||
rates,
|
||||
weights,
|
||||
laneKeys,
|
||||
outcomeKeys,
|
||||
});
|
||||
|
||||
return (
|
||||
<Card padding="loose" className="portal-pf">
|
||||
<header className="portal-pf__head">
|
||||
<div className="portal-pf__head-text">
|
||||
<span
|
||||
className={
|
||||
"portal-pf__live" + (animate ? " portal-pf__live--on" : "")
|
||||
}
|
||||
aria-hidden
|
||||
/>
|
||||
<h2 className="portal-pf__title">
|
||||
{t("portal.processorFlow.title")}
|
||||
</h2>
|
||||
<span className="portal-pf__connected">{statsLabel}</span>
|
||||
</div>
|
||||
<div className="portal-pf__head-actions">
|
||||
<StatusBadge tone={animate ? "success" : "neutral"} size="sm">
|
||||
{t("portal.processorFlow.liveBadge")}
|
||||
</StatusBadge>
|
||||
<SegmentedControl<Lens>
|
||||
size="xs"
|
||||
value={lens}
|
||||
onChange={setLens}
|
||||
ariaLabel={t("portal.processorFlow.lens.ariaLabel")}
|
||||
options={[
|
||||
{ label: t("portal.processorFlow.lens.flow"), value: "flow" },
|
||||
{ label: t("portal.processorFlow.lens.sankey"), value: "sankey" },
|
||||
]}
|
||||
/>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{isLoading ? (
|
||||
<div className="portal-pf__loading" aria-hidden>
|
||||
<Skeleton height="9rem" />
|
||||
</div>
|
||||
) : lens === "sankey" ? (
|
||||
<FlowSankey sources={sources} outcomes={outcomes} policies={policies} />
|
||||
) : (
|
||||
<div className="portal-pf__stage" ref={wrapRef}>
|
||||
<svg className="portal-pf__wires" aria-hidden>
|
||||
{wires}
|
||||
</svg>
|
||||
|
||||
<div className="portal-pf__cols">
|
||||
<FlowSources
|
||||
sources={sources}
|
||||
comingSoonSources={comingSoonSources}
|
||||
srcRefs={srcRefs}
|
||||
onOpen={() => setActiveView("sources")}
|
||||
/>
|
||||
<FlowPolicies
|
||||
policies={policies}
|
||||
activeCount={activeCount}
|
||||
coreRef={coreRef}
|
||||
laneRefs={laneRefs}
|
||||
onSetup={openPolicySetup}
|
||||
/>
|
||||
<FlowOutcomes
|
||||
outcomes={outcomes}
|
||||
outRefs={outRefs}
|
||||
onOpen={openAuditLog}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<svg className="portal-pf__particles" aria-hidden>
|
||||
<g ref={pGroupRef} />
|
||||
</svg>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<p className="portal-pf__foot">{t("portal.processorFlow.footnote")}</p>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -18,7 +18,7 @@
|
||||
gap: 1rem;
|
||||
flex-wrap: wrap;
|
||||
padding: 0.875rem 1.25rem;
|
||||
background: var(--color-hero-navy);
|
||||
background: #16213e;
|
||||
}
|
||||
|
||||
.portal-welcome__brand {
|
||||
@@ -90,7 +90,7 @@
|
||||
.portal-welcome__header .portal-welcome__cta.sui-btn {
|
||||
background: #ffffff;
|
||||
border-color: #ffffff;
|
||||
color: var(--color-hero-navy);
|
||||
color: #16213e;
|
||||
}
|
||||
.portal-welcome__header .portal-welcome__cta.sui-btn:hover {
|
||||
background: rgba(255, 255, 255, 0.88);
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
import { useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { StatusBadge, Tabs, type TabItem } from "@app/ui";
|
||||
import {
|
||||
type Agent,
|
||||
AGENT_STATUS_LABEL,
|
||||
AGENT_STATUS_TONE,
|
||||
} from "@portal/api/agents";
|
||||
import { type Agent, AGENT_STATUS_TONE } from "@portal/api/agents";
|
||||
import { ScenariosPanel } from "@portal/components/agent-builder/ScenariosPanel";
|
||||
import { ToolsPanel } from "@portal/components/agent-builder/ToolsPanel";
|
||||
import { EvalsPanel } from "@portal/components/agent-builder/EvalsPanel";
|
||||
@@ -56,7 +52,7 @@ export function AgentBuilderPanel({
|
||||
</div>
|
||||
<div className="portal-agents__builder-meta">
|
||||
<StatusBadge tone={AGENT_STATUS_TONE[agent.status]} size="sm">
|
||||
{t(AGENT_STATUS_LABEL[agent.status])}
|
||||
{agent.status}
|
||||
</StatusBadge>
|
||||
<code className="portal-agents__builder-version">
|
||||
{agent.version}
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
import { useTranslation } from "react-i18next";
|
||||
import {
|
||||
type Agent,
|
||||
AGENT_STATUS_LABEL,
|
||||
AGENT_STATUS_TONE,
|
||||
} from "@portal/api/agents";
|
||||
import { type Agent, AGENT_STATUS_TONE } from "@portal/api/agents";
|
||||
import { Button, StatusBadge } from "@app/ui";
|
||||
import "@portal/views/AgentBuilder.css";
|
||||
|
||||
@@ -44,7 +40,7 @@ export function AgentSelector({
|
||||
</span>
|
||||
<span className="portal-agents__selector-meta">
|
||||
<StatusBadge tone={AGENT_STATUS_TONE[a.status]} size="sm">
|
||||
{t(AGENT_STATUS_LABEL[a.status])}
|
||||
{a.status}
|
||||
</StatusBadge>
|
||||
<code className="portal-agents__selector-version">{a.version}</code>
|
||||
</span>
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button, StatusBadge } from "@app/ui";
|
||||
import {
|
||||
type Agent,
|
||||
AGENT_STATUS_LABEL,
|
||||
AGENT_STATUS_TONE,
|
||||
} from "@portal/api/agents";
|
||||
import { type Agent, AGENT_STATUS_TONE } from "@portal/api/agents";
|
||||
import "@portal/views/AgentBuilder.css";
|
||||
|
||||
interface VersionsPanelProps {
|
||||
@@ -56,7 +52,7 @@ export function VersionsPanel({ agent, historyUnlocked }: VersionsPanelProps) {
|
||||
{v.version}
|
||||
</code>
|
||||
<StatusBadge tone={AGENT_STATUS_TONE[v.status]} size="sm">
|
||||
{t(AGENT_STATUS_LABEL[v.status])}
|
||||
{v.status}
|
||||
</StatusBadge>
|
||||
{isCurrent && (
|
||||
<StatusBadge tone="info" size="sm" showDot={false}>
|
||||
|
||||
@@ -12,11 +12,9 @@ import {
|
||||
} from "@app/ui";
|
||||
import {
|
||||
type SdkComponent,
|
||||
BILLING_UNIT_LABEL,
|
||||
MATURITY_META,
|
||||
formatPrice,
|
||||
} from "@portal/api/sdkComponents";
|
||||
import { TIER_INFO } from "@portal/contexts/TierContext";
|
||||
import { ComponentPropsTable } from "@portal/components/catalogue/ComponentPropsTable";
|
||||
import "@portal/views/Components.css";
|
||||
|
||||
@@ -115,7 +113,7 @@ export function ComponentDetailModal({
|
||||
title={t("portal.catalogue.detail.locked.title")}
|
||||
description={t("portal.catalogue.detail.locked.description", {
|
||||
name: component.name,
|
||||
plan: t(TIER_INFO[component.minTier].labelKey),
|
||||
tier: component.minTier,
|
||||
})}
|
||||
/>
|
||||
)}
|
||||
@@ -207,7 +205,7 @@ export function ComponentDetailModal({
|
||||
/>
|
||||
<StatTile
|
||||
label={t("portal.catalogue.detail.stats.billedOn")}
|
||||
value={t(BILLING_UNIT_LABEL[component.pricing.unit])}
|
||||
value={component.pricing.unit}
|
||||
/>
|
||||
<StatTile
|
||||
label={t("portal.catalogue.detail.stats.freeQuota")}
|
||||
@@ -222,7 +220,7 @@ export function ComponentDetailModal({
|
||||
</div>
|
||||
<p className="portal-components__pricing-note">
|
||||
{t("portal.catalogue.detail.pricing.note", {
|
||||
unit: t(BILLING_UNIT_LABEL[component.pricing.unit]),
|
||||
unit: component.pricing.unit,
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -157,7 +157,7 @@ export function ReviewQueue({ documents, loading }: ReviewQueueProps) {
|
||||
<Button
|
||||
variant="secondary"
|
||||
onClick={() =>
|
||||
navigate(`${toPortalPath(VIEW_PATHS.sources)}/new`)
|
||||
navigate(`${toPortalPath(VIEW_PATHS.sources)}?new`)
|
||||
}
|
||||
>
|
||||
{t("portal.documents.queue.empty.connectSource")}
|
||||
|
||||
@@ -23,7 +23,7 @@ export function ApiKeyCard({ apiKey }: { apiKey: ApiKey }) {
|
||||
rightSection={
|
||||
<span className="portal-infra__key-head-right">
|
||||
<StatusBadge tone={KEY_TONE[apiKey.status]} size="sm">
|
||||
{t(KEY_LABEL[apiKey.status])}
|
||||
{KEY_LABEL[apiKey.status]}
|
||||
</StatusBadge>
|
||||
<span
|
||||
className={"portal-infra__chevron" + (open ? " is-open" : "")}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import { MantineProvider } from "@mantine/core";
|
||||
import { Tooltip } from "@app/components/shared/Tooltip";
|
||||
import type { ToolRegistry } from "@app/data/toolsTaxonomy";
|
||||
import type { WorkingToolStep } from "@app/hooks/tools/shared/toolAutomation";
|
||||
import { PipelineStepSettings } from "@portal/components/pipelines/PipelineStepSettings";
|
||||
|
||||
vi.mock("react-i18next", () => ({
|
||||
useTranslation: () => ({
|
||||
t: (key: string, fallback?: string) => fallback ?? key,
|
||||
}),
|
||||
}));
|
||||
|
||||
// A stand-in tool-settings UI that uses the shared editor Tooltip. The Tooltip
|
||||
// pulls in the Preferences + Sidebar contexts, which the portal does not mount
|
||||
// app-wide — so this reproduces the "usePreferences must be used within a
|
||||
// PreferencesProvider" crash unless PipelineStepSettings supplies them.
|
||||
function TooltipSettings() {
|
||||
return (
|
||||
<Tooltip content="help">
|
||||
<button type="button">field</button>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
const step = {
|
||||
support: "editable",
|
||||
toolId: "compress",
|
||||
params: {},
|
||||
} as unknown as WorkingToolStep;
|
||||
|
||||
const registry = {
|
||||
compress: { automationSettings: TooltipSettings },
|
||||
} as unknown as Partial<ToolRegistry>;
|
||||
|
||||
describe("PipelineStepSettings", () => {
|
||||
it("renders reused editor tool settings (which use the shared Tooltip) without app-wide Preferences/Sidebar providers", () => {
|
||||
expect(() =>
|
||||
render(
|
||||
<MantineProvider>
|
||||
<PipelineStepSettings
|
||||
step={step}
|
||||
registry={registry}
|
||||
onChange={() => {}}
|
||||
/>
|
||||
</MantineProvider>,
|
||||
),
|
||||
).not.toThrow();
|
||||
expect(screen.getByText("field")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,6 @@
|
||||
import { Suspense } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Banner } from "@app/ui";
|
||||
import { PreferencesProvider } from "@app/contexts/PreferencesContext";
|
||||
import { SidebarProvider } from "@app/contexts/SidebarContext";
|
||||
import { type ToolRegistry } from "@app/data/toolsTaxonomy";
|
||||
import { type ErasedToolParams } from "@app/hooks/tools/shared/toolOperationTypes";
|
||||
import { type WorkingToolStep } from "@app/hooks/tools/shared/toolAutomation";
|
||||
@@ -48,18 +46,14 @@ export function PipelineStepSettings({
|
||||
}
|
||||
|
||||
return (
|
||||
<PreferencesProvider>
|
||||
<SidebarProvider>
|
||||
<Suspense fallback={null}>
|
||||
<Settings
|
||||
parameters={step.params}
|
||||
onParameterChange={(key, value) =>
|
||||
onChange({ ...step.params, [key]: value })
|
||||
}
|
||||
disabled={false}
|
||||
/>
|
||||
</Suspense>
|
||||
</SidebarProvider>
|
||||
</PreferencesProvider>
|
||||
<Suspense fallback={null}>
|
||||
<Settings
|
||||
parameters={step.params}
|
||||
onParameterChange={(key, value) =>
|
||||
onChange({ ...step.params, [key]: value })
|
||||
}
|
||||
disabled={false}
|
||||
/>
|
||||
</Suspense>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
fireEvent,
|
||||
render as baseRender,
|
||||
screen,
|
||||
waitFor,
|
||||
} from "@testing-library/react";
|
||||
import { MantineProvider } from "@mantine/core";
|
||||
import { PolicySetupWizard } from "@portal/components/policies/PolicySetupWizard";
|
||||
import {
|
||||
POLICY_CATEGORIES,
|
||||
POLICY_CONFIG,
|
||||
type CatalogueEntry,
|
||||
type DecoratedPolicy,
|
||||
type PolicySetupResult,
|
||||
type PipelineStep,
|
||||
} from "@portal/api/policies";
|
||||
|
||||
const render = (ui: Parameters<typeof baseRender>[0]) =>
|
||||
baseRender(ui, { wrapper: MantineProvider });
|
||||
|
||||
// Deterministic i18n: return the fallback when given, else the key. initReactI18next is stubbed
|
||||
// because the import graph pulls core/i18n.ts, which registers it as a plugin.
|
||||
vi.mock("react-i18next", () => ({
|
||||
useTranslation: () => ({
|
||||
// Second arg is a string fallback in some call sites and an interpolation object in others;
|
||||
// only treat a string as the fallback.
|
||||
t: (key: string, fallback?: unknown) =>
|
||||
typeof fallback === "string" ? fallback : key,
|
||||
i18n: { changeLanguage: vi.fn() },
|
||||
}),
|
||||
initReactI18next: { type: "3rdParty", init: vi.fn() },
|
||||
}));
|
||||
|
||||
const fetchSources = vi.fn();
|
||||
vi.mock("@portal/api/sources", () => ({
|
||||
fetchSources: () => fetchSources(),
|
||||
}));
|
||||
|
||||
const CONTINUE = "portal.policies.wizard.actions.continue";
|
||||
const SAVE_CHANGES = "portal.policies.wizard.actions.saveChanges";
|
||||
const ENABLE = "portal.policies.wizard.actions.enablePolicy";
|
||||
|
||||
const security = POLICY_CATEGORIES.find((c) => c.id === "security")!;
|
||||
const securityConfig = POLICY_CONFIG.security;
|
||||
|
||||
function editEntry(steps: PipelineStep[]): CatalogueEntry {
|
||||
const policy: DecoratedPolicy = {
|
||||
category: security,
|
||||
config: securityConfig,
|
||||
state: {
|
||||
configured: true,
|
||||
status: "active",
|
||||
sources: ["editor"],
|
||||
scopeTypes: [],
|
||||
reviewerEmail: "",
|
||||
fieldValues: {},
|
||||
runOn: "upload",
|
||||
outputMode: "new_version",
|
||||
outputName: "",
|
||||
outputNamePosition: "suffix",
|
||||
maxRetries: 0,
|
||||
retryDelayMinutes: 0,
|
||||
backendId: "pol-1",
|
||||
isDefault: true,
|
||||
},
|
||||
steps,
|
||||
stats: { enforced: 0, dataProcessed: "-", activeFor: "-" },
|
||||
activity: [],
|
||||
};
|
||||
return { category: security, config: securityConfig, policy };
|
||||
}
|
||||
|
||||
/** Advance the wizard from the workflow tab to the settings tab and submit. */
|
||||
async function submitWizard(saveLabel: string) {
|
||||
fireEvent.click(await screen.findByRole("button", { name: CONTINUE }));
|
||||
fireEvent.click(await screen.findByRole("button", { name: saveLabel }));
|
||||
}
|
||||
|
||||
describe("PolicySetupWizard", () => {
|
||||
beforeEach(() => {
|
||||
fetchSources.mockResolvedValue({ sources: [] });
|
||||
});
|
||||
|
||||
it("round-trips a saved step's backend params on edit", async () => {
|
||||
const onSubmit = vi.fn().mockResolvedValue(undefined);
|
||||
const entry = editEntry([
|
||||
{
|
||||
operation: "/api/v1/security/auto-redact",
|
||||
parameters: { listOfText: "foo\nbar", useRegex: true },
|
||||
},
|
||||
]);
|
||||
|
||||
render(
|
||||
<PolicySetupWizard entry={entry} onClose={vi.fn()} onSubmit={onSubmit} />,
|
||||
);
|
||||
await submitWizard(SAVE_CHANGES);
|
||||
|
||||
await waitFor(() => expect(onSubmit).toHaveBeenCalled());
|
||||
const result = onSubmit.mock.calls[0][1] as PolicySetupResult;
|
||||
// Only the saved tool is enabled on edit, and its patterns survive the wire -> UI -> wire trip.
|
||||
expect(result.steps).toEqual([
|
||||
expect.objectContaining({
|
||||
operation: "/api/v1/security/auto-redact",
|
||||
parameters: expect.objectContaining({ listOfText: "foo\nbar" }),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("seeds the preset chain for a new policy (redact + sanitize on, watermark off)", async () => {
|
||||
const onSubmit = vi.fn().mockResolvedValue(undefined);
|
||||
const entry: CatalogueEntry = {
|
||||
category: security,
|
||||
config: securityConfig,
|
||||
policy: null,
|
||||
};
|
||||
|
||||
render(
|
||||
<PolicySetupWizard entry={entry} onClose={vi.fn()} onSubmit={onSubmit} />,
|
||||
);
|
||||
await submitWizard(ENABLE);
|
||||
|
||||
await waitFor(() => expect(onSubmit).toHaveBeenCalled());
|
||||
const result = onSubmit.mock.calls[0][1] as PolicySetupResult;
|
||||
const endpoints = result.steps.map((s) => s.operation);
|
||||
expect(endpoints).toEqual([
|
||||
"/api/v1/security/auto-redact",
|
||||
"/api/v1/security/sanitize-pdf",
|
||||
]);
|
||||
// Redact carries the preset PII patterns as the backend's listOfText.
|
||||
const redact = result.steps[0].parameters as { listOfText?: string };
|
||||
expect(redact.listOfText).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -13,24 +13,23 @@ import {
|
||||
} from "@app/ui";
|
||||
import { SettingsRow } from "@app/ui/SettingsRow";
|
||||
import {
|
||||
TOOL_ENDPOINTS,
|
||||
humanizeEndpoint,
|
||||
type CatalogueEntry,
|
||||
type PipelineStep,
|
||||
type PolicySetupResult,
|
||||
} from "@portal/api/policies";
|
||||
import type { ToolRegistry, ToolRegistryEntry } from "@app/data/toolsTaxonomy";
|
||||
import {
|
||||
policyEndpoint,
|
||||
policyStepFromWire,
|
||||
policyStepToWire,
|
||||
type PolicyParams,
|
||||
type PolicyToolId,
|
||||
type PolicyToolStep,
|
||||
} from "@app/policies/operations";
|
||||
deserializeToolStep,
|
||||
serializeStepFromEndpoint,
|
||||
} from "@app/hooks/tools/shared/toolAutomation";
|
||||
import { fetchSources } from "@portal/api/sources";
|
||||
import { useAsync } from "@portal/hooks/useAsync";
|
||||
import { PolicyFieldRow } from "@portal/components/policies/PolicyFieldRow";
|
||||
import { policyIcon } from "@portal/components/policies/policyIcons";
|
||||
import { sourceTypeMeta } from "@portal/components/sources/sourceTypes";
|
||||
import { useToolRegistry } from "@app/contexts/ToolRegistryContext";
|
||||
import { PolicyRedactConfig } from "@app/components/policies/PolicyRedactConfig";
|
||||
import { PolicyWatermarkConfig } from "@app/components/policies/PolicyWatermarkConfig";
|
||||
import "@portal/views/Policies.css";
|
||||
@@ -48,8 +47,12 @@ interface PolicySetupWizardProps {
|
||||
|
||||
type Step = "workflow" | "settings";
|
||||
|
||||
/** A policy step plus whether it runs. */
|
||||
type ToolState = PolicyToolStep & { enabled: boolean };
|
||||
/** A configurable tool in the workflow step: whether it runs + its params. */
|
||||
interface ToolState {
|
||||
operation: string;
|
||||
enabled: boolean;
|
||||
parameters: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Resolve each field's effective value: saved override, else definition default. */
|
||||
function resolveFieldValues(
|
||||
@@ -66,8 +69,9 @@ function resolveFieldValues(
|
||||
* round-trips); otherwise the category preset's default chain. Each preset step
|
||||
* starts enabled — the user toggles tools off in the workflow.
|
||||
*/
|
||||
// Temporary until the catalogue carries a defaultEnabled flag.
|
||||
const DISABLED_BY_DEFAULT = new Set<PolicyToolId>(["watermark"]);
|
||||
// Temporary: tracks which tools start disabled until the tool registry lands in
|
||||
// the portal and can drive this via registry metadata or a defaultEnabled flag.
|
||||
const DISABLED_BY_DEFAULT = new Set(["/api/v1/security/add-watermark"]);
|
||||
|
||||
/**
|
||||
* Policy-facing framing for each capability a policy can include. Labels and
|
||||
@@ -77,43 +81,43 @@ const DISABLED_BY_DEFAULT = new Set<PolicyToolId>(["watermark"]);
|
||||
* the humanised endpoint name with no description.
|
||||
*/
|
||||
const CAPABILITY_META: Record<
|
||||
PolicyToolId,
|
||||
string,
|
||||
{ labelKey: string; labelEn: string; descKey: string; descEn: string }
|
||||
> = {
|
||||
redact: {
|
||||
[TOOL_ENDPOINTS.redact]: {
|
||||
labelKey: "portal.policies.wizard.capability.redact.label",
|
||||
labelEn: "Redact sensitive information",
|
||||
descKey: "portal.policies.wizard.capability.redact.desc",
|
||||
descEn:
|
||||
"Finds and blacks out sensitive details — like Social Security and card numbers — so they can't be read.",
|
||||
},
|
||||
sanitize: {
|
||||
[TOOL_ENDPOINTS.sanitize]: {
|
||||
labelKey: "portal.policies.wizard.capability.sanitize.label",
|
||||
labelEn: "Strip active content",
|
||||
descKey: "portal.policies.wizard.capability.sanitize.desc",
|
||||
descEn:
|
||||
"Removes hidden JavaScript so nothing can run automatically when the document is opened.",
|
||||
},
|
||||
watermark: {
|
||||
[TOOL_ENDPOINTS.watermark]: {
|
||||
labelKey: "portal.policies.wizard.capability.watermark.label",
|
||||
labelEn: "Apply a watermark",
|
||||
descKey: "portal.policies.wizard.capability.watermark.desc",
|
||||
descEn: "Stamps a visible mark (e.g. “Confidential”) across every page.",
|
||||
},
|
||||
ocr: {
|
||||
[TOOL_ENDPOINTS.ocr]: {
|
||||
labelKey: "portal.policies.wizard.capability.ocr.label",
|
||||
labelEn: "Make text searchable",
|
||||
descKey: "portal.policies.wizard.capability.ocr.desc",
|
||||
descEn: "Runs OCR so scanned pages become selectable, searchable text.",
|
||||
},
|
||||
flatten: {
|
||||
[TOOL_ENDPOINTS.flatten]: {
|
||||
labelKey: "portal.policies.wizard.capability.flatten.label",
|
||||
labelEn: "Flatten the document",
|
||||
descKey: "portal.policies.wizard.capability.flatten.desc",
|
||||
descEn:
|
||||
"Merges form fields and annotations into the page so they can't be edited.",
|
||||
},
|
||||
compress: {
|
||||
[TOOL_ENDPOINTS.compress]: {
|
||||
labelKey: "portal.policies.wizard.capability.compress.label",
|
||||
labelEn: "Reduce file size",
|
||||
descKey: "portal.policies.wizard.capability.compress.desc",
|
||||
@@ -121,24 +125,29 @@ const CAPABILITY_META: Record<
|
||||
},
|
||||
};
|
||||
|
||||
function seedTools(entry: CatalogueEntry): ToolState[] {
|
||||
function seedTools(
|
||||
entry: CatalogueEntry,
|
||||
registry: Partial<ToolRegistry>,
|
||||
): ToolState[] {
|
||||
const savedSteps = entry.policy?.steps ?? [];
|
||||
const savedByTool = new Map<PolicyToolId, PolicyToolStep>();
|
||||
for (const wire of savedSteps) {
|
||||
const step = policyStepFromWire(wire);
|
||||
if (step) savedByTool.set(step.toolId, step);
|
||||
}
|
||||
// defaultOperations is the canonical list (so tools added later still show on edit); a saved
|
||||
// step's params win over the preset.
|
||||
return entry.config.defaultOperations.map((preset) => {
|
||||
const saved = savedByTool.get(preset.toolId);
|
||||
const savedByOp = new Map(savedSteps.map((s) => [s.operation, s]));
|
||||
// Always use defaultOperations as the canonical list so tools added after a
|
||||
// policy was first saved still appear when editing.
|
||||
return entry.config.defaultOperations.map((s) => {
|
||||
const saved = savedByOp.get(s.operation);
|
||||
return {
|
||||
...(saved ?? preset),
|
||||
operation: s.operation,
|
||||
enabled: saved
|
||||
? true
|
||||
: savedSteps.length > 0
|
||||
? false
|
||||
: !DISABLED_BY_DEFAULT.has(preset.toolId),
|
||||
: !DISABLED_BY_DEFAULT.has(s.operation),
|
||||
// Saved steps are in the backend contract shape; map them back to the UI
|
||||
// shape the config controls edit (e.g. `listOfText` -> `wordsToRedact`).
|
||||
// Presets are already authored in the UI shape, so use them as-is.
|
||||
parameters: saved
|
||||
? deserializeToolStep(saved, registry).params
|
||||
: s.parameters,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -176,12 +185,26 @@ function PolicySetupWizardBody({
|
||||
onSubmit: (entry: CatalogueEntry, result: PolicySetupResult) => Promise<void>;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const { allTools: toolRegistry } = useToolRegistry();
|
||||
|
||||
// Portal tool operations are endpoint paths (/api/v1/…), not short registry IDs.
|
||||
// Build a reverse map so we can look up icons and display names by endpoint.
|
||||
const registryByEndpoint = useMemo(() => {
|
||||
const map = new Map<string, ToolRegistryEntry>();
|
||||
for (const entry of Object.values(toolRegistry)) {
|
||||
const ep = (entry as ToolRegistryEntry).operationConfig?.endpoint;
|
||||
if (typeof ep === "string") map.set(ep, entry as ToolRegistryEntry);
|
||||
}
|
||||
return map;
|
||||
}, [toolRegistry]);
|
||||
|
||||
const { category, config, policy } = entry;
|
||||
const isEdit = policy != null;
|
||||
|
||||
const [step, setStep] = useState<Step>("workflow");
|
||||
const [tools, setTools] = useState<ToolState[]>(() => seedTools(entry));
|
||||
const [tools, setTools] = useState<ToolState[]>(() =>
|
||||
seedTools(entry, toolRegistry),
|
||||
);
|
||||
const [fieldValues, setFieldValues] = useState(() =>
|
||||
resolveFieldValues(entry),
|
||||
);
|
||||
@@ -190,11 +213,22 @@ function PolicySetupWizardBody({
|
||||
);
|
||||
|
||||
const sourcesAsync = useAsync(() => fetchSources(), []);
|
||||
const availableSources = useMemo(
|
||||
() =>
|
||||
(sourcesAsync.data?.sources ?? []).filter((s) => s.status !== "disabled"),
|
||||
[sourcesAsync.data],
|
||||
);
|
||||
const availableSources = useMemo(() => {
|
||||
const backendSources = (sourcesAsync.data?.sources ?? []).filter(
|
||||
(s) => s.status !== "disabled",
|
||||
);
|
||||
const editorSource = {
|
||||
id: "editor",
|
||||
name: t("portal.sources.types.editor.label"),
|
||||
type: "editor",
|
||||
status: "active" as const,
|
||||
referenceCount: 0,
|
||||
referencingPolicies: [],
|
||||
config: [],
|
||||
docsTotal: null,
|
||||
};
|
||||
return [editorSource, ...backendSources];
|
||||
}, [sourcesAsync.data, t]);
|
||||
// Document-type scoping has no UI; preserve any saved scope on edit and
|
||||
// default new policies to all document types.
|
||||
const [scopeTypes] = useState<string[]>(policy?.state.scopeTypes ?? []);
|
||||
@@ -222,20 +256,9 @@ function PolicySetupWizardBody({
|
||||
|
||||
const enabledTools = useMemo(() => tools.filter((tl) => tl.enabled), [tools]);
|
||||
|
||||
function setToolEnabled(toolId: PolicyToolId, enabled: boolean) {
|
||||
function patchTool(operation: string, patch: Partial<ToolState>) {
|
||||
setTools((prev) =>
|
||||
prev.map((tl) => (tl.toolId === toolId ? { ...tl, enabled } : tl)),
|
||||
);
|
||||
}
|
||||
|
||||
function setToolParams<Id extends PolicyToolId>(
|
||||
toolId: Id,
|
||||
params: PolicyParams<Id>,
|
||||
) {
|
||||
setTools((prev) =>
|
||||
prev.map((tl) =>
|
||||
tl.toolId === toolId ? ({ ...tl, params } as ToolState) : tl,
|
||||
),
|
||||
prev.map((tl) => (tl.operation === operation ? { ...tl, ...patch } : tl)),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -254,8 +277,11 @@ function PolicySetupWizardBody({
|
||||
}
|
||||
setError(null);
|
||||
setSubmitting(true);
|
||||
// Map each tool's UI-shaped params (e.g. redact's `wordsToRedact`) into the
|
||||
// backend step contract (e.g. `listOfText`) via its `toApiParams`; saving the
|
||||
// UI shape verbatim would drop those fields and the step would run with none.
|
||||
const steps: PipelineStep[] = enabledTools.map((tl) =>
|
||||
policyStepToWire(tl),
|
||||
serializeStepFromEndpoint(tl.operation, tl.parameters, toolRegistry),
|
||||
);
|
||||
try {
|
||||
await onSubmit(entry, {
|
||||
@@ -360,16 +386,20 @@ function PolicySetupWizardBody({
|
||||
<Card padding="none">
|
||||
<div className="portal-policies__capabilities">
|
||||
{tools.map((tl) => {
|
||||
const meta = CAPABILITY_META[tl.toolId];
|
||||
const meta = CAPABILITY_META[tl.operation];
|
||||
const label = meta
|
||||
? t(meta.labelKey, meta.labelEn)
|
||||
: humanizeEndpoint(policyEndpoint(tl.toolId), t);
|
||||
: (registryByEndpoint.get(tl.operation)?.name ??
|
||||
humanizeEndpoint(tl.operation, t));
|
||||
const description = meta
|
||||
? t(meta.descKey, meta.descEn)
|
||||
: undefined;
|
||||
const hasConfig =
|
||||
tl.operation === TOOL_ENDPOINTS.redact ||
|
||||
tl.operation === TOOL_ENDPOINTS.watermark;
|
||||
return (
|
||||
<div
|
||||
key={tl.toolId}
|
||||
key={tl.operation}
|
||||
className="portal-policies__capability"
|
||||
data-on={tl.enabled || undefined}
|
||||
>
|
||||
@@ -381,27 +411,27 @@ function PolicySetupWizardBody({
|
||||
size="sm"
|
||||
checked={tl.enabled}
|
||||
onChange={(checked) =>
|
||||
setToolEnabled(tl.toolId, checked)
|
||||
patchTool(tl.operation, { enabled: checked })
|
||||
}
|
||||
label=""
|
||||
/>
|
||||
}
|
||||
/>
|
||||
{tl.enabled && (
|
||||
{tl.enabled && hasConfig && (
|
||||
<div className="portal-policies__capability-config">
|
||||
{tl.toolId === "redact" && (
|
||||
{tl.operation === TOOL_ENDPOINTS.redact && (
|
||||
<PolicyRedactConfig
|
||||
parameters={tl.params}
|
||||
onChange={(params) =>
|
||||
setToolParams("redact", params)
|
||||
parameters={tl.parameters}
|
||||
onChange={(parameters) =>
|
||||
patchTool(tl.operation, { parameters })
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{tl.toolId === "watermark" && (
|
||||
{tl.operation === TOOL_ENDPOINTS.watermark && (
|
||||
<PolicyWatermarkConfig
|
||||
parameters={tl.params}
|
||||
onChange={(params) =>
|
||||
setToolParams("watermark", params)
|
||||
parameters={tl.parameters}
|
||||
onChange={(parameters) =>
|
||||
patchTool(tl.operation, { parameters })
|
||||
}
|
||||
/>
|
||||
)}
|
||||
@@ -445,8 +475,9 @@ function PolicySetupWizardBody({
|
||||
{t("portal.policies.wizard.sources.loading")}
|
||||
</p>
|
||||
) : (
|
||||
// The backend always returns the editor as a virtual source, so the
|
||||
// loaded list is never empty - no "no sources" state exists.
|
||||
// The editor is always an available source (unconditionally prepended
|
||||
// to availableSources), so the list is never empty — no "no sources"
|
||||
// state exists.
|
||||
<div className="portal-policies__sources">
|
||||
{availableSources.map((src) => (
|
||||
// A selectable multi-line tile (icon + name + type + check).
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
import LocalIcon from "@app/components/shared/LocalIcon";
|
||||
import type { FlowOutcomeKey } from "@portal/api/processorFlow";
|
||||
import {
|
||||
EDITOR_TYPE,
|
||||
ICON_SIZE,
|
||||
} from "@portal/components/processor-flow/flowTypes";
|
||||
|
||||
/** Real Material Symbols icon for a live source node, keyed off its `type`. */
|
||||
export function SourceIcon({ type }: { type: string }) {
|
||||
switch (type) {
|
||||
case EDITOR_TYPE:
|
||||
return <LocalIcon icon="edit-document" width={ICON_SIZE} />;
|
||||
case "s3":
|
||||
return <LocalIcon icon="cloud" width={ICON_SIZE} />;
|
||||
case "folder":
|
||||
return <LocalIcon icon="folder" width={ICON_SIZE} />;
|
||||
default:
|
||||
return <LocalIcon icon="database" width={ICON_SIZE} />;
|
||||
}
|
||||
}
|
||||
|
||||
/** Real Material Symbols icon for an audit outcome node. Literal icon names
|
||||
* (not a ternary) so the icon extractor bundles them. */
|
||||
export function OutcomeIcon({ outcome }: { outcome: FlowOutcomeKey }) {
|
||||
if (outcome === "success")
|
||||
return <LocalIcon icon="check-circle" width={ICON_SIZE} />;
|
||||
return <LocalIcon icon="cancel" width={ICON_SIZE} />;
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
import { type RefObject } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@app/ui";
|
||||
import type { FlowOutcome } from "@portal/api/processorFlow";
|
||||
import { OutcomeIcon } from "@portal/components/processor-flow/FlowIcons";
|
||||
|
||||
interface FlowOutcomesProps {
|
||||
outcomes: FlowOutcome[];
|
||||
/** One ref slot per outcome, in order, for geometry measurement. */
|
||||
outRefs: RefObject<(HTMLElement | null)[]>;
|
||||
onOpen: () => void;
|
||||
}
|
||||
|
||||
/** Right column: terminal audit outcomes (delivered / failed). */
|
||||
export function FlowOutcomes({ outcomes, outRefs, onOpen }: FlowOutcomesProps) {
|
||||
const { t } = useTranslation();
|
||||
return (
|
||||
<section
|
||||
className="portal-pf__col portal-pf__col--outcomes"
|
||||
aria-label={t("portal.processorFlow.outcomes.heading")}
|
||||
>
|
||||
<span className="portal-pf__col-head">
|
||||
{t("portal.processorFlow.outcomes.heading")}
|
||||
</span>
|
||||
{outcomes.map((outcome, j) => (
|
||||
<Button
|
||||
key={outcome.key}
|
||||
variant="quiet"
|
||||
justify="start"
|
||||
fullWidth
|
||||
px="sm"
|
||||
py="sm"
|
||||
className={
|
||||
"portal-pf__node portal-pf__node--outcome portal-pf__node--" +
|
||||
outcome.key
|
||||
}
|
||||
onClick={onOpen}
|
||||
ref={(el: HTMLButtonElement | null) => {
|
||||
outRefs.current[j] = el;
|
||||
}}
|
||||
leftSection={
|
||||
<span className="portal-pf__node-icon" aria-hidden>
|
||||
<OutcomeIcon outcome={outcome.key} />
|
||||
</span>
|
||||
}
|
||||
>
|
||||
<span className="portal-pf__node-text">
|
||||
<strong>{t(outcome.labelKey)}</strong>
|
||||
<span>
|
||||
{t("portal.processorFlow.outcomes.count", {
|
||||
n: outcome.count24h,
|
||||
})}
|
||||
</span>
|
||||
</span>
|
||||
</Button>
|
||||
))}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
import { type RefObject } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@app/ui";
|
||||
import LocalIcon from "@app/components/shared/LocalIcon";
|
||||
import type { FlowPolicy } from "@portal/api/processorFlow";
|
||||
import { ICON_SIZE } from "@portal/components/processor-flow/flowTypes";
|
||||
|
||||
interface FlowPoliciesProps {
|
||||
policies: FlowPolicy[];
|
||||
activeCount: number;
|
||||
/** Ref for the core card (measured as the particle waist). */
|
||||
coreRef: RefObject<HTMLDivElement | null>;
|
||||
/** Per-policy lane-line refs, keyed by policy id, for particle threading. */
|
||||
laneRefs: RefObject<Record<string, HTMLElement>>;
|
||||
/** Deep-link into that policy's setup wizard. */
|
||||
onSetup: (key: string) => void;
|
||||
}
|
||||
|
||||
/** Centre column: the standing-policy catalogue — the particle "waist". */
|
||||
export function FlowPolicies({
|
||||
policies,
|
||||
activeCount,
|
||||
coreRef,
|
||||
laneRefs,
|
||||
onSetup,
|
||||
}: FlowPoliciesProps) {
|
||||
const { t } = useTranslation();
|
||||
return (
|
||||
<div className="portal-pf__policies" ref={coreRef}>
|
||||
<div className="portal-pf__policies-head">
|
||||
<span>{t("portal.processorFlow.policies.heading")}</span>
|
||||
<span className="portal-pf__policies-active">
|
||||
{t("portal.processorFlow.policies.activeCount", { n: activeCount })}
|
||||
</span>
|
||||
</div>
|
||||
{policies.map((policy) => (
|
||||
<div
|
||||
key={policy.key}
|
||||
className={"portal-pf__policy portal-pf__policy--" + policy.state}
|
||||
>
|
||||
<div
|
||||
className="portal-pf__policy-line"
|
||||
ref={(el: HTMLDivElement | null) => {
|
||||
if (el) laneRefs.current[policy.key] = el;
|
||||
else delete laneRefs.current[policy.key];
|
||||
}}
|
||||
>
|
||||
<span className="portal-pf__policy-icon" aria-hidden>
|
||||
<LocalIcon icon={policy.icon} width={ICON_SIZE} />
|
||||
</span>
|
||||
<span className="portal-pf__policy-label">
|
||||
{t(policy.labelKey)}
|
||||
</span>
|
||||
{policy.state === "active" ? (
|
||||
<span className="portal-pf__policy-count">
|
||||
{t("portal.processorFlow.policies.count", {
|
||||
n: policy.runs24h,
|
||||
})}
|
||||
</span>
|
||||
) : policy.state === "off" ? (
|
||||
<Button
|
||||
size="sm"
|
||||
py="xs"
|
||||
variant="primary"
|
||||
className="portal-pf__setup"
|
||||
onClick={() => onSetup(policy.key)}
|
||||
>
|
||||
{t("portal.processorFlow.policies.setUp")}
|
||||
</Button>
|
||||
) : (
|
||||
<span className="portal-pf__policy-soon">
|
||||
{t("portal.processorFlow.policies.soon")}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
import { type ReactNode } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { EmptyState } from "@app/ui";
|
||||
import type {
|
||||
FlowOutcome,
|
||||
FlowOutcomeKey,
|
||||
FlowPolicy,
|
||||
FlowSource,
|
||||
} from "@portal/api/processorFlow";
|
||||
import {
|
||||
EDITOR_TYPE,
|
||||
OUTCOME_FILL,
|
||||
} from "@portal/components/processor-flow/flowTypes";
|
||||
|
||||
interface FlowSankeyProps {
|
||||
sources: FlowSource[];
|
||||
outcomes: FlowOutcome[];
|
||||
policies: FlowPolicy[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sankey lens: sources → policies waist → outcomes, ribbon width ∝ 24h volume.
|
||||
* The waist splits into one segment per active policy. Shows a friendly empty
|
||||
* state when nothing has flowed yet.
|
||||
*/
|
||||
export function FlowSankey({ sources, outcomes, policies }: FlowSankeyProps) {
|
||||
const { t } = useTranslation();
|
||||
const activePolicies = policies.filter((p) => p.state === "active");
|
||||
const activeCount = activePolicies.length;
|
||||
|
||||
const flows = sources.filter((s) => s.docs24h > 0);
|
||||
const srcSum = flows.reduce((sum, s) => sum + s.docs24h, 0);
|
||||
if (!srcSum) {
|
||||
return (
|
||||
<div className="portal-pf__sankey-empty">
|
||||
<EmptyState
|
||||
size="compact"
|
||||
title={t("portal.processorFlow.sankey.empty.title")}
|
||||
description={t("portal.processorFlow.sankey.empty.description")}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const SW = 720;
|
||||
const SH = 220;
|
||||
const padY = 22;
|
||||
const xL = 180;
|
||||
const xR = 560;
|
||||
const xM = (xL + xR) / 2;
|
||||
const barW = 9;
|
||||
const midW = 11;
|
||||
const gap = 12;
|
||||
const H = SH - padY * 2;
|
||||
|
||||
const k = (H - (flows.length - 1) * gap) / srcSum;
|
||||
const lt = flows.map((s) => Math.max(3, s.docs24h * k));
|
||||
const midH = lt.reduce((a, b) => a + b, 0);
|
||||
const y0L = padY + (H - (midH + (flows.length - 1) * gap)) / 2;
|
||||
const midY = padY + (H - midH) / 2;
|
||||
|
||||
const outSum = outcomes.reduce((a, o) => a + o.count24h, 0);
|
||||
const rawRt = outcomes.map((o) =>
|
||||
outSum > 0
|
||||
? Math.max(3, midH * (o.count24h / outSum))
|
||||
: midH / outcomes.length,
|
||||
);
|
||||
const rtSum = rawRt.reduce((a, b) => a + b, 0);
|
||||
const rt = rawRt.map((v) => (v * midH) / rtSum);
|
||||
const y0R = padY + (H - (midH + (outcomes.length - 1) * gap)) / 2;
|
||||
|
||||
const outFill = (key: FlowOutcomeKey) => OUTCOME_FILL[key];
|
||||
const srcFill = "var(--color-blue)";
|
||||
const waistFill = "var(--color-text-4)";
|
||||
|
||||
const ribbon = (
|
||||
x0: number,
|
||||
t0: number,
|
||||
b0: number,
|
||||
x1: number,
|
||||
t1: number,
|
||||
b1: number,
|
||||
fill: string,
|
||||
key: string,
|
||||
) => {
|
||||
const mx = (x0 + x1) / 2;
|
||||
return (
|
||||
<path
|
||||
key={key}
|
||||
d={`M ${x0} ${t0} C ${mx} ${t0}, ${mx} ${t1}, ${x1} ${t1} L ${x1} ${b1} C ${mx} ${b1}, ${mx} ${b0}, ${x0} ${b0} Z`}
|
||||
style={{ fill }}
|
||||
opacity={0.3}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const wires: ReactNode[] = [];
|
||||
const bars: ReactNode[] = [];
|
||||
const texts: ReactNode[] = [];
|
||||
|
||||
// Left stage: sources → waist.
|
||||
let accL = y0L;
|
||||
let accM = midY;
|
||||
flows.forEach((s, i) => {
|
||||
wires.push(
|
||||
ribbon(
|
||||
xL + barW,
|
||||
accL,
|
||||
accL + lt[i],
|
||||
xM,
|
||||
accM,
|
||||
accM + lt[i],
|
||||
srcFill,
|
||||
"wl" + i,
|
||||
),
|
||||
);
|
||||
bars.push(
|
||||
<rect
|
||||
key={"bl" + i}
|
||||
x={xL}
|
||||
y={accL}
|
||||
width={barW}
|
||||
height={lt[i]}
|
||||
rx={2}
|
||||
style={{ fill: srcFill }}
|
||||
opacity={0.9}
|
||||
/>,
|
||||
);
|
||||
const label =
|
||||
s.type === EDITOR_TYPE
|
||||
? t("portal.processorFlow.sources.editor")
|
||||
: s.name;
|
||||
texts.push(
|
||||
<text
|
||||
key={"tl" + i}
|
||||
x={xL - 10}
|
||||
y={accL + lt[i] / 2 + 4}
|
||||
textAnchor="end"
|
||||
className="portal-pf__sankey-label"
|
||||
>
|
||||
{label} · {s.docs24h}
|
||||
</text>,
|
||||
);
|
||||
accL += lt[i] + gap;
|
||||
accM += lt[i];
|
||||
});
|
||||
|
||||
// Waist: one segment per active policy (sized by its 24h runs) so the centre
|
||||
// reads as distinct policies rather than a single bar.
|
||||
const segGap = 4;
|
||||
const nSeg = Math.max(activePolicies.length, 1);
|
||||
const segAvail = midH - (nSeg - 1) * segGap;
|
||||
const polSum = activePolicies.reduce((a, p) => a + p.runs24h, 0);
|
||||
const segPolicies = activePolicies.length ? activePolicies : [null];
|
||||
let segY = midY;
|
||||
segPolicies.forEach((p, i) => {
|
||||
const wgt = p && polSum > 0 ? p.runs24h / polSum : 1 / nSeg;
|
||||
const h = Math.max(2, segAvail * wgt);
|
||||
bars.push(
|
||||
<rect
|
||||
key={"waist" + i}
|
||||
x={xM}
|
||||
y={segY}
|
||||
width={midW}
|
||||
height={h}
|
||||
rx={2}
|
||||
style={{ fill: waistFill }}
|
||||
opacity={0.9}
|
||||
/>,
|
||||
);
|
||||
segY += h + segGap;
|
||||
});
|
||||
texts.push(
|
||||
<text
|
||||
key="waist-cap"
|
||||
x={xM + midW / 2}
|
||||
y={midY - 8}
|
||||
textAnchor="middle"
|
||||
className="portal-pf__sankey-caption"
|
||||
>
|
||||
{t("portal.processorFlow.sankey.waist", { n: activeCount })}
|
||||
</text>,
|
||||
);
|
||||
|
||||
// Right stage: waist → outcomes.
|
||||
let accWaist = midY;
|
||||
let accR = y0R;
|
||||
outcomes.forEach((o, j) => {
|
||||
wires.push(
|
||||
ribbon(
|
||||
xM + midW,
|
||||
accWaist,
|
||||
accWaist + rt[j],
|
||||
xR,
|
||||
accR,
|
||||
accR + rt[j],
|
||||
outFill(o.key),
|
||||
"wr" + j,
|
||||
),
|
||||
);
|
||||
bars.push(
|
||||
<rect
|
||||
key={"br" + j}
|
||||
x={xR}
|
||||
y={accR}
|
||||
width={barW}
|
||||
height={rt[j]}
|
||||
rx={2}
|
||||
style={{ fill: outFill(o.key) }}
|
||||
opacity={0.9}
|
||||
/>,
|
||||
);
|
||||
texts.push(
|
||||
<text
|
||||
key={"tr" + j}
|
||||
x={xR + barW + 10}
|
||||
y={accR + rt[j] / 2 + 4}
|
||||
textAnchor="start"
|
||||
className="portal-pf__sankey-label"
|
||||
>
|
||||
{t(o.labelKey)} · {o.count24h}
|
||||
</text>,
|
||||
);
|
||||
accWaist += rt[j];
|
||||
accR += rt[j] + gap;
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="portal-pf__sankey">
|
||||
<svg viewBox={`0 0 ${SW} ${SH}`} width="100%" aria-hidden>
|
||||
{wires}
|
||||
{bars}
|
||||
{texts}
|
||||
</svg>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
import { type RefObject } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@app/ui";
|
||||
import LocalIcon from "@app/components/shared/LocalIcon";
|
||||
import type {
|
||||
FlowComingSoonSource,
|
||||
FlowSource,
|
||||
} from "@portal/api/processorFlow";
|
||||
import {
|
||||
EDITOR_TYPE,
|
||||
ICON_SIZE,
|
||||
} from "@portal/components/processor-flow/flowTypes";
|
||||
import { SourceIcon } from "@portal/components/processor-flow/FlowIcons";
|
||||
|
||||
interface FlowSourcesProps {
|
||||
sources: FlowSource[];
|
||||
comingSoonSources: FlowComingSoonSource[];
|
||||
/** One ref slot per live source, in order, for geometry measurement. */
|
||||
srcRefs: RefObject<(HTMLElement | null)[]>;
|
||||
onOpen: () => void;
|
||||
}
|
||||
|
||||
/** Left column: live source cards (measured) + coming-soon connect cards. */
|
||||
export function FlowSources({
|
||||
sources,
|
||||
comingSoonSources,
|
||||
srcRefs,
|
||||
onOpen,
|
||||
}: FlowSourcesProps) {
|
||||
const { t } = useTranslation();
|
||||
return (
|
||||
<section
|
||||
className="portal-pf__col portal-pf__col--sources"
|
||||
aria-label={t("portal.processorFlow.sources.heading")}
|
||||
>
|
||||
<span className="portal-pf__col-head">
|
||||
{t("portal.processorFlow.sources.heading")}
|
||||
</span>
|
||||
{sources.map((source, i) => (
|
||||
<Button
|
||||
key={source.id}
|
||||
variant="quiet"
|
||||
justify="start"
|
||||
fullWidth
|
||||
px="sm"
|
||||
py="sm"
|
||||
className="portal-pf__node portal-pf__node--source"
|
||||
onClick={onOpen}
|
||||
ref={(el: HTMLButtonElement | null) => {
|
||||
srcRefs.current[i] = el;
|
||||
}}
|
||||
leftSection={
|
||||
<span className="portal-pf__node-icon" aria-hidden>
|
||||
<SourceIcon type={source.type} />
|
||||
</span>
|
||||
}
|
||||
>
|
||||
<span className="portal-pf__node-text">
|
||||
<strong>
|
||||
{source.type === EDITOR_TYPE
|
||||
? t("portal.processorFlow.sources.editor")
|
||||
: source.name}
|
||||
</strong>
|
||||
<span>
|
||||
{t("portal.processorFlow.sources.perDay", { n: source.docs24h })}
|
||||
</span>
|
||||
</span>
|
||||
</Button>
|
||||
))}
|
||||
{comingSoonSources.map((cs) => (
|
||||
<Button
|
||||
key={cs.key}
|
||||
variant="quiet"
|
||||
justify="start"
|
||||
fullWidth
|
||||
px="sm"
|
||||
py="sm"
|
||||
className="portal-pf__node portal-pf__node--soon"
|
||||
onClick={onOpen}
|
||||
leftSection={
|
||||
<span className="portal-pf__node-icon" aria-hidden>
|
||||
<LocalIcon icon="add" width={ICON_SIZE} />
|
||||
</span>
|
||||
}
|
||||
>
|
||||
<span className="portal-pf__node-text">
|
||||
<strong>{t(cs.labelKey)}</strong>
|
||||
<span>{t("portal.processorFlow.sources.comingSoonTag")}</span>
|
||||
</span>
|
||||
</Button>
|
||||
))}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
import type { FlowOutcomeKey } from "@portal/api/processorFlow";
|
||||
|
||||
/** Which lens the visualiser is showing. */
|
||||
export type Lens = "flow" | "sankey";
|
||||
|
||||
/**
|
||||
* DEV ONLY: force the flow animation on even when nothing is set up / no
|
||||
* activity has taken place, using synthetic rates. Off by design — an idle
|
||||
* machine shows no animated dots; flip to true only to preview the motion
|
||||
* while iterating on an empty workspace.
|
||||
*/
|
||||
export const DEV_KEEP_FLOWING = false;
|
||||
|
||||
export const EDITOR_TYPE = "editor";
|
||||
|
||||
/** Emission tuning: particles/sec for a source ≈ rate / 86400 × SPEED. */
|
||||
export const SPEED = 300;
|
||||
/** Synthetic per-source rate used only while DEV_KEEP_FLOWING forces the flow. */
|
||||
export const DEV_SYNTH_RATE = 320;
|
||||
/** Hard cap on live particles (matches the reference). */
|
||||
export const MAX_PARTICLES = 36;
|
||||
/** No two dots leave the same source within this window (ms). */
|
||||
export const MIN_EMIT_GAP = 200;
|
||||
|
||||
export const ICON_SIZE = "1.125rem";
|
||||
|
||||
/** SVG `fill` (a CSS property, so var() resolves per-theme) for each outcome. */
|
||||
export const OUTCOME_FILL: Record<FlowOutcomeKey, string> = {
|
||||
success: "var(--color-green)",
|
||||
failed: "var(--color-red)",
|
||||
};
|
||||
|
||||
/* ── Measured geometry ──────────────────────────────────────────────────── */
|
||||
|
||||
export interface Rect {
|
||||
l: number;
|
||||
r: number;
|
||||
t: number;
|
||||
b: number;
|
||||
cy: number;
|
||||
}
|
||||
|
||||
export interface Lane {
|
||||
key: string;
|
||||
cy: number;
|
||||
el: HTMLElement;
|
||||
}
|
||||
|
||||
export interface Geo {
|
||||
w: number;
|
||||
h: number;
|
||||
srcs: (Rect | undefined)[];
|
||||
outs: (Rect | undefined)[];
|
||||
core: Rect | null;
|
||||
lanes: Lane[];
|
||||
}
|
||||
|
||||
export interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface Particle {
|
||||
el: SVGCircleElement;
|
||||
src: number;
|
||||
out: number;
|
||||
lane: string | null;
|
||||
phase: 0 | 1 | 2;
|
||||
t: number;
|
||||
d0: number;
|
||||
d1: number;
|
||||
d2: number;
|
||||
pulsed: boolean;
|
||||
}
|
||||
|
||||
/** Cubic bézier point at t. */
|
||||
export function cbez(a: Point, b: Point, c: Point, d: Point, t: number): Point {
|
||||
const m = 1 - t;
|
||||
return {
|
||||
x:
|
||||
m * m * m * a.x +
|
||||
3 * m * m * t * b.x +
|
||||
3 * m * t * t * c.x +
|
||||
t * t * t * d.x,
|
||||
y:
|
||||
m * m * m * a.y +
|
||||
3 * m * m * t * b.y +
|
||||
3 * m * t * t * c.y +
|
||||
t * t * t * d.y,
|
||||
};
|
||||
}
|
||||
|
||||
/** Smoothstep easing used for the in-card lane glide. */
|
||||
export function smooth(t: number): number {
|
||||
return t * t * (3 - 2 * t);
|
||||
}
|
||||
|
||||
/** Where each source's wire enters the core (spread across its height). */
|
||||
export function coreEntryY(g: Geo, i: number): number {
|
||||
if (!g.core) return 0;
|
||||
const n = Math.max(g.srcs.length, 2);
|
||||
return g.core.t + 34 + (g.core.b - g.core.t - 68) * (i / (n - 1));
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
import {
|
||||
useEffect,
|
||||
useLayoutEffect,
|
||||
useRef,
|
||||
useState,
|
||||
type ReactNode,
|
||||
} from "react";
|
||||
import {
|
||||
coreEntryY,
|
||||
type Geo,
|
||||
type Rect,
|
||||
} from "@portal/components/processor-flow/flowTypes";
|
||||
|
||||
/**
|
||||
* Owns the measured-geometry seam for the flow visualiser: refs for the source,
|
||||
* outcome and core cards (plus the per-policy lane lines), a `measure()` that
|
||||
* projects their edges into a wrapper-relative {@link Geo}, and the SVG wires
|
||||
* drawn between them. The particle loop reads the same `geoRef` live.
|
||||
*
|
||||
* Callers spread the returned refs onto the cards and render `wires` inside the
|
||||
* underlay `<svg>`; geometry re-measures on every layout + on resize, and the
|
||||
* wires re-render only when the measured signature actually changes.
|
||||
*/
|
||||
export function useFlowGeometry() {
|
||||
const wrapRef = useRef<HTMLDivElement>(null);
|
||||
const srcRefs = useRef<(HTMLElement | null)[]>([]);
|
||||
const outRefs = useRef<(HTMLElement | null)[]>([]);
|
||||
const coreRef = useRef<HTMLDivElement>(null);
|
||||
const laneRefs = useRef<Record<string, HTMLElement>>({});
|
||||
const geoRef = useRef<Geo | null>(null);
|
||||
const geoSigRef = useRef("");
|
||||
const [, setGeoTick] = useState(0);
|
||||
|
||||
const measure = () => {
|
||||
const w = wrapRef.current;
|
||||
if (!w) return;
|
||||
const wr = w.getBoundingClientRect();
|
||||
if (!wr.width) return;
|
||||
const rel = (r: DOMRect): Rect => ({
|
||||
l: r.left - wr.left,
|
||||
r: r.right - wr.left,
|
||||
t: r.top - wr.top,
|
||||
b: r.bottom - wr.top,
|
||||
cy: r.top - wr.top + r.height / 2,
|
||||
});
|
||||
const g: Geo = {
|
||||
w: wr.width,
|
||||
h: wr.height,
|
||||
srcs: [],
|
||||
outs: [],
|
||||
core: null,
|
||||
lanes: [],
|
||||
};
|
||||
srcRefs.current.forEach((el, i) => {
|
||||
if (el) g.srcs[i] = rel(el.getBoundingClientRect());
|
||||
});
|
||||
outRefs.current.forEach((el, j) => {
|
||||
if (el) g.outs[j] = rel(el.getBoundingClientRect());
|
||||
});
|
||||
if (coreRef.current) g.core = rel(coreRef.current.getBoundingClientRect());
|
||||
Object.entries(laneRefs.current).forEach(([key, el]) => {
|
||||
if (el && el.isConnected)
|
||||
g.lanes.push({ key, cy: rel(el.getBoundingClientRect()).cy, el });
|
||||
});
|
||||
geoRef.current = g;
|
||||
|
||||
let cySum = 0;
|
||||
g.srcs.forEach((s) => s && (cySum += s.cy));
|
||||
g.outs.forEach((o) => o && (cySum += o.cy));
|
||||
const sig = [
|
||||
Math.round(g.w),
|
||||
Math.round(g.h),
|
||||
g.srcs.length,
|
||||
g.outs.length,
|
||||
g.core ? Math.round(g.core.t) + ":" + Math.round(g.core.b) : 0,
|
||||
Math.round(cySum),
|
||||
].join(":");
|
||||
if (sig !== geoSigRef.current) {
|
||||
geoSigRef.current = sig;
|
||||
setGeoTick((n) => n + 1);
|
||||
}
|
||||
};
|
||||
|
||||
useLayoutEffect(measure);
|
||||
|
||||
useEffect(() => {
|
||||
const onResize = () => measure();
|
||||
window.addEventListener("resize", onResize);
|
||||
return () => window.removeEventListener("resize", onResize);
|
||||
}, []);
|
||||
|
||||
// Wires (SVG underlay); recomputed whenever the geometry signature changes.
|
||||
const g = geoRef.current;
|
||||
let wires: ReactNode = null;
|
||||
if (g && g.core) {
|
||||
const core = g.core;
|
||||
const paths: ReactNode[] = [];
|
||||
g.srcs.forEach((s, i) => {
|
||||
if (!s) return;
|
||||
const ty = coreEntryY(g, i);
|
||||
paths.push(
|
||||
<path
|
||||
key={"ws" + i}
|
||||
className="portal-pf__wire-path"
|
||||
d={`M ${s.r} ${s.cy} C ${s.r + 44} ${s.cy}, ${core.l - 44} ${ty}, ${core.l} ${ty}`}
|
||||
/>,
|
||||
);
|
||||
});
|
||||
g.outs.forEach((o, j) => {
|
||||
if (!o) return;
|
||||
paths.push(
|
||||
<path
|
||||
key={"wo" + j}
|
||||
className="portal-pf__wire-path"
|
||||
d={`M ${core.r} ${o.cy} C ${core.r + 44} ${o.cy}, ${o.l - 44} ${o.cy}, ${o.l} ${o.cy}`}
|
||||
/>,
|
||||
);
|
||||
});
|
||||
wires = paths;
|
||||
}
|
||||
|
||||
return { wrapRef, srcRefs, outRefs, coreRef, laneRefs, geoRef, wires };
|
||||
}
|
||||
@@ -1,261 +0,0 @@
|
||||
import { useEffect, useRef, type RefObject } from "react";
|
||||
import type { FlowOutcomeKey } from "@portal/api/processorFlow";
|
||||
import {
|
||||
cbez,
|
||||
coreEntryY,
|
||||
smooth,
|
||||
MAX_PARTICLES,
|
||||
MIN_EMIT_GAP,
|
||||
OUTCOME_FILL,
|
||||
SPEED,
|
||||
type Geo,
|
||||
type Lens,
|
||||
type Particle,
|
||||
type Point,
|
||||
} from "@portal/components/processor-flow/flowTypes";
|
||||
|
||||
interface FlowParticlesOptions {
|
||||
geoRef: RefObject<Geo | null>;
|
||||
animate: boolean;
|
||||
lens: Lens;
|
||||
/** Per-source emission rate (docs/24h, or synthetic while dev-forcing). */
|
||||
rates: number[];
|
||||
/** Outcome share for the weighted round-robin destination picker. */
|
||||
weights: number[];
|
||||
/** Policy lane keys a dot may thread through the core. */
|
||||
laneKeys: string[];
|
||||
/** Outcome keys, index-aligned with `weights`, for recolouring on arrival. */
|
||||
outcomeKeys: FlowOutcomeKey[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Drives the rAF particle loop: emits dots per source on a jittered schedule
|
||||
* (min-gap floored), routes each to an outcome via a weighted round-robin so
|
||||
* the split matches the counts, threads it through a policy lane (blinking that
|
||||
* row's LED), and recolours it to the outcome on arrival. Reads geometry live
|
||||
* from `geoRef`, so it tracks card movement without restarting.
|
||||
*
|
||||
* Returns the `<g>` ref the caller mounts inside the particle overlay `<svg>`.
|
||||
* The loop only runs on the flow lens, when `animate` is set, and outside
|
||||
* reduced-motion; browsers pause rAF for hidden tabs (desirable).
|
||||
*/
|
||||
export function useFlowParticles({
|
||||
geoRef,
|
||||
animate,
|
||||
lens,
|
||||
rates,
|
||||
weights,
|
||||
laneKeys,
|
||||
outcomeKeys,
|
||||
}: FlowParticlesOptions): RefObject<SVGGElement | null> {
|
||||
const pGroupRef = useRef<SVGGElement>(null);
|
||||
|
||||
// Restart the loop only when the meaningful inputs change.
|
||||
const flowSig = [
|
||||
animate,
|
||||
lens,
|
||||
rates.join(","),
|
||||
laneKeys.join(","),
|
||||
weights.map((w) => w.toFixed(3)).join(","),
|
||||
outcomeKeys.join(","),
|
||||
].join("|");
|
||||
|
||||
useEffect(() => {
|
||||
if (!animate || lens !== "flow") return;
|
||||
const reduced =
|
||||
typeof window !== "undefined" &&
|
||||
window.matchMedia &&
|
||||
window.matchMedia("(prefers-reduced-motion: reduce)").matches;
|
||||
if (reduced) return;
|
||||
const pg = pGroupRef.current;
|
||||
if (!pg) return;
|
||||
const NS = "http://www.w3.org/2000/svg";
|
||||
|
||||
const particles: Particle[] = [];
|
||||
let last = performance.now();
|
||||
let raf = 0;
|
||||
const glowTimers: Record<string, ReturnType<typeof setTimeout>> = {};
|
||||
|
||||
// Per-source emission schedule. Mean interval keeps each source's share of
|
||||
// the flow proportional to its rate; the scheduled model (vs. a steady
|
||||
// accumulator) is what lets us jitter departures and floor the gap.
|
||||
const meanInterval = rates.map((r) => {
|
||||
const perSec = (r / 86400) * SPEED;
|
||||
return perSec > 0 ? 1000 / perSec : Infinity;
|
||||
});
|
||||
// Stagger the first emission so sources don't all fire together at t=0.
|
||||
const nextEmit = meanInterval.map((mi) =>
|
||||
Number.isFinite(mi) ? last + Math.random() * mi : Infinity,
|
||||
);
|
||||
// Random departure within [0.5×, 1.5×] the mean, but never closer than the
|
||||
// minimum gap — a random flow with no two dots out at once per source.
|
||||
const scheduleNext = (i: number, now: number): number =>
|
||||
now + Math.max(MIN_EMIT_GAP, meanInterval[i] * (0.5 + Math.random()));
|
||||
|
||||
// Weighted round-robin so the outcome split visibly matches the counts
|
||||
// (e.g. 3 failed / 30 delivered → ~1 in 11 dots to Failed), interleaved
|
||||
// rather than clustered like independent random draws.
|
||||
const outAcc = weights.map(() => 0);
|
||||
const pickOut = (): number => {
|
||||
if (!weights.length) return 0;
|
||||
for (let j = 0; j < weights.length; j++) outAcc[j] += weights[j];
|
||||
let best = 0;
|
||||
for (let j = 1; j < weights.length; j++) {
|
||||
if (outAcc[j] > outAcc[best]) best = j;
|
||||
}
|
||||
outAcc[best] -= 1;
|
||||
return best;
|
||||
};
|
||||
const pickLane = (): string | null => {
|
||||
if (!laneKeys.length) return null;
|
||||
return laneKeys[Math.floor(Math.random() * laneKeys.length)];
|
||||
};
|
||||
const laneY = (g: Geo, key: string | null): number | null => {
|
||||
if (!key) return null;
|
||||
const l = g.lanes.find((x) => x.key === key);
|
||||
return l ? l.cy : null;
|
||||
};
|
||||
// Blink the row's leading LED (its icon) for 150ms as a particle threads it.
|
||||
const pulseLane = (g: Geo, key: string | null) => {
|
||||
if (!key) return;
|
||||
const lane = g.lanes.find((x) => x.key === key);
|
||||
const led = lane?.el.firstElementChild;
|
||||
if (!led || !led.classList.contains("portal-pf__policy-icon")) return;
|
||||
led.classList.add("is-pulse");
|
||||
if (glowTimers[key]) clearTimeout(glowTimers[key]);
|
||||
glowTimers[key] = setTimeout(() => led.classList.remove("is-pulse"), 150);
|
||||
};
|
||||
|
||||
const frame = (now: number) => {
|
||||
const g = geoRef.current;
|
||||
const dt = Math.min(now - last, 200);
|
||||
last = now;
|
||||
if (g && g.core) {
|
||||
// At most one dot per source per frame, once its scheduled (jittered)
|
||||
// departure time is reached — guarantees the per-source minimum gap.
|
||||
for (let i = 0; i < meanInterval.length; i++) {
|
||||
if (!Number.isFinite(meanInterval[i]) || !g.srcs[i]) continue;
|
||||
if (now >= nextEmit[i] && particles.length < MAX_PARTICLES) {
|
||||
const c = document.createElementNS(
|
||||
NS,
|
||||
"circle",
|
||||
) as SVGCircleElement;
|
||||
c.setAttribute("r", "2.5");
|
||||
c.setAttribute("opacity", "0.75");
|
||||
c.style.fill = "var(--color-blue)";
|
||||
pg.appendChild(c);
|
||||
particles.push({
|
||||
el: c,
|
||||
src: i,
|
||||
out: pickOut(),
|
||||
lane: pickLane(),
|
||||
phase: 0,
|
||||
t: 0,
|
||||
d0: 900 + Math.random() * 300,
|
||||
d1: 760,
|
||||
d2: 780 + Math.random() * 200,
|
||||
pulsed: false,
|
||||
});
|
||||
nextEmit[i] = scheduleNext(i, now);
|
||||
}
|
||||
}
|
||||
|
||||
for (let k = particles.length - 1; k >= 0; k--) {
|
||||
const p = particles[k];
|
||||
p.t += dt;
|
||||
const s = g.srcs[p.src];
|
||||
if (!s) {
|
||||
p.el.remove();
|
||||
particles.splice(k, 1);
|
||||
continue;
|
||||
}
|
||||
let pos: Point;
|
||||
if (p.phase === 0) {
|
||||
const ey = coreEntryY(g, p.src);
|
||||
const f0 = Math.min(1, p.t / p.d0);
|
||||
pos = cbez(
|
||||
{ x: s.r, y: s.cy },
|
||||
{ x: s.r + 44, y: s.cy },
|
||||
{ x: g.core.l - 44, y: ey },
|
||||
{ x: g.core.l, y: ey },
|
||||
f0,
|
||||
);
|
||||
if (f0 >= 1) {
|
||||
p.phase = 1;
|
||||
p.t = 0;
|
||||
p.pulsed = false;
|
||||
p.el.setAttribute("r", "2");
|
||||
p.el.setAttribute("opacity", "0.45");
|
||||
}
|
||||
} else if (p.phase === 1) {
|
||||
const o1 = g.outs[p.out];
|
||||
if (!o1) {
|
||||
p.el.remove();
|
||||
particles.splice(k, 1);
|
||||
continue;
|
||||
}
|
||||
const f1 = Math.min(1, p.t / p.d1);
|
||||
const entY = coreEntryY(g, p.src);
|
||||
const exitY = o1.cy;
|
||||
const ly1 = laneY(g, p.lane);
|
||||
let yy: number;
|
||||
if (ly1 == null) {
|
||||
yy = entY + (exitY - entY) * f1;
|
||||
} else if (f1 < 0.25) {
|
||||
yy = entY + (ly1 - entY) * smooth(f1 / 0.25);
|
||||
} else if (f1 < 0.75) {
|
||||
yy = ly1;
|
||||
if (!p.pulsed) {
|
||||
p.pulsed = true;
|
||||
pulseLane(g, p.lane);
|
||||
}
|
||||
} else {
|
||||
yy = ly1 + (exitY - ly1) * smooth((f1 - 0.75) / 0.25);
|
||||
}
|
||||
pos = { x: g.core.l + (g.core.r - g.core.l) * f1, y: yy };
|
||||
if (f1 >= 1) {
|
||||
p.phase = 2;
|
||||
p.t = 0;
|
||||
p.el.style.fill =
|
||||
OUTCOME_FILL[outcomeKeys[p.out]] ?? "var(--color-blue)";
|
||||
p.el.setAttribute("r", "2.5");
|
||||
p.el.setAttribute("opacity", "0.75");
|
||||
}
|
||||
} else {
|
||||
const o2 = g.outs[p.out];
|
||||
if (!o2) {
|
||||
p.el.remove();
|
||||
particles.splice(k, 1);
|
||||
continue;
|
||||
}
|
||||
const f2 = Math.min(1, p.t / p.d2);
|
||||
pos = cbez(
|
||||
{ x: g.core.r, y: o2.cy },
|
||||
{ x: g.core.r + 44, y: o2.cy },
|
||||
{ x: o2.l - 44, y: o2.cy },
|
||||
{ x: o2.l, y: o2.cy },
|
||||
f2,
|
||||
);
|
||||
if (f2 >= 1) {
|
||||
p.el.remove();
|
||||
particles.splice(k, 1);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
p.el.setAttribute("cx", String(pos.x));
|
||||
p.el.setAttribute("cy", String(pos.y));
|
||||
}
|
||||
}
|
||||
raf = requestAnimationFrame(frame);
|
||||
};
|
||||
raf = requestAnimationFrame(frame);
|
||||
|
||||
return () => {
|
||||
cancelAnimationFrame(raf);
|
||||
Object.values(glowTimers).forEach(clearTimeout);
|
||||
while (pg.firstChild) pg.removeChild(pg.firstChild);
|
||||
};
|
||||
}, [flowSig, geoRef]);
|
||||
|
||||
return pGroupRef;
|
||||
}
|
||||
@@ -75,7 +75,7 @@ export function DocumentLedger({
|
||||
className="portal-proc__stage-label"
|
||||
data-current={cur || undefined}
|
||||
>
|
||||
{t(group.label)}
|
||||
{group.label}
|
||||
</span>
|
||||
{blurb && (
|
||||
<span className="portal-proc__stage-hint">
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user