Compare commits

..
Author SHA1 Message Date
posthog-eu[bot]andGitHub 1fcc958460 Fix Prettier formatting in PdfEngineBoundary test
frontend:format:check failed CI because the new test file wasn't
Prettier-formatted. No logic change.

Generated-By: PostHog Code
Task-Id: 3aa5a9d0-e661-4860-8e51-4610b2599780
2026-07-15 11:40:54 +00:00
posthog-eu[bot]andGitHub a835e68ac2 Recover from PDF engine load failures instead of spinning forever
The viewer gated on `isLoading || !engine || !pdfUrl` and only rendered an
error when the `usePdfiumEngine` hook rejected. When the WASM engine load
hung (never resolving or rejecting), the center pane stayed on
"Loading PDF Engine..." indefinitely with no error and no way to recover, so
no selected/dropped PDF ever rendered for the whole session.

- Add `PdfEngineBoundary`, which owns the engine load, times it out, and
  surfaces an error state with a Retry button (nudging the user to update the
  app) instead of an infinite spinner. Retry remounts the boundary so the
  load re-runs from scratch.
- Harden `wasmPrecompiler` to fall back to fetch + `arrayBuffer()` +
  `WebAssembly.compile` when `compileStreaming` is unavailable or fails
  (e.g. wrong MIME type / content-encoding under the `tauri://` protocol).
- Add component tests reproducing the hang → error + retry transition.

Generated-By: PostHog Code
Task-Id: 3aa5a9d0-e661-4860-8e51-4610b2599780
2026-07-15 11:17:55 +00:00
EthanHealy01andGitHub 0570c4c4d9 Create-PDF engine: render from a structured document (#7018) 2026-07-14 12:30:33 +00:00
James BruntonandGitHub 776749277c Redesign policies to use typed mappings properly (#7017)
# Description of Changes
The Policies page and all the frontend logic for running Policies is not
making use of the bidirectional type mappings that we now have to safely
convert from frontend to backend param models and vice versa. This
changes the way we track the types throughout so we use the mappings
properly.

Because of this, the Add Watermark settings in Policies now actually
pre-populate with the defaults instead of with nothing like they
previously did.

<img width="791" height="725" alt="image"
src="https://github.com/user-attachments/assets/cbdf4ae0-35af-4792-bf64-89216e48d304"
/>
2026-07-14 09:58:04 +00:00
James BruntonandGitHub 41b1b89fcb Fix Policies page showing the Editor as a source twice (#7022)
# Description of Changes
The Policies page currently hard-codes the Editor to be available as a
source, but we now also have a virtual Editor source on the backend,
which the Policies page also renders. This removes the now-unnecessary
hard-coded Editor source.

## Before

<img width="842" height="640" alt="image"
src="https://github.com/user-attachments/assets/d78b33a3-fed4-4bb0-a02f-489ca2ae0614"
/>

## After

<img width="785" height="586" alt="image"
src="https://github.com/user-attachments/assets/37aa664f-74f2-41c7-b89a-9b483bffc3a2"
/>
2026-07-14 09:43:07 +00:00
James BruntonandGitHub 4d4e994562 Fix crash in Processor when loading tool settings with tooltips (#7015)
# Description of Changes
Some of the tool settings make use of editor preferences indirectly, but
the Processor never gets that provider, so it crashes when trying to
load them.
2026-07-14 09:09:23 +00:00
60 changed files with 2272 additions and 4804 deletions
-92
View File
@@ -1,92 +0,0 @@
name: Sync Portal Docs
# Regenerates the portal Developer Docs manifest from the Stirling docs repo and
# opens a PR when it changes. Runs weekly, on manual dispatch, or when the docs
# repo fires a `docs-updated` repository_dispatch.
on:
schedule:
- cron: "0 6 * * 1"
workflow_dispatch:
inputs:
ref:
description: "Docs repo ref (branch or tag) to sync from"
required: false
default: "main"
repository_dispatch:
types: [docs-updated]
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: true
permissions:
contents: read
jobs:
sync:
name: Sync docs manifest
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: write
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
persist-credentials: false
- name: Setup GitHub App Bot
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Set up Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "22"
cache: "npm"
cache-dependency-path: frontend/package-lock.json
- name: Install frontend dependencies
working-directory: frontend
env:
NPM_CONFIG_IGNORE_SCRIPTS: "true"
run: npm ci --ignore-scripts --audit=false --fund=false
- name: Regenerate docs manifest
working-directory: frontend
env:
DOCS_REF: ${{ github.event.inputs.ref || github.event.client_payload.ref || 'main' }}
GITHUB_TOKEN: ${{ steps.setup-bot.outputs.token }}
run: npm run docs:sync
- name: Create Pull Request
id: cpr
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
with:
token: ${{ steps.setup-bot.outputs.token }}
commit-message: "Sync portal docs from docs repo"
committer: ${{ steps.setup-bot.outputs.committer }}
author: ${{ steps.setup-bot.outputs.committer }}
signoff: true
branch: sync-portal-docs
base: main
title: "Sync portal docs from docs repo"
body: |
Auto-generated by ${{ steps.setup-bot.outputs.app-slug }}[bot].
Regenerates `frontend/editor/src/portal/generated/docsManifest.json`
from the Stirling docs repo via `npm run docs:sync`.
labels: documentation,github-actions,frontend
add-paths: frontend/editor/src/portal/generated/docsManifest.json
delete-branch: true
sign-commits: true
+12
View File
@@ -208,6 +208,18 @@
"moduleName": ".*",
"moduleLicense": "The W3C License"
},
{
"moduleName": "com.google.re2j:re2j",
"moduleLicense": "Go License"
},
{
"moduleName": "com.hubspot:algebra",
"moduleLicense": null
},
{
"moduleName": "com.hubspot.immutables:immutables-exceptions",
"moduleLicense": null
},
{
"moduleName": ".*",
"moduleLicense": "UnRar License"
+14
View File
@@ -66,6 +66,20 @@ dependencies {
implementation "com.google.code.gson:gson:${gsonVersion}"
// jinjava/jjwt transitively request older Jackson 2 versions; declare the current
// version directly so it is selected consistently (root build.gradle pins are the fallback).
runtimeOnly "com.fasterxml.jackson.core:jackson-core:${jackson2Version}"
runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${jackson2Version}"
implementation("com.hubspot.jinjava:jinjava:${jinjavaVersion}") {
// Compile-time-only annotation artifacts (class-retention annotations, not needed at
// runtime) whose declared licences (LGPL / none) fail the licence compatibility check.
exclude group: 'com.google.code.findbugs', module: 'annotations'
exclude group: 'org.derive4j', module: 'derive4j-annotation'
exclude group: 'com.hubspot.immutables', module: 'hubspot-style'
exclude group: 'com.hubspot.immutables', module: 'immutable-collection-encodings'
}
api 'io.micrometer:micrometer-registry-prometheus'
api "io.jsonwebtoken:jjwt-api:${jwtVersion}"
@@ -8,12 +8,14 @@ import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import io.github.pixee.security.Filenames;
import io.swagger.v3.oas.annotations.Hidden;
@@ -24,18 +26,24 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.configuration.RuntimePathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ProcessExecutor;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
import stirling.software.proprietary.model.api.ai.create.AiDocument;
import stirling.software.proprietary.service.AiDocumentHtmlRenderer;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.ObjectMapper;
/**
* Dispatchable tool that converts an AI-generated HTML string to a PDF via WeasyPrint.
* Dispatchable tool that converts an AI-generated document model to a PDF via WeasyPrint.
*
* <p>Called by {@link stirling.software.proprietary.service.AiWorkflowService} when the engine
* emits a {@code CREATE_PDF_FROM_HTML_AGENT} plan step. The HTML comes from a trusted Jinja
* template so sanitization is intentionally skipped.
* emits a {@code CREATE_PDF_FROM_HTML_AGENT} plan step. The engine supplies the document as
* structured fields; the HTML is built here from a fixed template.
*/
@Slf4j
@Hidden
@@ -48,6 +56,9 @@ public class CreatePdfAgentController {
private final TempFileManager tempFileManager;
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final RuntimePathConfig runtimePathConfig;
private final ApplicationProperties applicationProperties;
private final ObjectMapper objectMapper;
private final AiDocumentHtmlRenderer htmlRenderer;
/**
* Returns true only when WeasyPrint is definitively unavailable — either the binary could not
@@ -74,32 +85,42 @@ public class CreatePdfAgentController {
value = "/create-pdf-from-html-agent",
consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@Operation(
summary = "Convert AI-generated HTML to a PDF",
summary = "Convert an AI-generated document to a PDF",
description =
"Accepts an HTML document as a plain-text parameter and returns a PDF."
+ " This endpoint is dispatched by the AI workflow orchestrator as a"
+ " plan step; it is not intended for direct client use.")
public ResponseEntity<Resource> createPdfFromHtml(
@RequestParam("htmlContent") String htmlContent,
@RequestParam("filename") String filename)
"Accepts a structured document as a JSON parameter and returns a PDF. This"
+ " endpoint is dispatched by the AI workflow orchestrator as a plan"
+ " step; it is not intended for direct client use.")
public ResponseEntity<Resource> createPdf(
@RequestParam("document") String document, @RequestParam("filename") String filename)
throws Exception {
if (!applicationProperties.getAiEngine().isEnabled()) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND);
}
AiDocument model;
try {
model = objectMapper.readValue(document, AiDocument.class);
} catch (JacksonException e) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST);
}
String html = htmlRenderer.render(model);
log.info(
"[create-pdf-agent] converting HTML to PDF via WeasyPrint — html_bytes={}",
htmlContent.length());
"[create-pdf-agent] converting document to PDF via WeasyPrint — html_bytes={}",
html.length());
try (TempFile htmlFile = tempFileManager.createManagedTempFile(".html");
TempFile pdfFile = tempFileManager.createManagedTempFile(".pdf")) {
Files.writeString(htmlFile.getPath(), htmlContent, StandardCharsets.UTF_8);
Files.writeString(htmlFile.getPath(), html, StandardCharsets.UTF_8);
List<String> command = new ArrayList<>();
command.add(runtimePathConfig.getWeasyPrintPath());
command.add("-e");
command.add("utf-8");
command.add("-v");
// SSRF: the HTML is self-contained and the engine validates style colours, so no
// external url() reaches WeasyPrint. For full isolation, run it network-isolated.
command.add(htmlFile.getAbsolutePath());
command.add(pdfFile.getAbsolutePath());
@@ -126,8 +147,8 @@ public class CreatePdfAgentController {
// avoids materialising the whole document as a byte[] twice (read-all + re-serialise),
// which matters for large generated documents.
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
try (PDDocument document = pdfDocumentFactory.load(pdfFile.getPath())) {
document.save(tempOut.getPath().toFile());
try (PDDocument pdDocument = pdfDocumentFactory.load(pdfFile.getPath())) {
pdDocument.save(tempOut.getPath().toFile());
} catch (Exception e) {
tempOut.close();
throw e;
@@ -0,0 +1,35 @@
package stirling.software.proprietary.model.api.ai.create;
import java.util.List;
import lombok.Data;
@Data
public class AiDocument {
private String title;
private String subtitle;
private String referenceNumber;
private Style style;
private List<Section> sections;
@Data
public static class Style {
private String primaryColor;
private String backgroundColor;
private String bodyTextColor;
}
@Data
public static class Section {
private String type;
private String heading;
private String body;
private List<List<String>> pairs;
private List<String> columns;
private List<List<String>> rows;
private List<String> totalRow;
private List<String> items;
private List<String> signatories;
}
}
@@ -0,0 +1,135 @@
package stirling.software.proprietary.service;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Component;
import com.hubspot.jinjava.Jinjava;
import com.hubspot.jinjava.JinjavaConfig;
import stirling.software.proprietary.model.api.ai.create.AiDocument;
/** Renders an {@link AiDocument} to HTML using a Jinja template loaded from the classpath. */
@Component
public class AiDocumentHtmlRenderer {
private static final String TEMPLATE_PATH = "templates/ai/create/document.html.jinja2";
private static final Pattern SAFE_COLOR = Pattern.compile("^#[0-9a-fA-F]{6}$");
private final Jinjava jinjava;
private final String template;
public AiDocumentHtmlRenderer() {
JinjavaConfig config =
JinjavaConfig.newBuilder().withNestedInterpretationEnabled(false).build();
this.jinjava = new Jinjava(config);
this.template = loadTemplate();
}
public String render(AiDocument doc) {
return jinjava.render(template, buildContext(doc));
}
private static Map<String, Object> buildContext(AiDocument doc) {
Map<String, Object> context = new LinkedHashMap<>();
context.put("title", doc.getTitle());
context.put("subtitle", doc.getSubtitle());
context.put("reference_number", doc.getReferenceNumber());
AiDocument.Style style = doc.getStyle();
if (style != null) {
context.put("style_primary", safeColor(style.getPrimaryColor()));
context.put("style_background", safeColor(style.getBackgroundColor()));
context.put("style_body", safeColor(style.getBodyTextColor()));
}
List<Map<String, Object>> sections = new ArrayList<>();
if (doc.getSections() != null) {
for (AiDocument.Section section : doc.getSections()) {
if (section != null && section.getType() != null) {
sections.add(buildSection(section));
}
}
}
context.put("sections", sections);
return context;
}
private static Map<String, Object> buildSection(AiDocument.Section section) {
Map<String, Object> node = new LinkedHashMap<>();
node.put("type", section.getType());
node.put("heading", section.getHeading());
switch (section.getType()) {
case "text" -> node.put("paragraphs", paragraphs(section.getBody()));
case "key_value" -> node.put("pairs", pairs(section.getPairs()));
case "line_items" -> {
node.put("columns", orEmpty(section.getColumns()));
node.put("rows", orEmptyRows(section.getRows()));
node.put("total_row", emptyToNull(section.getTotalRow()));
}
case "bullet_list" -> node.put("items", orEmpty(section.getItems()));
case "signature" -> node.put("signatories", orEmpty(section.getSignatories()));
default -> {}
}
return node;
}
private static List<String> paragraphs(String body) {
String text = body == null ? "" : body;
List<String> out = new ArrayList<>();
for (String paragraph : text.split("\n\n")) {
out.add(paragraph.replace("\n", " "));
}
return out;
}
private static List<Map<String, String>> pairs(List<List<String>> pairs) {
List<Map<String, String>> out = new ArrayList<>();
if (pairs != null) {
for (List<String> pair : pairs) {
Map<String, String> node = new LinkedHashMap<>();
node.put("label", pair.isEmpty() ? "" : pair.get(0));
node.put("value", pair.size() < 2 ? "" : pair.get(1));
out.add(node);
}
}
return out;
}
private static List<String> orEmpty(List<String> values) {
return values == null ? List.of() : values;
}
private static List<List<String>> orEmptyRows(List<List<String>> rows) {
return rows == null ? List.of() : rows;
}
private static List<String> emptyToNull(List<String> values) {
return values == null || values.isEmpty() ? null : values;
}
private static String safeColor(String value) {
if (value == null) {
return null;
}
String trimmed = value.trim();
return SAFE_COLOR.matcher(trimmed).matches() ? trimmed : null;
}
private static String loadTemplate() {
try {
return new ClassPathResource(TEMPLATE_PATH).getContentAsString(StandardCharsets.UTF_8);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
@@ -1,3 +1,4 @@
{%- autoescape true -%}
<!DOCTYPE html>
<html lang="en">
<head>
@@ -175,18 +176,18 @@
color: var(--color-label);
}
</style>
{%- if doc.style %}
{%- if style_primary or style_background or style_body %}
<style>
:root {
{%- if doc.style.primary_color %}
--color-primary: {{ doc.style.primary_color }};
{%- if style_primary %}
--color-primary: {{ style_primary }};
{%- endif %}
{%- if doc.style.background_color %}
--color-bg: {{ doc.style.background_color }};
{%- if style_background %}
--color-bg: {{ style_background }};
{%- endif %}
{%- if doc.style.body_text_color %}
--color-body: {{ doc.style.body_text_color }};
--color-label: {{ doc.style.body_text_color }};
{%- if style_body %}
--color-body: {{ style_body }};
--color-label: {{ style_body }};
{%- endif %}
}
</style>
@@ -195,16 +196,16 @@
<body>
<div class="doc-header">
<div class="doc-title">{{ doc.title }}</div>
{%- if doc.subtitle %}
<div class="doc-subtitle">{{ doc.subtitle }}</div>
<div class="doc-title">{{ title }}</div>
{%- if subtitle %}
<div class="doc-subtitle">{{ subtitle }}</div>
{%- endif %}
{%- if doc.reference_number %}
<div class="doc-reference">{{ doc.reference_number }}</div>
{%- if reference_number %}
<div class="doc-reference">{{ reference_number }}</div>
{%- endif %}
</div>
{%- for section in doc.sections %}
{%- for section in sections %}
{%- if section.type == "text" %}
<section>
@@ -212,8 +213,8 @@
<h2>{{ section.heading }}</h2>
{%- endif %}
<div class="text-body">
{%- for para in section.body.split('\n\n') %}
<p>{{ para | replace('\n', ' ') }}</p>
{%- for para in section.paragraphs %}
<p>{{ para }}</p>
{%- endfor %}
</div>
</section>
@@ -225,10 +226,10 @@
{%- endif %}
<table class="kv-table">
<tbody>
{%- for label, value in section.pairs %}
{%- for pair in section.pairs %}
<tr>
<td class="kv-label">{{ label }}</td>
<td class="kv-value">{{ value }}</td>
<td class="kv-label">{{ pair.label }}</td>
<td class="kv-value">{{ pair.value }}</td>
</tr>
{%- endfor %}
</tbody>
@@ -299,3 +300,4 @@
</body>
</html>
{%- endautoescape %}
@@ -166,8 +166,8 @@ class PolicyExecutorTest {
new PipelineStep(
createPdf,
Map.of(
"htmlContent",
"<p>hi</p>",
"document",
"{\"title\":\"PO\",\"sections\":[]}",
"filename",
"purchase-order.pdf"))),
PolicyInputs.of(List.of()),
@@ -0,0 +1,139 @@
package stirling.software.proprietary.service;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.junit.jupiter.api.Test;
import stirling.software.proprietary.model.api.ai.create.AiDocument;
class AiDocumentHtmlRendererTest {
private final AiDocumentHtmlRenderer renderer = new AiDocumentHtmlRenderer();
private static AiDocument.Section section(String type) {
AiDocument.Section s = new AiDocument.Section();
s.setType(type);
return s;
}
private static AiDocument document(String title, List<AiDocument.Section> sections) {
AiDocument doc = new AiDocument();
doc.setTitle(title);
doc.setSections(sections);
return doc;
}
@Test
void rendersAllSectionTypes() {
AiDocument.Section text = section("text");
text.setBody("Some prose text.");
AiDocument.Section kv = section("key_value");
kv.setPairs(List.of(List.of("Key", "Value")));
AiDocument.Section items = section("line_items");
items.setColumns(List.of("A", "B"));
items.setRows(List.of(List.of("1", "2")));
AiDocument.Section bullets = section("bullet_list");
bullets.setItems(List.of("item one"));
AiDocument.Section sign = section("signature");
sign.setSignatories(List.of("Alice"));
String html = renderer.render(document("All", List.of(text, kv, items, bullets, sign)));
assertTrue(html.contains("<!DOCTYPE html>"));
assertTrue(html.contains("Some prose text."));
assertTrue(html.contains("Key") && html.contains("Value"));
assertTrue(html.contains("<th>"));
assertTrue(html.contains("item one"));
assertTrue(html.contains("Alice"));
}
@Test
void rendersMarkupCharactersAsText() {
AiDocument.Section text = section("text");
text.setBody("a <b>x</b> & y");
String html = renderer.render(document("Doc", List.of(text)));
assertFalse(html.contains("<b>"));
assertTrue(html.contains("&lt;b&gt;"));
}
@Test
void totalRowRenderedWhenPresent() {
AiDocument.Section items = section("line_items");
items.setColumns(List.of("Item", "Total"));
items.setRows(List.of(List.of("Widget", "$10")));
items.setTotalRow(List.of("Total", "$10"));
assertTrue(
renderer.render(document("Table", List.of(items)))
.contains("<tr class=\"total-row\">"));
}
@Test
void totalRowAbsentWhenNotProvided() {
AiDocument.Section items = section("line_items");
items.setColumns(List.of("Item"));
items.setRows(List.of(List.of("Widget")));
assertFalse(
renderer.render(document("Table", List.of(items)))
.contains("<tr class=\"total-row\">"));
}
@Test
void rendersSubtitleAndReference() {
AiDocument doc = document("My Doc", List.of());
doc.setSubtitle("Subtitle Here");
doc.setReferenceNumber("REF-42");
String html = renderer.render(doc);
assertTrue(html.contains("Subtitle Here"));
assertTrue(html.contains("REF-42"));
}
@Test
void appliesHexColourOverride() {
AiDocument doc = document("Styled", List.of());
AiDocument.Style style = new AiDocument.Style();
style.setPrimaryColor("#ff00ff");
style.setBackgroundColor("#111111");
doc.setStyle(style);
String html = renderer.render(doc);
assertTrue(html.contains("--color-primary: #ff00ff"));
assertTrue(html.contains("--color-bg: #111111"));
}
@Test
void ignoresColourWithDisallowedCharacters() {
AiDocument doc = document("Styled", List.of());
AiDocument.Style style = new AiDocument.Style();
style.setPrimaryColor("rgb(255, 0, 0)");
doc.setStyle(style);
String html = renderer.render(doc);
assertFalse(html.contains("rgb("));
assertTrue(html.contains("<!DOCTYPE html>"));
}
@Test
void ignoresNonHexColour() {
AiDocument doc = document("Styled", List.of());
AiDocument.Style style = new AiDocument.Style();
style.setPrimaryColor("magenta");
style.setBackgroundColor("#fff");
doc.setStyle(style);
String html = renderer.render(doc);
assertFalse(html.contains("--color-primary: magenta"));
assertFalse(html.contains("--color-bg: #fff;"));
}
}
+9
View File
@@ -36,6 +36,8 @@ ext {
okhttpBomVersion = "5.3.2"
gsonVersion = "2.14.0"
guavaVersion = "33.6.0-jre"
jinjavaVersion = "2.8.3"
jackson2Version = "2.21.2"
bucket4jVersion = "8.19.0"
archunitVersion = "1.4.2"
batikVersion = "1.19"
@@ -222,6 +224,13 @@ subprojects {
resolutionStrategy.force "org.apache.commons:commons-lang3:${commonsLang3}"
// CVE-2024-47554: commons-io DoS prevention
resolutionStrategy.force "commons-io:commons-io:${commonsIoVersion}"
// Jackson 2 is transitive-only here (jinjava, opensaml, jjwt request older versions);
// pin the family to a current release and keep modules aligned.
resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson2Version}"
resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson2Version}"
resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jackson2Version}"
resolutionStrategy.force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2Version}"
resolutionStrategy.force "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2Version}"
// Keep BouncyCastle modules aligned to avoid runtime linkage errors
resolutionStrategy.force "org.bouncycastle:bcprov-jdk18on:${bouncycastleVersion}"
resolutionStrategy.force "org.bouncycastle:bcpkix-jdk18on:${bouncycastleVersion}"
-1
View File
@@ -5,7 +5,6 @@ description = "AI Document Engine"
requires-python = ">=3.13"
dependencies = [
"fastapi>=0.116.0",
"jinja2>=3.1.0",
"pgvector>=0.3.6",
"psycopg[binary,pool]>=3.2",
"pydantic>=2.0.0",
+12 -29
View File
@@ -10,7 +10,7 @@ Flow:
4. SectionWriterAgents (smart_model) run in parallel via asyncio.gather.
Each returns a WrittenSections with fully populated DocumentSection objects.
5. The assembler collects sections in plan order → GeneratedDocument.
6. Jinja renders the document to HTML. The LLM never writes HTML.
6. The assembled document is emitted as structured fields. The LLM never writes HTML.
The planner is split into two calls (meta then sections) so each LLM output schema
stays small enough for grammar compilation on all model tiers including Haiku.
@@ -22,9 +22,7 @@ import asyncio
import logging
import re
from dataclasses import dataclass
from pathlib import Path
from jinja2 import Environment, FileSystemLoader
from pydantic_ai import Agent
from pydantic_ai.output import NativeOutput
@@ -51,8 +49,6 @@ from stirling.services import AppRuntime
logger = logging.getLogger(__name__)
_TEMPLATES_DIR = Path(__file__).parent / "templates"
# ── Token budget ──────────────────────────────────────────────────────────────────────────────────
# Conservative per-section token estimates mapped from planner-assigned depth.
@@ -166,10 +162,13 @@ Analyse the user's request and produce a DocumentMeta with:
document, if the user provides one. Leave empty if the user provides no such context.
- style_primary_color: accent and heading colour. Set ONLY when the user explicitly names a
colour or colour scheme (e.g. "make it red", "use navy blue"). Use CSS named colours
(e.g. "magenta", "navy", "crimson") or hex values. Leave null if no colour is stated.
- style_background_color: page background colour. Set only if explicitly requested.
- style_body_text_color: body text colour. Set only if explicitly requested.
colour or colour scheme (e.g. "make it red", "use navy blue"). Express it as a 6-digit hex
code in #RRGGBB format (map any named colour to its hex value yourself, e.g. "navy" →
"#000080"). No other format is accepted. Leave null if no colour is stated.
- style_background_color: page background colour, same #RRGGBB format. Set only if explicitly
requested.
- style_body_text_color: body text colour, same #RRGGBB format. Set only if explicitly
requested.
- cannot_do_reason: set this ONLY when the request is not asking to create a document at all
(e.g. a question, a greeting, an edit request to an existing document). Never set it
@@ -299,15 +298,6 @@ def _build_writer_prompt(plan: DocumentPlan, chunk: _Chunk) -> str:
# ── Helpers ───────────────────────────────────────────────────────────────────────────────────────
def _build_jinja_env() -> Environment:
return Environment(
loader=FileSystemLoader(str(_TEMPLATES_DIR)),
autoescape=True,
trim_blocks=True,
lstrip_blocks=True,
)
def _safe_filename(title: str) -> str:
slug = re.sub(r"[^\w\s-]", "", title.lower())
slug = re.sub(r"[\s_-]+", "-", slug).strip("-")
@@ -320,7 +310,6 @@ def _safe_filename(title: str) -> str:
class PdfCreateAgent:
def __init__(self, runtime: AppRuntime) -> None:
self.runtime = runtime
self._jinja_env = _build_jinja_env()
self._meta_planner: Agent[None, DocumentMeta] = Agent(
model=runtime.smart_model,
@@ -401,14 +390,12 @@ class PdfCreateAgent:
sections=all_sections,
)
# ── Phase 6: render ────────────────────────────────────────────────────
logger.info("[pdf-create] phase 6/6: rendering HTML")
html = self._render(doc)
# ── Phase 6: emit ──────────────────────────────────────────────────────
filename = _safe_filename(plan.title)
logger.info(
"[pdf-create] done — filename=%r html_bytes=%d",
"[pdf-create] done — filename=%r sections=%d",
filename,
len(html),
len(all_sections),
)
return EditPlanResponse(
@@ -417,7 +404,7 @@ class PdfCreateAgent:
ToolOperationStep(
tool=AgentToolId.CREATE_PDF_FROM_HTML_AGENT,
parameters=CreatePdfFromHtmlAgentParams(
html_content=html,
document=doc.model_dump_json(),
filename=filename,
),
)
@@ -437,7 +424,3 @@ class PdfCreateAgent:
len(result.output.sections),
)
return result.output
def _render(self, doc: GeneratedDocument) -> str:
template = self._jinja_env.get_template("document.html.jinja2")
return template.render(doc=doc)
+6 -8
View File
@@ -1,14 +1,14 @@
"""Contracts for the PDF Create Agent.
The agent accepts a natural-language prompt and returns a single
CREATE_PDF_FROM_HTML_AGENT plan step carrying the rendered HTML.
CREATE_PDF_FROM_HTML_AGENT plan step carrying the assembled document.
Pipeline:
1. PlannerAgent (smart_model) → DocumentPlan: structured skeleton, no body text.
2. Python chunks the plan by token budget.
3. SectionWriterAgents (smart_model, parallel) → WrittenSections per chunk.
4. Assembler collects sections in plan order → GeneratedDocument.
5. Jinja renders GeneratedDocument → HTML. The LLM never writes HTML.
5. The document is emitted as structured fields. The LLM never writes HTML.
"""
from __future__ import annotations
@@ -81,14 +81,12 @@ type DocumentSection = Annotated[
]
# Named colour or hex only — anything else is dropped so a colour can't inject CSS into the
# <style> block (which would let WeasyPrint fetch an attacker-controlled url() → SSRF).
_SAFE_COLOR_RE = re.compile(r"^#[0-9a-fA-F]{3,8}$|^[a-zA-Z]{1,30}$")
# Colours must be a 6-digit hex code (#RRGGBB); anything else is dropped to None.
_SAFE_COLOR_RE = re.compile(r"^#[0-9a-fA-F]{6}$")
class DocumentStyle(ApiModel):
"""Document colours, inferred by the meta planner and rendered into the engine's Jinja
template (never sent to Java). Unsafe colours are dropped to ``None``."""
"""Document colours, inferred by the meta planner. Non-hex values are dropped to ``None``."""
primary_color: str | None = Field(default=None)
background_color: str | None = Field(default=None)
@@ -103,7 +101,7 @@ class DocumentStyle(ApiModel):
class GeneratedDocument(ApiModel):
"""The full document model passed to Jinja for HTML rendering."""
"""The full document model emitted for rendering."""
title: str
subtitle: str | None = None
@@ -29,7 +29,7 @@ class PdfCommentAgentParams(ApiModel):
class CreatePdfFromHtmlAgentParams(ApiModel):
html_content: str
document: str
filename: str = Field(pattern=r"^.+\.pdf$")
+29 -148
View File
@@ -2,7 +2,7 @@
Coverage:
1. Section model validation (each section type round-trips correctly)
2. Jinja rendering (_render produces valid HTML for each section type)
2. orchestrate() emits the assembled document as structured JSON
3. _safe_filename produces clean slugs
4. _make_chunks groups sections correctly by token budget
5. orchestrate() produces the correct EditPlanResponse via planner + writer mocks
@@ -12,6 +12,8 @@ Coverage:
from __future__ import annotations
import json
import pytest
from conftest import build_app_settings
from pydantic_ai.models.test import TestModel
@@ -64,37 +66,6 @@ def agent(runtime: AppRuntime) -> PdfCreateAgent:
# ── Helpers ───────────────────────────────────────────────────────────────────────────────────────
def _invoice_doc() -> GeneratedDocument:
return GeneratedDocument(
title="Invoice",
subtitle="Acme Corp",
reference_number="Invoice #INV-001",
sections=[
KeyValueSection(
heading="Details",
pairs=[("Date", "2026-05-06"), ("Due", "2026-06-06"), ("Currency", "USD")],
),
LineItemsSection(
heading="Line Items",
columns=["Description", "Qty", "Unit Price", "Total"],
rows=[
["Consulting services", "10", "$500.00", "$5,000.00"],
["Expenses", "1", "$200.00", "$200.00"],
],
total_row=["Total", "", "", "$5,200.00"],
),
TextSection(
heading="Payment Terms",
body="Payment is due within 30 days.\n\nPlease reference the invoice number.",
),
SignatureSection(
heading="Authorised By",
signatories=["Jane Smith, CEO", "Bob Jones, CFO"],
),
],
)
def _simple_meta() -> DocumentMeta:
return DocumentMeta(
title="Invoice",
@@ -199,82 +170,6 @@ def test_generated_document_optional_fields() -> None:
assert doc.reference_number is None
# ── Jinja rendering ───────────────────────────────────────────────────────────────────────────────
def test_render_produces_html(agent: PdfCreateAgent) -> None:
doc = _invoice_doc()
html = agent._render(doc)
assert "<!DOCTYPE html>" in html
assert "Invoice" in html
assert "INV-001" in html
def test_render_includes_all_section_types(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="All Sections",
sections=[
TextSection(body="Some prose text."),
KeyValueSection(pairs=[("Key", "Value")]),
LineItemsSection(columns=["A", "B"], rows=[["1", "2"]]),
BulletListSection(items=["item one"]),
SignatureSection(signatories=["Alice"]),
],
)
html = agent._render(doc)
assert "Some prose text." in html
assert "Key" in html and "Value" in html
assert "<th>" in html
assert "item one" in html
assert "Alice" in html
def test_render_escapes_html_in_content(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="XSS Test",
sections=[TextSection(body="<script>alert('xss')</script>")],
)
html = agent._render(doc)
assert "<script>" not in html
assert "&lt;script&gt;" in html
def test_render_total_row_present(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="Table",
sections=[
LineItemsSection(
columns=["Item", "Total"],
rows=[["Widget", "$10"]],
total_row=["Total", "$10"],
)
],
)
html = agent._render(doc)
assert "total-row" in html
def test_render_no_total_row_skips_tfoot(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="Table",
sections=[LineItemsSection(columns=["Item"], rows=[["Widget"]])],
)
html = agent._render(doc)
assert "<tfoot>" not in html
def test_render_subtitle_and_reference(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="My Doc",
subtitle="Subtitle Here",
reference_number="REF-42",
sections=[TextSection(body="Content.")],
)
html = agent._render(doc)
assert "Subtitle Here" in html
assert "REF-42" in html
# ── _safe_filename ────────────────────────────────────────────────────────────────────────────────
@@ -396,8 +291,9 @@ async def test_orchestrate_returns_plan_step(agent: PdfCreateAgent) -> None:
assert step.tool == AgentToolId.CREATE_PDF_FROM_HTML_AGENT
assert isinstance(step.parameters, CreatePdfFromHtmlAgentParams)
assert step.parameters.filename.endswith(".pdf")
assert "<!DOCTYPE html>" in step.parameters.html_content
assert "Invoice" in step.parameters.html_content
parsed = json.loads(step.parameters.document)
assert parsed["title"] == "Invoice"
assert parsed["sections"]
@pytest.mark.anyio
@@ -468,9 +364,9 @@ async def test_orchestrate_assembles_multiple_chunks(agent: PdfCreateAgent) -> N
result = await agent.orchestrate(_orchestrator_request("Create a multi-chunk doc"))
assert isinstance(result, EditPlanResponse)
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
assert "Introduction text." in html
assert "Details" in html
document = result.steps[0].parameters.document # type: ignore[union-attr]
assert "Introduction text." in document
assert "Details" in document
# ── Style inference ───────────────────────────────────────────────────────────────────────────────
@@ -482,7 +378,7 @@ async def test_orchestrate_applies_planner_inferred_style(agent: PdfCreateAgent)
meta = DocumentMeta(
title="Styled Doc",
tone_brief="Professional.",
style_primary_color="magenta",
style_primary_color="#ff00ff",
)
sections = _simple_sections()
written = _written_sections()
@@ -499,50 +395,35 @@ async def test_orchestrate_applies_planner_inferred_style(agent: PdfCreateAgent)
result = await agent.orchestrate(_orchestrator_request("Make an invoice, magenta styling"))
assert isinstance(result, EditPlanResponse)
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
assert "magenta" in html
document = result.steps[0].parameters.document # type: ignore[union-attr]
assert json.loads(document)["style"]["primaryColor"] == "#ff00ff"
def test_render_applies_style(agent: PdfCreateAgent) -> None:
"""DocumentStyle fields are injected as CSS custom properties in the rendered HTML."""
doc = GeneratedDocument(
title="Styled",
sections=[TextSection(body="Content.")],
style=DocumentStyle(primary_color="magenta", background_color="#111111"),
)
html = agent._render(doc)
assert "--color-primary: magenta" in html
assert "--color-bg: #111111" in html
def test_document_style_drops_unsafe_colors() -> None:
"""Unsafe colours (not named/hex) are dropped, closing the <style> url() injection."""
safe = DocumentStyle(primary_color="navy", background_color="#1e3a5f", body_text_color="#fff")
def test_document_style_keeps_only_six_digit_hex() -> None:
"""Only #RRGGBB hex is kept; named colours and other formats drop to None."""
safe = DocumentStyle(primary_color="#1e3a5f", background_color="#ffffff", body_text_color="#1A1A1A")
assert (safe.primary_color, safe.background_color, safe.body_text_color) == (
"navy",
"#1e3a5f",
"#fff",
"#ffffff",
"#1A1A1A",
)
unsafe = DocumentStyle(
primary_color="red; background: url(http://evil.test/steal)",
background_color="expression(alert(1))",
body_text_color="navy; }",
)
assert unsafe.primary_color is None
assert unsafe.background_color is None
assert unsafe.body_text_color is None
assert DocumentStyle(primary_color="navy").primary_color is None
assert DocumentStyle(primary_color="#fff").primary_color is None
assert DocumentStyle(primary_color="#1e3a5f00").primary_color is None
assert DocumentStyle(primary_color="rgb(255, 0, 0)").primary_color is None
assert DocumentStyle(background_color="teal darken-2").background_color is None
# A trailing newline must not slip a value through (fullmatch, not $-before-newline).
assert DocumentStyle(primary_color="navy\n").primary_color is None
assert DocumentStyle(primary_color="#1e3a5f\n").primary_color is None
@pytest.mark.anyio
async def test_orchestrate_drops_unsafe_planner_color(agent: PdfCreateAgent) -> None:
"""An unsafe colour inferred by the meta planner never reaches the rendered HTML."""
async def test_orchestrate_drops_non_hex_planner_colour(agent: PdfCreateAgent) -> None:
"""A non-hex colour inferred by the meta planner is dropped before the document is emitted."""
meta = DocumentMeta(
title="Doc",
tone_brief="Professional.",
style_primary_color="blue; background: url(http://evil.test/)",
style_primary_color="rgb(0, 0, 255)",
)
sections = _simple_sections()
written = _written_sections()
@@ -559,6 +440,6 @@ async def test_orchestrate_drops_unsafe_planner_color(agent: PdfCreateAgent) ->
result = await agent.orchestrate(_orchestrator_request("make it blue"))
assert isinstance(result, EditPlanResponse)
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
assert "evil.test" not in html
assert "url(" not in html
document = result.steps[0].parameters.document # type: ignore[union-attr]
assert "rgb(" not in document
assert json.loads(document)["style"]["primaryColor"] is None
-2
View File
@@ -604,7 +604,6 @@ version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "fastapi" },
{ name = "jinja2" },
{ name = "opentelemetry-sdk" },
{ name = "pgvector" },
{ name = "posthog" },
@@ -631,7 +630,6 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "fastapi", specifier = ">=0.116.0" },
{ name = "jinja2", specifier = ">=3.1.0" },
{ name = "opentelemetry-sdk", specifier = ">=1.39.0" },
{ name = "pgvector", specifier = ">=0.3.6" },
{ name = "posthog", specifier = ">=3.0.0" },
-2
View File
@@ -12,8 +12,6 @@ editor/public/mockServiceWorker.js
# Auto-generated OG/social-preview metadata (scripts/generate-og-metadata.mjs); regenerated verbatim.
editor/public/og-metadata.json
editor/src/core/data/ogImageMap.json
# Auto-generated portal docs manifest (scripts/sync-portal-docs.mts); regenerated verbatim.
editor/src/portal/generated/docsManifest.json
editor/public/pdfjs*/
editor/public/js/thirdParty/
editor/public/css/cookieconsent.css
@@ -6603,10 +6603,6 @@ title = "No components available"
description = "GA components are available on Pay-as-you-go; a few Beta components are enterprise-only. Locked cards show an upgrade nudge."
title = "Some components need a paid plan"
[portal.docs]
browse = "Browse docs"
viewSource = "View source on GitHub"
[portal.docs.authentication]
codeCaption = "every request"
eyebrow = "GETTING STARTED"
@@ -6692,19 +6688,11 @@ title = "Official SDKs"
beta = "Beta"
deprecated = "Deprecated"
[portal.docs.search]
empty = "No matching docs"
placeholder = "Search docs"
results = "{{count}} results"
[portal.docs.skills]
eyebrow = "SKILLS"
lead = "Bundled, named capabilities your agent invokes as a single tool. Each skill is a deterministic op chain with evals attached."
title = "Agent skills"
[portal.docs.toc]
title = "On this page"
[portal.docs.webhooks]
codeCaption = "document.processed"
eyebrow = "API REFERENCE"
@@ -7258,7 +7246,7 @@ storage = "Storage"
[portal.nav]
agent-builder = "Agent Builder"
components = "Components"
docs = "Documentation"
docs = "Developer Docs"
documents = "Documents"
editor = "Editor"
home = "Home"
@@ -6563,10 +6563,6 @@ bucket = "Bucket"
name = "Name"
region = "Region"
[portal.docs]
browse = "Browse docs"
viewSource = "View source on GitHub"
[portal.docs.authentication]
codeCaption = "every request"
eyebrow = "GETTING STARTED"
@@ -6652,19 +6648,11 @@ title = "Official SDKs"
beta = "Beta"
deprecated = "Deprecated"
[portal.docs.search]
empty = "No matching docs"
placeholder = "Search docs"
results = "{{count}} results"
[portal.docs.skills]
eyebrow = "SKILLS"
lead = "Bundled, named capabilities your agent invokes as a single tool. Each skill is a deterministic op chain with evals attached."
title = "Agent skills"
[portal.docs.toc]
title = "On this page"
[portal.docs.webhooks]
codeCaption = "document.processed"
eyebrow = "API REFERENCE"
@@ -7329,7 +7317,7 @@ storage = "Storage"
[portal.nav]
agent-builder = "Agent Builder"
components = "Components"
docs = "Documentation"
docs = "Developer Docs"
documents = "Documents"
editor = "Editor"
home = "Home"
@@ -10424,6 +10412,9 @@ disableColorFilter = "Disable Color Filter"
dualPageView = "Dual Page View"
enableDarkFilter = "Enable Dark Filter"
enableSepiaFilter = "Enable Sepia Filter"
engineLoadErrorBody = "The PDF engine couldn't be loaded. If this keeps happening, make sure the app is up to date, then try again."
engineLoadErrorRetry = "Retry"
engineLoadErrorTitle = "Couldn't load the PDF viewer"
firstPage = "First Page"
lastPage = "Last Page"
moreOptions = "More"
@@ -1,159 +0,0 @@
/**
* Sync the portal Developer Docs from the Stirling docs repo.
*
* Fetches the docs repo tarball, extracts `docs/**` in-process (no external tar
* binary, no per-file GitHub rate limits), shapes it with the pure transforms in
* src/portal/docs/manifest/transform.ts, and writes the committed manifest that
* the portal docs view renders. Re-run with `npm run docs:sync`.
*
* Env: DOCS_REPO, DOCS_REF, DOCS_ROOT override the defaults below.
*/
import { gunzipSync } from "node:zlib";
import { mkdirSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
// tsx/node16 can't resolve the @portal alias here, so import by relative .ts path.
// eslint-disable-next-line no-restricted-imports
import {
buildManifest,
type CategoryMap,
type RawDoc,
} from "../src/portal/docs/manifest/transform.ts";
const REPO = process.env.DOCS_REPO ?? "Stirling-Tools/Stirling-Tools.github.io";
const REF = process.env.DOCS_REF ?? "main";
const ROOT = process.env.DOCS_ROOT ?? "docs";
const SITE = "https://docs.stirlingpdf.com";
const HERE = dirname(fileURLToPath(import.meta.url));
const OUT = resolve(HERE, "../src/portal/generated/docsManifest.json");
/* ── Minimal tar reader (ustar + pax/GNU long names) ─────────────────────── */
interface TarEntry {
name: string;
type: string;
data: Buffer;
}
function readTar(buf: Buffer): TarEntry[] {
const entries: TarEntry[] = [];
let offset = 0;
let longName: string | null = null;
let paxPath: string | null = null;
const str = (start: number, len: number) => {
const slice = buf.subarray(start, start + len);
const end = slice.indexOf(0);
return slice.toString("utf8", 0, end === -1 ? len : end);
};
while (offset + 512 <= buf.length) {
const header = buf.subarray(offset, offset + 512);
// Two consecutive zero blocks mark the end of the archive.
if (header.every((b) => b === 0)) break;
const name = str(offset, 100);
const prefix = str(offset + 345, 155);
const sizeStr = str(offset + 124, 12).trim();
const size = parseInt(sizeStr, 8) || 0;
const type = String.fromCharCode(header[156]);
const dataStart = offset + 512;
const data = buf.subarray(dataStart, dataStart + size);
let fullName = prefix ? `${prefix}/${name}` : name;
if (longName) {
fullName = longName;
longName = null;
}
if (paxPath) {
fullName = paxPath;
paxPath = null;
}
if (type === "L") {
// GNU long name: the payload is the real name of the next entry.
longName = data.toString("utf8").replace(/\0+$/, "");
} else if (type === "x") {
// pax extended header: pull a `path=` record for the next entry.
const record = /(?:^|\n)\d+ path=([^\n]+)\n/.exec(data.toString("utf8"));
if (record) paxPath = record[1];
} else if (type === "0" || type === "\0" || type === "") {
entries.push({ name: fullName, type, data: Buffer.from(data) });
}
offset = dataStart + Math.ceil(size / 512) * 512;
}
return entries;
}
/* ── Fetch + shape ───────────────────────────────────────────────────────── */
async function main(): Promise<void> {
const url = `https://api.github.com/repos/${REPO}/tarball/${REF}`;
console.log(`Fetching ${REPO}@${REF} …`);
const res = await fetch(url, {
headers: {
Accept: "application/vnd.github+json",
"User-Agent": "stirling-portal-docs-sync",
...(process.env.GITHUB_TOKEN
? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` }
: {}),
},
});
if (!res.ok) {
throw new Error(
`GitHub tarball fetch failed: ${res.status} ${res.statusText}`,
);
}
const gz = Buffer.from(await res.arrayBuffer());
const entries = readTar(gunzipSync(gz));
// Strip the "<repo>-<sha>/" wrapper dir and keep only files under docs root.
const prefix = `${ROOT}/`;
const rawDocs: RawDoc[] = [];
const categories: CategoryMap = {};
for (const entry of entries) {
const rel = entry.name.replace(/^[^/]+\//, "");
if (!rel.startsWith(prefix)) continue;
const inner = rel.slice(prefix.length);
if (!inner) continue;
if (inner.endsWith("/_category_.json")) {
const dir = inner.slice(0, -"/_category_.json".length);
try {
categories[dir] = JSON.parse(entry.data.toString("utf8"));
} catch {
console.warn(` skipping unparseable _category_.json in ${dir}`);
}
} else if (/\.mdx?$/i.test(inner)) {
rawDocs.push({ relPath: inner, content: entry.data.toString("utf8") });
}
}
if (rawDocs.length === 0) {
throw new Error(`No markdown found under ${ROOT}/ — wrong repo/ref/root?`);
}
const manifest = buildManifest(rawDocs, categories, {
repo: REPO,
ref: REF,
root: ROOT,
siteBaseUrl: SITE,
});
mkdirSync(dirname(OUT), { recursive: true });
writeFileSync(OUT, JSON.stringify(manifest, null, 2) + "\n", "utf8");
const items = manifest.nav.reduce((n, s) => n + s.items.length, 0);
console.log(
`Wrote ${manifest.nav.length} sections, ${items} docs → ${OUT.replace(/.*[/\\]frontend[/\\]/, "frontend/")}`,
);
for (const s of manifest.nav) {
console.log(` ${s.icon} ${s.label} (${s.items.length})`);
}
}
main().catch((err) => {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
});
+1 -3
View File
@@ -4,9 +4,7 @@
"module": "node16",
"moduleResolution": "node16",
"types": ["node"],
"noEmit": true,
// sync-portal-docs.mts imports the shared transform by its .ts path (run via tsx).
"allowImportingTsExtensions": true
"noEmit": true
},
"include": ["./**/*.ts", "./**/*.mts"]
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,101 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { render, screen, act, fireEvent } from "@testing-library/react";
import { MantineProvider } from "@mantine/core";
import type { ReactNode } from "react";
// Controllable stand-in for the library engine hook so we can simulate the
// three states that matter: still loading, loaded, and errored.
let engineState: { engine: unknown; isLoading: boolean; error: Error | null } =
{
engine: null,
isLoading: true,
error: null,
};
vi.mock("@embedpdf/engines/react", () => ({
usePdfiumEngine: () => engineState,
}));
import { PdfEngineBoundary } from "@app/components/viewer/PdfEngineBoundary";
const renderBoundary = (onRetry = vi.fn(), timeoutMs = 1000) => {
const utils = render(
<MantineProvider>
<PdfEngineBoundary
wasmUrl="pdfium.wasm"
onRetry={onRetry}
timeoutMs={timeoutMs}
>
{() => <div>PDF CONTENT</div>}
</PdfEngineBoundary>
</MantineProvider>,
);
return { onRetry, ...utils };
};
const wrap = (node: ReactNode) => <MantineProvider>{node}</MantineProvider>;
describe("PdfEngineBoundary", () => {
beforeEach(() => {
engineState = { engine: null, isLoading: true, error: null };
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it("shows the loading fallback while the engine is initialising", () => {
renderBoundary();
expect(screen.getByText("Loading PDF Engine...")).toBeInTheDocument();
expect(screen.queryByText("viewer.engineLoadErrorTitle")).toBeNull();
});
it("surfaces an error with a retry after the load times out (no infinite spinner)", () => {
const { onRetry } = renderBoundary(vi.fn(), 1000);
// Still spinning before the timeout elapses.
expect(screen.getByText("Loading PDF Engine...")).toBeInTheDocument();
act(() => {
vi.advanceTimersByTime(1000);
});
// The spinner is replaced by an actionable error state.
expect(screen.queryByText("Loading PDF Engine...")).toBeNull();
expect(screen.getByText("viewer.engineLoadErrorTitle")).toBeInTheDocument();
const retry = screen.getByText("viewer.engineLoadErrorRetry");
fireEvent.click(retry);
expect(onRetry).toHaveBeenCalledTimes(1);
});
it("surfaces an error immediately when the engine load rejects", () => {
engineState = {
engine: null,
isLoading: false,
error: new Error("boom"),
};
render(
wrap(
<PdfEngineBoundary wasmUrl="pdfium.wasm" onRetry={vi.fn()}>
{() => <div>PDF CONTENT</div>}
</PdfEngineBoundary>,
),
);
expect(screen.getByText("viewer.engineLoadErrorTitle")).toBeInTheDocument();
});
it("renders children once the engine is ready", () => {
engineState = { engine: {}, isLoading: false, error: null };
render(
wrap(
<PdfEngineBoundary wasmUrl="pdfium.wasm" onRetry={vi.fn()}>
{() => <div>PDF CONTENT</div>}
</PdfEngineBoundary>,
),
);
expect(screen.getByText("PDF CONTENT")).toBeInTheDocument();
expect(screen.queryByText("Loading PDF Engine...")).toBeNull();
});
});
@@ -0,0 +1,82 @@
import React, { useEffect, useState } from "react";
import { usePdfiumEngine } from "@embedpdf/engines/react";
import { Center, Stack, Text } from "@mantine/core";
import { useTranslation } from "react-i18next";
import { Button } from "@app/ui/Button";
import ToolLoadingFallback from "@app/components/tools/ToolLoadingFallback";
/** Engine instance produced by usePdfiumEngine once initialisation succeeds. */
type PdfiumEngine = NonNullable<ReturnType<typeof usePdfiumEngine>["engine"]>;
/**
* How long to wait for the PDFium WASM engine to initialise before treating the
* load as failed. The engine normally initialises in well under a second; if it
* has not resolved after this window it has almost certainly hung (e.g. the WASM
* fetch stalled or the worker never reported back), so we surface an error with
* a retry instead of spinning forever.
*/
const DEFAULT_ENGINE_LOAD_TIMEOUT_MS = 30_000;
interface PdfEngineBoundaryProps {
/** Absolute URL of the pdfium.wasm binary to load. */
wasmUrl: string;
/** Invoked when the user asks to retry after a failed/hung load. */
onRetry: () => void;
/** Override the load timeout (mainly for tests). */
timeoutMs?: number;
/** Rendered once the engine is ready. */
children: (engine: PdfiumEngine) => React.ReactNode;
}
/**
* Loads the PDFium WASM engine and gates its children on success.
*
* The underlying `usePdfiumEngine` hook only re-initialises when its `wasmUrl`
* changes, so retrying is handled by the parent remounting this component via a
* `key`. Because the boundary owns the hook, a fresh mount runs the load again
* from scratch.
*
* Without this boundary a failed or hung WASM load left the viewer showing an
* infinite "Loading PDF Engine..." spinner with no error and no way to recover.
*/
export function PdfEngineBoundary({
wasmUrl,
onRetry,
timeoutMs = DEFAULT_ENGINE_LOAD_TIMEOUT_MS,
children,
}: PdfEngineBoundaryProps) {
const { t } = useTranslation();
const { engine, isLoading, error } = usePdfiumEngine({ wasmUrl });
const [timedOut, setTimedOut] = useState(false);
useEffect(() => {
if (engine || error) return;
const timer = setTimeout(() => setTimedOut(true), timeoutMs);
return () => clearTimeout(timer);
}, [engine, error, isLoading, timeoutMs]);
if (engine) {
return <>{children(engine)}</>;
}
if (error || timedOut) {
return (
<Center h="100%" w="100%">
<Stack align="center" gap="md" style={{ maxWidth: 420, padding: 16 }}>
<div style={{ fontSize: "32px" }}>⚠️</div>
<Text fw={600} size="md" style={{ textAlign: "center" }}>
{t("viewer.engineLoadErrorTitle")}
</Text>
<Text c="dimmed" size="sm" style={{ textAlign: "center" }}>
{t("viewer.engineLoadErrorBody")}
</Text>
<Button onClick={onRetry} variant="primary">
{t("viewer.engineLoadErrorRetry")}
</Button>
</Stack>
</Center>
);
}
return <ToolLoadingFallback toolName="PDF Engine" />;
}
@@ -13,7 +13,6 @@ import {
import {
deserializeToolStep,
getExecutableTools,
serializeStepFromEndpoint,
serializeToolStep,
stepRequiresUpload,
type WorkingToolStep,
@@ -199,41 +198,6 @@ describe("serialize/deserialize round-trip", () => {
});
});
describe("serializeStepFromEndpoint", () => {
test("maps a wizard step's UI params to the backend contract, filling defaults", () => {
// The shape the policy setup wizard holds: an endpoint plus UI-shaped params
// (redact's `wordsToRedact`), with several fields left to their defaults.
const api = serializeStepFromEndpoint(
"/api/v1/security/auto-redact",
{ mode: "automatic", useRegex: true, wordsToRedact: ["ssn", "card"] },
dynamicRegistry,
);
expect(api.operation).toBe("/api/v1/security/auto-redact");
// wordsToRedact -> listOfText (the field the backend actually reads), and the
// frontend-only `mode` is dropped.
expect(api.parameters).toMatchObject({ listOfText: "ssn\ncard" });
expect(api.parameters).not.toHaveProperty("wordsToRedact");
expect(api.parameters).not.toHaveProperty("mode");
// Fields the wizard never set still get their defaults so the body is complete.
expect(api.parameters).toHaveProperty("wholeWordSearch");
expect(api.parameters).toHaveProperty("customPadding");
});
test("passes an unmapped endpoint's params through unchanged", () => {
expect(
serializeStepFromEndpoint(
"/api/v1/unknown/thing",
{ keep: true },
dynamicRegistry,
),
).toEqual({
operation: "/api/v1/unknown/thing",
parameters: { keep: true },
});
});
});
describe("stepRequiresUpload", () => {
const step = (params: Record<string, unknown>): WorkingToolStep => ({
toolId: "compress" as ToolId,
@@ -197,31 +197,6 @@ export function serializeToolStep(
return { operation, parameters };
}
/**
* Serialize a step held as an endpoint path plus frontend-shaped params - the form the policy setup
* wizard keeps, where params match the tool's UI shape (e.g. redact's `wordsToRedact`) rather than
* the backend contract - into the backend step contract, mapping params through the tool's
* `toApiParams` (merged over its defaults, so fields the wizard never set still get their defaults).
* The endpoint maps to a tool by path, so this works for dynamic-endpoint tools whose config
* endpoint is a function. Endpoints that map to no known tool pass through unchanged.
*/
export function serializeStepFromEndpoint(
operation: string,
params: ErasedToolParams,
registry: Partial<ToolRegistry>,
): ToolApiStep {
const match = findToolByEndpoint({ operation, parameters: params }, registry);
const config = match?.[1].operationConfig;
if (!config) return { operation, parameters: params };
const merged = { ...(config.defaultParameters ?? {}), ...params };
return {
operation: resolveEndpoint(config, merged) ?? operation,
parameters: config.toApiParams
? (config.toApiParams(merged) as Record<string, unknown>)
: {},
};
}
/**
* Find the registry tool for a stored step's endpoint: exact match for static endpoints, else
* membership in a dynamic tool's declared `endpoints` set (replaying its function can't recover a
@@ -0,0 +1,37 @@
import { describe, expect, test } from "vitest";
import { describeToolOperation } from "@app/hooks/tools/shared/toolOperationDescriptor";
interface Params {
a: number;
}
// A minimal config that type-checks against the flatten endpoint's model.
const CONFIG = {
endpoint: "/api/v1/misc/flatten" as const,
defaultParameters: { a: 1 } satisfies Params,
toApiParams: (p: Params) => ({ renderDpi: p.a }),
fromApiParams: (api: { renderDpi?: number }) => ({ a: api.renderDpi ?? 0 }),
};
describe("describeToolOperation", () => {
test("wraps the config's mappers and endpoint into a descriptor", () => {
const d = describeToolOperation("/api/v1/misc/flatten", CONFIG);
expect(d.endpoint).toBe("/api/v1/misc/flatten");
expect(d.toApi({ a: 200 })).toEqual({ renderDpi: 200 });
});
test("fromApi merges the mapped values over the defaults", () => {
const d = describeToolOperation("/api/v1/misc/flatten", CONFIG);
expect(d.fromApi({ renderDpi: 72 })).toEqual({ a: 72 });
});
test("throws when the config lacks a mapper", () => {
expect(() =>
describeToolOperation("/api/v1/misc/flatten", {
endpoint: "/api/v1/misc/flatten" as const,
defaultParameters: { a: 1 },
toApiParams: (p: Params) => ({ renderDpi: p.a }),
}),
).toThrow(/mappers/);
});
});
@@ -0,0 +1,59 @@
/**
* Typed wrapper over a tool's `toApiParams`/`fromApiParams` mappers, binding one endpoint to safe
* frontend<->backend parameter conversion.
*/
import type { ToolApiParams, ToolEndpoint } from "@app/types/toolApiTypes";
export interface ToolOperationDescriptor<E extends ToolEndpoint, TParams> {
readonly endpoint: E;
readonly defaultParameters: TParams;
toApi(params: TParams): ToolApiParams[E];
/** Backend model -> full frontend params (defaults merged under the mapped values). */
fromApi(api: ToolApiParams[E]): TParams;
}
/**
* Structural subset of a tool's config. `CE` is the config's declared endpoint type, inferred from
* the `endpoint` field: the literal for static tools, or the whole `ToolEndpoint` union for
* dynamic-endpoint tools (whose endpoint is a function typed against the union).
*/
export interface BidirectionalToolConfig<TParams, CE extends ToolEndpoint> {
endpoint: CE | null | ((params: TParams) => CE | null);
defaultParameters?: TParams;
toApiParams?(params: TParams): ToolApiParams[CE];
fromApiParams?(api: ToolApiParams[CE]): Partial<TParams>;
}
/**
* Pin a config to `endpoint` (passed explicitly, since dynamic-endpoint tools declare `endpoint` as
* a function). `E extends CE` rejects pairing a static tool's config with the wrong endpoint, while
* allowing a dynamic tool whose `CE` is the full union. Throws when mappers or defaults are missing.
*/
export function describeToolOperation<
E extends CE,
CE extends ToolEndpoint,
TParams,
>(
endpoint: E,
config: BidirectionalToolConfig<TParams, CE>,
): ToolOperationDescriptor<E, TParams> {
const { toApiParams, fromApiParams, defaultParameters } = config;
if (!toApiParams || !fromApiParams || defaultParameters === undefined) {
throw new Error(
`describeToolOperation: "${endpoint}" is missing mappers or defaults`,
);
}
return {
endpoint,
defaultParameters,
// A dynamic tool's mapper is typed against the union; narrow to this endpoint (sound - the
// runtime mapper produces this endpoint's model).
toApi: (params) => toApiParams(params) as ToolApiParams[E],
fromApi: (api) =>
({
...defaultParameters,
...fromApiParams(api as ToolApiParams[CE]),
}) as TParams,
};
}
@@ -28,24 +28,53 @@ export const pdfiumWasmModulePromise = new Promise<WebAssembly.Module | null>(
},
);
/**
* Compile the WASM without streaming by fetching the whole binary first.
*
* `compileStreaming` requires the response to be served with the
* `application/wasm` MIME type and no incompatible `Content-Encoding`. Under the
* `tauri://` asset protocol (and behind some proxies) those headers aren't
* guaranteed, which makes streaming compilation throw. Fetching the bytes and
* compiling them directly sidesteps the MIME/encoding requirement entirely.
*/
async function compileFromArrayBuffer(): Promise<WebAssembly.Module | null> {
try {
const response = await fetch(pdfiumWasmUrl);
if (!response.ok) {
throw new Error(`Unexpected response ${response.status} for pdfium.wasm`);
}
const bytes = await response.arrayBuffer();
return await WebAssembly.compile(bytes);
} catch (err) {
console.warn("Eager WASM ArrayBuffer compilation failed:", err);
return null;
}
}
export function startEagerWasmCompilation(): void {
if (compilationStarted) return;
compilationStarted = true;
if (
typeof WebAssembly === "object" &&
typeof WebAssembly.compileStreaming === "function"
) {
if (typeof WebAssembly !== "object") {
resolvePromise(null);
return;
}
// Prefer streaming compilation, but fall back to fetching the bytes and
// compiling them directly when streaming isn't available or fails (e.g. wrong
// MIME type / content-encoding under the tauri:// protocol). Resolving null on
// total failure lets pdfiumService fall back to its own instantiation path.
if (typeof WebAssembly.compileStreaming === "function") {
WebAssembly.compileStreaming(fetch(pdfiumWasmUrl))
.then(resolvePromise)
.catch((err) => {
console.warn(
"Eager WASM compilation failed or not supported in this environment:",
"Eager WASM streaming compilation failed, falling back to ArrayBuffer:",
err,
);
resolvePromise(null);
compileFromArrayBuffer().then(resolvePromise);
});
} else {
resolvePromise(null);
compileFromArrayBuffer().then(resolvePromise);
}
}
+2 -16
View File
@@ -1,4 +1,3 @@
import { lazy, Suspense } from "react";
import { Navigate, Route, Routes } from "react-router-dom";
import { Home } from "@portal/views/Home";
import { Users } from "@portal/views/Users";
@@ -13,16 +12,10 @@ import { Components } from "@portal/views/Components";
import { EditorAdmin } from "@portal/views/EditorAdmin";
import { Infrastructure } from "@portal/views/Infrastructure";
import { PortalBillingGate } from "@portal/components/billing/PortalBillingGate";
import { DeveloperDocs } from "@portal/views/DeveloperDocs";
import { Procurement } from "@portal/views/Procurement";
import { VIEW_PATHS, toPortalPath } from "@portal/contexts/ViewContext";
// Lazy so the generated docs manifest (bundled JSON) lands in its own chunk.
const DeveloperDocs = lazy(() =>
import("@portal/views/DeveloperDocs").then((m) => ({
default: m.DeveloperDocs,
})),
);
// The portal mounts as a route-set under /processor/* in the editor app, so these
// child routes are relative to that base: strip the leading slash from the
// logical VIEW_PATHS, and home is the index route. Redirects use toPortalPath
@@ -66,14 +59,7 @@ export function ViewRouter() {
/>
<Route path={rel(VIEW_PATHS.usage)} element={<PortalBillingGate />} />
<Route path={rel(VIEW_PATHS.procurement)} element={<Procurement />} />
<Route
path={rel(VIEW_PATHS.docs)}
element={
<Suspense fallback={null}>
<DeveloperDocs />
</Suspense>
}
/>
<Route path={rel(VIEW_PATHS.docs)} element={<DeveloperDocs />} />
{/* Account-link is now a Settings panel; redirect legacy bookmarks home. */}
<Route
path="account-link"
+22 -50
View File
@@ -13,6 +13,8 @@ import type { TFunction } from "i18next";
import { apiClient } from "@portal/api/http";
import { fromWirePolicy, toWirePolicy } from "@app/policies/codec";
import { runsToActivity, runsToStats } from "@app/policies/runs";
import { policyStep, type PolicyToolStep } from "@app/policies/operations";
import type { ToolEndpoint } from "@app/types/toolApiTypes";
import type {
PolicyDecodedState,
PolicyRunView,
@@ -66,7 +68,7 @@ export interface PolicyConfigDef {
rules: string[];
scopeLabel: string;
fields: PolicyField[];
defaultOperations: WirePipelineStep[];
defaultOperations: PolicyToolStep[];
}
export interface PolicyState {
@@ -128,20 +130,11 @@ export interface CatalogueEntry {
}
/* ──────────────────────────────────────────────────────────────────────── */
/* Tool → endpoint registry */
/* Endpoint display labels */
/* ──────────────────────────────────────────────────────────────────────── */
export const TOOL_ENDPOINTS: Record<string, string> = {
redact: "/api/v1/security/auto-redact",
sanitize: "/api/v1/security/sanitize-pdf",
watermark: "/api/v1/security/add-watermark",
ocr: "/api/v1/misc/ocr-pdf",
flatten: "/api/v1/misc/flatten",
compress: "/api/v1/misc/compress-pdf",
};
/** Values are i18n keys — render with t(). */
export const ENDPOINT_LABELS: Record<string, string> = {
/** i18n keys keyed by {@link ToolEndpoint}; labels stored steps in the detail view. */
export const ENDPOINT_LABELS: Partial<Record<ToolEndpoint, string>> = {
"/api/v1/security/auto-redact": "portal.policies.endpoints.autoRedact",
"/api/v1/security/sanitize-pdf": "portal.policies.endpoints.sanitizePdf",
"/api/v1/security/add-watermark": "portal.policies.endpoints.addWatermark",
@@ -154,7 +147,8 @@ export function humanizeEndpoint(
path: string,
t: (key: string) => string,
): string {
if (ENDPOINT_LABELS[path]) return t(ENDPOINT_LABELS[path]);
const label = ENDPOINT_LABELS[path as ToolEndpoint];
if (label) return t(label);
const last = path.split("/").filter(Boolean).pop() ?? path;
return last
.replace(/-/g, " ")
@@ -230,10 +224,7 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
"portal.policies.config.ingestion.rules.3",
],
scopeLabel: "portal.policies.config.scopeAll",
defaultOperations: [
{ operation: TOOL_ENDPOINTS.ocr, parameters: {} },
{ operation: TOOL_ENDPOINTS.flatten, parameters: {} },
],
defaultOperations: [policyStep("ocr"), policyStep("flatten")],
fields: [
{
label: "portal.policies.config.ingestion.fields.minConfidence",
@@ -260,32 +251,16 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
],
scopeLabel: "portal.policies.config.scopeAll",
defaultOperations: [
{
operation: TOOL_ENDPOINTS.redact,
parameters: {
mode: "automatic",
useRegex: true,
convertPDFToImage: true,
wordsToRedact: DEFAULT_PII_PATTERNS,
},
},
{
operation: TOOL_ENDPOINTS.sanitize,
parameters: {
removeJavaScript: true,
removeEmbeddedFiles: false,
removeMetadata: false,
removeLinks: false,
removeFonts: false,
},
},
{
operation: TOOL_ENDPOINTS.watermark,
// convertPDFToImage bakes the watermark in so it can't be stripped
parameters: {
convertPDFToImage: true,
},
},
// Flatten to image so redactions can't be lifted off.
policyStep("redact", {
useRegex: true,
convertPDFToImage: true,
wordsToRedact: DEFAULT_PII_PATTERNS,
}),
// JavaScript removal only; the tool enables removeEmbeddedFiles by default, so turn it off.
policyStep("sanitize", { removeEmbeddedFiles: false }),
// Bake in via image so it can't be stripped.
policyStep("watermark", { convertPDFToImage: true }),
],
fields: [],
},
@@ -297,10 +272,7 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
"portal.policies.config.compliance.rules.2",
],
scopeLabel: "portal.policies.config.scopeAll",
defaultOperations: [
{ operation: TOOL_ENDPOINTS.sanitize, parameters: {} },
{ operation: TOOL_ENDPOINTS.flatten, parameters: {} },
],
defaultOperations: [policyStep("sanitize"), policyStep("flatten")],
fields: [
{
label: "portal.policies.config.compliance.fields.frameworks",
@@ -343,7 +315,7 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
"portal.policies.config.routing.rules.2",
],
scopeLabel: "portal.policies.config.scopeAll",
defaultOperations: [{ operation: TOOL_ENDPOINTS.compress, parameters: {} }],
defaultOperations: [policyStep("compress")],
fields: [
{
label: "portal.policies.config.routing.fields.destination",
@@ -374,7 +346,7 @@ export const POLICY_CONFIG: Record<string, PolicyConfigDef> = {
"portal.policies.config.retention.rules.2",
],
scopeLabel: "portal.policies.config.scopeAll",
defaultOperations: [{ operation: TOOL_ENDPOINTS.compress, parameters: {} }],
defaultOperations: [policyStep("compress")],
fields: [
{
label: "portal.policies.config.retention.fields.keepFor",
@@ -1,45 +1,8 @@
import { useEffect, useMemo, useRef, useState } from "react";
import { Button, Skeleton, StatusBadge } from "@app/ui";
import { useTranslation } from "react-i18next";
import type { DocsNavSection } from "@portal/api/docs";
/**
* Left-hand documentation nav: a hierarchical accordion. Section ids encode their
* path ("functionality/security" is a child of "functionality"), so sub-sections
* nest under their parent. The root "Overview" section is static (always open, no
* toggle); every other section collapses, and only the branch leading to the
* active doc opens by default. (Search lives in DocsSearch above this.)
*/
// Matches the generator's ROOT_SECTION_ID: the intro section is never collapsible.
const STATIC_SECTION_ID = "overview";
interface NavNode {
section: DocsNavSection;
children: NavNode[];
}
/** Split "a/b/c" → "a/b"; null for a top-level id. */
function parentId(id: string): string | null {
const i = id.lastIndexOf("/");
return i === -1 ? null : id.slice(0, i);
}
/** Build the section tree from the flat, pre-sorted section list. */
function buildTree(sections: DocsNavSection[]): NavNode[] {
const byId = new Map<string, NavNode>(
sections.map((s) => [s.id, { section: s, children: [] }]),
);
const roots: NavNode[] = [];
for (const node of byId.values()) {
const pid = parentId(node.section.id);
const parent = pid ? byId.get(pid) : undefined;
if (parent) parent.children.push(node);
else roots.push(node);
}
return roots;
}
/** Left-hand documentation nav tree; each leaf selects an in-page section. */
export function DocsNav({
sections,
active,
@@ -50,125 +13,50 @@ export function DocsNav({
onSelect: (id: string) => void;
}) {
const { t } = useTranslation();
// Per-section manual open/close, overriding the "active branch only" default.
const [toggled, setToggled] = useState<Record<string, boolean>>({});
const activeRef = useRef<HTMLButtonElement>(null);
const activeSectionId = useMemo(
() => sections.find((s) => s.items.some((i) => i.id === active))?.id,
[sections, active],
);
const tree = useMemo(() => buildTree(sections), [sections]);
// Keep the active item in view when navigating (e.g. via a cross-link).
useEffect(() => {
activeRef.current?.scrollIntoView?.({ block: "nearest" });
}, [active]);
const isOpen = (id: string): boolean => {
if (id === STATIC_SECTION_ID) return true;
// Default-open the branch containing the active doc (self or ancestor).
const onActivePath =
!!activeSectionId &&
(activeSectionId === id || activeSectionId.startsWith(id + "/"));
return toggled[id] ?? onActivePath;
};
const renderNode = (node: NavNode) => {
const { section, children } = node;
const isStatic = section.id === STATIC_SECTION_ID;
const open = isOpen(section.id);
return (
<div key={section.id} className="portal-docs__nav-group">
{isStatic ? (
<div className="portal-docs__nav-head">{section.label}</div>
) : (
<Button
variant="tertiary"
fullWidth
justify="between"
className="portal-docs__nav-head portal-docs__nav-head--button"
aria-expanded={open}
onClick={() =>
setToggled((prev) => ({ ...prev, [section.id]: !open }))
}
rightSection={
<span className="portal-docs__nav-count">
{section.items.length}
</span>
}
>
<span className="portal-docs__nav-head-main">
<span
className={
"portal-docs__nav-chevron" + (open ? " is-open" : "")
}
aria-hidden
>
▸
</span>
<span className="portal-docs__nav-headlabel">
{section.label}
</span>
</span>
</Button>
)}
{open && (
<>
{section.items.length > 0 && (
<ul className="portal-docs__nav-list">
{section.items.map((item) => {
const isActive = item.id === active;
return (
<li key={item.id}>
<Button
ref={isActive ? activeRef : undefined}
variant="tertiary"
justify="start"
fullWidth
className={
"portal-docs__nav-link" +
(isActive ? " is-active" : "")
}
aria-current={isActive ? "page" : undefined}
onClick={() => onSelect(item.id)}
>
<span className="portal-docs__nav-label">
{item.label}
</span>
{item.badge && (
<StatusBadge
tone={item.badge === "New" ? "success" : "info"}
size="sm"
>
{item.badge}
</StatusBadge>
)}
</Button>
</li>
);
})}
</ul>
)}
{children.length > 0 && (
<div className="portal-docs__nav-children">
{children.map(renderNode)}
</div>
)}
</>
)}
</div>
);
};
return (
<nav
className="portal-docs__nav"
aria-label={t("portal.docs.nav.ariaLabel")}
>
{tree.map(renderNode)}
{sections.map((section) => (
<div key={section.id} className="portal-docs__nav-group">
<div className="portal-docs__nav-head">
<span className="portal-docs__nav-icon" aria-hidden>
{section.icon}
</span>
{section.label}
</div>
<ul className="portal-docs__nav-list">
{section.items.map((item) => {
const isActive = item.id === active;
return (
<li key={item.id}>
<Button
variant="tertiary"
justify="start"
fullWidth
className={
"portal-docs__nav-link" + (isActive ? " is-active" : "")
}
aria-current={isActive ? "page" : undefined}
onClick={() => onSelect(item.id)}
>
<span className="portal-docs__nav-label">{item.label}</span>
{item.badge && (
<StatusBadge
tone={item.badge === "New" ? "success" : "info"}
size="sm"
>
{item.badge}
</StatusBadge>
)}
</Button>
</li>
);
})}
</ul>
</div>
))}
</nav>
);
}
@@ -176,16 +64,14 @@ export function DocsNav({
export function DocsNavSkeleton() {
return (
<nav className="portal-docs__nav" aria-hidden>
{Array.from({ length: 5 }).map((_, gi) => (
{Array.from({ length: 4 }).map((_, gi) => (
<div key={gi} className="portal-docs__nav-group">
<Skeleton width="7rem" height="0.75rem" />
{gi === 0 && (
<div className="portal-docs__nav-list">
{Array.from({ length: 4 }).map((_, li) => (
<Skeleton key={li} width="80%" height="0.875rem" />
))}
</div>
)}
<div className="portal-docs__nav-list">
{Array.from({ length: 3 }).map((_, li) => (
<Skeleton key={li} width="80%" height="0.875rem" />
))}
</div>
</div>
))}
</nav>
@@ -1,137 +0,0 @@
import { useEffect, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
import { Button } from "@app/ui";
import type { SearchResult, Segment } from "@portal/docs/search";
/** Render highlighted segments, wrapping matched runs in <mark>. */
function Highlighted({ segments }: { segments: Segment[] }) {
return (
<>
{segments.map((s, i) =>
s.hit ? (
<mark key={i} className="portal-docs__hl">
{s.text}
</mark>
) : (
<span key={i}>{s.text}</span>
),
)}
</>
);
}
/**
* Docs search box + results. While a query is active it shows a ranked list of
* matching docs — each with its section, a highlighted title, and a content
* snippet — that navigates on click (or Enter). Arrow keys move the selection.
*/
export function DocsSearch({
query,
onQueryChange,
results,
onSelect,
}: {
query: string;
onQueryChange: (q: string) => void;
results: SearchResult[];
onSelect: (docId: string) => void;
}) {
const { t } = useTranslation();
// -1 = nothing pre-selected; arrow keys drive this, the mouse uses CSS :hover.
const [activeIndex, setActiveIndex] = useState(-1);
const listRef = useRef<HTMLUListElement>(null);
const hasQuery = query.trim().length > 0;
useEffect(() => setActiveIndex(-1), [query]);
useEffect(() => {
listRef.current
?.querySelector<HTMLElement>('[data-active="true"]')
?.scrollIntoView?.({ block: "nearest" });
}, [activeIndex]);
const onKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
if (e.key === "Escape") {
onQueryChange("");
return;
}
if (!results.length) return;
if (e.key === "ArrowDown") {
e.preventDefault();
setActiveIndex((i) => Math.min(i + 1, results.length - 1));
} else if (e.key === "ArrowUp") {
e.preventDefault();
setActiveIndex((i) => Math.max(i - 1, 0));
} else if (e.key === "Enter") {
e.preventDefault();
const hit = results[activeIndex >= 0 ? activeIndex : 0];
if (hit) onSelect(hit.id);
}
};
return (
<div className="portal-docs__search">
<div className="portal-docs__search-box">
<span className="portal-docs__search-icon" aria-hidden>
⌕
</span>
<input
type="search"
className="portal-docs__search-input"
placeholder={t("portal.docs.search.placeholder")}
value={query}
onChange={(e) => onQueryChange(e.target.value)}
onKeyDown={onKeyDown}
aria-label={t("portal.docs.search.placeholder")}
/>
</div>
{hasQuery && (
<div className="portal-docs__results">
{results.length === 0 ? (
<p className="portal-docs__nav-empty">
{t("portal.docs.search.empty")}
</p>
) : (
<>
<div className="portal-docs__results-count">
{t("portal.docs.search.results", { count: results.length })}
</div>
<ul ref={listRef} className="portal-docs__results-list">
{results.map((r, i) => (
<li key={r.id}>
<Button
variant="tertiary"
fullWidth
justify="start"
className={
"portal-docs__result" +
(i === activeIndex ? " is-active" : "")
}
data-active={i === activeIndex}
onClick={() => onSelect(r.id)}
>
<span className="portal-docs__result-body">
<span className="portal-docs__result-head">
<span className="portal-docs__result-title">
<Highlighted segments={r.titleSegments} />
</span>
<span className="portal-docs__result-section">
{r.sectionLabel}
</span>
</span>
<span className="portal-docs__result-snippet">
<Highlighted segments={r.snippet} />
</span>
</span>
</Button>
</li>
))}
</ul>
</>
)}
</div>
)}
</div>
);
}
@@ -1,80 +0,0 @@
import { useEffect, useState, type RefObject } from "react";
import { useTranslation } from "react-i18next";
import type { Heading } from "@portal/docs/headings";
/**
* "On this page" table of contents. Lists the current doc's H2/H3 headings,
* scrolls the reading pane to a heading on click, and highlights the section
* currently in view (scroll-spy against the pane's scroll container).
*/
export function DocsToc({
headings,
scrollRef,
}: {
headings: Heading[];
scrollRef: RefObject<HTMLElement | null>;
}) {
const { t } = useTranslation();
const [active, setActive] = useState<string>(headings[0]?.slug ?? "");
useEffect(() => {
const root = scrollRef.current;
if (!root || headings.length === 0) return;
setActive(headings[0].slug);
const visible = new Set<string>();
const observer = new IntersectionObserver(
(entries) => {
for (const e of entries) {
if (e.isIntersecting) visible.add(e.target.id);
else visible.delete(e.target.id);
}
// The topmost heading currently within the active zone wins.
const current = headings.find((h) => visible.has(h.slug));
if (current) setActive(current.slug);
},
// Active zone = the top ~30% of the reading pane.
{ root, rootMargin: "0px 0px -70% 0px", threshold: 0 },
);
const els = headings
.map((h) => root.querySelector(`[id="${h.slug}"]`))
.filter((el): el is Element => el !== null);
els.forEach((el) => observer.observe(el));
return () => observer.disconnect();
}, [headings, scrollRef]);
const onSelect = (slug: string) => {
scrollRef.current
?.querySelector(`[id="${slug}"]`)
?.scrollIntoView({ block: "start", behavior: "smooth" });
setActive(slug);
};
return (
<nav className="portal-docs__toc" aria-label={t("portal.docs.toc.title")}>
<div className="portal-docs__toc-title">{t("portal.docs.toc.title")}</div>
<ul className="portal-docs__toc-list">
{headings.map((h) => (
<li key={h.slug}>
<a
href={`#${h.slug}`}
className={
"portal-docs__toc-link" +
(h.level === 3 ? " is-sub" : "") +
(active === h.slug ? " is-active" : "")
}
aria-current={active === h.slug ? "location" : undefined}
onClick={(e) => {
e.preventDefault();
onSelect(h.slug);
}}
>
{h.text}
</a>
</li>
))}
</ul>
</nav>
);
}
@@ -1,131 +0,0 @@
import { isValidElement, useState, type ReactNode } from "react";
import ReactMarkdown, {
defaultUrlTransform,
type Components,
} from "react-markdown";
import remarkGfm from "remark-gfm";
import { Button } from "@app/ui";
import { makeSlugger } from "@portal/docs/headings";
/** Flatten a heading's React children to plain text for its anchor id. */
function childText(node: ReactNode): string {
if (typeof node === "string" || typeof node === "number") return String(node);
if (Array.isArray(node)) return node.map(childText).join("");
if (isValidElement(node)) {
return childText((node.props as { children?: ReactNode }).children);
}
return "";
}
// Keep our internal `doc:` scheme; sanitize every other URL as react-markdown
// would by default (it strips unknown protocols, which would kill doc: links).
function urlTransform(url: string): string {
return url.startsWith("doc:") ? url : defaultUrlTransform(url);
}
/**
* Renders a doc's normalised markdown. Internal cross-doc links carry the
* `doc:` scheme (see the sync transform) and are intercepted here so they
* navigate within the portal instead of leaving the app.
*/
function CopyButton({ text }: { text: string }) {
const [copied, setCopied] = useState(false);
return (
<Button
variant="tertiary"
size="sm"
className="portal-docs__md-copy"
onClick={() =>
void navigator.clipboard.writeText(text).then(() => {
setCopied(true);
setTimeout(() => setCopied(false), 1500);
})
}
>
{copied ? "✓ Copied" : "Copy"}
</Button>
);
}
function buildComponents(
onNavigate: (docId: string) => void,
slug: (text: string) => string,
): Components {
return {
h2: ({ children }) => <h2 id={slug(childText(children))}>{children}</h2>,
h3: ({ children }) => <h3 id={slug(childText(children))}>{children}</h3>,
a: ({ href, children }) => {
if (href?.startsWith("doc:")) {
const id = href.slice(4);
return (
<a
href={`#${id}`}
onClick={(e) => {
e.preventDefault();
onNavigate(id);
}}
>
{children}
</a>
);
}
const external = /^https?:/i.test(href ?? "");
return (
<a
href={href}
target={external ? "_blank" : undefined}
rel={external ? "noopener noreferrer" : undefined}
>
{children}
</a>
);
},
// Eager, not lazy: lazy-loading inside the docs' own scroll container isn't
// reliably triggered, and docs pages have only a handful of images.
img: ({ node: _node, ...props }) => (
<img {...props} className="portal-docs__md-img" />
),
pre: ({ children }) => {
const code = isValidElement(children)
? String(
(children.props as { children?: unknown }).children ?? "",
).replace(/\n$/, "")
: String(children ?? "");
return (
<div className="portal-docs__md-pre">
<pre>{children}</pre>
<CopyButton text={code} />
</div>
);
},
table: ({ children }) => (
<div className="portal-docs__md-tablewrap">
<table>{children}</table>
</div>
),
};
}
export function MarkdownDoc({
markdown,
onNavigate,
}: {
markdown: string;
onNavigate: (docId: string) => void;
}) {
// A fresh de-duping slugger per render; react-markdown invokes h2/h3 in
// document order, so ids line up with the TOC's extractHeadings slugs.
const slug = makeSlugger();
return (
<div className="portal-docs__md">
<ReactMarkdown
remarkPlugins={[remarkGfm]}
urlTransform={urlTransform}
components={buildComponents(onNavigate, slug)}
>
{markdown}
</ReactMarkdown>
</div>
);
}
@@ -0,0 +1,52 @@
import { describe, expect, it, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import { MantineProvider } from "@mantine/core";
import { Tooltip } from "@app/components/shared/Tooltip";
import type { ToolRegistry } from "@app/data/toolsTaxonomy";
import type { WorkingToolStep } from "@app/hooks/tools/shared/toolAutomation";
import { PipelineStepSettings } from "@portal/components/pipelines/PipelineStepSettings";
vi.mock("react-i18next", () => ({
useTranslation: () => ({
t: (key: string, fallback?: string) => fallback ?? key,
}),
}));
// A stand-in tool-settings UI that uses the shared editor Tooltip. The Tooltip
// pulls in the Preferences + Sidebar contexts, which the portal does not mount
// app-wide — so this reproduces the "usePreferences must be used within a
// PreferencesProvider" crash unless PipelineStepSettings supplies them.
function TooltipSettings() {
return (
<Tooltip content="help">
<button type="button">field</button>
</Tooltip>
);
}
const step = {
support: "editable",
toolId: "compress",
params: {},
} as unknown as WorkingToolStep;
const registry = {
compress: { automationSettings: TooltipSettings },
} as unknown as Partial<ToolRegistry>;
describe("PipelineStepSettings", () => {
it("renders reused editor tool settings (which use the shared Tooltip) without app-wide Preferences/Sidebar providers", () => {
expect(() =>
render(
<MantineProvider>
<PipelineStepSettings
step={step}
registry={registry}
onChange={() => {}}
/>
</MantineProvider>,
),
).not.toThrow();
expect(screen.getByText("field")).toBeInTheDocument();
});
});
@@ -1,6 +1,8 @@
import { Suspense } from "react";
import { useTranslation } from "react-i18next";
import { Banner } from "@app/ui";
import { PreferencesProvider } from "@app/contexts/PreferencesContext";
import { SidebarProvider } from "@app/contexts/SidebarContext";
import { type ToolRegistry } from "@app/data/toolsTaxonomy";
import { type ErasedToolParams } from "@app/hooks/tools/shared/toolOperationTypes";
import { type WorkingToolStep } from "@app/hooks/tools/shared/toolAutomation";
@@ -46,14 +48,18 @@ export function PipelineStepSettings({
}
return (
<Suspense fallback={null}>
<Settings
parameters={step.params}
onParameterChange={(key, value) =>
onChange({ ...step.params, [key]: value })
}
disabled={false}
/>
</Suspense>
<PreferencesProvider>
<SidebarProvider>
<Suspense fallback={null}>
<Settings
parameters={step.params}
onParameterChange={(key, value) =>
onChange({ ...step.params, [key]: value })
}
disabled={false}
/>
</Suspense>
</SidebarProvider>
</PreferencesProvider>
);
}
@@ -0,0 +1,134 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
fireEvent,
render as baseRender,
screen,
waitFor,
} from "@testing-library/react";
import { MantineProvider } from "@mantine/core";
import { PolicySetupWizard } from "@portal/components/policies/PolicySetupWizard";
import {
POLICY_CATEGORIES,
POLICY_CONFIG,
type CatalogueEntry,
type DecoratedPolicy,
type PolicySetupResult,
type PipelineStep,
} from "@portal/api/policies";
const render = (ui: Parameters<typeof baseRender>[0]) =>
baseRender(ui, { wrapper: MantineProvider });
// Deterministic i18n: return the fallback when given, else the key. initReactI18next is stubbed
// because the import graph pulls core/i18n.ts, which registers it as a plugin.
vi.mock("react-i18next", () => ({
useTranslation: () => ({
// Second arg is a string fallback in some call sites and an interpolation object in others;
// only treat a string as the fallback.
t: (key: string, fallback?: unknown) =>
typeof fallback === "string" ? fallback : key,
i18n: { changeLanguage: vi.fn() },
}),
initReactI18next: { type: "3rdParty", init: vi.fn() },
}));
const fetchSources = vi.fn();
vi.mock("@portal/api/sources", () => ({
fetchSources: () => fetchSources(),
}));
const CONTINUE = "portal.policies.wizard.actions.continue";
const SAVE_CHANGES = "portal.policies.wizard.actions.saveChanges";
const ENABLE = "portal.policies.wizard.actions.enablePolicy";
const security = POLICY_CATEGORIES.find((c) => c.id === "security")!;
const securityConfig = POLICY_CONFIG.security;
function editEntry(steps: PipelineStep[]): CatalogueEntry {
const policy: DecoratedPolicy = {
category: security,
config: securityConfig,
state: {
configured: true,
status: "active",
sources: ["editor"],
scopeTypes: [],
reviewerEmail: "",
fieldValues: {},
runOn: "upload",
outputMode: "new_version",
outputName: "",
outputNamePosition: "suffix",
maxRetries: 0,
retryDelayMinutes: 0,
backendId: "pol-1",
isDefault: true,
},
steps,
stats: { enforced: 0, dataProcessed: "-", activeFor: "-" },
activity: [],
};
return { category: security, config: securityConfig, policy };
}
/** Advance the wizard from the workflow tab to the settings tab and submit. */
async function submitWizard(saveLabel: string) {
fireEvent.click(await screen.findByRole("button", { name: CONTINUE }));
fireEvent.click(await screen.findByRole("button", { name: saveLabel }));
}
describe("PolicySetupWizard", () => {
beforeEach(() => {
fetchSources.mockResolvedValue({ sources: [] });
});
it("round-trips a saved step's backend params on edit", async () => {
const onSubmit = vi.fn().mockResolvedValue(undefined);
const entry = editEntry([
{
operation: "/api/v1/security/auto-redact",
parameters: { listOfText: "foo\nbar", useRegex: true },
},
]);
render(
<PolicySetupWizard entry={entry} onClose={vi.fn()} onSubmit={onSubmit} />,
);
await submitWizard(SAVE_CHANGES);
await waitFor(() => expect(onSubmit).toHaveBeenCalled());
const result = onSubmit.mock.calls[0][1] as PolicySetupResult;
// Only the saved tool is enabled on edit, and its patterns survive the wire -> UI -> wire trip.
expect(result.steps).toEqual([
expect.objectContaining({
operation: "/api/v1/security/auto-redact",
parameters: expect.objectContaining({ listOfText: "foo\nbar" }),
}),
]);
});
it("seeds the preset chain for a new policy (redact + sanitize on, watermark off)", async () => {
const onSubmit = vi.fn().mockResolvedValue(undefined);
const entry: CatalogueEntry = {
category: security,
config: securityConfig,
policy: null,
};
render(
<PolicySetupWizard entry={entry} onClose={vi.fn()} onSubmit={onSubmit} />,
);
await submitWizard(ENABLE);
await waitFor(() => expect(onSubmit).toHaveBeenCalled());
const result = onSubmit.mock.calls[0][1] as PolicySetupResult;
const endpoints = result.steps.map((s) => s.operation);
expect(endpoints).toEqual([
"/api/v1/security/auto-redact",
"/api/v1/security/sanitize-pdf",
]);
// Redact carries the preset PII patterns as the backend's listOfText.
const redact = result.steps[0].parameters as { listOfText?: string };
expect(redact.listOfText).toBeTruthy();
});
});
@@ -13,23 +13,24 @@ import {
} from "@app/ui";
import { SettingsRow } from "@app/ui/SettingsRow";
import {
TOOL_ENDPOINTS,
humanizeEndpoint,
type CatalogueEntry,
type PipelineStep,
type PolicySetupResult,
} from "@portal/api/policies";
import type { ToolRegistry, ToolRegistryEntry } from "@app/data/toolsTaxonomy";
import {
deserializeToolStep,
serializeStepFromEndpoint,
} from "@app/hooks/tools/shared/toolAutomation";
policyEndpoint,
policyStepFromWire,
policyStepToWire,
type PolicyParams,
type PolicyToolId,
type PolicyToolStep,
} from "@app/policies/operations";
import { fetchSources } from "@portal/api/sources";
import { useAsync } from "@portal/hooks/useAsync";
import { PolicyFieldRow } from "@portal/components/policies/PolicyFieldRow";
import { policyIcon } from "@portal/components/policies/policyIcons";
import { sourceTypeMeta } from "@portal/components/sources/sourceTypes";
import { useToolRegistry } from "@app/contexts/ToolRegistryContext";
import { PolicyRedactConfig } from "@app/components/policies/PolicyRedactConfig";
import { PolicyWatermarkConfig } from "@app/components/policies/PolicyWatermarkConfig";
import "@portal/views/Policies.css";
@@ -47,12 +48,8 @@ interface PolicySetupWizardProps {
type Step = "workflow" | "settings";
/** A configurable tool in the workflow step: whether it runs + its params. */
interface ToolState {
operation: string;
enabled: boolean;
parameters: Record<string, unknown>;
}
/** A policy step plus whether it runs. */
type ToolState = PolicyToolStep & { enabled: boolean };
/** Resolve each field's effective value: saved override, else definition default. */
function resolveFieldValues(
@@ -69,9 +66,8 @@ function resolveFieldValues(
* round-trips); otherwise the category preset's default chain. Each preset step
* starts enabled — the user toggles tools off in the workflow.
*/
// Temporary: tracks which tools start disabled until the tool registry lands in
// the portal and can drive this via registry metadata or a defaultEnabled flag.
const DISABLED_BY_DEFAULT = new Set(["/api/v1/security/add-watermark"]);
// Temporary until the catalogue carries a defaultEnabled flag.
const DISABLED_BY_DEFAULT = new Set<PolicyToolId>(["watermark"]);
/**
* Policy-facing framing for each capability a policy can include. Labels and
@@ -81,43 +77,43 @@ const DISABLED_BY_DEFAULT = new Set(["/api/v1/security/add-watermark"]);
* the humanised endpoint name with no description.
*/
const CAPABILITY_META: Record<
string,
PolicyToolId,
{ labelKey: string; labelEn: string; descKey: string; descEn: string }
> = {
[TOOL_ENDPOINTS.redact]: {
redact: {
labelKey: "portal.policies.wizard.capability.redact.label",
labelEn: "Redact sensitive information",
descKey: "portal.policies.wizard.capability.redact.desc",
descEn:
"Finds and blacks out sensitive details — like Social Security and card numbers — so they can't be read.",
},
[TOOL_ENDPOINTS.sanitize]: {
sanitize: {
labelKey: "portal.policies.wizard.capability.sanitize.label",
labelEn: "Strip active content",
descKey: "portal.policies.wizard.capability.sanitize.desc",
descEn:
"Removes hidden JavaScript so nothing can run automatically when the document is opened.",
},
[TOOL_ENDPOINTS.watermark]: {
watermark: {
labelKey: "portal.policies.wizard.capability.watermark.label",
labelEn: "Apply a watermark",
descKey: "portal.policies.wizard.capability.watermark.desc",
descEn: "Stamps a visible mark (e.g. “Confidential”) across every page.",
},
[TOOL_ENDPOINTS.ocr]: {
ocr: {
labelKey: "portal.policies.wizard.capability.ocr.label",
labelEn: "Make text searchable",
descKey: "portal.policies.wizard.capability.ocr.desc",
descEn: "Runs OCR so scanned pages become selectable, searchable text.",
},
[TOOL_ENDPOINTS.flatten]: {
flatten: {
labelKey: "portal.policies.wizard.capability.flatten.label",
labelEn: "Flatten the document",
descKey: "portal.policies.wizard.capability.flatten.desc",
descEn:
"Merges form fields and annotations into the page so they can't be edited.",
},
[TOOL_ENDPOINTS.compress]: {
compress: {
labelKey: "portal.policies.wizard.capability.compress.label",
labelEn: "Reduce file size",
descKey: "portal.policies.wizard.capability.compress.desc",
@@ -125,29 +121,24 @@ const CAPABILITY_META: Record<
},
};
function seedTools(
entry: CatalogueEntry,
registry: Partial<ToolRegistry>,
): ToolState[] {
function seedTools(entry: CatalogueEntry): ToolState[] {
const savedSteps = entry.policy?.steps ?? [];
const savedByOp = new Map(savedSteps.map((s) => [s.operation, s]));
// Always use defaultOperations as the canonical list so tools added after a
// policy was first saved still appear when editing.
return entry.config.defaultOperations.map((s) => {
const saved = savedByOp.get(s.operation);
const savedByTool = new Map<PolicyToolId, PolicyToolStep>();
for (const wire of savedSteps) {
const step = policyStepFromWire(wire);
if (step) savedByTool.set(step.toolId, step);
}
// defaultOperations is the canonical list (so tools added later still show on edit); a saved
// step's params win over the preset.
return entry.config.defaultOperations.map((preset) => {
const saved = savedByTool.get(preset.toolId);
return {
operation: s.operation,
...(saved ?? preset),
enabled: saved
? true
: savedSteps.length > 0
? false
: !DISABLED_BY_DEFAULT.has(s.operation),
// Saved steps are in the backend contract shape; map them back to the UI
// shape the config controls edit (e.g. `listOfText` -> `wordsToRedact`).
// Presets are already authored in the UI shape, so use them as-is.
parameters: saved
? deserializeToolStep(saved, registry).params
: s.parameters,
: !DISABLED_BY_DEFAULT.has(preset.toolId),
};
});
}
@@ -185,26 +176,12 @@ function PolicySetupWizardBody({
onSubmit: (entry: CatalogueEntry, result: PolicySetupResult) => Promise<void>;
}) {
const { t } = useTranslation();
const { allTools: toolRegistry } = useToolRegistry();
// Portal tool operations are endpoint paths (/api/v1/…), not short registry IDs.
// Build a reverse map so we can look up icons and display names by endpoint.
const registryByEndpoint = useMemo(() => {
const map = new Map<string, ToolRegistryEntry>();
for (const entry of Object.values(toolRegistry)) {
const ep = (entry as ToolRegistryEntry).operationConfig?.endpoint;
if (typeof ep === "string") map.set(ep, entry as ToolRegistryEntry);
}
return map;
}, [toolRegistry]);
const { category, config, policy } = entry;
const isEdit = policy != null;
const [step, setStep] = useState<Step>("workflow");
const [tools, setTools] = useState<ToolState[]>(() =>
seedTools(entry, toolRegistry),
);
const [tools, setTools] = useState<ToolState[]>(() => seedTools(entry));
const [fieldValues, setFieldValues] = useState(() =>
resolveFieldValues(entry),
);
@@ -213,22 +190,11 @@ function PolicySetupWizardBody({
);
const sourcesAsync = useAsync(() => fetchSources(), []);
const availableSources = useMemo(() => {
const backendSources = (sourcesAsync.data?.sources ?? []).filter(
(s) => s.status !== "disabled",
);
const editorSource = {
id: "editor",
name: t("portal.sources.types.editor.label"),
type: "editor",
status: "active" as const,
referenceCount: 0,
referencingPolicies: [],
config: [],
docsTotal: null,
};
return [editorSource, ...backendSources];
}, [sourcesAsync.data, t]);
const availableSources = useMemo(
() =>
(sourcesAsync.data?.sources ?? []).filter((s) => s.status !== "disabled"),
[sourcesAsync.data],
);
// Document-type scoping has no UI; preserve any saved scope on edit and
// default new policies to all document types.
const [scopeTypes] = useState<string[]>(policy?.state.scopeTypes ?? []);
@@ -256,9 +222,20 @@ function PolicySetupWizardBody({
const enabledTools = useMemo(() => tools.filter((tl) => tl.enabled), [tools]);
function patchTool(operation: string, patch: Partial<ToolState>) {
function setToolEnabled(toolId: PolicyToolId, enabled: boolean) {
setTools((prev) =>
prev.map((tl) => (tl.operation === operation ? { ...tl, ...patch } : tl)),
prev.map((tl) => (tl.toolId === toolId ? { ...tl, enabled } : tl)),
);
}
function setToolParams<Id extends PolicyToolId>(
toolId: Id,
params: PolicyParams<Id>,
) {
setTools((prev) =>
prev.map((tl) =>
tl.toolId === toolId ? ({ ...tl, params } as ToolState) : tl,
),
);
}
@@ -277,11 +254,8 @@ function PolicySetupWizardBody({
}
setError(null);
setSubmitting(true);
// Map each tool's UI-shaped params (e.g. redact's `wordsToRedact`) into the
// backend step contract (e.g. `listOfText`) via its `toApiParams`; saving the
// UI shape verbatim would drop those fields and the step would run with none.
const steps: PipelineStep[] = enabledTools.map((tl) =>
serializeStepFromEndpoint(tl.operation, tl.parameters, toolRegistry),
policyStepToWire(tl),
);
try {
await onSubmit(entry, {
@@ -386,20 +360,16 @@ function PolicySetupWizardBody({
<Card padding="none">
<div className="portal-policies__capabilities">
{tools.map((tl) => {
const meta = CAPABILITY_META[tl.operation];
const meta = CAPABILITY_META[tl.toolId];
const label = meta
? t(meta.labelKey, meta.labelEn)
: (registryByEndpoint.get(tl.operation)?.name ??
humanizeEndpoint(tl.operation, t));
: humanizeEndpoint(policyEndpoint(tl.toolId), t);
const description = meta
? t(meta.descKey, meta.descEn)
: undefined;
const hasConfig =
tl.operation === TOOL_ENDPOINTS.redact ||
tl.operation === TOOL_ENDPOINTS.watermark;
return (
<div
key={tl.operation}
key={tl.toolId}
className="portal-policies__capability"
data-on={tl.enabled || undefined}
>
@@ -411,27 +381,27 @@ function PolicySetupWizardBody({
size="sm"
checked={tl.enabled}
onChange={(checked) =>
patchTool(tl.operation, { enabled: checked })
setToolEnabled(tl.toolId, checked)
}
label=""
/>
}
/>
{tl.enabled && hasConfig && (
{tl.enabled && (
<div className="portal-policies__capability-config">
{tl.operation === TOOL_ENDPOINTS.redact && (
{tl.toolId === "redact" && (
<PolicyRedactConfig
parameters={tl.parameters}
onChange={(parameters) =>
patchTool(tl.operation, { parameters })
parameters={tl.params}
onChange={(params) =>
setToolParams("redact", params)
}
/>
)}
{tl.operation === TOOL_ENDPOINTS.watermark && (
{tl.toolId === "watermark" && (
<PolicyWatermarkConfig
parameters={tl.parameters}
onChange={(parameters) =>
patchTool(tl.operation, { parameters })
parameters={tl.params}
onChange={(params) =>
setToolParams("watermark", params)
}
/>
)}
@@ -475,9 +445,8 @@ function PolicySetupWizardBody({
{t("portal.policies.wizard.sources.loading")}
</p>
) : (
// The editor is always an available source (unconditionally prepended
// to availableSources), so the list is never empty — no "no sources"
// state exists.
// The backend always returns the editor as a virtual source, so the
// loaded list is never empty - no "no sources" state exists.
<div className="portal-policies__sources">
{availableSources.map((src) => (
// A selectable multi-line tile (icon + name + type + check).
@@ -20,6 +20,10 @@ export interface NavEntry {
externalUrl?: string;
}
// Developer docs has no built-in portal page yet, so the tab opens the hosted docs
// site in a new tab rather than routing to an empty page.
const DEVELOPER_DOCS_URL = "https://docs.stirlingpdf.com/";
// Sidebar nav groups. This is a flavor seam: the SaaS build shadows this file to
// drop sections not yet shipped there (see src/portal-saas/components/sidebarGroups).
export const GROUP_PRIMARY: NavEntry[] = [{ id: "home", icon: <HomeIcon /> }];
@@ -36,5 +40,5 @@ export const GROUP_OPERATIONAL: NavEntry[] = [
export const GROUP_PLATFORM: NavEntry[] = [
{ id: "infrastructure", icon: <InfrastructureIcon /> },
{ id: "usage", icon: <UsageIcon /> },
{ id: "docs", icon: <DocsIcon /> },
{ id: "docs", icon: <DocsIcon />, externalUrl: DEVELOPER_DOCS_URL },
];
@@ -30,7 +30,7 @@ export const VIEW_LABELS: Record<ViewId, string> = {
components: "Components",
infrastructure: "Infrastructure",
usage: "Usage & Billing",
docs: "Documentation",
docs: "Developer Docs",
procurement: "Procurement",
settings: "Settings",
};
@@ -1,47 +0,0 @@
import { describe, expect, it } from "vitest";
import { extractHeadings, slugify } from "@portal/docs/headings";
describe("slugify", () => {
it("lowercases, hyphenates, and trims punctuation", () => {
expect(slugify("How it Works!")).toBe("how-it-works");
expect(slugify(" Trailing & spaces ")).toBe("trailing-spaces");
});
});
describe("extractHeadings", () => {
it("extracts H2/H3 only, with slugs matching the rendered ids", () => {
const md = [
"# Page Title",
"## How it Works",
"text",
"### Sub Section",
"#### Too Deep",
"## Operations",
].join("\n");
expect(extractHeadings(md)).toEqual([
{ level: 2, text: "How it Works", slug: "how-it-works" },
{ level: 3, text: "Sub Section", slug: "sub-section" },
{ level: 2, text: "Operations", slug: "operations" },
]);
});
it("de-duplicates repeated heading text into unique slugs", () => {
const md = ["## What Changed", "### What Changed", "## What Changed"].join(
"\n",
);
expect(extractHeadings(md).map((h) => h.slug)).toEqual([
"what-changed",
"what-changed-1",
"what-changed-2",
]);
});
it("ignores headings inside fenced code and strips inline marks", () => {
const md = ["```", "## not a heading", "```", "## `Code` and *em*"].join(
"\n",
);
expect(extractHeadings(md)).toEqual([
{ level: 2, text: "Code and em", slug: "code-and-em" },
]);
});
});
@@ -1,48 +0,0 @@
/**
* Heading extraction for the "On this page" table of contents. The same
* `slugify` is used here and in MarkdownDoc's heading renderer, so the TOC links
* and the rendered heading ids always match.
*/
export interface Heading {
/** 2 or 3 (H2/H3). */
level: number;
text: string;
slug: string;
}
/** "How it Works!" → "how-it-works" (the base id, before de-duplication). */
export function slugify(text: string): string {
return text
.toLowerCase()
.trim()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "");
}
/**
* A stateful slugger that de-duplicates: repeated heading text gets `-1`, `-2`, …
* The TOC extraction and MarkdownDoc's heading renderer each make one and feed it
* headings in document order, so their slugs (and thus link ↔ id) always match.
*/
export function makeSlugger(): (text: string) => string {
const seen = new Map<string, number>();
return (text: string) => {
const base = slugify(text) || "section";
const n = seen.get(base) ?? 0;
seen.set(base, n + 1);
return n === 0 ? base : `${base}-${n}`;
};
}
/** Extract H2/H3 headings from a doc body, skipping fenced code blocks. */
export function extractHeadings(markdown: string): Heading[] {
const noCode = markdown.replace(/^(```|~~~)[\s\S]*?^\1[ \t]*$/gm, "");
const slug = makeSlugger();
const headings: Heading[] = [];
for (const m of noCode.matchAll(/^ {0,3}(#{2,3})[ \t]+(.+?)[ \t]*#*$/gm)) {
const text = m[2].replace(/[`*_]/g, "").trim();
if (text) headings.push({ level: m[1].length, text, slug: slug(text) });
}
return headings;
}
@@ -1,38 +0,0 @@
/**
* Runtime accessors over the generated docs manifest. This is the only module
* that imports the (large) JSON, so lazy-loading the docs view keeps it in its
* own chunk. Regenerate the JSON with `npm run docs:sync`.
*/
// Imported as a raw string (not a JSON module) so tsc doesn't infer a ~half-MB
// literal type; parsed once here into the typed manifest.
import manifestRaw from "@portal/generated/docsManifest.json?raw";
import type {
DocEntry,
DocsManifest,
DocsNavSection,
} from "@portal/docs/manifest/transform";
const manifest = JSON.parse(manifestRaw) as DocsManifest;
/** Provenance of the current manifest (repo + ref it was generated from). */
export const docsSource = manifest.source;
/** The auto-sorted nav tree (sections → items). */
export function loadDocsNav(): DocsNavSection[] {
return manifest.nav;
}
/** A single doc by id, or undefined if it isn't in the manifest. */
export function loadDoc(id: string): DocEntry | undefined {
return manifest.docs[id];
}
/** Every doc, for building the search index. */
export function allDocs(): DocEntry[] {
return Object.values(manifest.docs);
}
/** The first doc id (first item of the first section) — the default landing. */
export function firstDocId(): string | undefined {
return manifest.nav[0]?.items[0]?.id;
}
@@ -1,208 +0,0 @@
import { describe, expect, it } from "vitest";
import {
buildManifest,
convertAdmonitions,
demoteHeadings,
docIdForPath,
humanize,
parseFrontmatter,
resolveRelative,
rewriteReferences,
sectionIcon,
stripJsxTags,
stripMdxImports,
stripRedundantH1,
type CategoryMap,
type RawDoc,
} from "@portal/docs/manifest/transform";
const OPTS = {
repo: "Owner/Repo",
ref: "main",
root: "docs",
siteBaseUrl: "https://docs.example.com",
};
describe("parseFrontmatter", () => {
it("splits scalar YAML frontmatter from the body", () => {
const { data, body } = parseFrontmatter(
"---\ntitle: OCR\nsidebar_position: 7\n---\n# Heading\ntext",
);
expect(data.title).toBe("OCR");
expect(data.sidebar_position).toBe(7);
expect(body).toBe("# Heading\ntext");
});
it("returns the whole content as body when there is no frontmatter", () => {
const { data, body } = parseFrontmatter("# Just a doc\nbody");
expect(data).toEqual({});
expect(body).toBe("# Just a doc\nbody");
});
it("normalises CRLF line endings", () => {
const { data } = parseFrontmatter("---\r\nid: x\r\n---\r\nbody");
expect(data.id).toBe("x");
});
});
describe("id + label helpers", () => {
it("slugifies nested paths", () => {
expect(docIdForPath("Configuration/OCR.md")).toBe("configuration/ocr");
expect(docIdForPath("Getting Started.md")).toBe("getting-started");
});
it("humanises file/dir names", () => {
expect(humanize("Getting-Started.md")).toBe("Getting Started");
});
it("picks a section icon from the label", () => {
expect(sectionIcon("Configuration")).toBe("⚙");
expect(sectionIcon("Totally Unknown")).toBe("◇");
});
});
describe("MDX normalisation", () => {
it("converts admonitions to titled blockquotes", () => {
const out = convertAdmonitions(":::tip Upgrading?\nread this\n:::");
expect(out).toContain("> **💡 Tip: Upgrading?**");
expect(out).toContain("> read this");
});
it("strips import/export statements", () => {
const out = stripMdxImports("import Tabs from '@theme/Tabs';\n# Keep");
expect(out).toBe("# Keep");
});
it("removes JSX component tags but keeps inner content", () => {
expect(
stripJsxTags("<Tabs>\n<TabItem value='a'>keep</TabItem>\n</Tabs>"),
).toContain("keep");
expect(stripJsxTags("<TabItem>x</TabItem>")).not.toMatch(/<TabItem/);
});
it("demotes body H1 to H2 but leaves code comments alone", () => {
const md = "# Title\n\n```bash\n# a shell comment\n```";
const out = demoteHeadings(md);
expect(out).toContain("## Title");
expect(out).toContain("# a shell comment");
});
it("strips a leading H1 that duplicates the page title", () => {
expect(stripRedundantH1("# OCR\nbody", "OCR")).toBe("body");
expect(stripRedundantH1("# Other\nbody", "OCR")).toBe("# Other\nbody");
});
});
describe("resolveRelative", () => {
it("collapses ./ and ../ against a base dir", () => {
expect(resolveRelative("Configuration", "./OCR.md")).toBe(
"Configuration/OCR.md",
);
expect(
resolveRelative("Configuration", "../Functionality/Compare.md"),
).toBe("Functionality/Compare.md");
});
});
describe("rewriteReferences", () => {
const ctx = {
dir: "Configuration",
// Keys are lowercased file paths (spaces preserved), as buildManifest builds them.
pathToId: new Map([
[
"configuration/system and security",
"configuration/system-and-security",
],
]),
rawBase: "https://raw.example.com/Owner/Repo/main",
siteBaseUrl: "https://docs.example.com",
};
it("rewrites resolvable internal links to the doc: scheme (decoded + case-insensitive)", () => {
const out = rewriteReferences(
"see [sec](./System%20and%20Security.md)",
ctx,
"docs",
);
expect(out).toBe("see [sec](doc:configuration/system-and-security)");
});
it("falls back to the live docs site for unresolved internal links", () => {
const out = rewriteReferences("[x](./Missing.md)", ctx, "docs");
expect(out).toBe("[x](https://docs.example.com/Configuration/Missing)");
});
it("leaves absolute and anchor links untouched", () => {
const md = "[a](https://x.com) and [b](#top)";
expect(rewriteReferences(md, ctx, "docs")).toBe(md);
});
it("rewrites relative images to absolute raw URLs", () => {
expect(rewriteReferences("![a](./img/x.png)", ctx, "docs")).toBe(
"![a](https://raw.example.com/Owner/Repo/main/docs/Configuration/img/x.png)",
);
expect(rewriteReferences("![a](/img/y.png)", ctx, "docs")).toBe(
"![a](https://raw.example.com/Owner/Repo/main/static/img/y.png)",
);
});
it("does not rewrite inside fenced code blocks", () => {
const md = "```\n[x](./y.md)\n```";
expect(rewriteReferences(md, ctx, "docs")).toBe(md);
});
});
describe("buildManifest", () => {
const rawDocs: RawDoc[] = [
{
relPath: "Getting Started.md",
content: "---\nsidebar_position: 0\n---\nintro",
},
{
relPath: "Configuration/OCR.md",
content: "---\ntitle: OCR\nsidebar_position: 7\n---\n# OCR\nbody",
},
{
relPath: "Configuration/DATABASE.md",
content: "---\nsidebar_position: 1\n---\n# Database\nsee [ocr](./OCR.md)",
},
];
const categories: CategoryMap = {
Configuration: { label: "Configuration", position: 5 },
};
it("auto-sorts sections (root Overview first, then by category position)", () => {
const m = buildManifest(rawDocs, categories, OPTS);
expect(m.nav.map((s) => s.id)).toEqual(["overview", "configuration"]);
expect(m.nav[0].label).toBe("Overview");
expect(m.nav[1].label).toBe("Configuration");
});
it("orders nav items by sidebar_position", () => {
const m = buildManifest(rawDocs, categories, OPTS);
const config = m.nav.find((s) => s.id === "configuration")!;
expect(config.items.map((i) => i.id)).toEqual([
"configuration/database",
"configuration/ocr",
]);
});
it("derives titles from frontmatter, heading, then filename", () => {
const m = buildManifest(rawDocs, categories, OPTS);
expect(m.docs["configuration/ocr"].title).toBe("OCR");
expect(m.docs["getting-started"].title).toBe("Getting Started");
});
it("resolves cross-doc links and records source/edit urls", () => {
const m = buildManifest(rawDocs, categories, OPTS);
expect(m.docs["configuration/database"].markdown).toContain(
"[ocr](doc:configuration/ocr)",
);
expect(m.docs["configuration/ocr"].sourcePath).toBe(
"docs/Configuration/OCR.md",
);
expect(m.docs["configuration/ocr"].editUrl).toBe(
"https://github.com/Owner/Repo/blob/main/docs/Configuration/OCR.md",
);
});
});
@@ -1,481 +0,0 @@
/**
* Pure transforms that turn the Docusaurus docs repo into the portal docs
* manifest. No I/O and no external deps so `tsx` (the sync CLI) and vitest can
* both use it. The sync CLI does the fetching; this module does the shaping.
*
* The auto-sort rules:
* - every directory that directly holds markdown becomes a nav section,
* labelled + ordered by its `_category_.json` (root files → "Overview"),
* - each `.md`/`.mdx` file becomes a nav item, ordered by frontmatter
* `sidebar_position` then title,
* - Docusaurus MDX is normalised to plain GitHub-flavoured markdown that
* react-markdown can render (admonitions, JSX, relative links, images).
*/
/* ──────────────────────────────────────────────────────────────────────── */
/* Manifest shape (mirrored structurally by @portal/api/docs) */
/* ──────────────────────────────────────────────────────────────────────── */
export interface DocsNavItem {
id: string;
label: string;
badge?: string;
}
export interface DocsNavSection {
id: string;
label: string;
icon: string;
items: DocsNavItem[];
}
export interface DocEntry {
id: string;
title: string;
description?: string;
section: string;
markdown: string;
sourcePath: string;
editUrl: string;
}
export interface DocsManifest {
source: { repo: string; ref: string; root: string };
nav: DocsNavSection[];
docs: Record<string, DocEntry>;
}
/** One markdown file read from the repo, before shaping. */
export interface RawDoc {
/** Posix path relative to the docs root, e.g. "Configuration/OCR.md". */
relPath: string;
content: string;
}
/** `_category_.json` contents, keyed by posix dir path relative to docs root. */
export type CategoryMap = Record<string, { label?: string; position?: number }>;
export interface BuildOptions {
repo: string;
ref: string;
/** Docs root within the repo, e.g. "docs". */
root: string;
/** Live docs site base, used as the fallback for unresolved internal links. */
siteBaseUrl: string;
}
/* ──────────────────────────────────────────────────────────────────────── */
/* Small helpers */
/* ──────────────────────────────────────────────────────────────────────── */
const SECTION_ICONS: Array<[RegExp, string]> = [
[/overview|getting started|start/i, "▶"],
[/config/i, "⚙"],
[/function|tool|feature/i, "▤"],
[/install|deploy/i, "⤓"],
[/migrat|upgrade/i, "⇄"],
[/security|sign|auth/i, "🛡"],
[/convert/i, "⇋"],
[/page/i, "▦"],
[/api|develop/i, "{ }"],
];
/** A single-glyph icon for a section, chosen from its label. */
export function sectionIcon(label: string): string {
for (const [re, glyph] of SECTION_ICONS) if (re.test(label)) return glyph;
return "◇";
}
/** "Getting-Started_Guide" → "Getting Started Guide". */
export function humanize(name: string): string {
return name
.replace(/\.mdx?$/i, "")
.replace(/[-_]+/g, " ")
.replace(/\s+/g, " ")
.trim()
.replace(/\b\w/g, (c) => c.toUpperCase());
}
/** Lowercase, hyphenated, url-safe id for a path segment. */
export function slugifySegment(name: string): string {
return name
.replace(/\.mdx?$/i, "")
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "");
}
/** Docs-root-relative posix path → stable doc id, e.g. "configuration/ocr". */
export function docIdForPath(relPath: string): string {
return relPath.split("/").map(slugifySegment).filter(Boolean).join("/");
}
/** Posix dirname ("" for a root-level file). */
export function dirOf(relPath: string): string {
const i = relPath.lastIndexOf("/");
return i === -1 ? "" : relPath.slice(0, i);
}
/* ──────────────────────────────────────────────────────────────────────── */
/* Frontmatter */
/* ──────────────────────────────────────────────────────────────────────── */
export interface Frontmatter {
data: Record<string, string | number>;
body: string;
}
/** Split leading `--- ... ---` YAML frontmatter (scalar keys only) from body. */
export function parseFrontmatter(content: string): Frontmatter {
const normalised = content.replace(/\r\n/g, "\n");
if (!normalised.startsWith("---\n")) return { data: {}, body: normalised };
const end = normalised.indexOf("\n---", 4);
if (end === -1) return { data: {}, body: normalised };
const raw = normalised.slice(4, end);
const rest = normalised.slice(end + 4).replace(/^\n/, "");
const data: Record<string, string | number> = {};
for (const line of raw.split("\n")) {
const m = /^([A-Za-z0-9_]+):\s*(.*)$/.exec(line);
if (!m) continue;
let value: string | number = m[2].trim().replace(/^["']|["']$/g, "");
if (/^-?\d+(\.\d+)?$/.test(value)) value = Number(value);
data[m[1]] = value;
}
return { data, body: rest };
}
/** First `# H1` heading text in a body, if any. */
export function firstHeading(body: string): string | undefined {
const m = /^#\s+(.+?)\s*$/m.exec(stripCodeFences(body));
return m ? m[1].trim() : undefined;
}
/** Blank out fenced code blocks so heading/link scans ignore their contents. */
function stripCodeFences(md: string): string {
return md.replace(/^(```|~~~)[\s\S]*?^\1\s*$/gm, "");
}
/* ──────────────────────────────────────────────────────────────────────── */
/* Body normalisation (MDX → plain markdown) */
/* ──────────────────────────────────────────────────────────────────────── */
/** Run `fn` over the non-fenced-code spans of `md`, leaving code blocks intact. */
function mapOutsideCode(md: string, fn: (text: string) => string): string {
const parts = md.split(/(^(?:```|~~~)[\s\S]*?^(?:```|~~~)\s*$)/gm);
return parts.map((part, i) => (i % 2 === 0 ? fn(part) : part)).join("");
}
const ADMONITION_META: Record<string, { icon: string; label: string }> = {
tip: { icon: "💡", label: "Tip" },
note: { icon: "📝", label: "Note" },
info: { icon: "ℹ️", label: "Info" },
warning: { icon: "⚠️", label: "Warning" },
caution: { icon: "⚠️", label: "Caution" },
danger: { icon: "🚫", label: "Danger" },
};
/** `:::tip Title\n…\n:::` → a blockquote with a bold titled first line. */
export function convertAdmonitions(md: string): string {
const re = /^:::(\w+)[ \t]*(.*)\n([\s\S]*?)^:::[ \t]*$/gm;
return md.replace(re, (_all, type: string, title: string, body: string) => {
const meta = ADMONITION_META[type.toLowerCase()] ?? {
icon: "•",
label: humanize(type),
};
const heading = title.trim()
? `${meta.label}: ${title.trim()}`
: meta.label;
const quoted = body
.replace(/\s+$/, "")
.split("\n")
.map((l) => (l ? `> ${l}` : ">"))
.join("\n");
return `> **${meta.icon} ${heading}**\n>\n${quoted}\n`;
});
}
/** Drop MDX `import`/`export` statement lines (outside code). */
export function stripMdxImports(md: string): string {
return md
.split("\n")
.filter((l) => !/^\s*(import|export)\s.+from\s.+;?\s*$/.test(l))
.filter((l) => !/^\s*import\s+['"][^'"]+['"];?\s*$/.test(l))
.join("\n");
}
/** Remove JSX component tags (Capitalised), keeping any inner content. */
export function stripJsxTags(md: string): string {
return md.replace(/<\/?[A-Z][A-Za-z0-9.]*(?:\s[^>]*?)?\/?>/g, "");
}
/** Resolve a relative posix path against a base dir, collapsing `.`/`..`. */
export function resolveRelative(baseDir: string, target: string): string {
const stack = baseDir ? baseDir.split("/") : [];
for (const seg of target.split("/")) {
if (seg === "" || seg === ".") continue;
if (seg === "..") stack.pop();
else stack.push(seg);
}
return stack.join("/");
}
interface LinkContext {
dir: string;
pathToId: Map<string, string>;
rawBase: string;
siteBaseUrl: string;
}
/** Split "path#anchor" → [path, "#anchor" | ""]. */
function splitAnchor(target: string): [string, string] {
const i = target.indexOf("#");
return i === -1 ? [target, ""] : [target.slice(0, i), target.slice(i)];
}
/** Percent-decode a link path, tolerating malformed escapes. */
function decodePath(p: string): string {
try {
return decodeURIComponent(p);
} catch {
return p;
}
}
/**
* Look up the doc id for a relative link target. Keys in pathToId are lowercased
* so links work whether they use the filename or a slug, in any case, and with
* percent-encoded spaces (the repo links to "System%20and%20Security").
*/
function resolveDocId(ctx: LinkContext, rawPath: string): string | undefined {
const resolved = resolveRelative(ctx.dir, decodePath(rawPath)).toLowerCase();
const noExt = resolved.replace(/\.mdx?$/i, "");
const candidates = [
resolved,
noExt,
`${noExt.replace(/\/$/, "")}/index`,
noExt.replace(/\/index$/i, ""),
];
for (const c of candidates) {
const id = ctx.pathToId.get(c);
if (id) return id;
}
return undefined;
}
/** Rewrite a single markdown link target to a portal-usable href. */
function rewriteLinkTarget(ctx: LinkContext, target: string): string {
const trimmed = target.trim();
if (/^(https?:|mailto:|tel:|#|doc:)/i.test(trimmed)) return trimmed;
const [path, anchor] = splitAnchor(trimmed);
if (!path) return trimmed;
const id = resolveDocId(ctx, path);
if (id) return `doc:${id}`;
// Unresolved internal link → fall back to the live docs site (encode spaces).
const slug = resolveRelative(ctx.dir, decodePath(path)).replace(
/\.mdx?$/i,
"",
);
const encoded = slug.split("/").map(encodeURIComponent).join("/");
return `${ctx.siteBaseUrl}/${encoded}${anchor}`;
}
/** Rewrite a relative image src to an absolute raw-content URL. */
function rewriteImageSrc(ctx: LinkContext, src: string, root: string): string {
const trimmed = src.trim();
if (/^(https?:|data:)/i.test(trimmed)) return trimmed;
if (trimmed.startsWith("/")) return `${ctx.rawBase}/static${trimmed}`;
const resolved = resolveRelative(`${root}/${ctx.dir}`, trimmed);
return `${ctx.rawBase}/${resolved}`;
}
/** Rewrite markdown links + images (outside code) to portal/absolute targets. */
export function rewriteReferences(
md: string,
ctx: LinkContext,
root: string,
): string {
return mapOutsideCode(md, (text) => {
// Images first so their `!` prefix isn't eaten by the link pattern.
let out = text.replace(
/!\[([^\]]*)\]\(([^)\s]+)([^)]*)\)/g,
(_m, alt: string, src: string, tail: string) =>
`![${alt}](${rewriteImageSrc(ctx, src, root)}${tail})`,
);
out = out.replace(
/(^|[^!])\[([^\]]+)\]\(([^)\s]+)([^)]*)\)/g,
(_m, pre: string, label: string, href: string, tail: string) =>
`${pre}[${label}](${rewriteLinkTarget(ctx, href)}${tail})`,
);
return out;
});
}
/** Drop a leading `# H1` whose text equals the page title (avoids a dup head). */
export function stripRedundantH1(md: string, title: string): string {
const m = /^\s*#\s+(.+?)\s*(\n|$)/.exec(md);
if (m && m[1].trim().toLowerCase() === title.trim().toLowerCase()) {
return md.slice(m[0].length).replace(/^\n+/, "");
}
return md;
}
/** Demote body `# H1` headings to `## H2` so the page title is the sole H1. */
export function demoteHeadings(md: string): string {
return mapOutsideCode(md, (text) => text.replace(/^# (?=\S)/gm, "## "));
}
/* ──────────────────────────────────────────────────────────────────────── */
/* Section ordering */
/* ──────────────────────────────────────────────────────────────────────── */
const ROOT_SECTION_ID = "overview";
/** Composite order key: `_category_.json.position` down the dir path. */
function sectionOrderKey(dir: string, categories: CategoryMap): number[] {
if (dir === "") return [-1];
const key: number[] = [];
const segs = dir.split("/");
for (let i = 0; i < segs.length; i++) {
const sub = segs.slice(0, i + 1).join("/");
key.push(categories[sub]?.position ?? 999);
}
return key;
}
function compareKeys(a: number[], b: number[]): number {
const n = Math.max(a.length, b.length);
for (let i = 0; i < n; i++) {
const d = (a[i] ?? 0) - (b[i] ?? 0);
if (d !== 0) return d;
}
return 0;
}
/* ──────────────────────────────────────────────────────────────────────── */
/* Build */
/* ──────────────────────────────────────────────────────────────────────── */
interface ShapedDoc extends DocEntry {
navLabel: string;
sidebarPosition: number;
orderKey: number[];
sectionLabel: string;
}
/** Turn raw docs + category metadata into the full portal docs manifest. */
export function buildManifest(
rawDocs: RawDoc[],
categories: CategoryMap,
opts: BuildOptions,
): DocsManifest {
const rawBase = `https://raw.githubusercontent.com/${opts.repo}/${opts.ref}`;
const editBase = `https://github.com/${opts.repo}/blob/${opts.ref}`;
// First pass: assign stable ids so links between docs can resolve. Keys are
// lowercased (case-insensitive link matching); both with and without ext.
const pathToId = new Map<string, string>();
for (const doc of rawDocs) {
const id = docIdForPath(doc.relPath);
const lower = doc.relPath.toLowerCase();
pathToId.set(lower, id);
pathToId.set(lower.replace(/\.mdx?$/i, ""), id);
}
const shaped: ShapedDoc[] = rawDocs.map((doc) => {
const dir = dirOf(doc.relPath);
const { data, body } = parseFrontmatter(doc.content);
const title =
(typeof data.title === "string" && data.title) ||
firstHeading(body) ||
humanize(doc.relPath.split("/").pop() ?? doc.relPath);
const navLabel =
(typeof data.sidebar_label === "string" && data.sidebar_label) || title;
const description =
typeof data.description === "string" ? data.description : undefined;
const ctx: LinkContext = {
dir,
pathToId,
rawBase,
siteBaseUrl: opts.siteBaseUrl.replace(/\/$/, ""),
};
let markdown = body;
markdown = stripMdxImports(markdown);
markdown = convertAdmonitions(markdown);
markdown = stripJsxTags(markdown);
markdown = rewriteReferences(markdown, ctx, opts.root);
markdown = stripRedundantH1(markdown, title);
markdown = demoteHeadings(markdown).trim();
const sectionId = dir === "" ? ROOT_SECTION_ID : docIdForPath(dir);
const sectionLabel =
dir === ""
? "Overview"
: (categories[dir]?.label ?? humanize(dir.split("/").pop() ?? dir));
return {
id: docIdForPath(doc.relPath),
title,
navLabel,
description,
section: sectionId,
markdown,
sourcePath: `${opts.root}/${doc.relPath}`,
editUrl: `${editBase}/${opts.root}/${doc.relPath}`,
sidebarPosition:
typeof data.sidebar_position === "number" ? data.sidebar_position : 999,
orderKey: sectionOrderKey(dir, categories),
sectionLabel,
};
});
// Group into sections and order everything deterministically.
const bySection = new Map<string, ShapedDoc[]>();
for (const doc of shaped) {
const list = bySection.get(doc.section) ?? [];
list.push(doc);
bySection.set(doc.section, list);
}
const nav: DocsNavSection[] = [...bySection.entries()]
.map(([id, docs]) => {
const first = docs[0];
const items = [...docs]
.sort(
(a, b) =>
a.sidebarPosition - b.sidebarPosition ||
a.navLabel.localeCompare(b.navLabel),
)
.map<DocsNavItem>((d) => ({ id: d.id, label: d.navLabel }));
return {
id,
label: first.sectionLabel,
icon: sectionIcon(first.sectionLabel),
items,
_key: first.orderKey,
};
})
.sort(
(a, b) => compareKeys(a._key, b._key) || a.label.localeCompare(b.label),
)
.map(({ _key, ...section }) => section);
const docs: Record<string, DocEntry> = {};
for (const d of shaped) {
docs[d.id] = {
id: d.id,
title: d.title,
description: d.description,
section: d.section,
markdown: d.markdown,
sourcePath: d.sourcePath,
editUrl: d.editUrl,
};
}
return {
source: { repo: opts.repo, ref: opts.ref, root: opts.root },
nav,
docs,
};
}
@@ -1,100 +0,0 @@
import { describe, expect, it } from "vitest";
import {
buildSnippet,
highlight,
searchDocs,
toPlainText,
type SearchDoc,
} from "@portal/docs/search";
const DOCS: SearchDoc[] = [
{
id: "ocr",
title: "OCR Guide",
sectionLabel: "Configuration",
text: "Stirling PDF uses Tesseract for its text recognition and language packs.",
},
{
id: "docker",
title: "Docker Install",
sectionLabel: "Installation",
text: "Run Stirling with docker compose up to start the container.",
},
{
id: "ranky",
title: "Something else",
sectionLabel: "Misc",
text: "docker docker docker appears many times in the body here.",
},
];
describe("toPlainText", () => {
it("strips headings, links, inline code, and code fences", () => {
const md =
"# Title\n\nSee [the guide](doc:x) and run `npm i`.\n\n```bash\nnpm run build\n```";
const out = toPlainText(md);
expect(out).toContain("Title");
expect(out).toContain("the guide");
expect(out).toContain("npm i");
expect(out).toContain("npm run build"); // code text kept, fences dropped
expect(out).not.toContain("#");
expect(out).not.toContain("```");
expect(out).not.toContain("](");
});
});
describe("highlight", () => {
it("splits text into hit/non-hit segments", () => {
expect(highlight("Hello world", ["world"])).toEqual([
{ text: "Hello ", hit: false },
{ text: "world", hit: true },
]);
});
it("is safe against regex metacharacters in terms", () => {
expect(() => highlight("a (b) c", ["("])).not.toThrow();
});
});
describe("buildSnippet", () => {
it("windows around the first match and marks it", () => {
const text =
"lorem ipsum ".repeat(20) + "the TARGET keyword " + "dolor ".repeat(20);
const segs = buildSnippet(text, ["target"]);
expect(segs.some((s) => s.hit && /target/i.test(s.text))).toBe(true);
// Windowed, so it should be far shorter than the full text.
expect(segs.map((s) => s.text).join("").length).toBeLessThan(text.length);
});
});
describe("searchDocs", () => {
it("returns nothing for an empty query", () => {
expect(searchDocs(DOCS, " ")).toEqual([]);
});
it("matches body content, not just titles (with a snippet)", () => {
const res = searchDocs(DOCS, "tesseract");
expect(res.map((r) => r.id)).toEqual(["ocr"]);
expect(res[0].snippet.some((s) => s.hit && /tesseract/i.test(s.text))).toBe(
true,
);
});
it("ranks title matches above body matches", () => {
const res = searchDocs(DOCS, "docker");
// "Docker Install" (title hit) outranks "Something else" (body-only hits).
expect(res[0].id).toBe("docker");
expect(res.map((r) => r.id)).toContain("ranky");
});
it("requires every term to match (AND)", () => {
expect(searchDocs(DOCS, "docker compose").map((r) => r.id)).toEqual([
"docker",
]);
expect(searchDocs(DOCS, "docker tesseract")).toEqual([]);
});
it("does not throw on regex-special-character queries", () => {
expect(() => searchDocs(DOCS, "a(b")).not.toThrow();
});
});
-153
View File
@@ -1,153 +0,0 @@
/**
* Full-text search over the docs manifest. Pure + dependency-free so it's unit
* testable. Indexes each doc's title + plaintext body; ranks title matches above
* body matches; returns highlighted title + a content snippet per hit.
*/
export interface SearchDoc {
id: string;
title: string;
sectionLabel: string;
/** Plaintext body (markdown stripped), original case. */
text: string;
}
/** A run of result text, flagged when it matches a query term (for <mark>). */
export interface Segment {
text: string;
hit: boolean;
}
export interface SearchResult {
id: string;
title: string;
sectionLabel: string;
titleSegments: Segment[];
snippet: Segment[];
score: number;
}
/** Strip markdown/MDX down to readable plaintext for indexing + snippets. */
export function toPlainText(md: string): string {
return md
.replace(/^(```|~~~).*$/gm, " ") // fence delimiters (keep the code text)
.replace(/`([^`]+)`/g, "$1") // inline code
.replace(/!\[[^\]]*\]\([^)]*\)/g, " ") // images
.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") // links → their text
.replace(/^\s{0,3}>\s?/gm, "") // blockquotes
.replace(/^\s{0,3}#{1,6}\s+/gm, "") // headings
.replace(/^\s*[-*+]\s+/gm, "") // list bullets
.replace(/[*_~]/g, "") // emphasis marks
.replace(/\|/g, " ") // table pipes
.replace(/\s+/g, " ") // collapse whitespace
.trim();
}
function escapeRegExp(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function countOccurrences(haystack: string, needle: string): number {
let count = 0;
let i = haystack.indexOf(needle);
while (i !== -1) {
count++;
i = haystack.indexOf(needle, i + needle.length);
}
return count;
}
/** Split `text` into segments, flagging any run that matches a query term. */
export function highlight(text: string, terms: string[]): Segment[] {
const cleaned = terms.map(escapeRegExp).filter(Boolean);
if (!cleaned.length) return [{ text, hit: false }];
const re = new RegExp(`(${cleaned.join("|")})`, "gi");
const segments: Segment[] = [];
let last = 0;
let m: RegExpExecArray | null;
while ((m = re.exec(text)) !== null) {
if (m.index > last) {
segments.push({ text: text.slice(last, m.index), hit: false });
}
segments.push({ text: m[0], hit: true });
last = m.index + m[0].length;
if (m.index === re.lastIndex) re.lastIndex++; // guard against zero-width
}
if (last < text.length) segments.push({ text: text.slice(last), hit: false });
return segments.length ? segments : [{ text, hit: false }];
}
/** Build a ~context-window snippet around the earliest term match. */
export function buildSnippet(
text: string,
terms: string[],
radius = 90,
): Segment[] {
const lower = text.toLowerCase();
let pos = -1;
for (const term of terms) {
const i = lower.indexOf(term);
if (i !== -1 && (pos === -1 || i < pos)) pos = i;
}
if (pos === -1) {
const head = text.slice(0, radius * 2);
return highlight(head + (text.length > head.length ? "…" : ""), terms);
}
let start = Math.max(0, pos - radius);
let end = Math.min(text.length, pos + radius);
// Snap to word boundaries so we don't slice mid-word.
if (start > 0) {
const space = text.indexOf(" ", start);
if (space !== -1 && space < pos) start = space + 1;
}
if (end < text.length) {
const space = text.lastIndexOf(" ", end);
if (space > pos) end = space;
}
let snippet = text.slice(start, end).trim();
if (start > 0) snippet = "…" + snippet;
if (end < text.length) snippet = snippet + "…";
return highlight(snippet, terms);
}
/**
* Rank docs against a query. A doc matches when every term appears in its title
* or body; title hits score highest.
*/
export function searchDocs(
docs: SearchDoc[],
query: string,
limit = 40,
): SearchResult[] {
const terms = query.trim().toLowerCase().split(/\s+/).filter(Boolean);
if (!terms.length) return [];
const results: SearchResult[] = [];
for (const doc of docs) {
const titleLower = doc.title.toLowerCase();
const textLower = doc.text.toLowerCase();
const matchesAll = terms.every(
(term) => titleLower.includes(term) || textLower.includes(term),
);
if (!matchesAll) continue;
let score = 0;
for (const term of terms) {
if (titleLower.includes(term)) score += 10;
if (titleLower.startsWith(term)) score += 5;
score += Math.min(countOccurrences(textLower, term), 5);
}
results.push({
id: doc.id,
title: doc.title,
sectionLabel: doc.sectionLabel,
titleSegments: highlight(doc.title, terms),
snippet: buildSnippet(doc.text, terms),
score,
});
}
results.sort((a, b) => b.score - a.score || a.title.localeCompare(b.title));
return results.slice(0, limit);
}
File diff suppressed because one or more lines are too long
+23 -3
View File
@@ -4,13 +4,33 @@
* only builds seed data for the MSW handlers and tests.
*/
import type { PolicyRunView, WirePolicy } from "@app/policies/types";
import { POLICY_CONFIG } from "@portal/api/policies";
import type {
PolicyRunView,
WirePipelineStep,
WirePolicy,
} from "@app/policies/types";
/* ──────────────────────────────────────────────────────────────────────── */
/* Seed data — real backend wire format */
/* ──────────────────────────────────────────────────────────────────────── */
// Literal wire steps (not derived from the catalogue) so this fixtures module stays independent of
// @portal/api/policies and its heavy tool-operation import graph.
const SECURITY_STEPS: WirePipelineStep[] = [
{
operation: "/api/v1/security/auto-redact",
parameters: {
listOfText: "",
useRegex: true,
convertPDFToImage: true,
},
},
{
operation: "/api/v1/security/sanitize-pdf",
parameters: { removeJavaScript: true },
},
];
export function seedPolicies(): WirePolicy[] {
return [
{
@@ -19,7 +39,7 @@ export function seedPolicies(): WirePolicy[] {
owner: "security@acme.com",
enabled: true,
trigger: null,
steps: POLICY_CONFIG.security.defaultOperations,
steps: SECURITY_STEPS,
output: {
type: "inline",
options: {
@@ -1,254 +1,73 @@
/* Full-height docs surface: the sidebar and reading pane each scroll on their
own inside the portal shell's scroll container, so a 70+ item nav no longer
drags the whole page around. */
.portal-docs {
display: grid;
grid-template-columns: 16rem minmax(0, 1fr);
height: 100%;
min-height: 0;
overflow: hidden;
grid-template-columns: 15rem minmax(0, 1fr);
gap: 2rem;
padding: 1.5rem;
max-width: 84rem;
margin: 0 auto;
align-items: start;
}
/* With an "On this page" TOC, add a right-hand column. */
.portal-docs--with-toc {
grid-template-columns: 16rem minmax(0, 1fr) 15rem;
}
/* Not enough room for three columns → drop the TOC. */
@media (max-width: 72rem) {
.portal-docs--with-toc {
grid-template-columns: 16rem minmax(0, 1fr);
}
.portal-docs__toc-col {
display: none;
@media (max-width: 60rem) {
.portal-docs {
grid-template-columns: 1fr;
gap: 1.25rem;
}
}
.portal-docs--empty {
display: block;
padding: 3rem 1.5rem;
}
/* ── Left nav (own scroll column) ──────────────────────────────────────── */
/* ── Left nav ──────────────────────────────────────────────────────────── */
.portal-docs__sidebar {
min-height: 0;
height: 100%;
overflow-y: auto;
border-right: 1px solid var(--color-border-light);
padding: 1.25rem 0;
position: sticky;
top: 1.5rem;
align-self: start;
}
@media (max-width: 60rem) {
.portal-docs__sidebar {
position: static;
}
}
.portal-docs__nav {
display: flex;
flex-direction: column;
gap: 0.25rem;
}
/* Search */
.portal-docs__search {
margin-bottom: 0.75rem;
padding: 0 0.75rem;
}
.portal-docs__search-box {
position: relative;
}
.portal-docs__search-icon {
position: absolute;
left: 0.625rem;
top: 50%;
transform: translateY(-50%);
font-size: 0.9375rem;
color: var(--color-text-4);
pointer-events: none;
}
.portal-docs__search-input {
width: 100%;
padding: 0.4rem 0.6rem 0.4rem 1.9rem;
font-size: 0.8125rem;
color: var(--color-text-1);
background: var(--color-bg-subtle);
border: 1px solid var(--color-border-light);
border-radius: var(--radius-md);
outline: none;
}
.portal-docs__search-input:focus {
border-color: var(--color-blue);
}
.portal-docs__nav-empty {
font-size: 0.8125rem;
color: var(--color-text-4);
padding: 0.5rem 0.75rem;
margin: 0;
}
/* ── Search results ────────────────────────────────────────────────────── */
.portal-docs__results {
margin-top: 0.5rem;
}
.portal-docs__results-count {
font-size: 0.6875rem;
font-weight: 500;
color: var(--color-text-4);
padding: 0 0.75rem 0.5rem;
}
.portal-docs__results-list {
list-style: none;
margin: 0;
padding: 0;
display: flex;
flex-direction: column;
}
/* Hairline divider between results for clear, calm separation. */
.portal-docs__results-list li + li {
border-top: 1px solid var(--color-border-light);
}
.portal-docs__result {
height: auto;
padding: 0.5rem 0.75rem;
border-radius: 0;
}
.portal-docs__result:hover,
.portal-docs__result.is-active {
background: var(--color-bg-hover);
}
.portal-docs__result-body {
display: flex;
flex-direction: column;
gap: 0.125rem;
width: 100%;
min-width: 0;
text-align: left;
white-space: normal;
}
/* Title + section share one line; the title truncates before the section. */
.portal-docs__result-head {
display: flex;
align-items: baseline;
gap: 0.4rem;
min-width: 0;
}
.portal-docs__result-title {
font-size: 0.8125rem;
font-weight: 600;
color: var(--color-text-1);
line-height: 1.3;
min-width: 0;
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
}
.portal-docs__result-section {
flex-shrink: 0;
font-size: 0.6875rem;
color: var(--color-text-4);
}
.portal-docs__result-snippet {
font-size: 0.75rem;
line-height: 1.4;
color: var(--color-text-4);
display: -webkit-box;
-webkit-line-clamp: 1;
-webkit-box-orient: vertical;
overflow: hidden;
}
/* Subtle match emphasis — coloured text, not a filled block. */
.portal-docs__hl {
color: var(--color-blue);
font-weight: 600;
background: none;
gap: 1.25rem;
}
.portal-docs__nav-group {
display: flex;
flex-direction: column;
gap: 0.0625rem;
gap: 0.375rem;
}
/* Section label: shared typography for the static Overview label and the
collapsible accordion headers below. */
.portal-docs__nav-head {
padding: 0.4rem 0.75rem 0.2rem;
display: flex;
align-items: center;
gap: 0.5rem;
font-size: 0.6875rem;
font-weight: 700;
letter-spacing: 0.06em;
text-transform: uppercase;
color: var(--color-text-4);
padding: 0 0.5rem;
}
/* Collapsible section header (Overview stays a static label). */
.portal-docs__nav-head--button:hover {
background: var(--color-bg-hover);
color: var(--color-text-2);
}
.portal-docs__nav-head-main {
display: flex;
.portal-docs__nav-icon {
display: inline-flex;
align-items: center;
gap: 0.4rem;
min-width: 0;
}
.portal-docs__nav-headlabel {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.portal-docs__nav-chevron {
font-size: 0.625rem;
color: var(--color-text-4);
transition: transform var(--motion-fast);
}
.portal-docs__nav-chevron.is-open {
transform: rotate(90deg);
}
.portal-docs__nav-count {
font-size: 0.625rem;
font-weight: 600;
color: var(--color-text-4);
justify-content: center;
width: 1.125rem;
height: 1.125rem;
font-size: 0.6875rem;
border-radius: var(--radius-sm);
background: var(--color-bg-subtle);
border-radius: var(--radius-pill);
padding: 0.05rem 0.4rem;
}
/* Space consecutive sections apart (not before the first). */
.portal-docs__nav-group + .portal-docs__nav-group {
margin-top: 0.4rem;
}
/* Nested sub-sections: indented under their parent with a guide line. */
.portal-docs__nav-children {
margin: 0.125rem 0 0.25rem 0.85rem;
padding-left: 0.4rem;
border-left: 1px solid var(--color-border-light);
display: flex;
flex-direction: column;
gap: 0.0625rem;
color: var(--color-text-3);
}
.portal-docs__nav-list {
list-style: none;
margin: 0 0 0.375rem;
margin: 0;
padding: 0;
display: flex;
flex-direction: column;
@@ -261,10 +80,10 @@
justify-content: space-between;
gap: 0.5rem;
width: 100%;
padding: 0.35rem 0.75rem;
padding: 0.375rem 0.5rem;
border: none;
background: transparent;
border-radius: 0;
border-radius: var(--radius-md);
font-size: 0.8125rem;
color: var(--color-text-3);
cursor: pointer;
@@ -291,129 +110,10 @@
white-space: nowrap;
}
/* Mobile-only nav toggle (a drawer button); hidden on desktop. */
.portal-docs__nav-toggle {
display: none;
}
/* ── Reading pane (own scroll column) ──────────────────────────────────── */
/* ── Content shell ─────────────────────────────────────────────────────── */
.portal-docs__content {
min-width: 0;
min-height: 0;
height: 100%;
overflow-y: auto;
padding: 1.5rem 2rem 4rem;
}
.portal-docs__content-inner {
max-width: 46rem;
margin: 0 auto;
}
/* ── On this page (right rail) ─────────────────────────────────────────── */
.portal-docs__toc-col {
min-height: 0;
height: 100%;
overflow-y: auto;
border-left: 1px solid var(--color-border-light);
padding: 1.5rem 0.75rem;
}
.portal-docs__toc {
position: sticky;
top: 0;
}
.portal-docs__toc-title {
font-size: 0.6875rem;
font-weight: 700;
letter-spacing: 0.06em;
text-transform: uppercase;
color: var(--color-text-4);
padding: 0 0.5rem 0.5rem;
}
.portal-docs__toc-list {
list-style: none;
margin: 0;
padding: 0;
display: flex;
flex-direction: column;
gap: 0.0625rem;
}
.portal-docs__toc-link {
display: block;
padding: 0.25rem 0.5rem;
border-left: 2px solid transparent;
font-size: 0.8125rem;
line-height: 1.35;
color: var(--color-text-3);
text-decoration: none;
transition: color var(--motion-fast);
}
.portal-docs__toc-link.is-sub {
padding-left: 1.25rem;
font-size: 0.75rem;
}
.portal-docs__toc-link:hover {
color: var(--color-text-1);
}
.portal-docs__toc-link.is-active {
color: var(--color-blue);
border-left-color: var(--color-blue);
font-weight: 600;
}
/* Scroll-to-heading lands just below the pane top, not flush against it. */
.portal-docs__md h2,
.portal-docs__md h3 {
scroll-margin-top: 1rem;
}
/* ── Small screens: nav collapses into a drawer above the content ──────── */
@media (max-width: 60rem) {
.portal-docs {
grid-template-columns: 1fr;
grid-template-rows: auto auto minmax(0, 1fr);
}
.portal-docs__nav-toggle {
display: flex;
align-items: center;
gap: 0.5rem;
margin: 0.75rem 1rem 0;
padding: 0.5rem 0.75rem;
font-size: 0.8125rem;
font-weight: 600;
color: var(--color-text-1);
background: var(--color-bg-subtle);
border: 1px solid var(--color-border-light);
border-radius: var(--radius-md);
cursor: pointer;
}
.portal-docs__sidebar {
display: none;
height: auto;
max-height: 60vh;
border-right: none;
border-bottom: 1px solid var(--color-border-light);
}
.portal-docs__sidebar.is-open {
display: block;
}
.portal-docs__content {
padding: 1.25rem 1.25rem 3rem;
}
}
.portal-docs__section {
@@ -823,160 +523,3 @@
font-size: 0.75rem;
color: var(--color-text-4);
}
/* ── Markdown content ──────────────────────────────────────────────────── */
.portal-docs__md {
font-size: 0.9375rem;
line-height: 1.65;
color: var(--color-text-2);
}
.portal-docs__md h1,
.portal-docs__md h2,
.portal-docs__md h3,
.portal-docs__md h4 {
color: var(--color-text-1);
font-weight: 650;
line-height: 1.25;
margin: 1.75rem 0 0.75rem;
}
.portal-docs__md h2 {
font-size: 1.3125rem;
padding-bottom: 0.3rem;
border-bottom: 1px solid var(--color-border-light);
}
.portal-docs__md h3 {
font-size: 1.0625rem;
}
.portal-docs__md h4 {
font-size: 0.9375rem;
}
.portal-docs__md > :first-child {
margin-top: 0;
}
.portal-docs__md p,
.portal-docs__md ul,
.portal-docs__md ol {
margin: 0 0 0.875rem;
}
.portal-docs__md li {
margin: 0.25rem 0;
}
.portal-docs__md a {
color: var(--color-blue);
text-decoration: none;
}
.portal-docs__md a:hover {
text-decoration: underline;
}
.portal-docs__md code {
font-family: var(--font-mono, monospace);
font-size: 0.85em;
background: var(--color-bg-subtle);
padding: 0.1em 0.35em;
border-radius: var(--radius-sm);
}
.portal-docs__md blockquote {
margin: 0 0 0.875rem;
padding: 0.125rem 0.875rem;
border-left: 3px solid var(--color-blue);
background: var(--color-bg-subtle);
border-radius: 0 var(--radius-md) var(--radius-md) 0;
color: var(--color-text-3);
}
.portal-docs__md blockquote p {
margin: 0.5rem 0;
}
.portal-docs__md-img {
max-width: 100%;
height: auto;
border-radius: var(--radius-md);
border: 1px solid var(--color-border-light);
}
.portal-docs__md hr {
border: none;
border-top: 1px solid var(--color-border-light);
margin: 1.5rem 0;
}
.portal-docs__md-pre {
position: relative;
margin: 0 0 0.875rem;
}
.portal-docs__md-pre pre {
margin: 0;
padding: 0.875rem 4rem 0.875rem 0.875rem;
background: var(--color-bg-subtle);
border: 1px solid var(--color-border-light);
border-radius: var(--radius-md);
overflow-x: auto;
font-size: 0.8125rem;
line-height: 1.5;
}
.portal-docs__md-pre pre code {
background: none;
padding: 0;
font-size: inherit;
}
.portal-docs__md-copy {
position: absolute;
top: 0.5rem;
right: 0.5rem;
}
.portal-docs__md-tablewrap {
overflow-x: auto;
margin: 0 0 0.875rem;
}
.portal-docs__md-tablewrap table {
border-collapse: collapse;
width: 100%;
font-size: 0.8125rem;
}
.portal-docs__md-tablewrap th,
.portal-docs__md-tablewrap td {
border: 1px solid var(--color-border-light);
padding: 0.4rem 0.625rem;
text-align: left;
}
.portal-docs__md-tablewrap th {
background: var(--color-bg-subtle);
font-weight: 600;
}
.portal-docs__source {
margin-top: 2rem;
padding-top: 1rem;
border-top: 1px solid var(--color-border-light);
font-size: 0.8125rem;
}
.portal-docs__source a {
color: var(--color-text-3);
text-decoration: none;
}
.portal-docs__source a:hover {
color: var(--color-blue);
text-decoration: underline;
}
@@ -1,24 +0,0 @@
import type { Meta, StoryObj } from "@storybook/react-vite";
import { DeveloperDocs } from "@portal/views/DeveloperDocs";
// Renders the real docs manifest (generated by scripts/sync-portal-docs.mts), so
// this story is a live view of the auto-built Documentation browser. The view
// fills its container's height (independent sidebar/content scroll), so the
// decorator gives it a viewport-height frame.
const meta: Meta<typeof DeveloperDocs> = {
title: "Portal/DeveloperDocs/View",
component: DeveloperDocs,
parameters: { layout: "fullscreen" },
decorators: [
(Story) => (
<div style={{ height: "100vh" }}>
<Story />
</div>
),
],
};
export default meta;
type Story = StoryObj<typeof DeveloperDocs>;
export const Default: Story = {};
@@ -1,80 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
import { MantineProvider } from "@mantine/core";
import { MemoryRouter } from "react-router-dom";
import type { ReactElement } from "react";
// Deterministic labels; the view + DocsNav + MarkdownDoc all use useTranslation.
vi.mock("react-i18next", () => ({
useTranslation: () => ({
// 2nd arg may be a default string or i18next interpolation options ({count}).
t: (key: string, opts?: unknown) => (typeof opts === "string" ? opts : key),
i18n: { changeLanguage: vi.fn() },
}),
}));
import { DeveloperDocs } from "@portal/views/DeveloperDocs";
const renderDocs = (ui: ReactElement) =>
render(
<MemoryRouter initialEntries={["/processor/docs"]}>
<MantineProvider>{ui}</MantineProvider>
</MemoryRouter>,
);
describe("DeveloperDocs — markdown browser over the generated manifest", () => {
it("keeps Overview static (open, no toggle) and other sections collapsed", () => {
renderDocs(<DeveloperDocs />);
expect(screen.getByRole("searchbox")).toBeInTheDocument();
// Overview is static: its items show, and it has no toggle button.
expect(
screen.getByRole("button", { name: "Production Deployment Guide" }),
).toBeInTheDocument();
expect(
screen.queryByRole("button", { name: /^Overview/i }),
).not.toBeInTheDocument();
// Other sections collapse, so their items are hidden until expanded.
expect(
screen.queryByRole("button", { name: "Kubernetes Guide" }),
).not.toBeInTheDocument();
expect(
screen.getByText(/locally hosted web application/i),
).toBeInTheDocument();
});
it("expands a collapsible section when its header is clicked", () => {
renderDocs(<DeveloperDocs />);
fireEvent.click(screen.getByRole("button", { name: /Installation/i }));
expect(
screen.getByRole("button", { name: "Kubernetes Guide" }),
).toBeInTheDocument();
});
it("searches doc content (not just titles), shows a snippet, and navigates", async () => {
renderDocs(<DeveloperDocs />);
// "Tesseract" appears in the OCR doc body but in no doc title — a result
// whose snippet contains it proves full-text (content) search.
fireEvent.change(screen.getByRole("searchbox"), {
target: { value: "Tesseract" },
});
const hits = await screen.findAllByRole("button", { name: /Tesseract/i });
expect(hits.length).toBeGreaterThan(0);
fireEvent.click(hits[0]);
await waitFor(() =>
expect(
screen.queryByText(/locally hosted web application/i),
).not.toBeInTheDocument(),
);
});
it("follows an internal doc: link inside the rendered markdown", async () => {
renderDocs(<DeveloperDocs />);
// The Getting Started body links to the Migration guide via the doc: scheme.
fireEvent.click(screen.getByRole("link", { name: /Migration Guide/i }));
await waitFor(() =>
expect(
screen.queryByText(/locally hosted web application/i),
).not.toBeInTheDocument(),
);
});
});
@@ -1,140 +1,97 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useState } from "react";
import { useTranslation } from "react-i18next";
import { useLocation, useNavigate } from "react-router-dom";
import { Button, EmptyState } from "@app/ui";
import { DocsNav } from "@portal/components/docs/DocsNav";
import { DocsSearch } from "@portal/components/docs/DocsSearch";
import { DocsSection } from "@portal/components/docs/DocsSection";
import { DocsToc } from "@portal/components/docs/DocsToc";
import { MarkdownDoc } from "@portal/components/docs/MarkdownDoc";
import { extractHeadings } from "@portal/docs/headings";
import { EmptyState } from "@app/ui";
import { useTier } from "@portal/contexts/TierContext";
import { useAsync, useSectionFlags } from "@portal/hooks/useAsync";
import {
allDocs,
firstDocId,
loadDoc,
loadDocsNav,
} from "@portal/docs/manifest/registry";
import { searchDocs, toPlainText, type SearchDoc } from "@portal/docs/search";
fetchDocsContent,
fetchDocsNav,
type DocsContent,
type DocsNavSection,
} from "@portal/api/docs";
import { DocsNav, DocsNavSkeleton } from "@portal/components/docs/DocsNav";
import { GettingStartedSection } from "@portal/components/docs/GettingStartedSection";
import { AuthenticationSection } from "@portal/components/docs/AuthenticationSection";
import { RateLimitsSection } from "@portal/components/docs/RateLimitsSection";
import { EndpointReferenceSection } from "@portal/components/docs/EndpointReferenceSection";
import { ErrorsSection } from "@portal/components/docs/ErrorsSection";
import { WebhooksSection } from "@portal/components/docs/WebhooksSection";
import { SdksSection } from "@portal/components/docs/SdksSection";
import { ComponentsSection } from "@portal/components/docs/ComponentsSection";
import { PlaybooksSection } from "@portal/components/docs/PlaybooksSection";
import { SkillsSection } from "@portal/components/docs/SkillsSection";
import "@portal/views/DeveloperDocs.css";
/**
* Developer Docs — a markdown browser over the docs manifest generated from the
* Stirling docs repo (see scripts/sync-portal-docs.mts). The nav is auto-sorted
* from the repo's folders + frontmatter; content is the repo markdown, and the
* search box does full-text search across every doc.
*/
/** Renders the content pane for the active nav leaf against fetched content. */
function DocsContentPane({
active,
content,
}: {
active: string;
content: DocsContent;
}) {
switch (active) {
case "authentication":
return <AuthenticationSection />;
case "rate-limits":
return <RateLimitsSection rateLimit={content.rateLimit} />;
case "endpoints":
return <EndpointReferenceSection />;
case "errors":
return <ErrorsSection errors={content.errors} />;
case "webhooks":
return <WebhooksSection />;
case "sdk-overview":
return <SdksSection sdks={content.sdks} />;
case "component-library":
return <ComponentsSection components={content.components} />;
case "recipes":
return <PlaybooksSection playbooks={content.playbooks} />;
case "skill-catalog":
return <SkillsSection skills={content.skills} />;
default:
return (
<GettingStartedSection
samples={content.quickstartSamples}
response={content.quickstartResponse}
/>
);
}
}
export function DeveloperDocs() {
const { t } = useTranslation();
const { hash } = useLocation();
const navigate = useNavigate();
const contentRef = useRef<HTMLElement>(null);
const [navOpen, setNavOpen] = useState(false);
const [query, setQuery] = useState("");
const { tier } = useTier();
const [active, setActive] = useState("quickstart");
const nav = useMemo(() => loadDocsNav(), []);
const fallback = useMemo(() => firstDocId(), []);
const navState = useAsync<DocsNavSection[]>(() => fetchDocsNav(), []);
const { data: nav } = navState;
const { isLoading, isEmpty } = useSectionFlags(navState);
// Full-text index over every doc's plaintext body (built once).
const index = useMemo<SearchDoc[]>(() => {
const labels = new Map(nav.map((s) => [s.id, s.label]));
return allDocs().map((d) => ({
id: d.id,
title: d.title,
sectionLabel: labels.get(d.section) ?? "",
text: toPlainText(d.markdown),
}));
}, [nav]);
const results = useMemo(() => searchDocs(index, query), [index, query]);
const searching = query.trim().length > 0;
// Deep-link support: the active doc id lives in the URL hash.
const hashId = decodeURIComponent(hash.replace(/^#/, ""));
const activeId = hashId && loadDoc(hashId) ? hashId : fallback;
const doc = activeId ? loadDoc(activeId) : undefined;
const section = useMemo(
() => nav.find((s) => s.items.some((i) => i.id === activeId)),
[nav, activeId],
const { data: content } = useAsync<DocsContent>(
() => fetchDocsContent(tier),
[tier],
);
// "On this page" headings for the current doc.
const headings = useMemo(
() => (doc ? extractHeadings(doc.markdown) : []),
[doc],
);
// Navigating closes the mobile drawer, clears the search, and resets the pane.
const onSelect = useCallback(
(id: string) => {
navigate({ hash: id });
setNavOpen(false);
setQuery("");
},
[navigate],
);
useEffect(() => {
contentRef.current?.scrollTo?.({ top: 0 });
}, [activeId]);
if (nav.length === 0 || !doc) {
return (
<div className="portal-docs portal-docs--empty">
<EmptyState
title={t("portal.docs.nav.empty.title")}
description={t("portal.docs.nav.empty.description")}
/>
</div>
);
}
const hasToc = headings.length > 0;
return (
<div className={"portal-docs" + (hasToc ? " portal-docs--with-toc" : "")}>
<Button
variant="tertiary"
className="portal-docs__nav-toggle"
aria-expanded={navOpen}
onClick={() => setNavOpen((open) => !open)}
leftSection={<span aria-hidden>☰</span>}
>
{t("portal.docs.browse")}
</Button>
<aside className={"portal-docs__sidebar" + (navOpen ? " is-open" : "")}>
<DocsSearch
query={query}
onQueryChange={setQuery}
results={results}
onSelect={onSelect}
/>
{!searching && (
<DocsNav sections={nav} active={activeId ?? ""} onSelect={onSelect} />
<div className="portal-docs">
<aside className="portal-docs__sidebar">
{isLoading && <DocsNavSkeleton />}
{isEmpty && (
<EmptyState
size="compact"
title={t("portal.docs.nav.empty.title")}
description={t("portal.docs.nav.empty.description")}
/>
)}
{nav && nav.length > 0 && (
<DocsNav sections={nav} active={active} onSelect={setActive} />
)}
</aside>
<main className="portal-docs__content" ref={contentRef}>
<div className="portal-docs__content-inner">
<DocsSection
id={doc.id}
eyebrow={section?.label ?? ""}
title={doc.title}
lead={doc.description}
>
<MarkdownDoc markdown={doc.markdown} onNavigate={onSelect} />
<div className="portal-docs__source">
<a href={doc.editUrl} target="_blank" rel="noopener noreferrer">
{t("portal.docs.viewSource")}
</a>
</div>
</DocsSection>
</div>
<main className="portal-docs__content">
{content && <DocsContentPane active={active} content={content} />}
</main>
{hasToc && (
<aside className="portal-docs__toc-col">
<DocsToc headings={headings} scrollRef={contentRef} />
</aside>
)}
</div>
);
}
@@ -1,6 +1,7 @@
import { MultiSelect } from "@app/ui/MultiSelect";
import { useTranslation } from "react-i18next";
import { PII_PRESETS } from "@app/data/policyDefinitions";
import type { RedactParameters } from "@app/hooks/tools/redact/useRedactParameters";
/** The set of preset regexes — used to separate preset words from custom ones. */
export const PRESET_PATTERNS = new Set(PII_PRESETS.map((p) => p.pattern));
@@ -8,8 +9,8 @@ const PATTERN_BY_VALUE = new Map(PII_PRESETS.map((p) => [p.value, p.pattern]));
const VALUE_BY_PATTERN = new Map(PII_PRESETS.map((p) => [p.pattern, p.value]));
interface PolicyPiiFieldProps {
parameters: Record<string, unknown>;
onChange: (parameters: Record<string, unknown>) => void;
parameters: RedactParameters;
onChange: (parameters: RedactParameters) => void;
disabled?: boolean;
}
@@ -26,9 +27,7 @@ export function PolicyPiiField({
disabled,
}: PolicyPiiFieldProps) {
const { t } = useTranslation();
const words = Array.isArray(parameters.wordsToRedact)
? (parameters.wordsToRedact as string[])
: [];
const words = parameters.wordsToRedact;
const selected = words
.map((w) => VALUE_BY_PATTERN.get(w))
.filter((v): v is string => Boolean(v));
@@ -1,9 +1,10 @@
import { useEffect } from "react";
import { PolicyPiiField } from "@app/components/policies/PolicyPiiField";
import type { RedactParameters } from "@app/hooks/tools/redact/useRedactParameters";
interface PolicyRedactConfigProps {
parameters: Record<string, unknown>;
onChange: (parameters: Record<string, unknown>) => void;
parameters: RedactParameters;
onChange: (parameters: RedactParameters) => void;
disabled?: boolean;
}
@@ -3,8 +3,8 @@ import AddWatermarkSingleStepSettings from "@app/components/tools/addWatermark/A
import type { AddWatermarkParameters } from "@app/hooks/tools/addWatermark/useAddWatermarkParameters";
interface PolicyWatermarkConfigProps {
parameters: Record<string, unknown>;
onChange: (parameters: Record<string, unknown>) => void;
parameters: AddWatermarkParameters;
onChange: (parameters: AddWatermarkParameters) => void;
disabled?: boolean;
}
@@ -20,7 +20,7 @@ export function PolicyWatermarkConfig({
disabled,
}: PolicyWatermarkConfigProps) {
useEffect(() => {
const patch: Record<string, unknown> = {};
const patch: Partial<AddWatermarkParameters> = {};
if (parameters.convertPDFToImage !== true) patch.convertPDFToImage = true;
// Policies only support text watermarks.
if (parameters.watermarkType !== "text") patch.watermarkType = "text";
@@ -29,7 +29,7 @@ export function PolicyWatermarkConfig({
return (
<AddWatermarkSingleStepSettings
parameters={parameters as unknown as AddWatermarkParameters}
parameters={parameters}
onParameterChange={(key, value) =>
onChange({ ...parameters, [key]: value })
}
@@ -0,0 +1,112 @@
import { describe, expect, test } from "vitest";
import {
POLICY_OPERATIONS,
policyEndpoint,
policyStep,
policyStepFromWire,
policyStepToWire,
policyToolIdForEndpoint,
type PolicyToolId,
} from "@app/policies/operations";
const ALL_TOOL_IDS = Object.keys(POLICY_OPERATIONS) as PolicyToolId[];
describe("POLICY_OPERATIONS", () => {
test("every category operation is a typed descriptor with a known endpoint", () => {
// The catalogue uses these six across all categories; each must be wired.
expect(ALL_TOOL_IDS.sort()).toEqual([
"compress",
"flatten",
"ocr",
"redact",
"sanitize",
"watermark",
]);
for (const id of ALL_TOOL_IDS) {
expect(POLICY_OPERATIONS[id].endpoint).toBe(policyEndpoint(id));
expect(typeof POLICY_OPERATIONS[id].toApi).toBe("function");
expect(typeof POLICY_OPERATIONS[id].fromApi).toBe("function");
}
});
test("policyEndpoint returns the pinned endpoint literal", () => {
expect(policyEndpoint("redact")).toBe("/api/v1/security/auto-redact");
expect(policyEndpoint("sanitize")).toBe("/api/v1/security/sanitize-pdf");
expect(policyEndpoint("watermark")).toBe("/api/v1/security/add-watermark");
expect(policyEndpoint("ocr")).toBe("/api/v1/misc/ocr-pdf");
expect(policyEndpoint("flatten")).toBe("/api/v1/misc/flatten");
expect(policyEndpoint("compress")).toBe("/api/v1/misc/compress-pdf");
});
test("policyToolIdForEndpoint maps endpoints back, and rejects non-policy ones", () => {
for (const id of ALL_TOOL_IDS) {
expect(policyToolIdForEndpoint(policyEndpoint(id))).toBe(id);
}
expect(policyToolIdForEndpoint("/api/v1/misc/repair")).toBeNull();
expect(policyToolIdForEndpoint("not-an-endpoint")).toBeNull();
});
});
describe("policyStep", () => {
test("merges partial params over the tool's defaults", () => {
const step = policyStep("redact", {
useRegex: true,
wordsToRedact: ["ssn", "card"],
});
expect(step.toolId).toBe("redact");
// Overrides applied...
expect(step.params.useRegex).toBe(true);
expect(step.params.wordsToRedact).toEqual(["ssn", "card"]);
// ...and untouched fields fall back to the tool's defaults.
expect(step.params.mode).toBe("automatic");
expect(step.params.redactColor).toBe("#000000");
});
});
describe("wire conversion", () => {
test("redact maps frontend params to the backend request model (wordsToRedact -> listOfText)", () => {
const wire = policyStepToWire(
policyStep("redact", {
useRegex: true,
convertPDFToImage: true,
wordsToRedact: ["ssn", "card"],
}),
);
expect(wire.operation).toBe("/api/v1/security/auto-redact");
// The backend field the endpoint actually reads, and no frontend-only `mode`/`wordsToRedact`.
expect(wire.parameters).toMatchObject({
listOfText: "ssn\ncard",
useRegex: true,
convertPDFToImage: true,
});
expect(wire.parameters).not.toHaveProperty("wordsToRedact");
expect(wire.parameters).not.toHaveProperty("mode");
});
test("every policy operation round-trips through wire and back", () => {
for (const id of ALL_TOOL_IDS) {
const step = policyStep(id);
const back = policyStepFromWire(policyStepToWire(step));
expect(back?.toolId).toBe(id);
}
});
test("redact round-trip preserves the configured patterns", () => {
const step = policyStep("redact", {
useRegex: true,
wordsToRedact: ["ssn", "card"],
});
const back = policyStepFromWire(policyStepToWire(step));
expect(back?.toolId).toBe("redact");
if (back?.toolId === "redact") {
expect(back.params.wordsToRedact).toEqual(["ssn", "card"]);
expect(back.params.useRegex).toBe(true);
}
});
test("a non-policy endpoint decodes to null", () => {
expect(
policyStepFromWire({ operation: "/api/v1/misc/repair", parameters: {} }),
).toBeNull();
});
});
@@ -0,0 +1,131 @@
/**
* The tool operations the Policies feature can run, each a typed {@link ToolOperationDescriptor}.
* Source of truth for the catalogue, wizard, and wire conversion. Add a tool here to use it in a
* policy - the catalogue can't reference an untyped operation.
*/
import { describeToolOperation } from "@app/hooks/tools/shared/toolOperationDescriptor";
import { redactOperationConfig } from "@app/hooks/tools/redact/useRedactOperation";
import { sanitizeOperationConfig } from "@app/hooks/tools/sanitize/useSanitizeOperation";
import { addWatermarkOperationConfig } from "@app/hooks/tools/addWatermark/useAddWatermarkOperation";
import { ocrOperationConfig } from "@app/hooks/tools/ocr/useOCROperation";
import { flattenOperationConfig } from "@app/hooks/tools/flatten/useFlattenOperation";
import { compressOperationConfig } from "@app/hooks/tools/compress/useCompressOperation";
import type { ToolOperationDescriptor } from "@app/hooks/tools/shared/toolOperationDescriptor";
import type { ToolApiParams, ToolEndpoint } from "@app/types/toolApiTypes";
import type { WirePipelineStep } from "@app/policies/types";
export const POLICY_OPERATIONS = {
redact: describeToolOperation(
"/api/v1/security/auto-redact",
redactOperationConfig,
),
sanitize: describeToolOperation(
"/api/v1/security/sanitize-pdf",
sanitizeOperationConfig,
),
watermark: describeToolOperation(
"/api/v1/security/add-watermark",
addWatermarkOperationConfig,
),
ocr: describeToolOperation("/api/v1/misc/ocr-pdf", ocrOperationConfig),
flatten: describeToolOperation(
"/api/v1/misc/flatten",
flattenOperationConfig,
),
compress: describeToolOperation(
"/api/v1/misc/compress-pdf",
compressOperationConfig,
),
} as const;
export type PolicyToolId = keyof typeof POLICY_OPERATIONS;
export type PolicyParams<Id extends PolicyToolId> =
(typeof POLICY_OPERATIONS)[Id] extends ToolOperationDescriptor<
ToolEndpoint,
infer P
>
? P
: never;
/** Discriminated on `toolId` so `params` matches the tool. */
export type PolicyToolStep = {
[Id in PolicyToolId]: { toolId: Id; params: PolicyParams<Id> };
}[PolicyToolId];
export type PolicyToolStepOf<Id extends PolicyToolId> = Extract<
PolicyToolStep,
{ toolId: Id }
>;
const POLICY_TOOL_IDS = Object.keys(POLICY_OPERATIONS) as PolicyToolId[];
const TOOL_ID_BY_ENDPOINT = new Map<string, PolicyToolId>(
POLICY_TOOL_IDS.map((id) => [POLICY_OPERATIONS[id].endpoint, id]),
);
export function policyEndpoint(toolId: PolicyToolId): ToolEndpoint {
return POLICY_OPERATIONS[toolId].endpoint;
}
/** Tool id for an endpoint path, or null if it isn't a policy tool. */
export function policyToolIdForEndpoint(endpoint: string): PolicyToolId | null {
return TOOL_ID_BY_ENDPOINT.get(endpoint) ?? null;
}
/** A step for `toolId`, partial params merged over the tool's defaults. */
export function policyStep<Id extends PolicyToolId>(
toolId: Id,
params: Partial<PolicyParams<Id>> = {},
): PolicyToolStepOf<Id> {
const defaults = POLICY_OPERATIONS[toolId].defaultParameters as object;
return {
toolId,
params: { ...defaults, ...(params as object) },
} as PolicyToolStepOf<Id>;
}
export function policyStepToWire(step: PolicyToolStep): WirePipelineStep {
return serializeStep(step);
}
// Generic over the id so `params` stays correlated with the descriptor; TS can't do that through
// the union, so `op` is widened here (a contained cast at the wire boundary).
function serializeStep<Id extends PolicyToolId>(step: {
toolId: Id;
params: PolicyParams<Id>;
}): WirePipelineStep {
const op = POLICY_OPERATIONS[step.toolId] as ToolOperationDescriptor<
ToolEndpoint,
PolicyParams<Id>
>;
return {
operation: op.endpoint,
parameters: op.toApi(step.params) as Record<string, unknown>,
};
}
/** Wire step -> typed policy step, or null if the endpoint isn't a policy tool. */
export function policyStepFromWire(
wire: WirePipelineStep,
): PolicyToolStep | null {
const toolId = policyToolIdForEndpoint(wire.operation);
if (!toolId) return null;
return deserializeStep(toolId, wire.parameters);
}
function deserializeStep<Id extends PolicyToolId>(
toolId: Id,
parameters: Record<string, unknown>,
): PolicyToolStepOf<Id> {
const op = POLICY_OPERATIONS[toolId] as ToolOperationDescriptor<
ToolEndpoint,
PolicyParams<Id>
>;
// Wire params are untyped JSON; this is the one point they enter the typed model.
const params = op.fromApi(
parameters as unknown as ToolApiParams[ToolEndpoint],
);
return { toolId, params } as unknown as PolicyToolStepOf<Id>;
}
-1
View File
@@ -83,7 +83,6 @@
"web-vitals": "^5.1.0"
},
"scripts": {
"docs:sync": "tsx editor/scripts/sync-portal-docs.mts",
"update:minor": "node scripts/update-minor.js",
"update:major": "npx npm-check-updates -u && npm install",
"update:interactive": "npx npm-check-updates -i",