diff --git a/.github/workflows/docker-compose-tests-saas.yml b/.github/workflows/docker-compose-tests-saas.yml new file mode 100644 index 0000000000..c4e2a0834e --- /dev/null +++ b/.github/workflows/docker-compose-tests-saas.yml @@ -0,0 +1,136 @@ +name: Docker Compose Cucumber tests (saas / PAYG) + +# Self-contained CI job for the PAYG shadow-mode cucumber scenarios. +# Triggers only on PAYG-relevant paths so we don't add CI minutes to every PR +# that doesn't touch the saas flavour. +# +# Companion to `docker-compose-tests.yml` (which runs against the +# proprietary-flavour stack and skips features/payg via behave.ini's +# exclude_re). Kept as a separate workflow so the saas matrix can fail and +# succeed independently without touching the main cucumber harness. + +on: + pull_request: + paths: + - "app/saas/**" + - "testing/cucumber/features/payg/**" + - "testing/cucumber/features/steps/payg_step_definitions.py" + - "testing/cucumber/requirements.txt" + - "testing/compose/docker-compose-saas.yml" + - "testing/compose/payg/**" + - "testing/test-payg.sh" + - ".github/workflows/docker-compose-tests-saas.yml" + push: + branches: [main] + paths: + - "app/saas/**" + - "testing/cucumber/features/payg/**" + - "testing/cucumber/features/steps/payg_step_definitions.py" + - "testing/cucumber/requirements.txt" + - "testing/compose/docker-compose-saas.yml" + - "testing/compose/payg/**" + - "testing/test-payg.sh" + - ".github/workflows/docker-compose-tests-saas.yml" + +permissions: + contents: read + +jobs: + pick: + uses: ./.github/workflows/_runner-pick.yml + + docker-compose-tests-saas: + needs: pick + runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }} + permissions: + actions: write + contents: read + checks: write + env: + DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + with: + egress-policy: audit + + - name: Checkout Repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up JDK 25 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: "25" + distribution: "temurin" + + - name: Cache Gradle dependency artifacts + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.gradle/wrapper + ~/.gradle/caches/modules-2/files-2.1 + ~/.gradle/caches/modules-2/metadata-2.* + key: gradle-deps-saas-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }} + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 + with: + gradle-version: 9.3.1 + cache-disabled: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Expose GitHub runtime for Buildx cache + uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0 + + # No "Install Docker Compose" step: Ubuntu runners ship with `docker compose` + # v2 (built into the Docker CLI). test-payg.sh uses the v2 form throughout + # (`docker compose …`, no hyphen), so the legacy v1 `docker-compose` binary + # isn't needed. Avoids a `curl | sudo install` without checksum verification + # (Aikido flagged this when copy-pasted from docker-compose-tests.yml). + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + cache: "pip" + cache-dependency-path: ./testing/cucumber/requirements.txt + + - name: Pip requirements + run: | + pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt + + - name: Run PAYG Cucumber Tests + env: + MAVEN_USER: ${{ secrets.MAVEN_USER }} + MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} + MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }} + run: | + chmod +x ./testing/test-payg.sh + ./testing/test-payg.sh + + - name: Dump saas container logs on failure + if: failure() + run: | + docker compose -f testing/compose/docker-compose-saas.yml logs --tail 500 stirling-pdf-saas || true + docker compose -f testing/compose/docker-compose-saas.yml logs --tail 200 postgres-saas || true + + - name: Upload PAYG Cucumber Report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: payg-cucumber-report + path: testing/cucumber/report-payg.html + retention-days: 7 + if-no-files-found: warn + + - name: PAYG Cucumber Test Report + if: always() + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0 + with: + name: PAYG Cucumber Tests + path: testing/cucumber/junit-payg/*.xml + reporter: java-junit + fail-on-error: false diff --git a/.taskfiles/desktop.yml b/.taskfiles/desktop.yml index ac4ccbe806..de6bfdebd2 100644 --- a/.taskfiles/desktop.yml +++ b/.taskfiles/desktop.yml @@ -62,21 +62,21 @@ tasks: deps: [prepare] dir: editor cmds: - - npx tauri build --bundles app + - npx tauri build --bundles app --config '{"bundle":{"createUpdaterArtifacts":false}}' build:dev:windows: desc: "Build Tauri desktop NSIS installer (Windows)" deps: [prepare] dir: editor cmds: - - npx tauri build --bundles nsis + - npx tauri build --bundles nsis --config '{"bundle":{"createUpdaterArtifacts":false}}' build:dev:linux: desc: "Build Tauri desktop AppImage (Linux)" deps: [prepare] dir: editor cmds: - - npx tauri build --bundles appimage + - npx tauri build --bundles appimage --config '{"bundle":{"createUpdaterArtifacts":false}}' test: desc: "Run Tauri/Cargo tests" diff --git a/app/common/src/main/java/stirling/software/common/service/ToolMetadataService.java b/app/common/src/main/java/stirling/software/common/service/ToolMetadataService.java index 662878b741..fb7a928d76 100644 --- a/app/common/src/main/java/stirling/software/common/service/ToolMetadataService.java +++ b/app/common/src/main/java/stirling/software/common/service/ToolMetadataService.java @@ -1,11 +1,21 @@ package stirling.software.common.service; +import java.util.List; + /** Provides metadata about tool endpoints for internal dispatch. */ public interface ToolMetadataService { /** Returns true if the given operation path accepts multiple input files. */ boolean isMultiInput(String operationPath); + /** + * Returns the file extensions (lowercase, no leading dot, e.g. {@code "pdf"}) that the + * operation accepts as input ({@code output=false}) or produces as output ({@code + * output=true}), derived from the endpoint's declared type. Returns {@code null} when the + * endpoint declares no specific type, which callers should treat as "any type accepted". + */ + List getExtensionTypes(boolean output, String operationPath); + /** * Returns true when the endpoint's ZIP response is a transport for multiple typed results and * should be unpacked: multi-output endpoints (Type:SIMO / Type:MIMO) and wrapper declarations diff --git a/app/common/src/main/java/stirling/software/common/util/OfficeDocumentSanitizer.java b/app/common/src/main/java/stirling/software/common/util/OfficeDocumentSanitizer.java new file mode 100644 index 0000000000..9cdf8cf53b --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/util/OfficeDocumentSanitizer.java @@ -0,0 +1,310 @@ +package stirling.software.common.util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.springframework.stereotype.Component; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import io.github.pixee.security.ZipSecurity; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.SsrfProtectionService; + +// Strips external refs from OOXML/ODF uploads so LibreOffice can't be made to fetch them. +@Component +@Slf4j +public class OfficeDocumentSanitizer { + + private static final Set OOXML_EXTENSIONS = + Set.of( + "docx", "docm", "dotx", "dotm", "xlsx", "xlsm", "xltx", "xltm", "pptx", "pptm", + "potx", "potm", "ppsx", "ppsm"); + + private static final Set ODF_EXTENSIONS = + Set.of( + "odt", "ott", "ods", "ots", "odp", "otp", "odg", "otg", "odf", "odc", "odi", + "odm"); + + private static final Set ODF_XML_PARTS = + Set.of("content.xml", "styles.xml", "meta.xml", "settings.xml"); + + private final SsrfProtectionService ssrfProtectionService; + private final ApplicationProperties applicationProperties; + + public OfficeDocumentSanitizer( + SsrfProtectionService ssrfProtectionService, + ApplicationProperties applicationProperties) { + this.ssrfProtectionService = ssrfProtectionService; + this.applicationProperties = applicationProperties; + } + + public boolean isSanitizableExtension(String extension) { + if (extension == null) { + return false; + } + String lower = extension.toLowerCase(Locale.ROOT); + return OOXML_EXTENSIONS.contains(lower) || ODF_EXTENSIONS.contains(lower); + } + + public byte[] sanitize(byte[] documentBytes, String extension) throws IOException { + if (documentBytes == null || documentBytes.length == 0) { + throw new IOException("Office document input is empty or null"); + } + if (applicationProperties.getSystem().isDisableSanitize()) { + log.debug("Office document sanitization disabled by configuration"); + return documentBytes; + } + if (!isSanitizableExtension(extension)) { + return documentBytes; + } + + ByteArrayOutputStream out = new ByteArrayOutputStream(documentBytes.length); + try (ZipInputStream zipIn = + ZipSecurity.createHardenedInputStream( + new ByteArrayInputStream(documentBytes)); + ZipOutputStream zipOut = new ZipOutputStream(out)) { + + ZipEntry entry; + while ((entry = zipIn.getNextEntry()) != null) { + String name = entry.getName(); + byte[] bytes = entry.isDirectory() ? new byte[0] : zipIn.readAllBytes(); + + if (!entry.isDirectory()) { + bytes = sanitizeEntry(name, bytes); + } + + ZipEntry outEntry = new ZipEntry(name); + if (entry.getComment() != null) { + outEntry.setComment(entry.getComment()); + } + if (entry.getExtra() != null) { + outEntry.setExtra(entry.getExtra()); + } + zipOut.putNextEntry(outEntry); + if (!entry.isDirectory()) { + zipOut.write(bytes); + } + zipOut.closeEntry(); + } + } + return out.toByteArray(); + } + + private byte[] sanitizeEntry(String entryName, byte[] entryBytes) { + String lower = entryName.toLowerCase(Locale.ROOT); + try { + if (lower.endsWith(".rels")) { + return sanitizeOoxmlRels(entryBytes); + } + if (isOdfXmlPart(lower)) { + return sanitizeOdfXml(entryBytes); + } + } catch (ParserConfigurationException + | SAXException + | IOException + | TransformerException e) { + log.warn( + "Failed to parse XML part '{}' for sanitization, leaving as-is: {}", + entryName, + e.getMessage()); + } + return entryBytes; + } + + private boolean isOdfXmlPart(String lowerName) { + int slash = lowerName.lastIndexOf('/'); + String base = slash >= 0 ? lowerName.substring(slash + 1) : lowerName; + return ODF_XML_PARTS.contains(base); + } + + private byte[] sanitizeOoxmlRels(byte[] xmlBytes) + throws IOException, ParserConfigurationException, SAXException, TransformerException { + Document doc = parseSecurely(xmlBytes); + Element root = doc.getDocumentElement(); + if (root == null) { + return xmlBytes; + } + NodeList relationships = root.getElementsByTagNameNS("*", "Relationship"); + List toRemove = new ArrayList<>(); + for (int i = 0; i < relationships.getLength(); i++) { + Node node = relationships.item(i); + NamedNodeMap attrs = node.getAttributes(); + if (attrs == null) { + continue; + } + Node targetMode = attrs.getNamedItem("TargetMode"); + if (targetMode == null || !"external".equalsIgnoreCase(targetMode.getNodeValue())) { + continue; + } + Node target = attrs.getNamedItem("Target"); + String targetValue = target == null ? "" : target.getNodeValue(); + if (isAdminAllowed(targetValue)) { + continue; + } + log.warn( + "Stripping OOXML external relationship target: {}", + truncateForLog(targetValue)); + toRemove.add(node); + } + if (toRemove.isEmpty()) { + return xmlBytes; + } + for (Node n : toRemove) { + n.getParentNode().removeChild(n); + } + return serializeDocument(doc); + } + + private byte[] sanitizeOdfXml(byte[] xmlBytes) + throws IOException, ParserConfigurationException, SAXException, TransformerException { + Document doc = parseSecurely(xmlBytes); + Element root = doc.getDocumentElement(); + if (root == null) { + return xmlBytes; + } + boolean modified = stripExternalHrefs(root); + if (!modified) { + return xmlBytes; + } + return serializeDocument(doc); + } + + private boolean stripExternalHrefs(Node node) { + boolean modified = false; + if (node.getNodeType() == Node.ELEMENT_NODE) { + NamedNodeMap attrs = node.getAttributes(); + List hrefAttrsToRemove = new ArrayList<>(); + for (int i = 0; i < attrs.getLength(); i++) { + Node attr = attrs.item(i); + String name = attr.getNodeName(); + if (name == null) { + continue; + } + String lower = name.toLowerCase(Locale.ROOT); + if (!(lower.equals("xlink:href") + || lower.endsWith(":href") + || lower.equals("href"))) { + continue; + } + String value = attr.getNodeValue(); + if (!isExternalUrl(value)) { + continue; + } + if (isAdminAllowed(value)) { + continue; + } + log.warn( + "Stripping ODF external href attribute ({}): {}", + name, + truncateForLog(value)); + hrefAttrsToRemove.add(name); + } + Element element = (Element) node; + for (String attrName : hrefAttrsToRemove) { + element.removeAttribute(attrName); + modified = true; + } + } + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + if (stripExternalHrefs(children.item(i))) { + modified = true; + } + } + return modified; + } + + private boolean isExternalUrl(String url) { + if (url == null) { + return false; + } + String trimmed = url.trim().toLowerCase(Locale.ROOT); + if (trimmed.isEmpty() || trimmed.startsWith("#") || trimmed.startsWith("../")) { + return false; + } + return trimmed.startsWith("http://") + || trimmed.startsWith("https://") + || trimmed.startsWith("ftp://") + || trimmed.startsWith("ftps://") + || trimmed.startsWith("file:") + || trimmed.startsWith("smb:") + || trimmed.startsWith("\\\\") + || trimmed.startsWith("//"); + } + + // Preserved only with an explicit allowedDomains entry; MEDIUM default would admit public URLs. + private boolean isAdminAllowed(String url) { + if (ssrfProtectionService == null || url == null || url.isBlank()) { + return false; + } + ApplicationProperties.Html.UrlSecurity config = + applicationProperties.getSystem().getHtml().getUrlSecurity(); + if (config == null + || config.getAllowedDomains() == null + || config.getAllowedDomains().isEmpty()) { + return false; + } + return ssrfProtectionService.isUrlAllowed(url); + } + + private Document parseSecurely(byte[] xmlBytes) + throws ParserConfigurationException, SAXException, IOException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + factory.setNamespaceAware(true); + DocumentBuilder builder = factory.newDocumentBuilder(); + return builder.parse(new ByteArrayInputStream(xmlBytes)); + } + + private byte[] serializeDocument(Document doc) throws TransformerException { + TransformerFactory tf = TransformerFactory.newInstance(); + tf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + transformer.setOutputProperty(OutputKeys.INDENT, "no"); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + transformer.transform(new DOMSource(doc), new StreamResult(baos)); + return baos.toByteArray(); + } + + private String truncateForLog(String value) { + if (value == null) { + return "null"; + } + return value.length() > 80 ? value.substring(0, 80) + "..." : value; + } +} diff --git a/app/common/src/test/java/stirling/software/common/util/OfficeDocumentSanitizerTest.java b/app/common/src/test/java/stirling/software/common/util/OfficeDocumentSanitizerTest.java new file mode 100644 index 0000000000..eb22919068 --- /dev/null +++ b/app/common/src/test/java/stirling/software/common/util/OfficeDocumentSanitizerTest.java @@ -0,0 +1,370 @@ +package stirling.software.common.util; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.SsrfProtectionService; + +class OfficeDocumentSanitizerTest { + + private static final String EXTERNAL_URL = "https://webhook.site/ssrf-callback"; + private static final String INTERNAL_TARGET = "media/image1.png"; + + private static final String DOCX_RELS = + "" + + "" + + "" + + "" + + ""; + + private static final String DOCX_DOCUMENT = + "" + + "" + + ""; + + private static final String ODF_CONTENT_EXTERNAL = + "" + + "" + + "" + + "" + + "" + + ""; + + private SsrfProtectionService ssrfProtectionService; + private ApplicationProperties applicationProperties; + private OfficeDocumentSanitizer sanitizer; + + @BeforeEach + void setUp() { + applicationProperties = new ApplicationProperties(); + ssrfProtectionService = mock(SsrfProtectionService.class); + sanitizer = new OfficeDocumentSanitizer(ssrfProtectionService, applicationProperties); + } + + @Test + void isSanitizableExtension_recognizesOoxmlAndOdf() { + assertTrue(sanitizer.isSanitizableExtension("docx")); + assertTrue(sanitizer.isSanitizableExtension("DOCX")); + assertTrue(sanitizer.isSanitizableExtension("xlsx")); + assertTrue(sanitizer.isSanitizableExtension("pptx")); + assertTrue(sanitizer.isSanitizableExtension("odt")); + assertTrue(sanitizer.isSanitizableExtension("ods")); + assertTrue(sanitizer.isSanitizableExtension("odp")); + assertFalse(sanitizer.isSanitizableExtension("pdf")); + assertFalse(sanitizer.isSanitizableExtension("html")); + assertFalse(sanitizer.isSanitizableExtension("")); + assertFalse(sanitizer.isSanitizableExtension(null)); + } + + @Test + void sanitize_stripsOoxmlExternalRelationship() throws IOException { + Map entries = new LinkedHashMap<>(); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + entries.put("word/document.xml", DOCX_DOCUMENT.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8); + assertFalse(rels.contains(EXTERNAL_URL), "External URL should be stripped from .rels"); + assertFalse( + rels.toLowerCase().contains("targetmode=\"external\""), + "TargetMode=External relationship should be removed"); + assertTrue(rels.contains(INTERNAL_TARGET), "Internal image target should be preserved"); + assertArrayEquals( + DOCX_DOCUMENT.getBytes(StandardCharsets.UTF_8), + result.get("word/document.xml"), + "Non-rels entries must be untouched"); + } + + @Test + void sanitize_pptxExternalImageRelStripped() throws IOException { + String pptxRels = + "" + + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put("ppt/slides/_rels/slide1.xml.rels", pptxRels.getBytes(StandardCharsets.UTF_8)); + byte[] pptx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(pptx, "pptx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("ppt/slides/_rels/slide1.xml.rels"), StandardCharsets.UTF_8); + assertFalse(rels.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_xlsxExternalImageRelStripped() throws IOException { + String xlsxRels = + "" + + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put( + "xl/drawings/_rels/drawing1.xml.rels", xlsxRels.getBytes(StandardCharsets.UTF_8)); + byte[] xlsx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(xlsx, "xlsx"); + + Map result = unzip(cleaned); + String rels = + new String( + result.get("xl/drawings/_rels/drawing1.xml.rels"), StandardCharsets.UTF_8); + assertFalse(rels.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_odtStripsExternalXlinkHrefButKeepsInternal() throws IOException { + Map entries = new LinkedHashMap<>(); + entries.put("content.xml", ODF_CONTENT_EXTERNAL.getBytes(StandardCharsets.UTF_8)); + String manifestXml = + ""; + entries.put("META-INF/manifest.xml", manifestXml.getBytes(StandardCharsets.UTF_8)); + byte[] odt = zip(entries); + + byte[] cleaned = sanitizer.sanitize(odt, "odt"); + + Map result = unzip(cleaned); + String content = new String(result.get("content.xml"), StandardCharsets.UTF_8); + assertFalse(content.contains(EXTERNAL_URL), "External xlink:href should be stripped"); + assertTrue(content.contains("Pictures/image1.png"), "Internal href should be preserved"); + } + + @Test + void sanitize_odsStripsExternalXlinkHref() throws IOException { + Map entries = new LinkedHashMap<>(); + entries.put("content.xml", ODF_CONTENT_EXTERNAL.getBytes(StandardCharsets.UTF_8)); + byte[] ods = zip(entries); + + byte[] cleaned = sanitizer.sanitize(ods, "ods"); + + Map result = unzip(cleaned); + String content = new String(result.get("content.xml"), StandardCharsets.UTF_8); + assertFalse(content.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_odpStripsExternalXlinkHrefInStylesXml() throws IOException { + String stylesXml = + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put("styles.xml", stylesXml.getBytes(StandardCharsets.UTF_8)); + byte[] odp = zip(entries); + + byte[] cleaned = sanitizer.sanitize(odp, "odp"); + + Map result = unzip(cleaned); + String content = new String(result.get("styles.xml"), StandardCharsets.UTF_8); + assertFalse(content.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_disabledByConfigReturnsOriginal() throws IOException { + applicationProperties.getSystem().setDisableSanitize(true); + Map entries = new LinkedHashMap<>(); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] result = sanitizer.sanitize(docx, "docx"); + assertArrayEquals(docx, result); + } + + @Test + void sanitize_unrecognizedExtensionReturnsOriginal() throws IOException { + byte[] original = "irrelevant".getBytes(StandardCharsets.UTF_8); + byte[] result = sanitizer.sanitize(original, "pdf"); + assertArrayEquals(original, result); + } + + @Test + void sanitize_emptyInputThrows() { + assertThrows(IOException.class, () -> sanitizer.sanitize(new byte[0], "docx")); + } + + @Test + void sanitize_nullInputThrows() { + assertThrows(IOException.class, () -> sanitizer.sanitize(null, "docx")); + } + + @Test + void sanitize_preservesEntryWithExternalRefWhenAdminAllowsDomain() throws IOException { + applicationProperties + .getSystem() + .getHtml() + .getUrlSecurity() + .getAllowedDomains() + .add("webhook.site"); + lenient().when(ssrfProtectionService.isUrlAllowed(eq(EXTERNAL_URL))).thenReturn(true); + + Map entries = new LinkedHashMap<>(); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8); + assertTrue(rels.contains(EXTERNAL_URL), "Allow-listed external URL should be preserved"); + } + + @Test + void sanitize_doesNotConsultSsrfServiceWhenAllowedDomainsEmpty() throws IOException { + // Even if mock would say allowed, we should not invoke it when there is no allow-list, + // because MEDIUM default would let public URLs through and re-introduce the vulnerability. + lenient().when(ssrfProtectionService.isUrlAllowed(eq(EXTERNAL_URL))).thenReturn(true); + + Map entries = new LinkedHashMap<>(); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8); + assertFalse(rels.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_handlesNonXmlEntriesSafely() throws IOException { + Map entries = new LinkedHashMap<>(); + byte[] imageBytes = new byte[] {(byte) 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a}; + entries.put("word/media/image1.png", imageBytes); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + assertArrayEquals(imageBytes, result.get("word/media/image1.png")); + } + + @Test + void sanitize_internalLinksKeptWhenNoExternalPresent() throws IOException { + String internalOnlyRels = + "" + + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put( + "word/_rels/document.xml.rels", internalOnlyRels.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8); + assertTrue(rels.contains("media/image1.png")); + } + + @Test + void sanitize_corruptZipProducesSafeOutput() throws IOException { + byte[] garbage = "this is not a zip file".getBytes(StandardCharsets.UTF_8); + byte[] result = sanitizer.sanitize(garbage, "docx"); + Map entries = unzip(result); + assertTrue(entries.isEmpty(), "Garbage input must not yield exploitable entries"); + } + + @Test + void sanitize_relativeOdfPathsArePreserved() throws IOException { + String content = + "" + + "" + + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put("content.xml", content.getBytes(StandardCharsets.UTF_8)); + byte[] odt = zip(entries); + + byte[] cleaned = sanitizer.sanitize(odt, "odt"); + + Map result = unzip(cleaned); + String out = new String(result.get("content.xml"), StandardCharsets.UTF_8); + assertTrue(out.contains("../Pictures/image1.png")); + assertTrue(out.contains("#anchor")); + } + + private static byte[] zip(Map entries) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + for (Map.Entry e : entries.entrySet()) { + ZipEntry entry = new ZipEntry(e.getKey()); + zos.putNextEntry(entry); + zos.write(e.getValue()); + zos.closeEntry(); + } + } + return baos.toByteArray(); + } + + private static Map unzip(byte[] data) throws IOException { + Map entries = new HashMap<>(); + try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(data))) { + ZipEntry e; + while ((e = zis.getNextEntry()) != null) { + entries.put(e.getName(), zis.readAllBytes()); + zis.closeEntry(); + } + } + return entries; + } +} diff --git a/app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java b/app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java index 205a0d5734..7c7fe115d0 100644 --- a/app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java +++ b/app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java @@ -18,6 +18,7 @@ import io.swagger.v3.oas.models.media.StringSchema; import io.swagger.v3.oas.models.security.SecurityRequirement; import io.swagger.v3.oas.models.security.SecurityScheme; import io.swagger.v3.oas.models.servers.Server; +import io.swagger.v3.oas.models.tags.Tag; import lombok.RequiredArgsConstructor; @@ -60,6 +61,15 @@ public class OpenApiConfig { OpenAPI openAPI = new OpenAPI().info(info).openapi("3.0.3"); + // Register a single global "AI" tag so every AI endpoint groups under it in the docs. + // The AI controllers are currently @Hidden, so they don't emit this tag themselves yet; + // defining it here keeps the grouping ready for when those endpoints are unhidden. + openAPI.addTagsItem( + new Tag() + .name("AI") + .description( + "AI-powered document creation, editing, and assistant endpoints.")); + // Add server configuration from environment variable String swaggerServerUrl = System.getenv("SWAGGER_SERVER_URL"); Server server; diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/PdfOverlayController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/PdfOverlayController.java index 50b42d89dc..4aab03ff55 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/PdfOverlayController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/PdfOverlayController.java @@ -10,6 +10,7 @@ import java.util.Map; import org.apache.pdfbox.Loader; import org.apache.pdfbox.multipdf.Overlay; +import org.apache.pdfbox.pdfwriter.compress.CompressParameters; import org.apache.pdfbox.pdmodel.PDDocument; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; @@ -157,7 +158,10 @@ public class PdfOverlayController { PDDocument singlePageDocument = new PDDocument()) { singlePageDocument.addPage(overlayPdf.getPage(pageCountInCurrentOverlay)); File tempFile = Files.createTempFile("overlay-page-", ".pdf").toFile(); - singlePageDocument.save(tempFile); + // NO_COMPRESSION: this single-page doc holds a page copied from overlayPdf. + // PDFBox 3.0.7's compressed writer (PDFBOX-6203) drops shared resources imported + // across documents, corrupting overlay fonts. Revert once on 3.0.8. + singlePageDocument.save(tempFile, CompressParameters.NO_COMPRESSION); overlayGuide.put(basePageIndex, tempFile.getAbsolutePath()); tempFiles.add(tempFile); // Keep track of the temporary file for cleanup diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/RearrangePagesPDFController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/RearrangePagesPDFController.java index 1e87a15d57..6dd7aacd79 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/RearrangePagesPDFController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/RearrangePagesPDFController.java @@ -6,11 +6,9 @@ import java.util.Collections; import java.util.List; import java.util.Locale; -import org.apache.pdfbox.cos.COSName; import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDDocumentCatalog; import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm; +import org.apache.pdfbox.pdmodel.PDPageTree; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -262,38 +260,31 @@ public class RearrangePagesPDFController { } log.info("newPageOrder = {}", newPageOrder); log.info("totalPages = {}", totalPages); - // Create a new list to hold the pages in the new order - List newPages = new ArrayList<>(); - for (int i = 0; i < newPageOrder.size(); i++) { - newPages.add(document.getPage(newPageOrder.get(i))); + + // Snapshot the desired pages before mutating the source document's page tree. + List newPages = new ArrayList<>(newPageOrder.size()); + for (Integer idx : newPageOrder) { + newPages.add(document.getPage(idx)); } - // Create a new document based on the original one - try (PDDocument rearrangedDocument = - pdfDocumentFactory.createNewDocumentBasedOnOldDocument(document)) { - - // Add the pages in the new order - for (PDPage page : newPages) { - rearrangedDocument.addPage(page); - } - - PDDocumentCatalog sourceCatalog = document.getDocumentCatalog(); - if (sourceCatalog != null) { - PDAcroForm sourceForm = sourceCatalog.getAcroForm(null); - if (sourceForm != null) { - rearrangedDocument - .getDocumentCatalog() - .getCOSObject() - .setItem(COSName.ACRO_FORM, sourceForm.getCOSObject()); - } - } - - return WebResponseUtils.pdfDocToWebResponse( - rearrangedDocument, - GeneralUtils.generateFilename( - pdfFile.getOriginalFilename(), "_rearranged.pdf"), - tempFileManager); + // Rearrange in-place on the source document rather than copying pages into a + // freshly-created PDDocument. Copying pages across documents triggers a PDFBox + // 3.0.7 compressed-save regression (PDFBOX-6203, fixed for 3.0.8) where shared + // resource objects (fonts, etc.) imported from the source can be silently + // dropped from the output, producing pages with "font not found" errors. + PDPageTree pages = document.getPages(); + for (int i = totalPages - 1; i >= 0; i--) { + pages.remove(i); } + for (PDPage page : newPages) { + pages.add(page); + } + + return WebResponseUtils.pdfDocToWebResponse( + document, + GeneralUtils.generateFilename( + pdfFile.getOriginalFilename(), "_rearranged.pdf"), + tempFileManager); } } catch (IOException e) { ExceptionUtils.logException("document rearrangement", e); diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java index 8bf53c79e1..4fc6669bc7 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java @@ -35,6 +35,7 @@ import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.CustomHtmlSanitizer; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.GeneralUtils; +import stirling.software.common.util.OfficeDocumentSanitizer; import stirling.software.common.util.ProcessExecutor; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; import stirling.software.common.util.RegexPatternUtils; @@ -50,6 +51,7 @@ public class ConvertOfficeController { private final CustomPDFDocumentFactory pdfDocumentFactory; private final RuntimePathConfig runtimePathConfig; private final CustomHtmlSanitizer customHtmlSanitizer; + private final OfficeDocumentSanitizer officeDocumentSanitizer; private final EndpointConfiguration endpointConfiguration; private final TempFileManager tempFileManager; @@ -83,14 +85,16 @@ public class ConvertOfficeController { Path inputPath = workDir.resolve(baseName + "." + extensionLower); Path outputPath = workDir.resolve(baseName + ".pdf"); - // Check if the file is HTML and apply sanitization if needed + // Sanitize input before LibreOffice sees it so embedded URLs can't trigger SSRF. if ("html".equals(extensionLower) || "htm".equals(extensionLower)) { - // Read and sanitize HTML content String htmlContent = new String(inputFile.getBytes(), StandardCharsets.UTF_8); String sanitizedHtml = customHtmlSanitizer.sanitize(htmlContent); Files.writeString(inputPath, sanitizedHtml, StandardCharsets.UTF_8); + } else if (officeDocumentSanitizer.isSanitizableExtension(extensionLower)) { + byte[] sanitized = + officeDocumentSanitizer.sanitize(inputFile.getBytes(), extensionLower); + Files.write(inputPath, sanitized); } else { - // copy file content Files.copy(inputFile.getInputStream(), inputPath, StandardCopyOption.REPLACE_EXISTING); } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java index d9714f0618..ad5776b3cd 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java @@ -14,6 +14,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdfwriter.compress.CompressParameters; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject; @@ -357,7 +358,10 @@ public class AutoSplitPdfController { for (int i = 0; i < splitDocuments.size(); i++) { String fileName = filename + "_" + (i + 1) + ".pdf"; zipOut.putNextEntry(new ZipEntry(fileName)); - splitDocuments.get(i).save(zipOut); + // NO_COMPRESSION: split docs are built by addPage()-ing pages copied from the + // source document. PDFBox 3.0.7's compressed writer (PDFBOX-6203) drops shared + // resources imported across documents, corrupting fonts. Revert once on 3.0.8. + splitDocuments.get(i).save(zipOut, CompressParameters.NO_COMPRESSION); zipOut.closeEntry(); } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java index 64ed5b79ea..4660d5f128 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java @@ -17,6 +17,7 @@ import javax.imageio.ImageIO; import org.apache.pdfbox.io.IOUtils; import org.apache.pdfbox.multipdf.PDFMergerUtility; +import org.apache.pdfbox.pdfwriter.compress.CompressParameters; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.rendering.PDFRenderer; @@ -427,7 +428,10 @@ public class OCRController { // Save original page without OCR as fallback try (PDDocument pageDoc = new PDDocument()) { pageDoc.addPage(page); - pageDoc.save(pageOutputPath); + // NO_COMPRESSION: page is copied from another document; + // PDFBox 3.0.7 compressed writer (PDFBOX-6203) drops shared + // resources, corrupting fonts. Revert once on 3.0.8. + pageDoc.save(pageOutputPath, CompressParameters.NO_COMPRESSION); } } @@ -437,7 +441,10 @@ public class OCRController { // Save original page without OCR try (PDDocument pageDoc = new PDDocument()) { pageDoc.addPage(page); - pageDoc.save(pageOutputPath); + // NO_COMPRESSION: page is copied from another document; PDFBox 3.0.7 + // compressed writer (PDFBOX-6203) drops shared resources, corrupting + // fonts on retained text pages. Revert once on 3.0.8. + pageDoc.save(pageOutputPath, CompressParameters.NO_COMPRESSION); merger.addSource(pageOutputPath); } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java index c55597b47d..557c77c76e 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java @@ -25,6 +25,7 @@ import stirling.software.common.annotations.api.MiscApi; import stirling.software.common.enumeration.ResourceWeight; import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.GeneralUtils; +import stirling.software.common.util.SvgSanitizer; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; import stirling.software.common.util.WebResponseUtils; @@ -36,6 +37,7 @@ public class OverlayImageController { private final CustomPDFDocumentFactory pdfDocumentFactory; private final TempFileManager tempFileManager; + private final SvgSanitizer svgSanitizer; @AutoJobPostMapping( consumes = MediaType.MULTIPART_FORM_DATA_VALUE, @@ -61,6 +63,9 @@ public class OverlayImageController { byte[] imageBytes = imageFile.getBytes(); boolean isSvg = SvgOverlayUtil.isSvgImage(imageBytes); + if (isSvg) { + imageBytes = svgSanitizer.sanitize(imageBytes); + } try (PDDocument document = pdfDocumentFactory.load(pdfBytes)) { int pages = document.getNumberOfPages(); diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java b/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java index ce0f5edaf0..d6a95e66d4 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java @@ -41,6 +41,8 @@ public class ReactRoutingController { private boolean indexHtmlExists = false; private boolean useExternalIndexHtml = false; private boolean loggedMissingIndex = false; + private String cachedSaasLandingHtml; + private boolean saasLandingExists = false; @PostConstruct public void init() { @@ -49,6 +51,20 @@ public class ReactRoutingController { // Always initialize callback HTML (used for OAuth desktop flow) this.cachedCallbackHtml = buildCallbackHtml(); + // SaaS landing page: only present on the classpath when the :saas module is bundled + // (app/saas/src/main/resources/static/saas-landing.html). When present it replaces the + // root page so the SaaS API host shows its own landing instead of the OSS API-only page. + ClassPathResource saasLanding = new ClassPathResource("static/saas-landing.html"); + if (saasLanding.exists()) { + try (InputStream in = saasLanding.getInputStream()) { + this.cachedSaasLandingHtml = new String(in.readAllBytes(), StandardCharsets.UTF_8); + this.saasLandingExists = true; + log.info("SaaS landing page detected; serving it at '/' and '/index.html'"); + } catch (Exception ex) { + log.warn("Failed to read saas-landing.html; falling back to index.html", ex); + } + } + // Check for external index.html first (customFiles/static/) Path externalIndexPath = Paths.get(InstallationPathConfig.getStaticPath(), "index.html"); log.debug("Checking for custom index.html at: {}", externalIndexPath); @@ -132,6 +148,18 @@ public class ReactRoutingController { @GetMapping( value = {"/", "/index.html"}, produces = MediaType.TEXT_HTML_VALUE) + public ResponseEntity serveRootPage(HttpServletRequest request) { + // Swap ONLY the root page for SaaS. SPA entry points that delegate to serveIndexHtml + // (/auth/callback, /share/{token}, forwarded routes) keep serving the normal shell. + if (saasLandingExists && cachedSaasLandingHtml != null) { + return ResponseEntity.ok() + .cacheControl(CacheControl.noCache().mustRevalidate()) + .contentType(MediaType.TEXT_HTML) + .body(cachedSaasLandingHtml); + } + return serveIndexHtml(request); + } + public ResponseEntity serveIndexHtml(HttpServletRequest request) { try { if (indexHtmlExists && cachedIndexHtml != null) { diff --git a/app/core/src/main/java/stirling/software/SPDF/exception/GlobalExceptionHandler.java b/app/core/src/main/java/stirling/software/SPDF/exception/GlobalExceptionHandler.java index 6b43b66ff6..504dd36f5b 100644 --- a/app/core/src/main/java/stirling/software/SPDF/exception/GlobalExceptionHandler.java +++ b/app/core/src/main/java/stirling/software/SPDF/exception/GlobalExceptionHandler.java @@ -24,6 +24,7 @@ import org.springframework.web.multipart.MaxUploadSizeExceededException; import org.springframework.web.multipart.support.MissingServletRequestPartException; import org.springframework.web.server.ResponseStatusException; import org.springframework.web.servlet.NoHandlerFoundException; +import org.springframework.web.servlet.resource.NoResourceFoundException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -993,6 +994,49 @@ public class GlobalExceptionHandler { .body(problemDetail); } + /** Unmapped path → clean 404 instead of falling through to the generic 500 catch-all. */ + @ExceptionHandler(NoResourceFoundException.class) + public ResponseEntity handleNoResourceFound( + NoResourceFoundException ex, HttpServletRequest request) { + // /api/* miss = likely missing controller (operator-relevant); other paths = favicons, + // robots.txt, scanner noise. Demote the latter so prod logs aren't flooded. + String uri = request.getRequestURI(); + if (uri != null && uri.startsWith("/api/")) { + log.warn("No resource at {}: {}", uri, ex.getMessage()); + } else { + log.debug("No resource at {}: {}", uri, ex.getMessage()); + } + + String title = getLocalizedMessage("error.notFound.title", ErrorTitles.NOT_FOUND_DEFAULT); + String detail = + getLocalizedMessage( + "error.notFound.detail", + String.format( + "No endpoint found for %s %s", + request.getMethod(), request.getRequestURI()), + request.getMethod(), + request.getRequestURI()); + + ProblemDetail problemDetail = + createBaseProblemDetail(HttpStatus.NOT_FOUND, detail, request); + problemDetail.setType(URI.create(ErrorTypes.NOT_FOUND)); + problemDetail.setTitle(title); + problemDetail.setProperty("title", title); + problemDetail.setProperty("method", request.getMethod()); + addStandardHints( + problemDetail, + "error.notFound.hints", + List.of( + "Verify the URL path and HTTP method are correct.", + "Check the API base path and version if applicable.", + "Ensure there are no typos in the endpoint path.")); + problemDetail.setProperty("actionRequired", "Use a valid endpoint URL and method."); + + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .contentType(PROBLEM_JSON) + .body(problemDetail); + } + /** * Handle IllegalArgumentException. * diff --git a/app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java b/app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java index b0d2e5a4a0..3a8d418aa6 100644 --- a/app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java +++ b/app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java @@ -63,6 +63,7 @@ public class ApiDocService implements stirling.software.common.service.ToolMetad return "http://localhost:" + port + contextPath + "/v1/api-docs"; } + @Override public List getExtensionTypes(boolean output, String operationName) { if (outputToFileTypes.isEmpty()) { outputToFileTypes.put("PDF", List.of("pdf")); diff --git a/app/core/src/main/java/stirling/software/SPDF/utils/SvgOverlayUtil.java b/app/core/src/main/java/stirling/software/SPDF/utils/SvgOverlayUtil.java index 253d9c687a..b58bb055fc 100644 --- a/app/core/src/main/java/stirling/software/SPDF/utils/SvgOverlayUtil.java +++ b/app/core/src/main/java/stirling/software/SPDF/utils/SvgOverlayUtil.java @@ -10,6 +10,7 @@ import org.apache.batik.bridge.GVTBuilder; import org.apache.batik.bridge.UserAgent; import org.apache.batik.bridge.UserAgentAdapter; import org.apache.batik.gvt.GraphicsNode; +import org.apache.batik.util.ParsedURL; import org.apache.batik.util.XMLResourceDescriptor; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; @@ -39,7 +40,16 @@ public class SvgOverlayUtil { svgDoc = factory.createSVGDocument("file:///overlay.svg", inputStream); } - UserAgent userAgent = new UserAgentAdapter(); + UserAgent userAgent = + new UserAgentAdapter() { + @Override + public void checkLoadExternalResource( + ParsedURL resourceURL, ParsedURL docURL) { + throw new SecurityException( + "External resource loading is disabled for SVG overlays: " + + resourceURL); + } + }; DocumentLoader loader = new DocumentLoader(userAgent); BridgeContext ctx = new BridgeContext(userAgent, loader); ctx.setDynamicState(BridgeContext.DYNAMIC); diff --git a/app/core/src/main/resources/application.properties b/app/core/src/main/resources/application.properties index fc2fda630b..da564e454b 100644 --- a/app/core/src/main/resources/application.properties +++ b/app/core/src/main/resources/application.properties @@ -93,6 +93,11 @@ posthog.host=https://eu.i.posthog.com spring.main.allow-bean-definition-overriding=true +# spring-data-redis is on the classpath only for the optional Valkey backplane (which wires its own +# factory); exclude Spring Boot's stock Redis auto-config so a default install doesn't create a dead +# localhost:6379 factory that flips /actuator/health to DOWN. +spring.autoconfigure.exclude=org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration + # Set up a consistent temporary directory location java.io.tmpdir=${stirling.tempfiles.directory:${java.io.tmpdir}/stirling-pdf} diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/RearrangePagesPDFControllerTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/RearrangePagesPDFControllerTest.java index c31cae2958..a225c3fc52 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/RearrangePagesPDFControllerTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/RearrangePagesPDFControllerTest.java @@ -4,10 +4,14 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.*; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import org.apache.pdfbox.Loader; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.junit.jupiter.api.BeforeEach; @@ -56,6 +60,38 @@ class RearrangePagesPDFControllerTest { "fileInput", "test.pdf", MediaType.APPLICATION_PDF_VALUE, new byte[] {1, 2, 3}); } + /** Build a real, in-memory PDDocument with the requested number of blank pages. */ + private PDDocument buildRealPdf(int pageCount) throws IOException { + PDDocument doc = new PDDocument(); + for (int i = 0; i < pageCount; i++) { + doc.addPage(new PDPage()); + } + return doc; + } + + /** + * Returns the underlying {@link org.apache.pdfbox.cos.COSDictionary} for each page in document + * order. PDPageTree returns a fresh PDPage wrapper per get(), so comparing wrappers with + * assertSame is unreliable - the COSDictionary identity is the stable handle. + */ + private List snapshotCosPages(PDDocument doc) { + List snapshot = new ArrayList<>(); + for (PDPage p : doc.getPages()) { + snapshot.add(p.getCOSObject()); + } + return snapshot; + } + + private List reloadAndSnapshot(ResponseEntity response) throws IOException { + try (var in = response.getBody().getInputStream(); + var baos = new ByteArrayOutputStream()) { + in.transferTo(baos); + try (PDDocument out = Loader.loadPDF(baos.toByteArray())) { + return snapshotCosPages(out); + } + } + } + @Test void testDeletePages_Success() throws IOException { MockMultipartFile file = createMockPdf(); @@ -83,27 +119,23 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("REVERSE_ORDER"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); - PDPage page2 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(3)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(3); - when(mockDoc.getPage(0)).thenReturn(page0); - when(mockDoc.getPage(1)).thenReturn(page1); - when(mockDoc.getPage(2)).thenReturn(page2); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); - verify(mockNewDoc).addPage(page2); - verify(mockNewDoc).addPage(page1); - verify(mockNewDoc).addPage(page0); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + List finalOrder = reloadAndSnapshot(response); + assertEquals(3, finalOrder.size()); + // We can no longer compare references after a save/reload, so compare via + // the in-memory snapshot taken *after* the controller mutated the source. + List mutatedSource = snapshotCosPages(realDoc); + assertSame(originals.get(2), mutatedSource.get(0)); + assertSame(originals.get(1), mutatedSource.get(1)); + assertSame(originals.get(0), mutatedSource.get(2)); + } } @Test @@ -114,25 +146,18 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("REMOVE_FIRST"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); - PDPage page2 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(3)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(3); - when(mockDoc.getPage(1)).thenReturn(page1); - when(mockDoc.getPage(2)).thenReturn(page2); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - verify(mockNewDoc).addPage(page1); - verify(mockNewDoc).addPage(page2); - verify(mockNewDoc, never()).addPage(page0); + assertNotNull(response); + List mutated = snapshotCosPages(realDoc); + assertEquals(2, mutated.size()); + assertSame(originals.get(1), mutated.get(0)); + assertSame(originals.get(2), mutated.get(1)); + } } @Test @@ -143,23 +168,18 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("REMOVE_LAST"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(3)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(3); - when(mockDoc.getPage(0)).thenReturn(page0); - when(mockDoc.getPage(1)).thenReturn(page1); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - verify(mockNewDoc).addPage(page0); - verify(mockNewDoc).addPage(page1); + assertNotNull(response); + List mutated = snapshotCosPages(realDoc); + assertEquals(2, mutated.size()); + assertSame(originals.get(0), mutated.get(0)); + assertSame(originals.get(1), mutated.get(1)); + } } @Test @@ -170,21 +190,19 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("REMOVE_FIRST_AND_LAST"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page1 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(1)).thenReturn(page1); - when(mockDoc.getPage(2)).thenReturn(page1); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + List mutated = snapshotCosPages(realDoc); + assertEquals(2, mutated.size()); + assertSame(originals.get(1), mutated.get(0)); + assertSame(originals.get(2), mutated.get(1)); + } } @Test @@ -195,23 +213,15 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("DUPLEX_SORT"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); - PDPage page2 = mock(PDPage.class); - PDPage page3 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(anyInt())).thenReturn(page0); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + assertEquals(4, realDoc.getNumberOfPages()); + } } @Test @@ -222,20 +232,15 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("BOOKLET_SORT"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(anyInt())).thenReturn(page); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + assertEquals(4, realDoc.getNumberOfPages()); + } } @Test @@ -246,20 +251,15 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("ODD_EVEN_SPLIT"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(anyInt())).thenReturn(page); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + assertEquals(4, realDoc.getNumberOfPages()); + } } @Test @@ -270,24 +270,20 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers("3,1,2"); request.setCustomMode("custom"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); - PDPage page2 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(3)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(3); - when(mockDoc.getPage(0)).thenReturn(page0); - when(mockDoc.getPage(1)).thenReturn(page1); - when(mockDoc.getPage(2)).thenReturn(page2); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + List mutated = snapshotCosPages(realDoc); + assertEquals(3, mutated.size()); + assertSame(originals.get(2), mutated.get(0)); + assertSame(originals.get(0), mutated.get(1)); + assertSame(originals.get(1), mutated.get(2)); + } } @Test @@ -298,21 +294,15 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers("3"); request.setCustomMode("DUPLICATE"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(2)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(2); - when(mockDoc.getPage(anyInt())).thenReturn(page); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - // 2 pages * 3 duplicates = 6 addPage calls - verify(mockNewDoc, times(6)).addPage(page); + assertNotNull(response); + // 2 pages * 3 duplicates = 6 final pages + assertEquals(6, realDoc.getNumberOfPages()); + } } @Test @@ -323,19 +313,14 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("SIDE_STITCH_BOOKLET_SORT"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(anyInt())).thenReturn(page); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + assertEquals(4, realDoc.getNumberOfPages()); + } } } diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/misc/OverlayImageControllerTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/misc/OverlayImageControllerTest.java index 308518e8f0..6a13fb3049 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/misc/OverlayImageControllerTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/misc/OverlayImageControllerTest.java @@ -32,6 +32,7 @@ import org.springframework.mock.web.MockMultipartFile; import stirling.software.SPDF.model.api.misc.OverlayImageRequest; import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.util.SvgSanitizer; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; import stirling.software.common.util.WebResponseUtils; @@ -52,6 +53,7 @@ class OverlayImageControllerTest { @Mock private CustomPDFDocumentFactory pdfDocumentFactory; @Mock private TempFileManager tempFileManager; + @Mock private SvgSanitizer svgSanitizer; @InjectMocks private OverlayImageController controller; @@ -205,6 +207,52 @@ class OverlayImageControllerTest { mockDoc.close(); } + @Test + void overlayImage_svgInput_sanitizedBeforeOverlay() throws Exception { + byte[] maliciousSvg = + ("" + + "" + + "") + .getBytes(); + byte[] sanitized = + ("" + + "" + + "") + .getBytes(); + when(svgSanitizer.sanitize(maliciousSvg)).thenReturn(sanitized); + + MockMultipartFile svgFile = + new MockMultipartFile("imageFile", "overlay.svg", "image/svg+xml", maliciousSvg); + OverlayImageRequest request = new OverlayImageRequest(); + request.setFileInput(pdfFile); + request.setImageFile(svgFile); + request.setX(0); + request.setY(0); + request.setEveryPage(false); + + PDDocument mockDoc = new PDDocument(); + mockDoc.addPage(new PDPage(PDRectangle.A4)); + when(pdfDocumentFactory.load(any(byte[].class))).thenReturn(mockDoc); + + try (MockedStatic mockedWebResponse = + mockStatic(WebResponseUtils.class)) { + mockedWebResponse + .when( + () -> + WebResponseUtils.pdfFileToWebResponse( + any(TempFile.class), anyString())) + .thenReturn(streamingOk("result".getBytes())); + + controller.overlayImage(request); + } + mockDoc.close(); + + verify(svgSanitizer).sanitize(maliciousSvg); + } + @Test void overlayImage_withCoordinates_usesXY() throws Exception { OverlayImageRequest request = new OverlayImageRequest(); diff --git a/app/core/src/test/java/stirling/software/SPDF/exception/GlobalExceptionHandlerTest.java b/app/core/src/test/java/stirling/software/SPDF/exception/GlobalExceptionHandlerTest.java index 8e2275a728..4de6398e43 100644 --- a/app/core/src/test/java/stirling/software/SPDF/exception/GlobalExceptionHandlerTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/exception/GlobalExceptionHandlerTest.java @@ -18,6 +18,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.context.MessageSource; import org.springframework.core.env.Environment; +import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ProblemDetail; import org.springframework.http.ResponseEntity; @@ -27,6 +28,7 @@ import org.springframework.web.bind.MissingServletRequestParameterException; import org.springframework.web.multipart.MaxUploadSizeExceededException; import org.springframework.web.multipart.support.MissingServletRequestPartException; import org.springframework.web.servlet.NoHandlerFoundException; +import org.springframework.web.servlet.resource.NoResourceFoundException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -208,6 +210,19 @@ class GlobalExceptionHandlerTest { assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode()); } + // ---- NoResourceFoundException ---- + // Regression guard: was falling through to the 500 catch-all. + + @Test + void handleNoResourceFound_returns_404_not_500() { + when(request.getMethod()).thenReturn("GET"); + NoResourceFoundException ex = + new NoResourceFoundException(HttpMethod.GET, "/api/v1/storage/folders", ""); + ResponseEntity resp = handler.handleNoResourceFound(ex, request); + assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode()); + assertEquals("GET", resp.getBody().getProperties().get("method")); + } + // ---- IllegalArgumentException ---- @Test diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java new file mode 100644 index 0000000000..0a94eef1fc --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -0,0 +1,319 @@ +package stirling.software.proprietary.policy.controller; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.util.MultiValueMap; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.MultipartHttpServletRequest; +import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import io.github.pixee.security.Filenames; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.job.JobResponse; +import stirling.software.common.util.TempFile; +import stirling.software.common.util.TempFileManager; +import stirling.software.proprietary.policy.engine.PolicyRunHandle; +import stirling.software.proprietary.policy.engine.PolicyRunRegistry; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.PolicyRunStatus; +import stirling.software.proprietary.policy.model.PolicyRunView; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; +import stirling.software.proprietary.policy.store.PolicyStore; +import stirling.software.proprietary.policy.trigger.ManualTrigger; +import stirling.software.proprietary.security.config.PremiumEndpoint; + +import tools.jackson.core.JacksonException; +import tools.jackson.databind.ObjectMapper; + +/** + * Manages policies and runs pipelines. The premium backend entry point: CRUD for stored {@code + * Policy} objects, running a stored policy by id, and running an ad-hoc pipeline (for AI/Automate + * one-offs). + * + *

Runs execute asynchronously and return a run id immediately. Poll {@code GET /run/{runId}} for + * status, and download outputs via the existing {@code GET /api/v1/general/files/{fileId}} using + * the file ids in the run view. + */ +@Slf4j +@RestController +@RequestMapping("/api/v1/policies") +@Hidden +@PremiumEndpoint +@RequiredArgsConstructor +@Tag(name = "Policies", description = "Run tool pipelines on the backend") +public class PolicyController { + + private final ManualTrigger manualTrigger; + private final PolicyRunRegistry runRegistry; + private final PolicyStore policyStore; + private final ObjectMapper objectMapper; + private final TempFileManager tempFileManager; + + /** SSE emitter timeout, generous enough for long multi-step runs on large files. */ + @Value("${stirling.policies.streamTimeoutMs:1800000}") + private long streamTimeoutMs; + + @PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Run a tool pipeline", + description = + "Accepts the documents to process (multipart field 'fileInput'), any supporting" + + " files (each under a multipart field named as its asset key, e.g." + + " 'company-logo'), and a JSON pipeline definition ('json'). Runs the" + + " steps in order asynchronously and returns a run id. Poll the run" + + " status endpoint and download outputs via /api/v1/general/files/{id}.") + public ResponseEntity> run( + @RequestParam("json") String json, MultipartHttpServletRequest request) + throws IOException { + PipelineDefinition definition = parseDefinition(json); + PolicyInputs inputs = collectInputs(request); + String runId = manualTrigger.fire(definition, inputs, PolicyProgressListener.NOOP).runId(); + return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); + } + + @PostMapping(value = "/run/stream", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Run a tool pipeline with live progress", + description = + "Same as /run, but returns Server-Sent Events: a 'step' event as each step" + + " starts and completes, then a terminal 'completed', 'failed'," + + " 'cancelled', or 'waiting' event carrying the final run view.") + public SseEmitter runStream( + @RequestParam("json") String json, MultipartHttpServletRequest request) + throws IOException { + PipelineDefinition definition = parseDefinition(json); + PolicyInputs inputs = collectInputs(request); + + SseEmitter emitter = new SseEmitter(streamTimeoutMs); + emitter.onError(e -> log.warn("Policy run SSE emitter error", e)); + + PolicyRunHandle handle = manualTrigger.fire(definition, inputs, streamListener(emitter)); + // Close the stream with a terminal event once the run finishes. whenComplete runs on the + // engine's worker thread after the run is done, so this never races the step events. + handle.completion() + .whenComplete( + (run, throwable) -> { + if (throwable != null) { + sendEvent( + emitter, + "failed", + Map.of("message", throwable.getMessage())); + } else { + sendEvent(emitter, terminalEventName(run), PolicyRunView.of(run)); + } + emitter.complete(); + }); + return emitter; + } + + @GetMapping("/run/{runId}") + @Operation( + summary = "Get pipeline run status", + description = "Returns the current status, step cursor, and output files of a run.") + public ResponseEntity status(@PathVariable String runId) { + PolicyRun run = runRegistry.get(runId); + if (run == null) { + return ResponseEntity.notFound().build(); + } + return ResponseEntity.ok(PolicyRunView.of(run)); + } + + // --- Policy management --- + + @PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE) + @Operation( + summary = "Create or update a policy", + description = + "Stores a policy (trigger config + steps + output + metadata). A blank id is" + + " assigned; returns the stored policy with its id.") + public ResponseEntity savePolicy(@RequestBody String json) { + return ResponseEntity.ok(policyStore.save(parsePolicy(json))); + } + + @GetMapping + @Operation(summary = "List policies") + public List listPolicies() { + return policyStore.all(); + } + + @GetMapping("/{policyId}") + @Operation(summary = "Get a policy by id") + public ResponseEntity getPolicy(@PathVariable String policyId) { + return policyStore + .get(policyId) + .map(ResponseEntity::ok) + .orElseGet(() -> ResponseEntity.notFound().build()); + } + + @DeleteMapping("/{policyId}") + @Operation(summary = "Delete a policy by id") + public ResponseEntity deletePolicy(@PathVariable String policyId) { + return policyStore.delete(policyId) + ? ResponseEntity.noContent().build() + : ResponseEntity.notFound().build(); + } + + @PostMapping(value = "/{policyId}/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Run a stored policy", + description = + "Runs the stored policy's pipeline on the supplied files (primary documents" + + " under 'fileInput', supporting files under their asset-key fields)." + + " Runs regardless of the policy's enabled flag, which only gates" + + " automatic triggering. Returns a run id.") + public ResponseEntity> runStoredPolicy( + @PathVariable String policyId, MultipartHttpServletRequest request) throws IOException { + Policy policy = + policyStore + .get(policyId) + .orElseThrow( + () -> + new ResponseStatusException( + HttpStatus.NOT_FOUND, "No policy: " + policyId)); + PolicyInputs inputs = collectInputs(request); + String runId = manualTrigger.run(policy, inputs, PolicyProgressListener.NOOP).runId(); + return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); + } + + private Policy parsePolicy(String json) { + try { + return objectMapper.readValue(json, Policy.class); + } catch (JacksonException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid policy JSON"); + } + } + + private PipelineDefinition parseDefinition(String json) { + PipelineDefinition definition; + try { + definition = objectMapper.readValue(json, PipelineDefinition.class); + } catch (JacksonException e) { + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, "Invalid pipeline definition JSON"); + } + if (definition.steps().isEmpty()) { + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, "Pipeline definition has no steps"); + } + return definition; + } + + /** + * Split the multipart file parts into the primary document stream ("fileInput") and the named + * supporting-file store: every other file field becomes an asset keyed by its field name, which + * a step references from {@code fileParameters}. + */ + private PolicyInputs collectInputs(MultipartHttpServletRequest request) throws IOException { + MultiValueMap fileMap = request.getMultiFileMap(); + List primary = toResources(fileMap.get("fileInput")); + Map> supportingFiles = new LinkedHashMap<>(); + for (Map.Entry> entry : fileMap.entrySet()) { + if ("fileInput".equals(entry.getKey())) { + continue; + } + List assets = toResources(entry.getValue()); + if (!assets.isEmpty()) { + supportingFiles.put(entry.getKey(), assets); + } + } + return new PolicyInputs(primary, supportingFiles); + } + + /** + * A progress listener that forwards each step transition to the SSE stream as a "step" event. + */ + private PolicyProgressListener streamListener(SseEmitter emitter) { + return new PolicyProgressListener() { + @Override + public void onStepStart(int stepIndex, int stepCount, String operation) { + sendEvent(emitter, "step", stepEvent("started", stepIndex, stepCount, operation)); + } + + @Override + public void onStepComplete(int stepIndex, int stepCount, String operation) { + sendEvent(emitter, "step", stepEvent("completed", stepIndex, stepCount, operation)); + } + }; + } + + private static Map stepEvent( + String phase, int stepIndex, int stepCount, String operation) { + return Map.of( + "phase", phase, + "stepIndex", stepIndex, + "stepCount", stepCount, + "operation", operation); + } + + private static String terminalEventName(PolicyRun run) { + PolicyRunStatus status = run.getStatus(); + return switch (status) { + case COMPLETED -> "completed"; + case FAILED -> "failed"; + case CANCELLED -> "cancelled"; + case WAITING_FOR_INPUT -> "waiting"; + default -> "ended"; + }; + } + + private void sendEvent(SseEmitter emitter, String name, Object data) { + try { + emitter.send(SseEmitter.event().name(name).data(data, MediaType.APPLICATION_JSON)); + } catch (IOException | IllegalStateException e) { + // Client disconnected or the emitter already closed. The run continues and its results + // remain downloadable via the job endpoints; nothing useful left to stream. + log.debug("Dropping policy SSE event '{}': {}", name, e.getMessage()); + } + } + + private List toResources(List files) throws IOException { + List resources = new ArrayList<>(); + if (files == null) { + return resources; + } + for (MultipartFile file : files) { + if (file == null || file.isEmpty()) { + continue; + } + TempFile tempFile = tempFileManager.createManagedTempFile("policy-run"); + file.transferTo(tempFile.getPath()); + final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename()); + resources.add( + new FileSystemResource(tempFile.getFile()) { + @Override + public String getFilename() { + return originalName; + } + }); + } + return resources; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java new file mode 100644 index 0000000000..8f41209c7e --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java @@ -0,0 +1,261 @@ +package stirling.software.proprietary.policy.engine; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.job.ResultFile; +import stirling.software.common.service.FileStorage; +import stirling.software.common.service.InternalApiTimeoutException; +import stirling.software.common.service.JobOwnershipService; +import stirling.software.common.service.JobQueue; +import stirling.software.common.service.ResourceMonitor; +import stirling.software.common.service.TaskManager; +import stirling.software.common.util.ExecutorFactory; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.WaitState; +import stirling.software.proprietary.policy.output.PolicyOutputSink; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +/** + * Runs pipelines asynchronously as tracked jobs. + * + *

Each run is the unit of async work: {@link #submit} returns a run id immediately and the + * pipeline executes on a virtual thread, so a step blocking on a slow tool does not tie up a + * platform thread. The run drives {@link PolicyExecutor} for the actual step loop, registers its + * outputs and progress with {@link TaskManager} (so the existing job status/download endpoints work + * unchanged), and keeps rich state in {@link PolicyRunRegistry}. + * + *

The engine deliberately manages its own virtual-thread execution rather than routing through + * {@code JobExecutorService}: that path force-completes a job once its work returns, which is + * incompatible with a run that suspends in {@code WAITING_FOR_INPUT}. It still applies the shared + * {@link ResourceMonitor}/{@link JobQueue} admission control, so heavy runs queue under load + * instead of oversubscribing. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class PolicyEngine { + + /** + * Resource weight of a pipeline run for admission control. A run chains many tools and holds + * intermediate files, so it is weighted as heavy work: the shared {@link ResourceMonitor} + * should let it start while the system is healthy but hold it back under memory/CPU pressure. + * See {@link ResourceMonitor#shouldQueueJob(int)} for how a weight maps to that decision. + */ + private static final int RUN_RESOURCE_WEIGHT = 50; + + private final PolicyExecutor stepExecutor; + private final TaskManager taskManager; + private final PolicyRunRegistry registry; + private final FileStorage fileStorage; + private final JobOwnershipService jobOwnershipService; + private final List outputSinks; + private final ResourceMonitor resourceMonitor; + private final JobQueue jobQueue; + + private final ExecutorService asyncExecutor = ExecutorFactory.newVirtualThreadExecutor(); + + /** + * Submit a pipeline to run asynchronously. The returned handle's run id scopes a job in {@link + * TaskManager}, so progress (notes), status, and result files are observable via the existing + * job endpoints as well as via {@link #getRun(String)}; its completion future resolves when the + * run reaches a terminal or paused state. + */ + public PolicyRunHandle submit( + PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) { + // Scope the run id to the current user (on this request thread) so the file-download + // ownership check passes; NoOpJobOwnershipService returns the id unchanged when security + // is off. + String runId = jobOwnershipService.createScopedJobKey(UUID.randomUUID().toString()); + taskManager.createTask(runId); + PolicyRun run = new PolicyRun(runId, definition); + registry.register(run); + CompletableFuture completion = new CompletableFuture<>(); + PolicyProgressListener tracking = trackingListener(runId, run, listener); + Runnable task = () -> runToCompletion(run, inputs, tracking, completion); + + // Each run is one admission unit; steps run synchronously within it, so this gates heavy + // work under load without the pool-within-pool risk of queueing each tool call. Under + // resource pressure the run waits in the shared JobQueue; otherwise it starts immediately. + if (resourceMonitor.shouldQueueJob(RUN_RESOURCE_WEIGHT)) { + log.debug("Queueing policy run {} under resource pressure", runId); + jobQueue.queueJob( + runId, + RUN_RESOURCE_WEIGHT, + () -> { + task.run(); + return null; + }, + 0L) + .exceptionally(ex -> failRejectedRun(run, completion, ex)); + } else { + asyncExecutor.execute(task); + } + return new PolicyRunHandle(runId, completion); + } + + /** + * Run a stored policy on demand. Builds the policy's pipeline and submits it. {@code enabled} + * gates automatic triggering, not explicit runs, so this runs regardless of that flag. + */ + public PolicyRunHandle runPolicy( + Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { + return submit(policy.toDefinition(), inputs, listener); + } + + public PolicyRun getRun(String runId) { + return registry.get(runId); + } + + /** + * Request cancellation of a run. Stage 1 marks the run cancelled in the registry if it has not + * already finished; interrupting an in-flight tool call lands in a later stage. + */ + public boolean cancel(String runId) { + PolicyRun run = registry.get(runId); + if (run == null) { + return false; + } + boolean cancelled = run.cancel(); + if (cancelled) { + taskManager.addNote(runId, "Run cancelled by request"); + } + return cancelled; + } + + /** + * Resume a run paused in {@code WAITING_FOR_INPUT}. Not yet implemented; the run shape and + * {@link WaitState} snapshot are in place so this can be added without reworking the engine. + */ + public String resume(String runId, List additionalInputs) { + throw new UnsupportedOperationException("Pause/resume is not yet implemented"); + } + + private void runToCompletion( + PolicyRun run, + PolicyInputs inputs, + PolicyProgressListener listener, + CompletableFuture completion) { + String runId = run.getRunId(); + try { + run.markRunning(); + PolicyExecutionResult result = + stepExecutor.execute(run.getDefinition(), inputs, listener); + OutputSpec output = run.getDefinition().output(); + List outputs = sinkFor(output).deliver(runId, result.files(), output); + taskManager.setMultipleFileResults(runId, outputs); + taskManager.setComplete(runId); + run.complete(outputs); + } catch (PolicyInputRequiredException e) { + // Designed-for path: suspend the run rather than fail it. Persist intermediates as + // fileIds so the run can resume after this worker thread is gone. + WaitState wait = suspend(e); + run.waitForInput(wait); + taskManager.addNote(runId, "Waiting for input: " + e.getMessage()); + } catch (InternalApiTimeoutException e) { + String message = toolTimeoutMessage(e); + log.error( + "Policy run {} timed out on {}: {}", + runId, + e.getEndpointPath(), + e.getMessage()); + run.fail(message); + taskManager.setError(runId, message); + } catch (Exception e) { + String message = "Policy run failed: " + e.getMessage(); + log.error("Policy run {} failed", runId, e); + run.fail(message); + taskManager.setError(runId, message); + } finally { + // Always resolve the handle with the run's final state so stream/await callers unblock. + completion.complete(run); + } + } + + private ResponseEntity failRejectedRun( + PolicyRun run, CompletableFuture completion, Throwable ex) { + // Only reached if the run never started (e.g. the queue was full). A run that started + // always resolves its own completion in runToCompletion. + if (!completion.isDone()) { + String message = "Policy run could not be queued: " + ex.getMessage(); + log.error("Policy run {} was not admitted: {}", run.getRunId(), ex.getMessage()); + run.fail(message); + taskManager.setError(run.getRunId(), message); + completion.complete(run); + } + return null; + } + + private WaitState suspend(PolicyInputRequiredException e) { + List fileIds = new ArrayList<>(); + for (Resource resource : e.getPendingFiles()) { + String name = resource.getFilename() != null ? resource.getFilename() : "pending"; + try (InputStream is = resource.getInputStream()) { + fileIds.add(fileStorage.storeInputStream(is, name).fileId()); + } catch (IOException io) { + log.warn("Failed to persist pending file for paused run: {}", io.getMessage()); + } + } + return new WaitState(e.getMessage(), e.getResumeStepIndex(), fileIds); + } + + private PolicyProgressListener trackingListener( + String runId, PolicyRun run, PolicyProgressListener delegate) { + return new PolicyProgressListener() { + @Override + public void onStepStart(int stepIndex, int stepCount, String operation) { + run.enterStep(stepIndex); + taskManager.addNote( + runId, + "Step " + stepIndex + "/" + stepCount + ": " + operation + " started"); + delegate.onStepStart(stepIndex, stepCount, operation); + } + + @Override + public void onStepComplete(int stepIndex, int stepCount, String operation) { + taskManager.addNote( + runId, + "Step " + stepIndex + "/" + stepCount + ": " + operation + " completed"); + delegate.onStepComplete(stepIndex, stepCount, operation); + } + + @Override + public void onHeartbeat() { + delegate.onHeartbeat(); + } + }; + } + + private PolicyOutputSink sinkFor(OutputSpec spec) { + return outputSinks.stream() + .filter(sink -> sink.supports(spec)) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "No output sink supports spec: " + + (spec == null ? "" : spec.type()))); + } + + private static String toolTimeoutMessage(InternalApiTimeoutException e) { + return String.format( + "The %s tool did not respond within %d seconds and was aborted.", + e.getEndpointPath(), e.getReadTimeout().toSeconds()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java new file mode 100644 index 0000000000..eb5f028bde --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java @@ -0,0 +1,17 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.List; + +import org.springframework.core.io.Resource; + +import tools.jackson.databind.JsonNode; + +/** + * Result of running a pipeline through {@link PolicyExecutor}. + * + *

{@code files} are the final output resources (temp files, not yet stored to {@code + * FileStorage}). {@code report} is the structured metadata payload captured from the last step that + * produced one (a JSON body, or an {@code X-Stirling-Tool-Report} header), with {@code reportTool} + * naming the step it came from; both are null when no step produced a report. + */ +public record PolicyExecutionResult(List files, JsonNode report, String reportTool) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java new file mode 100644 index 0000000000..92a436bdfd --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java @@ -0,0 +1,312 @@ +package stirling.software.proprietary.policy.engine; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.service.InternalApiClient; +import stirling.software.common.service.InternalApiTimeoutException; +import stirling.software.common.service.ToolMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.ZipExtractionUtils; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; +import stirling.software.proprietary.service.AiToolResponseHeaders; + +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +/** + * Runs an ordered chain of tool steps, chaining each step's output files into the next step's + * input. + * + *

This is the single execution loop for the proprietary surface (AI plans now; + * manually-triggered runs and watched folders later). Each step is dispatched synchronously via + * {@link InternalApiClient} loopback HTTP: the tool runs in its own handler and returns its file + * inline. The caller decides how to run the executor itself (the AI turn loop calls it directly; + * the engine runs it on a virtual thread for async runs). Files cross step boundaries as {@link + * Resource} temp files; they are only persisted to durable storage at the run boundaries by the + * caller. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class PolicyExecutor { + + private static final String FILTER_OPERATION_PREFIX = "/api/v1/filter/filter-"; + + private final InternalApiClient internalApiClient; + private final ToolMetadataService toolMetadataService; + private final TempFileManager tempFileManager; + private final ObjectMapper objectMapper; + + /** + * Internal value-class for tool responses. {@code files} holds any result files (typically one; + * multiple for ZIP-response tools). {@code report} holds an optional structured metadata + * payload the tool chose to surface alongside (or instead of) a file. + */ + private record ToolResult(List files, JsonNode report) {} + + /** + * Execute every step in {@code definition} in order, feeding each step's output into the next. + * Supporting files supplied in {@code inputs} are bound to steps' named file fields and never + * enter the document stream. + * + * @param definition the pipeline to run (must have at least one step) + * @param inputs the primary documents plus the named supporting-file store + * @param listener receives per-step progress + * @return the final output files plus the last structured report produced, if any + * @throws InternalApiTimeoutException if a tool does not respond within its read timeout + * @throws IOException if a tool returns a non-OK response, references a missing supporting + * file, or a file cannot be read + */ + public PolicyExecutionResult execute( + PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) + throws IOException { + List steps = definition.steps(); + if (steps.isEmpty()) { + throw new IllegalArgumentException("Pipeline definition has no steps"); + } + + List currentFiles = inputs.primary(); + Map> supportingFiles = inputs.supportingFiles(); + // Propagate the *last* non-null report; the terminal step defines the output. + JsonNode lastReport = null; + String lastReportTool = null; + + for (int i = 0; i < steps.size(); i++) { + PipelineStep step = steps.get(i); + String operation = step.operation(); + if (operation == null || operation.isBlank()) { + throw new IllegalArgumentException( + "Pipeline step " + (i + 1) + " has no operation"); + } + listener.onStepStart(i + 1, steps.size(), operation); + ToolResult stepResult = executeStep(step, currentFiles, supportingFiles); + currentFiles = stepResult.files(); + if (stepResult.report() != null) { + lastReport = stepResult.report(); + lastReportTool = operation; + } + listener.onStepComplete(i + 1, steps.size(), operation); + } + + return new PolicyExecutionResult(currentFiles, lastReport, lastReportTool); + } + + /** + * Execute a single tool step. If the endpoint accepts multiple files, all files are sent in one + * call. Otherwise, the endpoint is called once per file. ZIP responses are unpacked so each + * inner file is treated as its own result (e.g. split outputs a ZIP of pages). + * + *

A structured {@code report} may be returned alongside (or instead of) files; see {@link + * ToolResult}. For per-file dispatch (single-input endpoints called once per input), the first + * non-null report wins. + */ + private ToolResult executeStep( + PipelineStep step, + List inputFiles, + Map> supportingFiles) + throws IOException { + requireAcceptedTypes(step.operation(), inputFiles); + List files = new ArrayList<>(); + JsonNode report = null; + if (toolMetadataService.isMultiInput(step.operation())) { + ToolResult r = callEndpoint(step, inputFiles, supportingFiles); + files.addAll(r.files()); + report = r.report(); + } else { + for (Resource file : inputFiles) { + ToolResult r = callEndpoint(step, List.of(file), supportingFiles); + files.addAll(r.files()); + if (report == null) { + report = r.report(); + } + } + } + return new ToolResult(files, report); + } + + /** + * Call an endpoint and return its result files and optional report. + * + *

    + *
  • JSON body (Content-Type: application/json): the entire body is the report, no files are + * returned. + *
  • File body (PDF etc.): the file is returned; if an {@link + * AiToolResponseHeaders#TOOL_REPORT} header is present, its (minified JSON) value is + * parsed as the report. + *
  • ZIP responses declared by the tool metadata service are unpacked so callers always see + * a flat list of result files. + *
+ */ + private ToolResult callEndpoint( + PipelineStep step, List files, Map> supportingFiles) + throws IOException { + String endpointPath = step.operation(); + MultiValueMap body = new LinkedMultiValueMap<>(); + for (Resource file : files) { + body.add("fileInput", file); + } + // Bind supporting files to their named tool fields (e.g. stampImage, overlayFiles). These + // come from the run's named asset store, not the document stream. + for (Map.Entry binding : step.fileParameters().entrySet()) { + String fieldName = binding.getKey(); + String assetKey = binding.getValue(); + List assets = supportingFiles.get(assetKey); + if (assets == null || assets.isEmpty()) { + throw new IOException( + "Step " + + endpointPath + + " references supporting file '" + + assetKey + + "' for field '" + + fieldName + + "' but no such file was provided"); + } + for (Resource asset : assets) { + body.add(fieldName, asset); + } + } + for (Map.Entry entry : step.parameters().entrySet()) { + if (entry.getValue() instanceof List list) { + if (containsStructuredElements(list)) { + // Endpoints binding lists of structured objects (e.g. /security/redact's + // redactions, /general/edit-text's edits) parse a single JSON string field via + // a property editor. Pre-serialize the whole list so binding succeeds. + body.add(entry.getKey(), objectMapper.writeValueAsString(list)); + } else { + for (Object item : list) { + body.add(entry.getKey(), item); + } + } + } else { + body.add(entry.getKey(), entry.getValue()); + } + } + ResponseEntity response = internalApiClient.post(endpointPath, body); + if (!HttpStatus.OK.equals(response.getStatusCode()) || response.getBody() == null) { + throw new IOException( + "Tool returned HTTP " + response.getStatusCode() + " for " + endpointPath); + } + Resource resource = response.getBody(); + + // Filter operations return an empty body to signal the file was filtered out: drop it + // rather than forwarding a zero-byte document. + if (isFilterOperation(endpointPath) && isEmpty(resource)) { + return new ToolResult(List.of(), null); + } + + HttpHeaders headers = response.getHeaders(); + MediaType contentType = headers.getContentType(); + + // JSON-only response: the whole body is the structured report, no result file. + if (contentType != null && MediaType.APPLICATION_JSON.isCompatibleWith(contentType)) { + try (InputStream is = resource.getInputStream()) { + JsonNode report = objectMapper.readTree(is); + return new ToolResult(List.of(), report); + } + } + + JsonNode report = parseReportHeader(headers, endpointPath); + if (toolMetadataService.shouldUnpackZipResponse(endpointPath)) { + return new ToolResult(ZipExtractionUtils.extractZip(resource, tempFileManager), report); + } + return new ToolResult(List.of(resource), report); + } + + /** + * Parse the optional {@link AiToolResponseHeaders#TOOL_REPORT} header into a {@link JsonNode}, + * or return null. + */ + private JsonNode parseReportHeader(HttpHeaders headers, String endpointPath) { + String raw = headers.getFirst(AiToolResponseHeaders.TOOL_REPORT); + if (raw == null || raw.isBlank()) { + return null; + } + try { + return objectMapper.readTree(raw); + } catch (JacksonException e) { + log.warn( + "Ignoring malformed {} header from {}: {}", + AiToolResponseHeaders.TOOL_REPORT, + endpointPath, + e.getMessage()); + return null; + } + } + + private static boolean containsStructuredElements(List list) { + for (Object item : list) { + if (item instanceof Map || item instanceof List) { + return true; + } + } + return false; + } + + /** + * Fail the run if any document in the primary stream is not a file type the step accepts. An + * endpoint that declares no specific input type accepts anything. + */ + private void requireAcceptedTypes(String operation, List files) throws IOException { + List accepted = toolMetadataService.getExtensionTypes(false, operation); + if (accepted == null || accepted.isEmpty()) { + return; + } + for (Resource file : files) { + if (!matchesType(file, accepted)) { + throw new IOException( + "Step " + + operation + + " accepts " + + accepted + + " but received '" + + file.getFilename() + + "'"); + } + } + } + + private static boolean matchesType(Resource file, List acceptedExtensions) { + String filename = file.getFilename(); + if (filename == null) { + return false; + } + int dot = filename.lastIndexOf('.'); + if (dot < 0 || dot == filename.length() - 1) { + return false; + } + return acceptedExtensions.contains(filename.substring(dot + 1).toLowerCase(Locale.ROOT)); + } + + private static boolean isFilterOperation(String operation) { + return operation.startsWith(FILTER_OPERATION_PREFIX); + } + + private static boolean isEmpty(Resource resource) { + try { + return resource.contentLength() == 0; + } catch (IOException e) { + return false; + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyInputRequiredException.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyInputRequiredException.java new file mode 100644 index 0000000000..fa30373c57 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyInputRequiredException.java @@ -0,0 +1,32 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.List; + +import org.springframework.core.io.Resource; + +import lombok.Getter; + +/** + * Thrown by a step to signal that the run cannot proceed without further user input, pausing the + * run in {@code WAITING_FOR_INPUT} rather than failing it. + * + *

Carries everything needed to resume: a human-readable reason, the 0-based index of the step to + * resume from, and the intermediate files produced so far. The engine persists those files and + * suspends the run. + * + *

Defined now to fix the run shape; no step throws it yet, and the resume handshake is + * implemented in a later stage. + */ +@Getter +public class PolicyInputRequiredException extends RuntimeException { + + private final transient List pendingFiles; + private final int resumeStepIndex; + + public PolicyInputRequiredException( + String reason, int resumeStepIndex, List pendingFiles) { + super(reason); + this.resumeStepIndex = resumeStepIndex; + this.pendingFiles = pendingFiles == null ? List.of() : pendingFiles; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunHandle.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunHandle.java new file mode 100644 index 0000000000..eeda863ccc --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunHandle.java @@ -0,0 +1,16 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.concurrent.CompletableFuture; + +import stirling.software.proprietary.policy.model.PolicyRun; + +/** + * Returned by {@link PolicyEngine#submit}: the run id (for status polling and result download) plus + * a future that resolves when the run reaches a terminal or paused state. + * + *

The completion future lets callers react to the end of a run (e.g. an SSE endpoint sending a + * final event and closing the stream) without polling. It carries the {@link PolicyRun} whose + * status describes the outcome (completed, failed, cancelled, or waiting for input); it does not + * complete exceptionally for ordinary run failures. + */ +public record PolicyRunHandle(String runId, CompletableFuture completion) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java new file mode 100644 index 0000000000..0b613726d7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java @@ -0,0 +1,97 @@ +package stirling.software.proprietary.policy.engine; + +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import jakarta.annotation.PreDestroy; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.policy.model.PolicyRun; + +/** + * In-memory store of live {@link PolicyRun} state, keyed by runId. Holds the authoritative run + * state machine; durable status/files for download are projected separately into {@code + * TaskManager}. + * + *

Finished runs are evicted on a fixed interval once they age past {@code + * stirling.policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a + * run's rich in-memory state does not outlive the process. Only terminal runs are evicted; active + * and paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not + * touched here: a run shares its runId with a {@code TaskManager} job, which owns file-lifecycle + * cleanup, so eviction only frees this map's entry. + */ +@Slf4j +@Service +public class PolicyRunRegistry { + + private final Map runs = new ConcurrentHashMap<>(); + + private final Duration runExpiry; + private final ScheduledExecutorService cleanupExecutor = + Executors.newSingleThreadScheduledExecutor( + Thread.ofVirtual().name("policy-run-cleanup-", 0).factory()); + + public PolicyRunRegistry( + @Value("${stirling.policies.runExpiryMinutes:30}") int runExpiryMinutes) { + this.runExpiry = Duration.ofMinutes(runExpiryMinutes); + cleanupExecutor.scheduleAtFixedRate(this::evictExpiredRuns, 10, 10, TimeUnit.MINUTES); + log.debug( + "Policy run registry initialized with run expiry of {} minutes", runExpiryMinutes); + } + + public void register(PolicyRun run) { + runs.put(run.getRunId(), run); + } + + public PolicyRun get(String runId) { + return runs.get(runId); + } + + public Collection all() { + return runs.values(); + } + + /** Scheduled hook: evict terminal runs that finished before the expiry window. */ + private void evictExpiredRuns() { + try { + evictExpired(Instant.now().minus(runExpiry)); + } catch (Exception e) { + log.error("Error during policy run cleanup: {}", e.getMessage(), e); + } + } + + /** + * Remove every terminal run last updated before {@code cutoff}; active and paused runs are kept + * regardless of age. Returns the number evicted. Package-visible so the scheduled sweep and + * tests exercise the same path with an explicit cutoff. + */ + int evictExpired(Instant cutoff) { + int removed = 0; + for (Map.Entry entry : runs.entrySet()) { + PolicyRun run = entry.getValue(); + if (run.getStatus().isTerminal() && run.getUpdatedAt().isBefore(cutoff)) { + runs.remove(entry.getKey()); + removed++; + } + } + if (removed > 0) { + log.info("Evicted {} expired policy runs", removed); + } + return removed; + } + + @PreDestroy + void shutdown() { + cleanupExecutor.shutdownNow(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java new file mode 100644 index 0000000000..4759810bc8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java @@ -0,0 +1,21 @@ +package stirling.software.proprietary.policy.model; + +import java.util.Map; + +/** + * Describes where a pipeline run's output files should be delivered. {@code type} selects a {@code + * PolicyOutputSink} (e.g. "inline"); {@code options} carries sink-specific configuration. + * + *

New destinations (folder, S3) are added as new sink beans keyed on a new {@code type} without + * changing this shape or the engine. + */ +public record OutputSpec(String type, Map options) { + public OutputSpec { + options = options == null ? Map.of() : options; + } + + /** The default destination: store outputs and return them to the caller for download. */ + public static OutputSpec inline() { + return new OutputSpec("inline", Map.of()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineDefinition.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineDefinition.java new file mode 100644 index 0000000000..983c3dbca9 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineDefinition.java @@ -0,0 +1,16 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; + +/** + * An ordered chain of tool steps plus where the output should go. + * + *

This is the single shape executed by the policy engine, shared by AI plans, manually-triggered + * runs, and (later) watched folders. {@code output} may be null for callers that handle result + * files themselves (e.g. the AI workflow, which builds its own response payload). + */ +public record PipelineDefinition(String name, List steps, OutputSpec output) { + public PipelineDefinition { + steps = steps == null ? List.of() : steps; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineStep.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineStep.java new file mode 100644 index 0000000000..52c6c060a2 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineStep.java @@ -0,0 +1,30 @@ +package stirling.software.proprietary.policy.model; + +import java.util.Map; + +/** + * A single tool invocation in a pipeline: the API endpoint path to call and the inputs to pass. + * + *

{@code operation} is a Stirling tool endpoint path (e.g. {@code /api/v1/misc/compress-pdf}), + * matching the dispatch convention used by {@code InternalApiClient}. {@code parameters} are the + * tool-specific scalar form fields. + * + *

{@code fileParameters} binds a tool's named file fields (beyond the primary {@code fileInput} + * stream) to supporting files supplied with the run: it maps the form field name (e.g. {@code + * stampImage}, {@code overlayFiles}) to an asset key in the run's supporting-file store. This keeps + * supporting inputs (a stamp image, a certificate, an overlay) out of the document stream that + * flows step to step. + */ +public record PipelineStep( + String operation, Map parameters, Map fileParameters) { + + public PipelineStep { + parameters = parameters == null ? Map.of() : parameters; + fileParameters = fileParameters == null ? Map.of() : fileParameters; + } + + /** A step with no supporting-file bindings. */ + public PipelineStep(String operation, Map parameters) { + this(operation, parameters, Map.of()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java new file mode 100644 index 0000000000..3a2acaf0c8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java @@ -0,0 +1,38 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; + +/** + * A stored, owned automation: how it is triggered, the ordered tool steps to run, and where its + * output goes, plus identity and metadata. + * + *

This is the central object of the feature. Everything that runs a chain of tools is a use of a + * Policy: a watched folder is a Policy with a folder {@link TriggerConfig} and a folder {@link + * OutputSpec}; a scheduled job is a Policy with a schedule trigger; manual/Automate/AI runs execute + * a Policy (or an ad-hoc {@link PipelineDefinition}) on demand. The engine itself only ever + * executes the {@link PipelineDefinition} this exposes via {@link #toDefinition()} - it is + * trigger-agnostic. + * + *

{@code enabled} gates automatic triggering (a disabled policy is not picked up by its + * trigger); it does not block an explicit manual run. + */ +public record Policy( + String id, + String name, + String owner, + boolean enabled, + TriggerConfig trigger, + List steps, + OutputSpec output) { + + public Policy { + trigger = trigger == null ? TriggerConfig.manual() : trigger; + steps = steps == null ? List.of() : steps; + output = output == null ? OutputSpec.inline() : output; + } + + /** The engine-level, trigger-agnostic view of this policy's pipeline. */ + public PipelineDefinition toDefinition() { + return new PipelineDefinition(name, steps, output); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyInputs.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyInputs.java new file mode 100644 index 0000000000..4e6ddae834 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyInputs.java @@ -0,0 +1,32 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; +import java.util.Map; + +import org.springframework.core.io.Resource; + +/** + * The files a run operates on, split into two roles: + * + *

    + *
  • {@code primary} - the documents that flow through the pipeline, each step's output becoming + * the next step's input. + *
  • {@code supportingFiles} - a named store of auxiliary files (a stamp image, certificate, + * overlay, attachments) that steps bind to their named file fields via {@link + * PipelineStep#fileParameters()}. These never enter the document stream. + *
+ * + * Asset values are lists so a single key can carry multi-file fields (e.g. attachments). + */ +public record PolicyInputs(List primary, Map> supportingFiles) { + + public PolicyInputs { + primary = primary == null ? List.of() : primary; + supportingFiles = supportingFiles == null ? Map.of() : supportingFiles; + } + + /** Inputs with primary documents only and no supporting files. */ + public static PolicyInputs of(List primary) { + return new PolicyInputs(primary, Map.of()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRun.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRun.java new file mode 100644 index 0000000000..20621f547f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRun.java @@ -0,0 +1,87 @@ +package stirling.software.proprietary.policy.model; + +import java.time.Instant; +import java.util.List; + +import lombok.Getter; + +import stirling.software.common.model.job.ResultFile; + +/** + * Live, mutable state of a single pipeline run, held in memory by {@code PolicyRunRegistry}. + * + *

This carries the rich execution state (status, step cursor, wait state) that the job system's + * {@code JobResult} does not model. The run is also projected into {@code TaskManager} for + * cluster-visible status, progress notes, and file download; this object is the authoritative + * source of the state machine. + */ +@Getter +public class PolicyRun { + + private final String runId; + private final PipelineDefinition definition; + private final Instant createdAt = Instant.now(); + + private volatile PolicyRunStatus status = PolicyRunStatus.PENDING; + + /** 1-based index of the step currently running (0 before the run starts). */ + private volatile int currentStep = 0; + + private volatile WaitState waitState; + private volatile String error; + private volatile List outputs = List.of(); + private volatile Instant updatedAt = Instant.now(); + + public PolicyRun(String runId, PipelineDefinition definition) { + this.runId = runId; + this.definition = definition; + } + + public int stepCount() { + return definition.steps().size(); + } + + public synchronized void markRunning() { + this.status = PolicyRunStatus.RUNNING; + touch(); + } + + public synchronized void enterStep(int oneBasedStepIndex) { + this.currentStep = oneBasedStepIndex; + touch(); + } + + public synchronized void complete(List resultFiles) { + this.outputs = resultFiles == null ? List.of() : List.copyOf(resultFiles); + this.status = PolicyRunStatus.COMPLETED; + touch(); + } + + public synchronized void fail(String message) { + this.error = message; + this.status = PolicyRunStatus.FAILED; + touch(); + } + + public synchronized void waitForInput(WaitState wait) { + this.waitState = wait; + this.status = PolicyRunStatus.WAITING_FOR_INPUT; + touch(); + } + + /** + * Mark cancelled if the run has not already reached a terminal state. Returns whether it did. + */ + public synchronized boolean cancel() { + if (status.isTerminal()) { + return false; + } + this.status = PolicyRunStatus.CANCELLED; + touch(); + return true; + } + + private void touch() { + this.updatedAt = Instant.now(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunStatus.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunStatus.java new file mode 100644 index 0000000000..ee75f5e50f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunStatus.java @@ -0,0 +1,21 @@ +package stirling.software.proprietary.policy.model; + +/** + * Lifecycle states of a {@link PolicyRun}. + * + *

{@code WAITING_FOR_INPUT} is modelled now so the engine and run shape support pausing a run + * (e.g. a step that blocks for a human decision) without holding a thread; the resume handshake is + * implemented in a later stage. + */ +public enum PolicyRunStatus { + PENDING, + RUNNING, + WAITING_FOR_INPUT, + COMPLETED, + FAILED, + CANCELLED; + + public boolean isTerminal() { + return this == COMPLETED || this == FAILED || this == CANCELLED; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java new file mode 100644 index 0000000000..a3427835ce --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java @@ -0,0 +1,28 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; + +import stirling.software.common.model.job.ResultFile; + +/** + * Read-only view of a {@link PolicyRun} returned by the status endpoint. Output files are surfaced + * as {@link ResultFile} so the caller can download each via {@code GET /api/v1/general/files/{id}}. + */ +public record PolicyRunView( + String runId, + PolicyRunStatus status, + int currentStep, + int stepCount, + String error, + List outputs) { + + public static PolicyRunView of(PolicyRun run) { + return new PolicyRunView( + run.getRunId(), + run.getStatus(), + run.getCurrentStep(), + run.stepCount(), + run.getError(), + run.getOutputs()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java new file mode 100644 index 0000000000..848c48e4f5 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java @@ -0,0 +1,25 @@ +package stirling.software.proprietary.policy.model; + +import java.util.Map; + +/** + * How a {@link Policy} is automatically triggered. {@code type} selects a trigger kind ("manual", + * "folder", "schedule", "s3"); {@code options} carries type-specific configuration (a folder path, + * a cron expression, a bucket, ...). + * + *

Data-driven and parallel to {@link OutputSpec}: new trigger kinds are new {@code type} values + * handled by a new trigger bean, with no change to the model. {@code "manual"} means there is no + * automatic trigger - the policy is only ever run on demand. + */ +public record TriggerConfig(String type, Map options) { + + public TriggerConfig { + type = type == null || type.isBlank() ? "manual" : type; + options = options == null ? Map.of() : options; + } + + /** No automatic trigger; the policy is run on demand only. */ + public static TriggerConfig manual() { + return new TriggerConfig("manual", Map.of()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/WaitState.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/WaitState.java new file mode 100644 index 0000000000..7fff6c9fdd --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/WaitState.java @@ -0,0 +1,19 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; + +/** + * Captured when a run pauses in {@link PolicyRunStatus#WAITING_FOR_INPUT}. Together with the run's + * {@link PipelineDefinition} this is the resumable snapshot: {@code resumeStepIndex} is the 0-based + * step to continue from, and {@code pendingFileIds} are the intermediate files (stored in {@code + * FileStorage}, so they survive the worker thread ending or a node restart) that become the input + * to the resumed run. + * + *

Stored as fileIds rather than in-memory resources by design: a paused run must be resumable + * long after its worker thread has gone. + */ +public record WaitState(String reason, int resumeStepIndex, List pendingFileIds) { + public WaitState { + pendingFileIds = pendingFileIds == null ? List.of() : pendingFileIds; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java new file mode 100644 index 0000000000..bef5d46b37 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java @@ -0,0 +1,68 @@ +package stirling.software.proprietary.policy.output; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.MediaTypeFactory; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.model.job.ResultFile; +import stirling.software.common.service.FileStorage; +import stirling.software.proprietary.policy.model.OutputSpec; + +/** + * Default output sink: stores each output file in {@code FileStorage} so it is downloadable via + * {@code GET /api/v1/general/files/{fileId}}. This is the destination for manually-triggered runs + * whose results are returned to the caller. + */ +@Service +@RequiredArgsConstructor +public class InlineOutputSink implements PolicyOutputSink { + + private static final String TYPE = "inline"; + + private final FileStorage fileStorage; + + @Override + public String type() { + return TYPE; + } + + @Override + public boolean supports(OutputSpec spec) { + return spec == null || spec.type() == null || TYPE.equals(spec.type()); + } + + @Override + public List deliver(String runId, List outputs, OutputSpec spec) + throws IOException { + List results = new ArrayList<>(); + for (int i = 0; i < outputs.size(); i++) { + Resource resource = outputs.get(i); + String name = + resource.getFilename() != null ? resource.getFilename() : "result-" + (i + 1); + String contentType = + MediaTypeFactory.getMediaType(name) + .orElse(MediaType.APPLICATION_OCTET_STREAM) + .toString(); + FileStorage.StoredFile stored; + try (InputStream is = resource.getInputStream()) { + stored = fileStorage.storeInputStream(is, name); + } + results.add( + ResultFile.builder() + .fileId(stored.fileId()) + .fileName(name) + .contentType(contentType) + .fileSize(stored.size()) + .build()); + } + return results; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java new file mode 100644 index 0000000000..f6bf478e7f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java @@ -0,0 +1,35 @@ +package stirling.software.proprietary.policy.output; + +import java.io.IOException; +import java.util.List; + +import org.springframework.core.io.Resource; + +import stirling.software.common.model.job.ResultFile; +import stirling.software.proprietary.policy.model.OutputSpec; + +/** + * Delivers a finished run's output files to a destination, returning durable {@link ResultFile} + * descriptors (fileId + metadata) for the run record. + * + *

Implementations are Spring beans selected by {@link #supports(OutputSpec)}. New destinations + * (folder, S3) are added as new beans without changing the engine. + */ +public interface PolicyOutputSink { + + /** Stable identifier for this sink, matching {@code OutputSpec.type()} (e.g. "inline"). */ + String type(); + + /** Whether this sink can handle the given output spec. */ + boolean supports(OutputSpec spec); + + /** + * Persist/deliver the output files and return their descriptors. + * + * @param runId the run these outputs belong to + * @param outputs the final pipeline output resources + * @param spec the requested destination + */ + List deliver(String runId, List outputs, OutputSpec spec) + throws IOException; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/progress/PolicyProgressListener.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/progress/PolicyProgressListener.java new file mode 100644 index 0000000000..84eae284ef --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/progress/PolicyProgressListener.java @@ -0,0 +1,22 @@ +package stirling.software.proprietary.policy.progress; + +/** + * Receives live progress as a pipeline run executes. Implementations forward to an SSE stream, + * write job notes for polling, or both. Step indices are 1-based. + * + *

All methods default to no-ops so callers implement only what they surface. + */ +public interface PolicyProgressListener { + + /** A listener that ignores all progress. */ + PolicyProgressListener NOOP = new PolicyProgressListener() {}; + + /** Called immediately before step {@code stepIndex} of {@code stepCount} begins. */ + default void onStepStart(int stepIndex, int stepCount, String operation) {} + + /** Called immediately after step {@code stepIndex} of {@code stepCount} completes. */ + default void onStepComplete(int stepIndex, int stepCount, String operation) {} + + /** Called on a keep-alive tick so downstream connections can detect disconnects promptly. */ + default void onHeartbeat() {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java new file mode 100644 index 0000000000..df8fc93394 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java @@ -0,0 +1,61 @@ +package stirling.software.proprietary.policy.store; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import stirling.software.proprietary.policy.model.Policy; + +/** + * In-memory {@link PolicyStore}. Not the runtime bean - {@link JpaPolicyStore} is the durable + * store. Kept as a lightweight, dependency-free implementation for tests and for any future no- + * database mode. + */ +public class InProcessPolicyStore implements PolicyStore { + + private final Map policies = new ConcurrentHashMap<>(); + + @Override + public Policy save(Policy policy) { + String id = + policy.id() == null || policy.id().isBlank() + ? UUID.randomUUID().toString() + : policy.id(); + Policy stored = + new Policy( + id, + policy.name(), + policy.owner(), + policy.enabled(), + policy.trigger(), + policy.steps(), + policy.output()); + policies.put(id, stored); + return stored; + } + + @Override + public Optional get(String id) { + return Optional.ofNullable(policies.get(id)); + } + + @Override + public List all() { + return List.copyOf(policies.values()); + } + + @Override + public List findByTriggerType(String triggerType) { + return policies.values().stream() + .filter(Policy::enabled) + .filter(policy -> triggerType.equals(policy.trigger().type())) + .toList(); + } + + @Override + public boolean delete(String id) { + return policies.remove(id) != null; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java new file mode 100644 index 0000000000..977e8c9c12 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java @@ -0,0 +1,83 @@ +package stirling.software.proprietary.policy.store; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.policy.model.Policy; + +import tools.jackson.databind.ObjectMapper; + +/** + * Durable {@link PolicyStore} backed by JPA. The runtime store whenever the proprietary module runs + * (a datasource is always present). Policies are persisted as JSON via {@link PolicyEntity}; the + * scalar columns are kept in sync for querying. + */ +@Service +@RequiredArgsConstructor +public class JpaPolicyStore implements PolicyStore { + + private final PolicyRepository repository; + private final ObjectMapper objectMapper; + + @Override + public Policy save(Policy policy) { + String id = + policy.id() == null || policy.id().isBlank() + ? UUID.randomUUID().toString() + : policy.id(); + Policy stored = + new Policy( + id, + policy.name(), + policy.owner(), + policy.enabled(), + policy.trigger(), + policy.steps(), + policy.output()); + + PolicyEntity entity = new PolicyEntity(); + entity.setId(id); + entity.setName(stored.name()); + entity.setOwner(stored.owner()); + entity.setEnabled(stored.enabled()); + entity.setTriggerType(stored.trigger().type()); + entity.setPolicyJson(objectMapper.writeValueAsString(stored)); + repository.save(entity); + return stored; + } + + @Override + public Optional get(String id) { + return repository.findById(id).map(this::toPolicy); + } + + @Override + public List all() { + return repository.findAll().stream().map(this::toPolicy).toList(); + } + + @Override + public List findByTriggerType(String triggerType) { + return repository.findByTriggerTypeAndEnabledTrue(triggerType).stream() + .map(this::toPolicy) + .toList(); + } + + @Override + public boolean delete(String id) { + if (!repository.existsById(id)) { + return false; + } + repository.deleteById(id); + return true; + } + + private Policy toPolicy(PolicyEntity entity) { + return objectMapper.readValue(entity.getPolicyJson(), Policy.class); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java new file mode 100644 index 0000000000..d5aad61303 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java @@ -0,0 +1,50 @@ +package stirling.software.proprietary.policy.store; + +import java.io.Serializable; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * JPA row for a {@link stirling.software.proprietary.policy.model.Policy}. + * + *

The whole policy is stored as JSON in {@code policyJson} (authoritative on read, and the same + * serialization the API uses); the scalar columns are denormalized copies for querying - notably + * {@code triggerType} + {@code enabled} so background triggers can fetch their policies. Ownership + * is a plain {@code owner} string rather than a foreign key, to stay decoupled from the security + * entities; richer team scoping can be layered on later. + */ +@Entity +@Table(name = "policies") +@NoArgsConstructor +@Getter +@Setter +public class PolicyEntity implements Serializable { + + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "name") + private String name; + + @Column(name = "owner") + private String owner; + + @Column(name = "enabled") + private boolean enabled; + + @Column(name = "trigger_type") + private String triggerType; + + @Column(name = "policy_json", columnDefinition = "text") + private String policyJson; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java new file mode 100644 index 0000000000..ba6924f0f8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java @@ -0,0 +1,13 @@ +package stirling.software.proprietary.policy.store; + +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface PolicyRepository extends JpaRepository { + + /** Enabled policies of a given trigger type, for background triggers to activate. */ + List findByTriggerTypeAndEnabledTrue(String triggerType); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java new file mode 100644 index 0000000000..16b2ae8614 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java @@ -0,0 +1,28 @@ +package stirling.software.proprietary.policy.store; + +import java.util.List; +import java.util.Optional; + +import stirling.software.proprietary.policy.model.Policy; + +/** + * Stores {@link Policy} definitions. The in-memory implementation backs simple deployments now; a + * durable (JPA) implementation can replace it behind this interface without touching callers. + */ +public interface PolicyStore { + + /** Create or update a policy. A blank/absent id is assigned; returns the stored policy. */ + Policy save(Policy policy); + + Optional get(String id); + + List all(); + + /** + * Enabled policies whose automatic trigger is of the given type (used by background triggers). + */ + List findByTriggerType(String triggerType); + + /** Remove a policy; returns whether it existed. */ + boolean delete(String id); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java new file mode 100644 index 0000000000..2ae24671a8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java @@ -0,0 +1,41 @@ +package stirling.software.proprietary.policy.trigger; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.policy.engine.PolicyEngine; +import stirling.software.proprietary.policy.engine.PolicyRunHandle; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +/** + * Runs policies on demand, in response to a request (the {@code PolicyController} endpoints, an AI, + * or another automation). It is the request-driven trigger: no background lifecycle, it just + * forwards to the engine. Any policy can be run manually regardless of its configured trigger type. + */ +@Service +@RequiredArgsConstructor +public class ManualTrigger implements PolicyTrigger { + + private final PolicyEngine policyEngine; + + @Override + public String type() { + return "manual"; + } + + /** Run a stored policy immediately and return its run handle. */ + public PolicyRunHandle run( + Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { + return policyEngine.runPolicy(policy, inputs, listener); + } + + /** Run an ad-hoc pipeline (no stored policy), e.g. for AI or Automate one-offs. */ + public PolicyRunHandle fire( + PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) { + return policyEngine.submit(definition, inputs, listener); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java new file mode 100644 index 0000000000..c63e5a1da7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java @@ -0,0 +1,24 @@ +package stirling.software.proprietary.policy.trigger; + +/** + * Activates policies of one trigger type. A trigger owns a {@link #type()} (matching {@code + * TriggerConfig.type()}); when its condition fires it runs the relevant {@code Policy} through the + * {@code PolicyEngine}. + * + *

Background triggers (folder watcher, schedule) are driven by configuration: on {@link + * #start()} they begin watching/scheduling for the policies returned by {@code + * PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. Request-driven triggers + * (manual) have no background lifecycle and run a policy directly in response to a call. New + * trigger kinds are new beans of this type; the engine and the {@code Policy} model do not change. + */ +public interface PolicyTrigger { + + /** Stable identifier for this trigger kind, matching {@code TriggerConfig.type()}. */ + String type(); + + /** Begin activating policies of this type (e.g. start a folder watcher). No-op for manual. */ + default void start() {} + + /** Stop activating and release any resources. */ + default void stop() {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java index d81bf6a360..454df5f67e 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java @@ -31,13 +31,15 @@ import stirling.software.common.model.exception.UnsupportedProviderException; "stirling.software.proprietary.security.repository", "stirling.software.proprietary.repository", "stirling.software.proprietary.storage.repository", - "stirling.software.proprietary.workflow.repository" + "stirling.software.proprietary.workflow.repository", + "stirling.software.proprietary.policy.store" }) @EntityScan({ "stirling.software.proprietary.security.model", "stirling.software.proprietary.model", "stirling.software.proprietary.storage.model", - "stirling.software.proprietary.workflow.model" + "stirling.software.proprietary.workflow.model", + "stirling.software.proprietary.policy.store" }) public class DatabaseConfig { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java index 95f8afcea9..5683d634c1 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java @@ -14,14 +14,9 @@ import org.apache.pdfbox.pdmodel.PDDocument; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.MediaTypeFactory; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; import org.springframework.web.multipart.MultipartFile; import io.github.pixee.security.Filenames; @@ -32,14 +27,11 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.service.FileStorage; -import stirling.software.common.service.InternalApiClient; import stirling.software.common.service.InternalApiTimeoutException; -import stirling.software.common.service.ToolMetadataService; import stirling.software.common.service.UserServiceInterface; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; -import stirling.software.common.util.ZipExtractionUtils; import stirling.software.proprietary.model.api.ai.AiConversationMessage; import stirling.software.proprietary.model.api.ai.AiDocumentIngestRequest; import stirling.software.proprietary.model.api.ai.AiEngineProgressDetail; @@ -53,6 +45,13 @@ import stirling.software.proprietary.model.api.ai.AiWorkflowProgressEvent; import stirling.software.proprietary.model.api.ai.AiWorkflowRequest; import stirling.software.proprietary.model.api.ai.AiWorkflowResponse; import stirling.software.proprietary.model.api.ai.AiWorkflowResultFile; +import stirling.software.proprietary.policy.engine.PolicyExecutionResult; +import stirling.software.proprietary.policy.engine.PolicyExecutor; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; import stirling.software.proprietary.security.util.DesktopClientUtils; import stirling.software.proprietary.service.PdfContentExtractor.LoadedFile; import stirling.software.proprietary.service.PdfContentExtractor.PdfContentResult; @@ -72,12 +71,11 @@ public class AiWorkflowService { private final AiEngineClient aiEngineClient; private final PdfContentExtractor pdfContentExtractor; private final ObjectMapper objectMapper; - private final InternalApiClient internalApiClient; private final FileStorage fileStorage; - private final ToolMetadataService toolMetadataService; private final TempFileManager tempFileManager; private final FileIdStrategy fileIdStrategy; private final AiEngineEndpointResolver endpointResolver; + private final PolicyExecutor policyExecutor; private final UserServiceInterface userService; private final ApplicationProperties applicationProperties; @@ -86,24 +84,22 @@ public class AiWorkflowService { AiEngineClient aiEngineClient, PdfContentExtractor pdfContentExtractor, ObjectMapper objectMapper, - InternalApiClient internalApiClient, FileStorage fileStorage, - ToolMetadataService toolMetadataService, TempFileManager tempFileManager, FileIdStrategy fileIdStrategy, AiEngineEndpointResolver endpointResolver, + PolicyExecutor policyExecutor, @Autowired(required = false) UserServiceInterface userService, ApplicationProperties applicationProperties) { this.pdfDocumentFactory = pdfDocumentFactory; this.aiEngineClient = aiEngineClient; this.pdfContentExtractor = pdfContentExtractor; this.objectMapper = objectMapper; - this.internalApiClient = internalApiClient; this.fileStorage = fileStorage; - this.toolMetadataService = toolMetadataService; this.tempFileManager = tempFileManager; this.fileIdStrategy = fileIdStrategy; this.endpointResolver = endpointResolver; + this.policyExecutor = policyExecutor; this.userService = userService; this.applicationProperties = applicationProperties; } @@ -150,16 +146,6 @@ public class AiWorkflowService { record Terminal(AiWorkflowResponse response) implements WorkflowState {} } - /** - * Internal value-class for tool responses. {@code files} holds any result files (typically one; - * multiple for ZIP-response tools). {@code report} holds an optional structured metadata - * payload the tool chose to surface alongside (or instead of) a file. - * - *

Tools populate the report either by returning a JSON body (whole body → report) or by - * adding the {@link AiToolResponseHeaders#TOOL_REPORT} header alongside a file body. - */ - private record ToolResult(List files, JsonNode report) {} - public AiWorkflowResponse orchestrate(AiWorkflowRequest request) throws IOException { return orchestrate(request, NOOP_LISTENER); } @@ -377,7 +363,6 @@ public class AiWorkflowService { pages.size()); } - @SuppressWarnings("unchecked") private WorkflowState onToolCall( AiWorkflowResponse response, Map filesById, @@ -394,8 +379,14 @@ public class AiWorkflowService { try { List inputFiles = toResources(filesById); - listener.onProgress(AiWorkflowProgressEvent.executingTool(endpointPath, 1, 1)); - ToolResult result = executeStep(endpointPath, parameters, inputFiles); + PipelineDefinition definition = + new PipelineDefinition( + null, + List.of(new PipelineStep(endpointPath, parameters)), + OutputSpec.inline()); + PolicyExecutionResult result = + policyExecutor.execute( + definition, PolicyInputs.of(inputFiles), stepProgress(listener)); return new WorkflowState.Terminal( buildCompletedResponse( response.getRationale(), @@ -469,38 +460,32 @@ public class AiWorkflowService { cannotContinue("AI engine returned a plan with no steps.")); } - try { - List currentFiles = toResources(filesById); - // Propagate the *last* non-null report — the terminal step defines the output. - JsonNode lastReport = null; - String lastReportTool = null; - - for (int i = 0; i < steps.size(); i++) { - Map step = steps.get(i); - String endpointPath = (String) step.get("tool"); - Map parameters = - step.containsKey("parameters") - ? (Map) step.get("parameters") - : Map.of(); - - if (endpointPath == null || endpointPath.isBlank()) { - return new WorkflowState.Terminal( - cannotContinue("Plan step " + (i + 1) + " has no tool endpoint.")); - } - - listener.onProgress( - AiWorkflowProgressEvent.executingTool(endpointPath, i + 1, steps.size())); - ToolResult stepResult = executeStep(endpointPath, parameters, currentFiles); - currentFiles = stepResult.files(); - if (stepResult.report() != null) { - lastReport = stepResult.report(); - lastReportTool = endpointPath; - } + List pipelineSteps = new ArrayList<>(); + for (int i = 0; i < steps.size(); i++) { + Map step = steps.get(i); + String endpointPath = (String) step.get("tool"); + if (endpointPath == null || endpointPath.isBlank()) { + return new WorkflowState.Terminal( + cannotContinue("Plan step " + (i + 1) + " has no tool endpoint.")); } + Map parameters = + step.containsKey("parameters") + ? (Map) step.get("parameters") + : Map.of(); + pipelineSteps.add(new PipelineStep(endpointPath, parameters)); + } + + try { + List inputFiles = toResources(filesById); + PipelineDefinition definition = + new PipelineDefinition(summary, pipelineSteps, OutputSpec.inline()); + PolicyExecutionResult result = + policyExecutor.execute( + definition, PolicyInputs.of(inputFiles), stepProgress(listener)); // Multi-turn: if the plan was emitted with resume_with set, the delegate wants // Java to re-invoke the orchestrator with any captured report as an artifact. - if (resumeWith != null && !resumeWith.isBlank() && lastReport != null) { + if (resumeWith != null && !resumeWith.isBlank() && result.report() != null) { WorkflowTurnRequest resumeRequest = new WorkflowTurnRequest(); resumeRequest.setUserMessage(previousRequest.getUserMessage()); resumeRequest.setFiles(previousRequest.getFiles()); @@ -510,14 +495,14 @@ public class AiWorkflowService { .getArtifacts() .add( new PdfContentExtractor.ToolReportArtifact( - lastReportTool, lastReport)); + result.reportTool(), result.report())); resumeRequest.setResumeWith(resumeWith); return new WorkflowState.Pending(resumeRequest); } return new WorkflowState.Terminal( buildCompletedResponse( - summary, currentFiles, inputFileNames(filesById), lastReport)); + summary, result.files(), inputFileNames(filesById), result.report())); } catch (InternalApiTimeoutException e) { log.error("Plan step on tool {} timed out: {}", e.getEndpointPath(), e.getMessage()); return new WorkflowState.Terminal( @@ -548,123 +533,19 @@ public class AiWorkflowService { } /** - * Execute a single tool step. If the endpoint accepts multiple files, all files are sent in one - * call. Otherwise, the endpoint is called once per file. ZIP responses are unpacked so each - * inner file is treated as its own result (e.g. split outputs a ZIP of pages). - * - *

A structured {@code report} may be returned alongside (or instead of) files — see {@link - * ToolResult}. For per-file dispatch (single-input endpoints called once per input), the first - * non-null report wins. + * Adapt the AI workflow's {@link ProgressListener} to the engine's {@link + * PolicyProgressListener}: each step start maps to an {@code EXECUTING_TOOL} progress event + * carrying the tool path and 1-based step position, preserving the event shape the frontend + * already renders. */ - private ToolResult executeStep( - String endpointPath, Map parameters, List inputFiles) - throws IOException { - List files = new ArrayList<>(); - JsonNode report = null; - if (toolMetadataService.isMultiInput(endpointPath)) { - ToolResult r = callEndpoint(endpointPath, parameters, inputFiles); - files.addAll(r.files()); - report = r.report(); - } else { - for (Resource file : inputFiles) { - ToolResult r = callEndpoint(endpointPath, parameters, List.of(file)); - files.addAll(r.files()); - if (report == null) { - report = r.report(); - } + private static PolicyProgressListener stepProgress(ProgressListener listener) { + return new PolicyProgressListener() { + @Override + public void onStepStart(int stepIndex, int stepCount, String operation) { + listener.onProgress( + AiWorkflowProgressEvent.executingTool(operation, stepIndex, stepCount)); } - } - return new ToolResult(files, report); - } - - /** - * Call an endpoint and return its result files and optional report. - * - *

    - *
  • JSON body (Content-Type: application/json) → the entire body is the report, no files - * are returned. - *
  • File body (PDF etc.) → the file is returned; if an {@link - * AiToolResponseHeaders#TOOL_REPORT} header is present, its (minified JSON) value is - * parsed as the report. - *
  • ZIP responses declared by the tool metadata service are unpacked so callers always see - * a flat list of result files. - *
- */ - private ToolResult callEndpoint( - String endpointPath, Map parameters, List files) - throws IOException { - MultiValueMap body = new LinkedMultiValueMap<>(); - for (Resource file : files) { - body.add("fileInput", file); - } - for (Map.Entry entry : parameters.entrySet()) { - if (entry.getValue() instanceof List list) { - if (containsStructuredElements(list)) { - // Endpoints binding lists of structured objects (e.g. /security/redact's - // redactions, /general/edit-text's edits) parse a single JSON string field via - // a property editor. Pre-serialize the whole list so binding succeeds. - body.add(entry.getKey(), objectMapper.writeValueAsString(list)); - } else { - for (Object item : list) { - body.add(entry.getKey(), item); - } - } - } else { - body.add(entry.getKey(), entry.getValue()); - } - } - ResponseEntity response = internalApiClient.post(endpointPath, body); - if (!HttpStatus.OK.equals(response.getStatusCode()) || response.getBody() == null) { - throw new IOException( - "Tool returned HTTP " + response.getStatusCode() + " for " + endpointPath); - } - Resource resource = response.getBody(); - HttpHeaders headers = response.getHeaders(); - MediaType contentType = headers.getContentType(); - - // JSON-only response — the whole body is the structured report, no result file. - if (contentType != null && MediaType.APPLICATION_JSON.isCompatibleWith(contentType)) { - try (java.io.InputStream is = resource.getInputStream()) { - JsonNode report = objectMapper.readTree(is); - return new ToolResult(List.of(), report); - } - } - - JsonNode report = parseReportHeader(headers, endpointPath); - if (toolMetadataService.shouldUnpackZipResponse(endpointPath)) { - return new ToolResult(ZipExtractionUtils.extractZip(resource, tempFileManager), report); - } - return new ToolResult(List.of(resource), report); - } - - /** - * Parse the optional {@link AiToolResponseHeaders#TOOL_REPORT} header into a {@link JsonNode}, - * or return null. - */ - private JsonNode parseReportHeader(HttpHeaders headers, String endpointPath) { - String raw = headers.getFirst(AiToolResponseHeaders.TOOL_REPORT); - if (raw == null || raw.isBlank()) { - return null; - } - try { - return objectMapper.readTree(raw); - } catch (JacksonException e) { - log.warn( - "Ignoring malformed {} header from {}: {}", - AiToolResponseHeaders.TOOL_REPORT, - endpointPath, - e.getMessage()); - return null; - } - } - - private static boolean containsStructuredElements(List list) { - for (Object item : list) { - if (item instanceof Map || item instanceof List) { - return true; - } - } - return false; + }; } private List toResources(Map filesById) throws IOException { diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java new file mode 100644 index 0000000000..bf6442b423 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java @@ -0,0 +1,255 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.InputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.FileStorage; +import stirling.software.common.service.FileStorage.StoredFile; +import stirling.software.common.service.InternalApiClient; +import stirling.software.common.service.JobOwnershipService; +import stirling.software.common.service.JobQueue; +import stirling.software.common.service.ResourceMonitor; +import stirling.software.common.service.TaskManager; +import stirling.software.common.service.ToolMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.TempFileRegistry; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.PolicyRunStatus; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.output.InlineOutputSink; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +import tools.jackson.databind.json.JsonMapper; + +/** + * Tests for {@link PolicyEngine}: async submission runs the pipeline on a virtual thread, registers + * outputs and progress with {@link TaskManager}, and surfaces terminal state via {@link + * PolicyRunRegistry}. The step executor and inline sink are real (with mocked collaborators) so the + * full run path is exercised. + */ +@ExtendWith(MockitoExtension.class) +class PolicyEngineTest { + + private static final String ROTATE = "/api/v1/general/rotate-pdf"; + private static final String COMPRESS = "/api/v1/misc/compress-pdf"; + + @Mock private InternalApiClient internalApiClient; + @Mock private ToolMetadataService toolMetadataService; + @Mock private TaskManager taskManager; + @Mock private FileStorage fileStorage; + @Mock private JobOwnershipService jobOwnershipService; + @Mock private ResourceMonitor resourceMonitor; + @Mock private JobQueue jobQueue; + + @TempDir Path tempDir; + + private PolicyRunRegistry registry; + private PolicyEngine engine; + + @BeforeEach + void setUp() { + ApplicationProperties props = new ApplicationProperties(); + props.getSystem().getTempFileManagement().setBaseTmpDir(tempDir.toString()); + props.getSystem().getTempFileManagement().setPrefix("policy-engine-test-"); + TempFileManager tempFileManager = new TempFileManager(new TempFileRegistry(), props); + PolicyExecutor executor = + new PolicyExecutor( + internalApiClient, + toolMetadataService, + tempFileManager, + JsonMapper.builder().build()); + registry = new PolicyRunRegistry(30); + InlineOutputSink sink = new InlineOutputSink(fileStorage); + engine = + new PolicyEngine( + executor, + taskManager, + registry, + fileStorage, + jobOwnershipService, + List.of(sink), + resourceMonitor, + jobQueue); + + // Identity scoping: the run id is the generated UUID unchanged. Lenient because the + // resume/cancel tests do not submit a run. + lenient() + .when(jobOwnershipService.createScopedJobKey(anyString())) + .thenAnswer(inv -> inv.getArgument(0)); + // Default to running immediately; the queueing test overrides this. + lenient().when(resourceMonitor.shouldQueueJob(anyInt())).thenReturn(false); + } + + @Test + void submitRunsPipelineToCompletionAndRegistersOutputs() throws Exception { + when(toolMetadataService.isMultiInput(anyString())).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(anyString())).thenReturn(false); + stubEndpoint(ROTATE, pdf("rotated", "rotated.pdf")); + stubEndpoint(COMPRESS, pdf("compressed", "compressed.pdf")); + int[] counter = {0}; + when(fileStorage.storeInputStream(any(InputStream.class), anyString())) + .thenAnswer( + inv -> { + InputStream is = inv.getArgument(0); + long size = is.readAllBytes().length; + return new StoredFile("file-" + ++counter[0], size); + }); + + PolicyRunHandle handle = + engine.submit( + definition( + new PipelineStep(ROTATE, Map.of()), + new PipelineStep(COMPRESS, Map.of())), + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + PolicyProgressListener.NOOP); + + // The completion future resolves with the final run state, no polling needed. + String runId = handle.runId(); + PolicyRun run = handle.completion().get(10, TimeUnit.SECONDS); + assertEquals(PolicyRunStatus.COMPLETED, run.getStatus()); + assertEquals(1, run.getOutputs().size()); + assertEquals("compressed.pdf", run.getOutputs().get(0).getFileName()); + + // The run self-registers its results and completion with the job system. + verify(taskManager).createTask(runId); + verify(taskManager).setMultipleFileResults(eq(runId), any()); + verify(taskManager).setComplete(runId); + // Progress notes were written for each step. + verify(taskManager, atLeastOnce()).addNote(eq(runId), anyString()); + } + + @Test + void submitFailsRunWhenAToolErrors() throws Exception { + when(toolMetadataService.isMultiInput(ROTATE)).thenReturn(false); + when(internalApiClient.post(eq(ROTATE), any())).thenThrow(new RuntimeException("boom")); + + PolicyRunHandle handle = + engine.submit( + definition(new PipelineStep(ROTATE, Map.of())), + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + PolicyProgressListener.NOOP); + + String runId = handle.runId(); + PolicyRun run = handle.completion().get(10, TimeUnit.SECONDS); + assertEquals(PolicyRunStatus.FAILED, run.getStatus()); + verify(taskManager).setError(eq(runId), anyString()); + verify(taskManager, never()).setComplete(runId); + } + + @Test + void runPolicyExecutesThePolicysPipeline() throws Exception { + when(toolMetadataService.isMultiInput(anyString())).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(anyString())).thenReturn(false); + stubEndpoint(ROTATE, pdf("rotated", "rotated.pdf")); + int[] counter = {0}; + when(fileStorage.storeInputStream(any(InputStream.class), anyString())) + .thenAnswer( + inv -> { + InputStream is = inv.getArgument(0); + return new StoredFile("file-" + ++counter[0], is.readAllBytes().length); + }); + + Policy policy = + new Policy( + "p1", + "rotate", + "owner", + true, + TriggerConfig.manual(), + List.of(new PipelineStep(ROTATE, Map.of())), + OutputSpec.inline()); + + PolicyRunHandle handle = + engine.runPolicy( + policy, + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + PolicyProgressListener.NOOP); + + PolicyRun run = handle.completion().get(10, TimeUnit.SECONDS); + assertEquals(PolicyRunStatus.COMPLETED, run.getStatus()); + verify(internalApiClient).post(eq(ROTATE), any()); + } + + @Test + void runIsQueuedUnderResourcePressure() { + when(resourceMonitor.shouldQueueJob(anyInt())).thenReturn(true); + // Returning an already-completed future keeps the run parked: the queued work (which would + // start the run) is never executed by this mock, so it stays PENDING. + doReturn(CompletableFuture.completedFuture(null)) + .when(jobQueue) + .queueJob(anyString(), anyInt(), any(), anyLong()); + + PolicyRunHandle handle = + engine.submit( + definition(new PipelineStep(ROTATE, Map.of())), + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + PolicyProgressListener.NOOP); + + verify(jobQueue).queueJob(eq(handle.runId()), anyInt(), any(), anyLong()); + assertEquals(PolicyRunStatus.PENDING, registry.get(handle.runId()).getStatus()); + } + + @Test + void resumeIsNotYetImplemented() { + assertThrows(UnsupportedOperationException.class, () -> engine.resume("any", List.of())); + } + + @Test + void cancelUnknownRunReturnsFalse() { + assertFalse(engine.cancel("does-not-exist")); + } + + // --- helpers --- + + private static PipelineDefinition definition(PipelineStep... steps) { + return new PipelineDefinition("test", List.of(steps), OutputSpec.inline()); + } + + private void stubEndpoint(String endpoint, Resource body) { + when(internalApiClient.post(eq(endpoint), any())).thenReturn(ResponseEntity.ok(body)); + } + + private static ByteArrayResource pdf(String content, String filename) { + return new ByteArrayResource(content.getBytes()) { + @Override + public String getFilename() { + return filename; + } + }; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyExecutorTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyExecutorTest.java new file mode 100644 index 0000000000..d682c35cd7 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyExecutorTest.java @@ -0,0 +1,378 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.util.MultiValueMap; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.InternalApiClient; +import stirling.software.common.service.InternalApiTimeoutException; +import stirling.software.common.service.ToolMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.TempFileRegistry; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +/** + * Unit tests for {@link PolicyExecutor}, the shared pipeline step loop. Covers file chaining across + * steps, multi-input vs per-file dispatch, ZIP unpacking, structured-list parameter encoding, + * progress callbacks, and timeout propagation. External collaborators are mocked; {@link + * TempFileManager} is real so ZIP extraction exercises real code. + */ +@ExtendWith(MockitoExtension.class) +class PolicyExecutorTest { + + private static final String ROTATE = "/api/v1/general/rotate-pdf"; + private static final String COMPRESS = "/api/v1/misc/compress-pdf"; + private static final String SPLIT = "/api/v1/general/split-pages"; + private static final String MERGE = "/api/v1/general/merge-pdfs"; + + @Mock private InternalApiClient internalApiClient; + @Mock private ToolMetadataService toolMetadataService; + + @TempDir Path tempDir; + + private TempFileManager tempFileManager; + private PolicyExecutor executor; + + @BeforeEach + void setUp() { + ApplicationProperties props = new ApplicationProperties(); + props.getSystem().getTempFileManagement().setBaseTmpDir(tempDir.toString()); + props.getSystem().getTempFileManagement().setPrefix("policy-test-"); + tempFileManager = new TempFileManager(new TempFileRegistry(), props); + ObjectMapper objectMapper = JsonMapper.builder().build(); + executor = + new PolicyExecutor( + internalApiClient, toolMetadataService, tempFileManager, objectMapper); + } + + @Test + void executesStepsSequentiallyChainingOutputToInput() throws IOException { + when(toolMetadataService.isMultiInput(anyString())).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(anyString())).thenReturn(false); + stubEndpoint(ROTATE, pdf("rotated", "rotated.pdf")); + stubEndpoint(COMPRESS, pdf("compressed", "compressed.pdf")); + + List steps = new ArrayList<>(); + PolicyProgressListener listener = + new PolicyProgressListener() { + @Override + public void onStepStart(int stepIndex, int stepCount, String operation) { + steps.add(stepIndex); + } + }; + + PolicyExecutionResult result = + executor.execute( + definition( + new PipelineStep(ROTATE, Map.of()), + new PipelineStep(COMPRESS, Map.of())), + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + listener); + + assertEquals(1, result.files().size()); + assertEquals("compressed.pdf", result.files().get(0).getFilename()); + verify(internalApiClient, times(1)).post(eq(ROTATE), any()); + verify(internalApiClient, times(1)).post(eq(COMPRESS), any()); + // Progress fired once per step, in order. + assertEquals(List.of(1, 2), steps); + } + + @Test + void multiInputEndpointIsCalledOnceWithAllFiles() throws IOException { + when(toolMetadataService.isMultiInput(MERGE)).thenReturn(true); + when(toolMetadataService.shouldUnpackZipResponse(MERGE)).thenReturn(false); + stubEndpoint(MERGE, pdf("merged", "merged.pdf")); + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(MERGE, Map.of())), + PolicyInputs.of(List.of(pdf("a", "a.pdf"), pdf("b", "b.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(1, result.files().size()); + verify(internalApiClient, times(1)).post(eq(MERGE), any()); + } + + @Test + void singleInputEndpointIsCalledOncePerFile() throws IOException { + when(toolMetadataService.isMultiInput(ROTATE)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(ROTATE)).thenReturn(false); + stubEndpoint(ROTATE, pdf("rotated", "rotated.pdf")); + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(ROTATE, Map.of())), + PolicyInputs.of(List.of(pdf("a", "a.pdf"), pdf("b", "b.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(2, result.files().size()); + verify(internalApiClient, times(2)).post(eq(ROTATE), any()); + } + + @Test + void zipResponseIsUnpackedIntoIndividualFiles() throws IOException { + when(toolMetadataService.isMultiInput(SPLIT)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(SPLIT)).thenReturn(true); + stubEndpoint( + SPLIT, + zip( + "doc.zip", + List.of(new Entry("page-1.pdf", "one"), new Entry("page-2.pdf", "two")))); + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(SPLIT, Map.of())), + PolicyInputs.of(List.of(pdf("doc", "doc.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(2, result.files().size()); + assertEquals("page-1.pdf", result.files().get(0).getFilename()); + assertEquals("page-2.pdf", result.files().get(1).getFilename()); + } + + @Test + void structuredListParameterIsJsonEncodedAsSingleField() throws IOException { + String editText = "/api/v1/general/edit-text"; + when(toolMetadataService.isMultiInput(editText)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(editText)).thenReturn(false); + stubEndpoint(editText, pdf("edited", "edited.pdf")); + + // LinkedHashMap so the serialized key order is deterministic for the assertion below. + Map edit = new LinkedHashMap<>(); + edit.put("find", "foo"); + edit.put("replace", "bar"); + Map params = new LinkedHashMap<>(); + params.put("edits", List.of(edit)); + params.put("useRegex", false); + + executor.execute( + definition(new PipelineStep(editText, params)), + PolicyInputs.of(List.of(pdf("in", "in.pdf"))), + PolicyProgressListener.NOOP); + + @SuppressWarnings("unchecked") + ArgumentCaptor> bodyCaptor = + ArgumentCaptor.forClass(MultiValueMap.class); + verify(internalApiClient).post(eq(editText), bodyCaptor.capture()); + MultiValueMap body = bodyCaptor.getValue(); + + List edits = body.get("edits"); + assertNotNull(edits); + assertEquals(1, edits.size()); + assertEquals("[{\"find\":\"foo\",\"replace\":\"bar\"}]", edits.get(0)); + } + + @Test + void supportingFilesAreBoundToTheirNamedFields() throws IOException { + String addStamp = "/api/v1/misc/add-stamp-to-pdf"; + when(toolMetadataService.isMultiInput(addStamp)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(addStamp)).thenReturn(false); + stubEndpoint(addStamp, pdf("stamped", "stamped.pdf")); + + PipelineStep step = + new PipelineStep(addStamp, Map.of("opacity", 0.5), Map.of("stampImage", "logo")); + PolicyInputs inputs = + new PolicyInputs( + List.of(pdf("doc", "doc.pdf")), + Map.of("logo", List.of(pdf("logo-bytes", "logo.png")))); + + executor.execute( + new PipelineDefinition("stamp", List.of(step), OutputSpec.inline()), + inputs, + PolicyProgressListener.NOOP); + + @SuppressWarnings("unchecked") + ArgumentCaptor> bodyCaptor = + ArgumentCaptor.forClass(MultiValueMap.class); + verify(internalApiClient).post(eq(addStamp), bodyCaptor.capture()); + MultiValueMap body = bodyCaptor.getValue(); + // The document goes to fileInput; the supporting image is bound to its named field and is + // not part of the document stream. + assertEquals(1, body.get("fileInput").size()); + assertNotNull(body.get("stampImage")); + assertEquals(1, body.get("stampImage").size()); + } + + @Test + void missingSupportingFileFailsTheStep() { + String addStamp = "/api/v1/misc/add-stamp-to-pdf"; + when(toolMetadataService.isMultiInput(addStamp)).thenReturn(false); + PipelineStep step = new PipelineStep(addStamp, Map.of(), Map.of("stampImage", "logo")); + + IOException ex = + assertThrows( + IOException.class, + () -> + executor.execute( + new PipelineDefinition( + "stamp", List.of(step), OutputSpec.inline()), + PolicyInputs.of(List.of(pdf("doc", "doc.pdf"))), + PolicyProgressListener.NOOP)); + assertTrue(ex.getMessage().contains("logo")); + } + + @Test + void documentOfAnUnacceptedTypeFailsTheStep() { + String compress = "/api/v1/misc/compress-pdf"; + when(toolMetadataService.getExtensionTypes(false, compress)).thenReturn(List.of("pdf")); + + IOException ex = + assertThrows( + IOException.class, + () -> + executor.execute( + definition(new PipelineStep(compress, Map.of())), + PolicyInputs.of(List.of(pdf("img", "image.png"))), + PolicyProgressListener.NOOP)); + assertTrue(ex.getMessage().contains("image.png")); + // Type check happens before any dispatch. + verify(internalApiClient, never()).post(anyString(), any()); + } + + @Test + void documentOfAnAcceptedTypeProceeds() throws IOException { + String compress = "/api/v1/misc/compress-pdf"; + when(toolMetadataService.getExtensionTypes(false, compress)).thenReturn(List.of("pdf")); + when(toolMetadataService.isMultiInput(compress)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(compress)).thenReturn(false); + stubEndpoint(compress, pdf("compressed", "compressed.pdf")); + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(compress, Map.of())), + PolicyInputs.of(List.of(pdf("doc", "doc.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(1, result.files().size()); + verify(internalApiClient, times(1)).post(eq(compress), any()); + } + + @Test + void filterOperationWithEmptyResultDropsTheFile() throws IOException { + String filter = "/api/v1/filter/filter-page-count"; + when(toolMetadataService.isMultiInput(filter)).thenReturn(false); + stubEndpoint(filter, pdf("", "filtered.pdf")); // empty body => filtered out + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(filter, Map.of())), + PolicyInputs.of(List.of(pdf("doc", "doc.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(0, result.files().size()); + } + + @Test + void timeoutFromAStepPropagates() { + when(toolMetadataService.isMultiInput(ROTATE)).thenReturn(false); + when(internalApiClient.post(eq(ROTATE), any())) + .thenThrow( + new InternalApiTimeoutException( + ROTATE, + java.time.Duration.ofSeconds(300), + new IOException("Read timed out"))); + + assertThrows( + InternalApiTimeoutException.class, + () -> + executor.execute( + definition(new PipelineStep(ROTATE, Map.of())), + PolicyInputs.of(List.of(pdf("in", "in.pdf"))), + PolicyProgressListener.NOOP)); + } + + @Test + void emptyPipelineIsRejected() { + assertThrows( + IllegalArgumentException.class, + () -> + executor.execute( + new PipelineDefinition("empty", List.of(), OutputSpec.inline()), + PolicyInputs.of(List.of(pdf("in", "in.pdf"))), + PolicyProgressListener.NOOP)); + } + + // --- helpers --- + + private static PipelineDefinition definition(PipelineStep... steps) { + return new PipelineDefinition("test", List.of(steps), OutputSpec.inline()); + } + + private void stubEndpoint(String endpoint, Resource body) { + when(internalApiClient.post(eq(endpoint), any())).thenReturn(ResponseEntity.ok(body)); + } + + private static ByteArrayResource pdf(String content, String filename) { + return new ByteArrayResource(content.getBytes()) { + @Override + public String getFilename() { + return filename; + } + }; + } + + private static ByteArrayResource zip(String filename, List entries) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + for (Entry entry : entries) { + zos.putNextEntry(new ZipEntry(entry.name())); + zos.write(entry.content().getBytes()); + zos.closeEntry(); + } + } + byte[] zipBytes = baos.toByteArray(); + return new ByteArrayResource(zipBytes) { + @Override + public String getFilename() { + return filename; + } + + @Override + public InputStream getInputStream() { + return new ByteArrayInputStream(zipBytes); + } + }; + } + + private record Entry(String name, String content) {} +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java new file mode 100644 index 0000000000..823d53edcb --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java @@ -0,0 +1,101 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Instant; +import java.util.List; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.WaitState; + +/** + * Tests for {@link PolicyRunRegistry} eviction: terminal runs expire, active/paused runs persist. + */ +class PolicyRunRegistryTest { + + private PolicyRunRegistry registry; + + @BeforeEach + void setUp() { + registry = new PolicyRunRegistry(30); + } + + @AfterEach + void tearDown() { + registry.shutdown(); + } + + @Test + void evictsTerminalRunsPastTheCutoff() { + PolicyRun completed = register("completed"); + completed.complete(List.of()); + PolicyRun failed = register("failed"); + failed.fail("boom"); + PolicyRun cancelled = register("cancelled"); + cancelled.cancel(); + + // A cutoff in the future means every terminal run finished "before" it. + int removed = registry.evictExpired(Instant.now().plusSeconds(60)); + + assertEquals(3, removed); + assertNull(registry.get("completed")); + assertNull(registry.get("failed")); + assertNull(registry.get("cancelled")); + } + + @Test + void retainsActiveAndPausedRunsRegardlessOfAge() { + register("pending"); // PENDING: never started + PolicyRun running = register("running"); + running.markRunning(); + PolicyRun waiting = register("waiting"); + waiting.waitForInput(new WaitState("needs a signature", 1, List.of())); + + int removed = registry.evictExpired(Instant.now().plusSeconds(60)); + + assertEquals(0, removed); + assertNotNull(registry.get("pending")); + assertNotNull(registry.get("running")); + assertNotNull(registry.get("waiting")); + } + + @Test + void keepsTerminalRunsStillWithinTheExpiryWindow() { + PolicyRun completed = register("recent"); + completed.complete(List.of()); + + // A cutoff in the past means the run was updated "after" it: too young to evict. + int removed = registry.evictExpired(Instant.now().minusSeconds(60)); + + assertEquals(0, removed); + assertNotNull(registry.get("recent")); + } + + @Test + void evictionLeavesUnrelatedRunsInPlace() { + PolicyRun completed = register("done"); + completed.complete(List.of()); + PolicyRun running = register("busy"); + running.markRunning(); + + registry.evictExpired(Instant.now().plusSeconds(60)); + + assertNull(registry.get("done")); + assertNotNull(registry.get("busy")); + assertTrue(registry.all().stream().anyMatch(r -> r.getRunId().equals("busy"))); + } + + private PolicyRun register(String runId) { + PolicyRun run = new PolicyRun(runId, new PipelineDefinition(runId, List.of(), null)); + registry.register(run); + return run; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java new file mode 100644 index 0000000000..2e97019182 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java @@ -0,0 +1,87 @@ +package stirling.software.proprietary.policy.store; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; + +/** Tests for {@link InProcessPolicyStore}: id assignment, upsert, trigger-type lookup, delete. */ +class InProcessPolicyStoreTest { + + private PolicyStore store; + + @BeforeEach + void setUp() { + store = new InProcessPolicyStore(); + } + + @Test + void savedPolicyGetsAnIdAndIsRetrievable() { + Policy saved = store.save(policy(null, "compress", "manual", true)); + + assertNotNull(saved.id()); + assertFalse(saved.id().isBlank()); + assertEquals(saved, store.get(saved.id()).orElseThrow()); + } + + @Test + void savingWithAnExistingIdUpdatesInPlace() { + Policy created = store.save(policy(null, "before", "manual", true)); + + store.save( + new Policy( + created.id(), + "after", + "owner", + true, + TriggerConfig.manual(), + List.of(), + OutputSpec.inline())); + + assertEquals(1, store.all().size()); + assertEquals("after", store.get(created.id()).orElseThrow().name()); + } + + @Test + void findByTriggerTypeReturnsOnlyEnabledMatches() { + store.save(policy(null, "watch", "folder", true)); + store.save(policy(null, "watch-disabled", "folder", false)); + store.save(policy(null, "nightly", "schedule", true)); + + List folder = store.findByTriggerType("folder"); + + assertEquals(1, folder.size()); + assertEquals("watch", folder.get(0).name()); + } + + @Test + void deleteRemovesThePolicy() { + Policy saved = store.save(policy(null, "p", "manual", true)); + + assertTrue(store.delete(saved.id())); + assertTrue(store.get(saved.id()).isEmpty()); + assertFalse(store.delete(saved.id())); + } + + private static Policy policy(String id, String name, String triggerType, boolean enabled) { + return new Policy( + id, + name, + "owner", + enabled, + new TriggerConfig(triggerType, Map.of()), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java new file mode 100644 index 0000000000..2a8212e2d5 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java @@ -0,0 +1,129 @@ +package stirling.software.proprietary.policy.store; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; + +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +/** + * Tests for {@link JpaPolicyStore}'s entity mapping and query delegation. The repository is mocked; + * real Hibernate/H2 persistence is exercised at application boot (this module's convention is + * Mockito unit tests for store/service logic). + */ +@ExtendWith(MockitoExtension.class) +class JpaPolicyStoreTest { + + @Mock private PolicyRepository repository; + + private final ObjectMapper objectMapper = JsonMapper.builder().build(); + private JpaPolicyStore store; + + @BeforeEach + void setUp() { + store = new JpaPolicyStore(repository, objectMapper); + } + + @Test + void saveAssignsAnIdAndPersistsThePolicyAsJson() { + Policy saved = + store.save( + new Policy( + null, + "compress incoming", + "alice", + true, + new TriggerConfig("folder", Map.of("path", "/in")), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline())); + + assertNotNull(saved.id()); + ArgumentCaptor captor = ArgumentCaptor.forClass(PolicyEntity.class); + verify(repository).save(captor.capture()); + PolicyEntity entity = captor.getValue(); + assertEquals(saved.id(), entity.getId()); + assertEquals("folder", entity.getTriggerType()); + assertTrue(entity.isEnabled()); + // The stored JSON round-trips back to an equal policy. + assertEquals(saved, objectMapper.readValue(entity.getPolicyJson(), Policy.class)); + } + + @Test + void getDeserializesThePolicyFromJson() { + Policy policy = + new Policy( + "p1", + "rotate", + "alice", + true, + TriggerConfig.manual(), + List.of( + new PipelineStep( + "/api/v1/general/rotate-pdf", Map.of("angle", 90))), + OutputSpec.inline()); + when(repository.findById("p1")).thenReturn(Optional.of(entityFor(policy))); + + assertEquals(policy, store.get("p1").orElseThrow()); + } + + @Test + void findByTriggerTypeUsesTheEnabledQuery() { + Policy policy = + new Policy( + "p1", + "watch", + "alice", + true, + new TriggerConfig("folder", Map.of()), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + when(repository.findByTriggerTypeAndEnabledTrue("folder")) + .thenReturn(List.of(entityFor(policy))); + + List folder = store.findByTriggerType("folder"); + + assertEquals(1, folder.size()); + assertEquals("p1", folder.get(0).id()); + } + + @Test + void deleteReturnsWhetherThePolicyExisted() { + when(repository.existsById("p1")).thenReturn(true); + assertTrue(store.delete("p1")); + verify(repository).deleteById("p1"); + + when(repository.existsById("missing")).thenReturn(false); + assertFalse(store.delete("missing")); + } + + private PolicyEntity entityFor(Policy policy) { + PolicyEntity entity = new PolicyEntity(); + entity.setId(policy.id()); + entity.setName(policy.name()); + entity.setOwner(policy.owner()); + entity.setEnabled(policy.enabled()); + entity.setTriggerType(policy.trigger().type()); + entity.setPolicyJson(objectMapper.writeValueAsString(policy)); + return entity; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java index 5b73ea5cc6..81d1ccdfa4 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java @@ -57,6 +57,7 @@ import stirling.software.proprietary.model.api.ai.AiWorkflowFileInput; import stirling.software.proprietary.model.api.ai.AiWorkflowOutcome; import stirling.software.proprietary.model.api.ai.AiWorkflowRequest; import stirling.software.proprietary.model.api.ai.AiWorkflowResponse; +import stirling.software.proprietary.policy.engine.PolicyExecutor; import tools.jackson.databind.ObjectMapper; import tools.jackson.databind.json.JsonMapper; @@ -107,18 +108,20 @@ class AiWorkflowServiceTest { .when(fileIdStrategy.idFor(any(MultipartFile.class))) .thenAnswer(inv -> ((MultipartFile) inv.getArgument(0)).getOriginalFilename()); + PolicyExecutor policyExecutor = + new PolicyExecutor( + internalApiClient, toolMetadataService, tempFileManager, objectMapper); service = new AiWorkflowService( pdfDocumentFactory, aiEngineClient, pdfContentExtractor, objectMapper, - internalApiClient, fileStorage, - toolMetadataService, tempFileManager, fileIdStrategy, endpointResolver, + policyExecutor, null, new ApplicationProperties()); when(endpointResolver.getEnabledEndpointUrls()).thenReturn(List.of()); diff --git a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java index 85520a239b..848d596d8b 100644 --- a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java +++ b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java @@ -29,6 +29,9 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBo import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.tags.Tag; + import jakarta.servlet.http.HttpServletRequest; import lombok.RequiredArgsConstructor; @@ -48,6 +51,8 @@ import stirling.software.saas.util.CreditHeaderUtils; @RestController @Profile("saas") @RequestMapping("/api/v1/ai/create") +@Tag(name = "AI") +@Hidden @RequiredArgsConstructor @Slf4j public class AiCreateController { diff --git a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateInternalController.java b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateInternalController.java index 34c44176e5..bf2e4894e8 100644 --- a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateInternalController.java +++ b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateInternalController.java @@ -17,6 +17,9 @@ import org.springframework.web.server.ResponseStatusException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.tags.Tag; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,6 +30,8 @@ import stirling.software.saas.ai.service.AiCreateSessionService; @RestController @Profile("saas") @RequestMapping("/api/v1/ai/create/internal") +@Tag(name = "AI") +@Hidden @RequiredArgsConstructor @Slf4j public class AiCreateInternalController { diff --git a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiProxyController.java b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiProxyController.java index 074da8707c..8b6ae714a4 100644 --- a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiProxyController.java +++ b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiProxyController.java @@ -18,6 +18,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.tags.Tag; + import jakarta.servlet.http.HttpServletRequest; import lombok.extern.slf4j.Slf4j; @@ -33,6 +36,8 @@ import stirling.software.saas.util.CreditHeaderUtils; @RestController @Profile("saas") @RequestMapping("/api/v1/ai") +@Tag(name = "AI") +@Hidden @Slf4j public class AiProxyController { diff --git a/app/saas/src/main/java/stirling/software/saas/controller/CreditController.java b/app/saas/src/main/java/stirling/software/saas/controller/CreditController.java index 9459c8bbb4..2c1be26f45 100644 --- a/app/saas/src/main/java/stirling/software/saas/controller/CreditController.java +++ b/app/saas/src/main/java/stirling/software/saas/controller/CreditController.java @@ -36,6 +36,7 @@ public class CreditController { private final CreditService creditService; @GetMapping + @Hidden @Operation( summary = "Get user credit information", description = @@ -252,6 +253,7 @@ public class CreditController { } @GetMapping("/usage") + @Hidden @Operation( summary = "Get credit usage summary", description = "Get overview of credit usage (for authenticated user or admin view)") diff --git a/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygOutputExtractor.java b/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygOutputExtractor.java index a19a78e0a6..86b5cb6026 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygOutputExtractor.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygOutputExtractor.java @@ -49,8 +49,12 @@ public class PaygOutputExtractor { /** {@code %PDF-} in ASCII. */ private static final byte[] PDF_MAGIC = {0x25, 0x50, 0x44, 0x46, 0x2D}; + /** {@code PK\x03\x04} — local file header for any non-empty ZIP. */ + private static final byte[] ZIP_MAGIC = {0x50, 0x4B, 0x03, 0x04}; + private static final String ZIP_CONTENT_TYPE = "application/zip"; private static final String PDF_CONTENT_TYPE = "application/pdf"; + private static final String OCTET_STREAM_CONTENT_TYPE = "application/octet-stream"; private final TempFileManager tempFileManager; @@ -74,11 +78,31 @@ public class PaygOutputExtractor { return List.of(); } String mediaType = stripParameters(contentType); - if (PDF_CONTENT_TYPE.equalsIgnoreCase(mediaType)) { + + // Stirling-PDF tool endpoints sometimes set Content-Type to + // application/octet-stream (or no header at all) even when the body + // is a real PDF or ZIP — Spring's default StreamingResponseBody path + // doesn't always negotiate content type. When the declared + // Content-Type is missing or generic, sniff magic bytes to recover + // lineage capture. + boolean missingOrGeneric = + mediaType == null || OCTET_STREAM_CONTENT_TYPE.equalsIgnoreCase(mediaType); + boolean isPdf = PDF_CONTENT_TYPE.equalsIgnoreCase(mediaType); + boolean isZip = ZIP_CONTENT_TYPE.equalsIgnoreCase(mediaType); + + if (missingOrGeneric) { + if (isMagic(bodyPath, PDF_MAGIC)) { + isPdf = true; + } else if (isMagic(bodyPath, ZIP_MAGIC)) { + isZip = true; + } + } + + if (isPdf) { // Wrapper-owned path. Don't claim ownership. return List.of(new ExtractedPdf(bodyPath, null)); } - if (ZIP_CONTENT_TYPE.equalsIgnoreCase(mediaType)) { + if (isZip) { return extractZip(bodyPath); } return List.of(); @@ -126,20 +150,25 @@ public class PaygOutputExtractor { } private boolean isPdfMagic(Path path) { - byte[] head = new byte[PDF_MAGIC.length]; + return isMagic(path, PDF_MAGIC); + } + + /** Returns true if the first {@code magic.length} bytes of {@code path} equal {@code magic}. */ + private boolean isMagic(Path path, byte[] magic) { + byte[] head = new byte[magic.length]; try (InputStream in = Files.newInputStream(path)) { int read = in.read(head); - if (read != PDF_MAGIC.length) { + if (read != magic.length) { return false; } - for (int i = 0; i < PDF_MAGIC.length; i++) { - if (head[i] != PDF_MAGIC[i]) { + for (int i = 0; i < magic.length; i++) { + if (head[i] != magic[i]) { return false; } } return true; } catch (IOException e) { - log.debug("PDF magic-byte check failed for {}", path, e); + log.debug("Magic-byte check failed for {}", path, e); return false; } } diff --git a/app/saas/src/main/java/stirling/software/saas/payg/policy/PaygTeamExtensions.java b/app/saas/src/main/java/stirling/software/saas/payg/policy/PaygTeamExtensions.java index 2c0603fc80..7d6f1b4023 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/policy/PaygTeamExtensions.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/policy/PaygTeamExtensions.java @@ -58,6 +58,19 @@ public class PaygTeamExtensions implements Serializable { @Column(name = "stripe_customer_id", unique = true, length = 128) private String stripeCustomerId; + /** + * Stripe subscription id (sub_xxx) for this team's PAYG metered subscription. {@code null} + * means the team has not added a card yet — engine writes shadow rows only and free-tier gating + * applies. {@code non-null} means engine posts meter events to Stripe on every billable tool + * call. + * + *

This is the single switch that determines whether a team is billed. Mutated only by the + * {@code payg_link_subscription} / {@code payg_unlink_subscription} Postgres functions (V14) — + * never directly via JPA writes. Treat as read-only from Java. + */ + @Column(name = "payg_subscription_id", unique = true, length = 128) + private String paygSubscriptionId; + @CreationTimestamp @Column(name = "created_at", updatable = false) private LocalDateTime createdAt; diff --git a/app/saas/src/main/java/stirling/software/saas/payg/policy/PricingPolicy.java b/app/saas/src/main/java/stirling/software/saas/payg/policy/PricingPolicy.java index f1443664cc..743dcdc8ae 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/policy/PricingPolicy.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/policy/PricingPolicy.java @@ -73,6 +73,20 @@ public class PricingPolicy implements Serializable { @Column(name = "file_unit_cap", nullable = false) private Integer fileUnitCap = 1000; + /** + * Free-tier allowance — doc units a team on this policy can consume per cycle before they must + * add a card. {@code 0} (default) means no free tier; the team is blocked at 402 from their + * very first tool call until they pay. New-signup defaults will set this to a sensible positive + * value; the special launch policy used by the day-1 legacy migration script can override with + * a different size if product wants the original 10 customers to feel special. + * + *

Enforced by {@code PaygTeamUsageService} (PR-SB-4) on every tool call, gated on {@code + * payg_team_extensions.payg_subscription_id IS NULL} — teams with an active subscription bypass + * this check entirely. + */ + @Column(name = "free_tier_units_per_cycle", nullable = false) + private Long freeTierUnitsPerCycle = 0L; + /** * Max tool steps allowed in one process before it splits, keyed by the caller's {@link * JobSource}. Self-hosted teams typically get a higher limit via a per-team policy override. diff --git a/app/saas/src/main/java/stirling/software/saas/payg/test/PaygCucumberThrowController.java b/app/saas/src/main/java/stirling/software/saas/payg/test/PaygCucumberThrowController.java new file mode 100644 index 0000000000..8a9082e2c8 --- /dev/null +++ b/app/saas/src/main/java/stirling/software/saas/payg/test/PaygCucumberThrowController.java @@ -0,0 +1,78 @@ +package stirling.software.saas.payg.test; + +import org.springframework.context.annotation.Profile; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import io.swagger.v3.oas.annotations.Hidden; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.annotations.AutoJobPostMapping; +import stirling.software.common.enumeration.ResourceWeight; + +/** + * Cucumber-only force-5xx endpoint. Gated behind the {@code payg-cucumber} Spring profile so the + * bean never registers in production — only the PAYG cucumber compose stack activates the profile + * (via {@code SPRING_PROFILES_ACTIVE=saas,payg-cucumber} in {@code docker-compose-saas.yml}). + * + *

Purpose: drive the PAYG filter+interceptor's 5xx-first-step branch end-to-end. No reliably- + * 5xx-ing real tool endpoint exists in current Stirling — every malformed input is caught as 4xx by + * {@code GlobalExceptionHandler}. Without this stub the only way to exercise the refund path was a + * manual procedure (a temporary throw endpoint added, run, removed) documented in {@code + * notes/PAYG_DESIGN.md} §7.5.2 M1. This controller replaces that procedure with a profile- gated + * automated scenario. + * + *

{@link AutoJobPostMapping} consumes {@code multipart/form-data} so the filter's {@code + * MultipartHttpServletRequest} cast runs and the input lineage hash is computed before the + * controller throws. The PAYG filter chain therefore sees: preHandle → openProcess (CHARGED row + * written) → controller throws → afterCompletion observes status 500 → markFirstStepFailed (row → + * REFUNDED, job → CLOSED). + * + *

The thrown {@link IllegalStateException} is unwrapped by {@code GlobalExceptionHandler}'s + * RuntimeException handler — it has no IOException / IllegalArgument / BaseAppException cause, so + * it falls through to the "Unexpected RuntimeException" branch which sets HTTP 500. Don't use + * {@code ResponseStatusException} here — its dedicated handler is reached before the unwrapping + * branch but the upstream test still passes; {@link IllegalStateException} is the more honest mimic + * of a real bug-driven 500. + * + *

{@code @Hidden} keeps this endpoint out of OpenAPI / Swagger output even when the profile is + * active, so no docs leak to anyone running the cucumber stack interactively. + * + *

Return type matters: the method declares {@link ResponseEntity}{@code }, not + * {@code void}. Spring's {@code InvocableHandlerMethod} picks a return-value handler based on the + * declared return type; {@code void} matches the "no body, status 200" handler regardless of + * what the {@code @Around} advice on {@link AutoJobPostMapping} actually returns at runtime. The + * first version of this stub declared {@code void} and the 500 ResponseEntity from {@code + * JobExecutorService}'s catch block was silently discarded — the client saw a 200 with an empty + * body. Declaring {@code ResponseEntity} matches the real runtime type and lets the advice's + * 500 reach the wire. + */ +@Slf4j +@RestController +@Profile("payg-cucumber") +@RequestMapping("/api/v1/payg-cucumber") +@Hidden +public class PaygCucumberThrowController { + + @AutoJobPostMapping( + value = "/throw-500", + consumes = MediaType.MULTIPART_FORM_DATA_VALUE, + resourceWeight = ResourceWeight.SMALL_WEIGHT) + public ResponseEntity throw500( + @RequestParam(value = "fileInput", required = false) MultipartFile fileInput) { + // The file is read by the PAYG filter via getMultiFileMap() before we get here; the + // controller param is just to keep Spring's multipart binding happy. We don't touch it. + log.warn( + "PAYG cucumber forced 500 (fileInput name='{}' size={} bytes)", + fileInput != null ? fileInput.getOriginalFilename() : null, + fileInput != null ? fileInput.getSize() : 0); + throw new IllegalStateException("PAYG cucumber forced 500"); + // unreachable — kept as a type signature so AutoJobAspect's @Around return value (the + // 500 ResponseEntity from JobExecutorService) actually reaches the wire. + } +} diff --git a/app/saas/src/main/resources/db/migration/saas/V14__payg_subscription_state.sql b/app/saas/src/main/resources/db/migration/saas/V14__payg_subscription_state.sql new file mode 100644 index 0000000000..eca2665de7 --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V14__payg_subscription_state.sql @@ -0,0 +1,189 @@ +-- PAYG subscription state — the column + functions that let new customers reach Stripe billing. +-- +-- This migration is half of the Stripe/Supabase wire-up (PR-SB-1 in `notes/PAYG_DESIGN.md` +-- revision note + `payg-stripe-supabase-plan.html`). It's strictly additive: +-- * one new column on payg_team_extensions (payg_subscription_id) +-- * one new column on pricing_policy (free_tier_units_per_cycle) +-- * two RPC functions (payg_link_subscription, payg_unlink_subscription) — the only writers +-- of subscription state, called by stripe-webhook + create-payg-team-subscription edge fns +-- * an AFTER-INSERT trigger on teams that auto-creates the payg_team_extensions sidecar row +-- so every new signup is PAYG-by-default +-- * an RLS policy that lets team LEADERs (and the service role) link subscriptions +-- +-- No behaviour change for the running app until PR-SB-4 wires PaygMeterReportingService and +-- the free-tier gate into JobChargeService. Until then this just exposes new state for the +-- edge functions in PR-SB-2 to write through to. +-- +-- Design references: +-- * notes/PAYG_DESIGN.md (revision note 2026-06-03 — "subscription presence is the gate") +-- * payg-stripe-supabase-plan.html §3.1 — RPC functions; §3.5 — RLS policy + +-- --------------------------------------------------------------------------------------------- +-- 1. New columns +-- --------------------------------------------------------------------------------------------- + +ALTER TABLE stirling_pdf.payg_team_extensions + ADD COLUMN IF NOT EXISTS payg_subscription_id VARCHAR(128) UNIQUE; + +COMMENT ON COLUMN stirling_pdf.payg_team_extensions.payg_subscription_id IS + 'Stripe subscription id (sub_xxx) for this team''s PAYG metered subscription. ' + 'NULL = team has not added a card yet; engine writes shadow rows only. ' + 'NOT NULL = engine posts meter events to Stripe on every billable tool call. ' + 'Mutated exclusively by payg_link_subscription / payg_unlink_subscription RPC functions.'; + +ALTER TABLE stirling_pdf.pricing_policy + ADD COLUMN IF NOT EXISTS free_tier_units_per_cycle BIGINT NOT NULL DEFAULT 0; + +COMMENT ON COLUMN stirling_pdf.pricing_policy.free_tier_units_per_cycle IS + 'Doc units a team on this policy can consume per cycle before they must add a card. ' + 'Default 0 = no free tier (block immediately). The seeded default policy will set this ' + 'to the launch free-tier size; the special "launch" policy used by the day-1 legacy ' + 'migration script (see PAYG_DESIGN.md §3.10 revised) can override.'; + +-- --------------------------------------------------------------------------------------------- +-- 2. RPC: payg_link_subscription +-- +-- Called by: +-- * supabase/functions/create-payg-team-subscription/index.ts (post-Stripe-Checkout, with +-- either user JWT [normal path, RLS-enforced] or service-role [day-1 migration script]) +-- * supabase/functions/stripe-webhook/handlers/payg-subscription.ts on +-- customer.subscription.created (idempotent — second invocation with same args is a no-op) +-- --------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION stirling_pdf.payg_link_subscription( + p_team_id BIGINT, + p_customer_id TEXT, + p_subscription_id TEXT +) RETURNS VOID +LANGUAGE plpgsql +SECURITY INVOKER +AS $$ +BEGIN + UPDATE stirling_pdf.payg_team_extensions + SET stripe_customer_id = p_customer_id, + payg_subscription_id = p_subscription_id, + updated_at = now() + WHERE team_id = p_team_id; + + IF NOT FOUND THEN + RAISE EXCEPTION 'payg_team_extensions row missing for team %', p_team_id + USING ERRCODE = 'foreign_key_violation'; + END IF; + + INSERT INTO stirling_pdf.payg_subscription_change_log(team_id, action, subscription_id) + VALUES (p_team_id, 'LINKED', p_subscription_id); +END $$; + +COMMENT ON FUNCTION stirling_pdf.payg_link_subscription(BIGINT, TEXT, TEXT) IS + 'Idempotent link of a Stripe subscription to a team. SECURITY INVOKER means RLS applies — ' + 'the caller must be a LEADER of the team (or hold the service-role bypass). ' + 'Writes an audit row to payg_subscription_change_log.'; + +-- --------------------------------------------------------------------------------------------- +-- 3. RPC: payg_unlink_subscription +-- +-- Called by stripe-webhook handlers/payg-subscription.ts on customer.subscription.deleted +-- (after Stripe's own retries have given up). Drops the team back to free-tier-then-block. +-- --------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION stirling_pdf.payg_unlink_subscription( + p_team_id BIGINT, + p_reason TEXT +) RETURNS VOID +LANGUAGE plpgsql +SECURITY INVOKER +AS $$ +BEGIN + UPDATE stirling_pdf.payg_team_extensions + SET payg_subscription_id = NULL, + updated_at = now() + WHERE team_id = p_team_id; + -- We deliberately keep stripe_customer_id — the team may add a new card later and we'd + -- like to reuse the existing Stripe customer record rather than create a duplicate. + + INSERT INTO stirling_pdf.payg_subscription_change_log(team_id, action, reason) + VALUES (p_team_id, 'UNLINKED', p_reason); +END $$; + +COMMENT ON FUNCTION stirling_pdf.payg_unlink_subscription(BIGINT, TEXT) IS + 'Drops the team back to free-tier-then-block derived state. Reason is logged for audit ' + '(typically subscription_deleted | admin | card_removed).'; + +-- --------------------------------------------------------------------------------------------- +-- 4. Auto-create payg_team_extensions row when a team is created +-- +-- Every new signup gets a payg_team_extensions row with NULL pricing_policy_id (which the +-- backend's PricingPolicyService resolves to the default policy). The free-tier gate kicks in +-- from the very first tool call. +-- --------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION stirling_pdf.payg_create_team_extensions_trigger() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + INSERT INTO stirling_pdf.payg_team_extensions(team_id) + VALUES (NEW.team_id) + ON CONFLICT (team_id) DO NOTHING; + RETURN NEW; +END $$; + +DROP TRIGGER IF EXISTS trg_payg_create_team_extensions ON stirling_pdf.teams; +CREATE TRIGGER trg_payg_create_team_extensions + AFTER INSERT ON stirling_pdf.teams + FOR EACH ROW + EXECUTE FUNCTION stirling_pdf.payg_create_team_extensions_trigger(); + +COMMENT ON TRIGGER trg_payg_create_team_extensions ON stirling_pdf.teams IS + 'Ensures every team has a payg_team_extensions sidecar row from creation. New customers ' + 'are PAYG-default from minute one — they consume free-tier units until they add a card.'; + +-- --------------------------------------------------------------------------------------------- +-- 5. Backfill: any existing team without a sidecar row gets one now +-- --------------------------------------------------------------------------------------------- + +INSERT INTO stirling_pdf.payg_team_extensions(team_id) +SELECT t.team_id + FROM stirling_pdf.teams t + WHERE NOT EXISTS ( + SELECT 1 FROM stirling_pdf.payg_team_extensions x WHERE x.team_id = t.team_id + ); + +-- --------------------------------------------------------------------------------------------- +-- 6. RLS policy +-- +-- Service-role bypasses RLS (backend reads + day-1 migration script writes via the service-role +-- key). For user-initiated writes via the frontend Add-Card flow, only team LEADERs can link a +-- subscription. SELECT remains permissive — anyone in the team can see the row. +-- --------------------------------------------------------------------------------------------- + +ALTER TABLE stirling_pdf.payg_team_extensions ENABLE ROW LEVEL SECURITY; + +-- Read: any team member can see their team's payg row. +DROP POLICY IF EXISTS payg_team_ext_select ON stirling_pdf.payg_team_extensions; +CREATE POLICY payg_team_ext_select + ON stirling_pdf.payg_team_extensions + FOR SELECT + USING ( + team_id IN ( + SELECT tm.team_id + FROM stirling_pdf.team_memberships tm + JOIN stirling_pdf.users u ON u.user_id = tm.user_id + WHERE u.supabase_auth_id = auth.uid() + ) + ); + +-- Update: only LEADERs of the team can update (i.e. link / unlink a subscription). +DROP POLICY IF EXISTS payg_team_ext_leader_update ON stirling_pdf.payg_team_extensions; +CREATE POLICY payg_team_ext_leader_update + ON stirling_pdf.payg_team_extensions + FOR UPDATE + USING ( + team_id IN ( + SELECT tm.team_id + FROM stirling_pdf.team_memberships tm + JOIN stirling_pdf.users u ON u.user_id = tm.user_id + WHERE u.supabase_auth_id = auth.uid() + AND tm.role = 'LEADER' + ) + ); diff --git a/app/saas/src/main/resources/db/migration/saas/V15__payg_audit_logs.sql b/app/saas/src/main/resources/db/migration/saas/V15__payg_audit_logs.sql new file mode 100644 index 0000000000..1b7e61e9c1 --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V15__payg_audit_logs.sql @@ -0,0 +1,76 @@ +-- PAYG audit-log tables. Two append-only logs: +-- +-- * payg_meter_event_log — written by the backend's PaygMeterReportingService +-- on every Stripe meter event POST attempt. Gives us a +-- record independent of Stripe's own logs so we can +-- replay-after-24h-window (Stripe's idempotency window) +-- and run nightly reconciliation against Stripe's +-- meter-event list. +-- +-- * payg_subscription_change_log — written by V14's two RPC functions on every +-- subscription link / unlink. Independent of Stripe's +-- webhook log; lets us diagnose "why is this team in +-- free-tier-block when their Stripe sub is active?" +-- without leaving our DB. +-- +-- Both are pure additive; nothing reads them yet. PR-SB-5 (nightly reconcile) wires the +-- meter-event log; the subscription change log is queried only from admin tooling. +-- +-- Design references: +-- * payg-stripe-supabase-plan.html §3.10 — twin migrations +-- * payg-stripe-supabase-plan.html §8 H5 — 24h idempotency window mitigation + +-- --------------------------------------------------------------------------------------------- +-- 1. payg_meter_event_log — backend-side audit of every Stripe meter event we tried to post. +-- --------------------------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS stirling_pdf.payg_meter_event_log ( + event_id BIGSERIAL PRIMARY KEY, + team_id BIGINT NOT NULL REFERENCES stirling_pdf.teams(team_id) ON DELETE CASCADE, + job_id UUID, + idempotency_key VARCHAR(128) NOT NULL UNIQUE, + units INTEGER NOT NULL, + occurred_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + posted_to_stripe_at TIMESTAMP, + -- NULL while pending; set when the meter-payg-units edge fn returns success. NULL after + -- 24h means the event never made it to Stripe — nightly reconcile retries with a fresh + -- idempotency-key suffix (see §8 H5 mitigation). + stripe_error_code VARCHAR(64), + stripe_error_body TEXT, + metadata JSONB +); + +CREATE INDEX IF NOT EXISTS idx_payg_meter_event_team_time + ON stirling_pdf.payg_meter_event_log (team_id, occurred_at); + +CREATE INDEX IF NOT EXISTS idx_payg_meter_event_unposted + ON stirling_pdf.payg_meter_event_log (occurred_at) + WHERE posted_to_stripe_at IS NULL; + +COMMENT ON TABLE stirling_pdf.payg_meter_event_log IS + 'Backend audit of every Stripe meter event POST attempt. Independent of Stripe meter ' + 'history. idempotency_key is the same one passed to Stripe; the UNIQUE constraint here ' + 'gives us safe at-least-once semantics even on backend retry. Rows older than 24h with ' + 'posted_to_stripe_at IS NULL are stuck and retried by the nightly reconcile job.'; + +-- --------------------------------------------------------------------------------------------- +-- 2. payg_subscription_change_log — written by V14's RPC functions on every link / unlink. +-- --------------------------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS stirling_pdf.payg_subscription_change_log ( + change_id BIGSERIAL PRIMARY KEY, + team_id BIGINT NOT NULL REFERENCES stirling_pdf.teams(team_id) ON DELETE CASCADE, + action VARCHAR(32) NOT NULL, + -- LINKED — payg_link_subscription written subscription_id + -- UNLINKED — payg_unlink_subscription cleared the subscription + subscription_id VARCHAR(128), + reason TEXT, + changed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_payg_sub_change_team_time + ON stirling_pdf.payg_subscription_change_log (team_id, changed_at); + +COMMENT ON TABLE stirling_pdf.payg_subscription_change_log IS + 'Append-only log of every subscription link / unlink. Written by V14 RPC functions; ' + 'never updated. Diagnostic value when reconciling against Stripe webhook history.'; diff --git a/app/saas/src/main/resources/static/modern-logo.svg b/app/saas/src/main/resources/static/modern-logo.svg new file mode 100644 index 0000000000..a4a1a1f87e --- /dev/null +++ b/app/saas/src/main/resources/static/modern-logo.svg @@ -0,0 +1,4 @@ + + + + diff --git a/app/saas/src/main/resources/static/saas-landing.html b/app/saas/src/main/resources/static/saas-landing.html new file mode 100644 index 0000000000..3ffef1d7bf --- /dev/null +++ b/app/saas/src/main/resources/static/saas-landing.html @@ -0,0 +1,250 @@ + + + + + + + Stirling PDF - Cloud API + + + + + + + + + + + +

+
+ + +
+

+ You've reached the Stirling PDF Cloud API. This endpoint serves the REST API that powers our apps and your integrations. +

+
+ + Open API Documentation + Read the Developer Docs + +
+ +
+

Just looking to edit a PDF?

+

+ This page is the API endpoint - it doesn't have an editor. If you want the full interface: +

+ +
+ +
+ +
+

Want to run your own?

+

+ Stirling PDF is open source. You can self-host the same toolkit on your own hardware or run it inside your network: +

+ +
+ +
+ +
+

Support

+ +
+
+
+ +
+ Powered by + Stirling PDF +
+ + diff --git a/app/saas/src/test/java/stirling/software/saas/payg/filter/PaygOutputExtractorTest.java b/app/saas/src/test/java/stirling/software/saas/payg/filter/PaygOutputExtractorTest.java index 201a0d68ef..688377ad3a 100644 --- a/app/saas/src/test/java/stirling/software/saas/payg/filter/PaygOutputExtractorTest.java +++ b/app/saas/src/test/java/stirling/software/saas/payg/filter/PaygOutputExtractorTest.java @@ -102,6 +102,61 @@ class PaygOutputExtractorTest { assertThat(extractor.extract("application/zip", zip)).isEmpty(); } + @Test + void octetStreamWithPdfMagic_treatedAsPdf(@TempDir Path tmp) throws IOException { + // Stirling tool endpoints sometimes set Content-Type to + // application/octet-stream for streamed responses even when the body + // is a real PDF. The extractor must sniff magic bytes when the + // declared Content-Type is generic. + Path body = tmp.resolve("body.bin"); + Files.write(body, pdfBytes("real-pdf-content")); + List out = + extractor.extract("application/octet-stream", body); + assertThat(out).hasSize(1); + assertThat(out.get(0).path()).isEqualTo(body); + } + + @Test + void octetStreamWithZipMagic_unpackedAsZip(@TempDir Path tmp) throws IOException { + Path zip = tmp.resolve("body.bin"); + try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zip))) { + writeEntry(zos, "page1.pdf", pdfBytes("a")); + writeEntry(zos, "page2.pdf", pdfBytes("b")); + } + List out = + extractor.extract("application/octet-stream", zip); + try { + assertThat(out).hasSize(2); + } finally { + for (PaygOutputExtractor.ExtractedPdf p : out) { + p.close(); + } + } + } + + @Test + void nullContentTypeWithZipMagic_unpackedAsZip(@TempDir Path tmp) throws IOException { + Path zip = tmp.resolve("body.bin"); + try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zip))) { + writeEntry(zos, "page1.pdf", pdfBytes("a")); + } + List out = extractor.extract(null, zip); + try { + assertThat(out).hasSize(1); + } finally { + for (PaygOutputExtractor.ExtractedPdf p : out) { + p.close(); + } + } + } + + @Test + void octetStreamWithoutMagic_returnsEmpty(@TempDir Path tmp) throws IOException { + Path body = tmp.resolve("body.bin"); + Files.write(body, "neither pdf nor zip".getBytes(StandardCharsets.UTF_8)); + assertThat(extractor.extract("application/octet-stream", body)).isEmpty(); + } + private static void writeEntry(ZipOutputStream zos, String name, byte[] data) throws IOException { zos.putNextEntry(new ZipEntry(name)); diff --git a/frontend/editor/public/locales/en-GB/translation.toml b/frontend/editor/public/locales/en-GB/translation.toml index d1f4ecab84..ddc117e011 100644 --- a/frontend/editor/public/locales/en-GB/translation.toml +++ b/frontend/editor/public/locales/en-GB/translation.toml @@ -12,7 +12,6 @@ cancel = "Cancel" changedCredsMessage = "Credentials changed!" chooseFile = "Choose File" close = "Close" -color = "Colour" comingSoon = "Coming soon" confirm = "Confirm" confirmClose = "Confirm Close" @@ -199,6 +198,10 @@ ssoDescription = "Two-factor authentication is managed by your identity provider ssoManaged = "Configure MFA through your identity provider." title = "Two-factor authentication" +[account.overview] +logOut = "Log out" +signedInAs = "Signed in as: {{email}}" + [add-page-numbers] tags = "paginate,label,organize,index" @@ -226,6 +229,9 @@ title = "What it does" [AddAttachmentsRequest.tooltip.header] title = "About Add Attachments" +[addFileCard] +upload = "Upload" + [addImage] applySignatures = "Apply Images" header = "Add images to PDFs" @@ -606,6 +612,7 @@ label = "Temp File Management" [admin.settings.advanced.tempFileManagement.baseTmpDir] description = "Base directory for temporary files (leave empty for default: java.io.tmpdir/stirling-pdf)" label = "Base Temp Directory" +placeholder = "Default: java.io.tmpdir/stirling-pdf" [admin.settings.advanced.tempFileManagement.cleanupIntervalMinutes] description = "How often to run cleanup (in minutes)" @@ -618,6 +625,7 @@ label = "Cleanup System Temp" [admin.settings.advanced.tempFileManagement.libreofficeDir] description = "Directory for LibreOffice temp files (leave empty for default: baseTmpDir/libreoffice)" label = "LibreOffice Temp Directory" +placeholder = "Default: baseTmpDir/libreoffice" [admin.settings.advanced.tempFileManagement.maxAgeHours] description = "Maximum age in hours before temp files are cleaned up" @@ -634,6 +642,7 @@ label = "Startup Cleanup" [admin.settings.advanced.tempFileManagement.systemTempDir] description = "System temp directory to clean (only used if cleanupSystemTemp is enabled)" label = "System Temp Directory" +placeholder = "System temp directory path" [admin.settings.advanced.tessdataDir] description = "Path to the tessdata directory for OCR language files" @@ -839,6 +848,7 @@ label = "Database Name" [admin.settings.database.password] description = "Database authentication password" label = "Password" +placeholder = "Enter database password" [admin.settings.database.port] description = "Database server port (not used if custom URL is provided)" @@ -870,10 +880,12 @@ label = "Hide unavailable tools by default" [admin.settings.endpoints.groupsToRemove] description = "Select endpoint groups to disable" label = "Disabled Endpoint Groups" +placeholder = "Select groups to disable" [admin.settings.endpoints.toRemove] description = "Select individual endpoints to disable" label = "Disabled Endpoints" +placeholder = "Select endpoints to disable" [admin.settings.enterpriseRequired] message = "An Enterprise license is required to access {{featureName}}. You are viewing demo data for reference." @@ -1088,6 +1100,7 @@ label = "SMTP Host" [admin.settings.mail.password] description = "Password for SMTP authentication" label = "SMTP Password" +placeholder = "Enter SMTP password" [admin.settings.mail.port] description = "The port number for SMTP connection (typically 25, 465, or 587)" @@ -1097,6 +1110,10 @@ label = "SMTP Port" description = "Username for SMTP authentication" label = "SMTP Username" +[admin.settings.plan.noData] +message = "Plans data is not available at the moment." +title = "No data available" + [admin.settings.premium] description = "Configure your premium or enterprise license key." license = "License Configuration" @@ -1143,6 +1160,9 @@ info = "If you have a license key or certificate file from a direct purchase, yo toggle = "Got a license key or certificate file?" [admin.settings.premium.movedFeatures] +auditLogging = "<0>Audit Logging (ENTERPRISE) - Security" +customMetadata = "<0>Custom Metadata (PRO) - General" +databaseConfiguration = "<0>Database Configuration (ENTERPRISE) - Database" message = "Premium and Enterprise features are now organized in their respective sections:" title = "Premium Features Distributed" @@ -1526,10 +1546,10 @@ section_title = "Agents" show_less = "Show less" start_chat = "Start chatting" stirling_description = "Your general-purpose PDF assistant" -stirling_running = "Running..." stirling_full_name = "Stirling General Agent" stirling_long_description = "General purpose PDF assistant that can run tools, create PDFs and extract insights from your documents." stirling_name = "Stirling" +stirling_running = "Running..." stirling_tooltip = "Stirling agent" view_all = "View all agents" @@ -2126,11 +2146,31 @@ placeholder = "Number of pages" title = "Last N Pages" [bulkSelection.operators] +text = "AND has higher precedence than comma. NOT applies within the document range." +title = "Operators" + +[bulkSelection.operators.and] +title = "Combine selections (both conditions must be true)" + +[bulkSelection.operators.descriptions] and = "AND: & or \"and\" - require both conditions (e.g., 1-50 & even)" comma = "Comma: , or | - combine selections (e.g., 1-10, 20)" not = "NOT: ! or \"not\" - exclude pages (e.g., 3n & not 30)" -text = "AND has higher precedence than comma. NOT applies within the document range." -title = "Operators" + +[bulkSelection.operators.even] +title = "Select all even-numbered pages (2, 4, 6, 8...)" + +[bulkSelection.operators.not] +title = "Exclude from selection" + +[bulkSelection.operators.odd] +title = "Select all odd-numbered pages (1, 3, 5, 7...)" + +[bulkSelection.operators.or] +title = "Add to selection (either condition can be true)" + +[bulkSelection.pageSelection] +title = "Page Selection" [bulkSelection.range] fromPlaceholder = "From" @@ -2782,6 +2822,9 @@ unknownTool = "Unknown tool" [cloudBadge] tooltip = "This operation will use your cloud credits" +[color.eyeDropper] +tooltip = "Pick colour from screen" + [colorPicker] title = "Choose colour" @@ -3006,6 +3049,10 @@ title = "Compression Method" 1 = "1-3 PDF compression,
4-6 lite image compression,
7-9 intense image compression Will dramatically reduce image quality" _value = "Compression Settings" +[compress.settings] +desiredSize = "Desired File Size" +desiredSizePlaceholder = "Enter size" + [compress.tooltip.description] text = "Compression is an easy way to reduce your file size. Pick File Size to enter a target size and have us adjust quality for you. Pick Quality to set compression strength manually." title = "Description" @@ -3496,6 +3543,13 @@ docsLink = "View installation documentation" message = "Stirling-PDF does not have permission to update itself on this machine." title = "Administrator permissions required" +[dropdownList] +searchPlaceholder = "Search..." + +[editableSecretField] +edit = "Edit" +editSecretValue = "Edit secret value" + [editTableOfContents] submit = "Apply table of contents" @@ -3583,6 +3637,9 @@ tabTitle = "Outline workspace" description = "Select the Edit Table of Contents tool to load its workspace." title = "Open the tool to start editing" +[emptyFilesState] +upload = "Upload" + [encryptedPdfUnlock] description = "This PDF is password protected. Enter the password so you can continue working with it." emptyResponse = "Password removal did not produce a file." @@ -3697,6 +3754,10 @@ title = "Settings" [extractPages.tooltip] description = "Extracts the selected pages into a new PDF, preserving order." +[fileCard] +openInFileEditor = "Open in File Editor" +viewInViewer = "View in Viewer" + [fileChooser] click = "Click" dragAndDrop = "Drag & Drop" @@ -3706,6 +3767,9 @@ extractPDF = "Extracting..." hoveredDragAndDrop = "Drag & Drop file(s) here" or = "or" +[fileDropdownMenu] +closeFile = "Close file" + [fileEditor] addFiles = "Add Files" pageCount = "{{count}} pages" @@ -3872,6 +3936,10 @@ openSettings = "Open settings" search = "Search" searchPlaceholder = "Search files..." +[fileSidebar.fileItem] +closeViewer = "Close viewer" +openInViewer = "Open in viewer" + [filesPage] addToWorkspace = "Add to workspace" addToWorkspaceCount = "Add {{count}} to workspace" @@ -4178,6 +4246,15 @@ title = "About Flattening PDFs" discord = "Discord" issues = "GitHub" +[formFill] +flattenAfterFilling = "Flatten after filling" +rescanFields = "Re-scan fields" +rescanFormFields = "Re-scan form fields" +save = "Save" + +[formFill.sidebar] +close = "Close sidebar" + [getPdfInfo] downloadJson = "Download JSON" downloads = "Downloads" @@ -4737,6 +4814,11 @@ tags = "markup,web-content,transformation,convert" title = "HTML To PDF" zoom = "Zoom level for displaying the website." +[image.upload] +hint = "Upload a PNG, JPG, SVG, or other image file to place on the PDF. SVG files will be converted to PNG for compatibility." +label = "Upload Image" +placeholder = "Select image file" + [imageToPdf] tags = "conversion,img,jpg,picture,photo" @@ -4755,6 +4837,9 @@ title = "Image to PDF" 4 = "Merge into single PDF" 5 = "Convert to separate PDFs" +[infoBanner] +dismiss = "Dismiss" + [invite] acceptError = "Failed to create account" accountFor = "Creating account for" @@ -5090,6 +5175,7 @@ failed = "OCR operation failed" [ocr.languagePicker] additionalLanguages = "Looking for additional languages?" +loading = "Loading available languages..." viewSetupGuide = "View setup guide →" [ocr.operation] @@ -5226,6 +5312,10 @@ daysRemainingSingular = "{{days}} day remaining" title = "Your 30-Day Pro Trial" trialEnds = "Trial ends {{date}}" +[onboarding.mfa] +authenticationCode = "Authentication code" +qrCodeLoading = "Generating your QR code…" + [onboarding.planOverview] adminBodyLoginDisabled = "Once you enable login mode, you can manage users, configure settings, and monitor server health. The first {{freeTierLimit}} people on your server get to use Stirling free of charge." adminBodyLoginEnabled = "As an admin, you can manage users, configure settings, and monitor server health. The first {{freeTierLimit}} people on your server get to use Stirling free of charge." @@ -5235,6 +5325,9 @@ userTitle = "Plan Overview" [onboarding.securityCheck] message = "The application has undergone significant changes recently. Your server admin's attention may be required. Please confirm your role to continue." +roleAdmin = "Admin" +rolePlaceholder = "Confirm your role" +roleUser = "User" [onboarding.serverLicense] freeBody = "Our Open-Core licensing permits up to {{freeTierLimit}} users for free per server. To scale uninterrupted, we recommend the Stirling Server plan - unlimited seats and SSO support for $99/server/mo." @@ -5360,6 +5453,17 @@ title = "Page Editor" zoomIn = "Zoom In" zoomOut = "Zoom Out" +[pageEditor.emptyState] +body = "Add files to start editing pages" +title = "No PDF files loaded" + +[pageEditor.toolbar] +delete = "Delete Selected" +redo = "Redo" +rotateLeft = "Rotate Selected Left" +rotateRight = "Rotate Selected Right" +undo = "Undo" + [pageExtracter] header = "Extract Pages" placeholder = "(e.g. 1,2,8 or 4,7,12-16 or 2n-1)" @@ -5371,12 +5475,14 @@ addBorder = "Add Borders" borderWidth = "Border Thickness" bottom = "Bottom Margin" cols = "Columns" +colsPlaceholder = "Enter columns" header = "Multi Page Layout" innerMargin = "Inner Margin" left = "Left Margin" pagesPerSheet = "Pages per sheet:" right = "Right Margin" rows = "Rows" +rowsPlaceholder = "Enter rows" submit = "Submit" tags = "merge,composite,single-view,organize" title = "Multi Page Layout" @@ -5418,6 +5524,14 @@ innerMarginTooLarge = "Inner margin is too large for the selected layout." outerHorizontalMarginsTooLarge = "Left/Right margins are too large for this page size." outerVerticalMarginsTooLarge = "Top/Bottom margins are too large for this page size." +[pageLayout.margins] +borderThicknessPlaceholder = "Enter border thickness" +bottomPlaceholder = "Enter bottom margin" +innerPlaceholder = "Enter inner margin" +leftPlaceholder = "Enter left margin" +rightPlaceholder = "Enter right margin" +topPlaceholder = "Enter top margin" + [pageLayout.marginsBorders.tooltip.borders] text = "Enable borders to draw lines around each placed page. This can help visual separation or trimming." title = "Add Borders" @@ -6071,6 +6185,26 @@ pdfTools = "Basic PDF Tools" priority = "Priority Support" title = "Feature" +[plan.features] +allPdfOperations = "All PDF operations" +auditing = "Auditing" +communitySupport = "Community support" +customPdfMetadata = "Custom PDF metadata" +editingTextInPdfs = "Editing text in pdfs" +externalDatabase = "External Database" +googleDriveIntegration = "Google drive integration" +prometheusSupport = "Prometheus Support" +regularUpdates = "Regular updates" +saml = "SAML" +secureLoginSupport = "Secure Login Support" +selfHostedDeployment = "Self-hosted deployment" +sso = "SSO" +unlimitedUsers = "Unlimited users" +upToFiveUsers = "Up to 5 users" +upToFiveUsersLowercase = "up to 5 users" +usageTracking = "Usage tracking" +usersLimitedToSeats = "Users limited to seats" + [plan.free] forever = "Forever free" highlight1 = "Limited Tool Usage Per week" @@ -6079,6 +6213,22 @@ highlight3 = "Community support" included = "Included" name = "Free" +[plan.highlights] +advancedIntegrations = "Advanced integrations" +allBasicFeatures = "All basic features" +cancelAnytime = "Cancel anytime" +customPdfMetadata = "Custom PDF metadata" +editingTextInPdfsCaps = "Editing text in PDFs" +enterpriseFeatures = "Enterprise features (SAML, Auditing)" +perSeatLicensing = "Per-seat licensing" +saveWithAnnualBilling = "Save with annual billing" +selfHosted = "Self-hosted" +selfHostedOnInfrastructure = "Self-hosted on your infrastructure" +ssoOAuth = "SSO (OAuth2/OIDC)" +unlimitedUsers = "Unlimited users" +upToFiveUsers = "Up to 5 users" +usageTrackingPrometheus = "Usage tracking & Prometheus" + [plan.licenseWarning] body = "You have {{total}} users but the free tier only supports {{limit}} per server. Upgrade to keep Stirling PDF running smoothly." cta = "See plans" @@ -6389,6 +6539,7 @@ allSessions = "All Sessions" allTools = "Tools" automate = "Automate" back = "Back" +backToAllTools = "Back to all tools" certSign = "Certificate Sign" completedSessions = "Completed Sessions" completedTab = "Completed" @@ -7060,6 +7211,9 @@ bullet3 = "Critical for approval workflows and legal chains of custody" description = "The order you specify when creating the session determines who signs first." title = "Signature Order" +[settings] +close = "Close" + [settings.configuration] advanced = "Advanced" database = "Database" @@ -7542,6 +7696,7 @@ sharedDescription = "All users can see and use these signatures." sharedHeading = "Shared Signatures" tempStorageDescription = "Signatures are stored in your browser only. They will be lost if you clear browser data or switch browsers." tempStorageTitle = "Temporary browser storage" +use = "Use signature" [sign.saved.status] saved = "Saved" @@ -8122,9 +8277,18 @@ center = "Center" left = "Left" right = "Right" +[textInput] +clear = "Clear input" + [theme] toggle = "Toggle Theme" +[time.relative] +daysAgo = "{{count}}d ago" +hoursAgo = "{{count}}h ago" +justNow = "just now" +minutesAgo = "{{count}}m ago" + [timestampPdf] completed = "PDF timestamped successfully" desc = "Add an RFC 3161 document timestamp to your PDF using a trusted Time Stamp Authority (TSA) server." @@ -8151,11 +8315,9 @@ label = "Select a TSA server" [timestampPdf.steps] settings = "Settings" -[time.relative] -daysAgo = "{{count}}d ago" -hoursAgo = "{{count}}h ago" -justNow = "just now" -minutesAgo = "{{count}}m ago" +[toast] +dismiss = "Dismiss" +toggleDetails = "Toggle details" [tool] endpointUnavailable = "This tool is unavailable on your server." @@ -8243,6 +8405,9 @@ verification = "Verification" noSearchResults = "No tools found" noTools = "No tools available" +[tooltip] +close = "Close tooltip" + [unlockPDFForms] description = "This tool will remove read-only restrictions from PDF form fields, making them editable and fillable." filenamePrefix = "unlocked_forms" @@ -8269,6 +8434,7 @@ breakingChangesDefault = "This version contains breaking changes." breakingChangesDetected = "Breaking Changes Detected" breakingChangesMessage = "Some versions contain breaking changes. Please review the migration guides below before updating." close = "Close" +closeModal = "Close update modal" current = "Current Version" defaultRecommendation = "This update contains important fixes and improvements." downloadLatest = "Download Latest" @@ -8477,12 +8643,16 @@ enableDarkFilter = "Enable Dark Filter" enableSepiaFilter = "Enable Sepia Filter" firstPage = "First Page" lastPage = "Last Page" +moreOptions = "More" nextPage = "Next Page" onlyPdfSupported = "This file format is not supported for preview." +pageNavigation = "Page navigation" previousPage = "Previous Page" resetZoom = "Reset zoom" singlePageView = "Single Page View" unknownFile = "Unknown file" +view = "View" +zoom = "Zoom" zoomIn = "Zoom In" zoomOut = "Zoom Out" @@ -8520,6 +8690,9 @@ typeInsertText = "Insert Text" typeReplaceText = "Replace Text" viewComment = "View comment" +[viewer.error] +noFileProvided = "Error: No file provided to viewer" + [viewer.formBar] apply = "Apply Changes" dismiss = "Dismiss" @@ -8529,6 +8702,9 @@ title = "Form Fields" unsavedBadge = "Unsaved" unsavedDesc = "You have unsaved changes" +[viewer.link] +delete = "Delete link" + [viewer.nonPdf] columnDefault = "Column {{index}}" convertToPdf = "Convert to PDF" @@ -8544,6 +8720,19 @@ renderMarkdown = "Render markdown" sortedBy = "Sorted by: {{column}}" textStats = "{{lines}} lines · {{size}}" +[viewer.redaction] +removeMark = "Remove this mark" + +[viewer.search] +clear = "Clear search" +close = "Close search" +next = "Next result" +previous = "Previous result" +resultsOf = "of {{total}}" + +[viewer.signature] +delete = "Delete signature" + [viewPdf] header = "View PDF" tags = "view,read,annotate,text,image,highlight,edit" @@ -8792,6 +8981,37 @@ toggleSidebar = "Toggle Sidebar" toggleTheme = "Toggle Theme" viewer = "Viewer" +[workflow.participant] +accessExpired = "Your access to this document has expired." +certificateFile = "Certificate File" +certificateFilePlaceholder = "Select certificate file" +certificatePassword = "Certificate Password" +certificateType = "Certificate Type" +certTypeJks = "JKS Keystore" +certTypeP12 = "P12/PKCS12 Certificate" +certTypeServer = "Server Certificate (if available)" +completedDeclined = "You have declined this document." +completedSigned = "You have signed this document." +decline = "Decline" +downloadDocument = "Download Document" +dueDate = "Due Date: {{date}}" +errorTitle = "Error" +from = "From: {{name}}" +loadingSession = "Loading session..." +location = "Location" +locationPlaceholder = "e.g., San Francisco, CA" +pageNumber = "Page Number (optional)" +reason = "Reason" +reasonPlaceholder = "e.g., Document approval" +sessionNotFound = "Session not found or access denied." +signDocument = "Sign Document" +statusDeclined = "Declined" +statusNotified = "Notified" +statusPending = "Pending" +statusSigned = "Signed" +statusViewed = "Viewed" +submitSignature = "Submit Signature" + [workspace] title = "Workspace" diff --git a/frontend/editor/src-tauri/tauri.conf.json b/frontend/editor/src-tauri/tauri.conf.json index 5c2168f72b..c2e84b3ac1 100644 --- a/frontend/editor/src-tauri/tauri.conf.json +++ b/frontend/editor/src-tauri/tauri.conf.json @@ -17,6 +17,7 @@ "height": 800, "resizable": true, "fullscreen": false, + "dragDropEnabled": false, "additionalBrowserArgs": "--enable-features=CertVerifierBuiltinFeature" } ] diff --git a/frontend/editor/src/core/components/annotation/shared/ColorControl.tsx b/frontend/editor/src/core/components/annotation/shared/ColorControl.tsx index ff2a3ff458..ca41f0f6a2 100644 --- a/frontend/editor/src/core/components/annotation/shared/ColorControl.tsx +++ b/frontend/editor/src/core/components/annotation/shared/ColorControl.tsx @@ -8,6 +8,7 @@ import { Group, } from "@mantine/core"; import { useState, useCallback, useEffect } from "react"; +import { useTranslation } from "react-i18next"; import ColorizeIcon from "@mui/icons-material/Colorize"; // safari and firefox do not support the eye dropper API, only edge, chrome and opera do. @@ -33,6 +34,7 @@ export function ColorControl({ label, disabled = false, }: ColorControlProps) { + const { t } = useTranslation(); const [opened, setOpened] = useState(false); // Buffer the colour locally so the picker stays responsive during drag. // Only propagate to the parent (which triggers expensive annotation updates) @@ -111,7 +113,9 @@ export function ColorControl({ /> {supportsEyeDropper && ( - + = ({ onImageChange, disabled = false, }) => { + const { t } = useTranslation(); const [, setImageData] = useState(null); const handleImageUpload = async (file: File | null) => { @@ -57,9 +59,12 @@ export const ImageTool: React.FC = ({ diff --git a/frontend/editor/src/core/components/fileEditor/AddFileCard.tsx b/frontend/editor/src/core/components/fileEditor/AddFileCard.tsx index 5fc62680b3..00c9e4b396 100644 --- a/frontend/editor/src/core/components/fileEditor/AddFileCard.tsx +++ b/frontend/editor/src/core/components/fileEditor/AddFileCard.tsx @@ -104,44 +104,44 @@ const AddFileCard = ({ }} onMouseLeave={() => setIsUploadHover(false)} > - + + )} @@ -442,14 +511,24 @@ const ParticipantView: React.FC = ({ token }) => { {participant.hasCompleted && ( } color="green"> - You have {participant.status === "SIGNED" ? "signed" : "declined"}{" "} - this document. + {participant.status === "SIGNED" + ? t( + "workflow.participant.completedSigned", + "You have signed this document.", + ) + : t( + "workflow.participant.completedDeclined", + "You have declined this document.", + )} )} {participant.isExpired && ( } color="orange"> - Your access to this document has expired. + {t( + "workflow.participant.accessExpired", + "Your access to this document has expired.", + )} )} diff --git a/frontend/editor/src/proprietary/constants/planConstants.ts b/frontend/editor/src/proprietary/constants/planConstants.ts index 6a03a72b7e..bb8a66ac4f 100644 --- a/frontend/editor/src/proprietary/constants/planConstants.ts +++ b/frontend/editor/src/proprietary/constants/planConstants.ts @@ -1,100 +1,219 @@ +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; import type { PlanFeature } from "@app/types/license"; /** * Shared plan feature definitions for Stirling PDF Self-Hosted - * Used by both dynamic (Stripe) and static (fallback) plan displays + * Used by both dynamic (Stripe) and static (fallback) plan displays. + * + * These are exposed as hooks so that every feature/highlight string can be + * localized via `t()` while preserving the original shape. */ -export const PLAN_FEATURES = { - FREE: [ - { name: "Self-hosted deployment", included: true }, - { name: "All PDF operations", included: true }, - { name: "Secure Login Support", included: true }, - { name: "Community support", included: true }, - { name: "Regular updates", included: true }, - { name: "up to 5 users", included: true }, - { name: "Unlimited users", included: false }, - { name: "Google drive integration", included: false }, - { name: "External Database", included: false }, - { name: "Editing text in pdfs", included: false }, - { name: "Users limited to seats", included: false }, - { name: "SSO", included: false }, - { name: "SAML", included: false }, - { name: "Auditing", included: false }, - { name: "Usage tracking", included: false }, - { name: "Prometheus Support", included: false }, - { name: "Custom PDF metadata", included: false }, - ] as PlanFeature[], +export interface PlanFeaturesMap { + FREE: PlanFeature[]; + SERVER: PlanFeature[]; + ENTERPRISE: PlanFeature[]; +} - SERVER: [ - { name: "Self-hosted deployment", included: true }, - { name: "All PDF operations", included: true }, - { name: "Secure Login Support", included: true }, - { name: "Community support", included: true }, - { name: "Regular updates", included: true }, - { name: "Up to 5 users", included: false }, - { name: "Unlimited users", included: true }, - { name: "Google drive integration", included: true }, - { name: "External Database", included: true }, - { name: "Editing text in pdfs", included: true }, - { name: "Users limited to seats", included: false }, - { name: "SSO", included: true }, - { name: "SAML", included: false }, - { name: "Auditing", included: false }, - { name: "Usage tracking", included: false }, - { name: "Prometheus Support", included: false }, - { name: "Custom PDF metadata", included: false }, - ] as PlanFeature[], +export interface PlanHighlightsMap { + FREE: string[]; + SERVER_MONTHLY: string[]; + SERVER_YEARLY: string[]; + ENTERPRISE_MONTHLY: string[]; + ENTERPRISE_YEARLY: string[]; +} - ENTERPRISE: [ - { name: "Self-hosted deployment", included: true }, - { name: "All PDF operations", included: true }, - { name: "Secure Login Support", included: true }, - { name: "Community support", included: true }, - { name: "Regular updates", included: true }, - { name: "up to 5 users", included: false }, - { name: "Unlimited users", included: false }, - { name: "Google drive integration", included: true }, - { name: "External Database", included: true }, - { name: "Editing text in pdfs", included: true }, - { name: "Users limited to seats", included: true }, - { name: "SSO", included: true }, - { name: "SAML", included: true }, - { name: "Auditing", included: true }, - { name: "Usage tracking", included: true }, - { name: "Prometheus Support", included: true }, - { name: "Custom PDF metadata", included: true }, - ] as PlanFeature[], -} as const; +export const usePlanFeatures = (): PlanFeaturesMap => { + const { t } = useTranslation(); -export const PLAN_HIGHLIGHTS = { - FREE: ["Up to 5 users", "Self-hosted", "All basic features"], - SERVER_MONTHLY: [ - "Self-hosted on your infrastructure", - "Unlimited users", - "Advanced integrations", - "SSO (OAuth2/OIDC)", - "Editing text in PDFs", - "Cancel anytime", - ], - SERVER_YEARLY: [ - "Self-hosted on your infrastructure", - "Unlimited users", - "Advanced integrations", - "SSO (OAuth2/OIDC)", - "Editing text in PDFs", - "Save with annual billing", - ], - ENTERPRISE_MONTHLY: [ - "Enterprise features (SAML, Auditing)", - "Usage tracking & Prometheus", - "Custom PDF metadata", - "Per-seat licensing", - ], - ENTERPRISE_YEARLY: [ - "Enterprise features (SAML, Auditing)", - "Usage tracking & Prometheus", - "Custom PDF metadata", - "Save with annual billing", - ], -} as const; + return useMemo(() => { + const selfHostedDeployment = t( + "plan.features.selfHostedDeployment", + "Self-hosted deployment", + ); + const allPdfOperations = t( + "plan.features.allPdfOperations", + "All PDF operations", + ); + const secureLoginSupport = t( + "plan.features.secureLoginSupport", + "Secure Login Support", + ); + const communitySupport = t( + "plan.features.communitySupport", + "Community support", + ); + const regularUpdates = t("plan.features.regularUpdates", "Regular updates"); + const upToFiveUsersLowercase = t( + "plan.features.upToFiveUsersLowercase", + "up to 5 users", + ); + const upToFiveUsers = t("plan.features.upToFiveUsers", "Up to 5 users"); + const unlimitedUsers = t("plan.features.unlimitedUsers", "Unlimited users"); + const googleDriveIntegration = t( + "plan.features.googleDriveIntegration", + "Google drive integration", + ); + const externalDatabase = t( + "plan.features.externalDatabase", + "External Database", + ); + const editingTextInPdfs = t( + "plan.features.editingTextInPdfs", + "Editing text in pdfs", + ); + const usersLimitedToSeats = t( + "plan.features.usersLimitedToSeats", + "Users limited to seats", + ); + const sso = t("plan.features.sso", "SSO"); + const saml = t("plan.features.saml", "SAML"); + const auditing = t("plan.features.auditing", "Auditing"); + const usageTracking = t("plan.features.usageTracking", "Usage tracking"); + const prometheusSupport = t( + "plan.features.prometheusSupport", + "Prometheus Support", + ); + const customPdfMetadata = t( + "plan.features.customPdfMetadata", + "Custom PDF metadata", + ); + + return { + FREE: [ + { name: selfHostedDeployment, included: true }, + { name: allPdfOperations, included: true }, + { name: secureLoginSupport, included: true }, + { name: communitySupport, included: true }, + { name: regularUpdates, included: true }, + { name: upToFiveUsersLowercase, included: true }, + { name: unlimitedUsers, included: false }, + { name: googleDriveIntegration, included: false }, + { name: externalDatabase, included: false }, + { name: editingTextInPdfs, included: false }, + { name: usersLimitedToSeats, included: false }, + { name: sso, included: false }, + { name: saml, included: false }, + { name: auditing, included: false }, + { name: usageTracking, included: false }, + { name: prometheusSupport, included: false }, + { name: customPdfMetadata, included: false }, + ], + SERVER: [ + { name: selfHostedDeployment, included: true }, + { name: allPdfOperations, included: true }, + { name: secureLoginSupport, included: true }, + { name: communitySupport, included: true }, + { name: regularUpdates, included: true }, + { name: upToFiveUsers, included: false }, + { name: unlimitedUsers, included: true }, + { name: googleDriveIntegration, included: true }, + { name: externalDatabase, included: true }, + { name: editingTextInPdfs, included: true }, + { name: usersLimitedToSeats, included: false }, + { name: sso, included: true }, + { name: saml, included: false }, + { name: auditing, included: false }, + { name: usageTracking, included: false }, + { name: prometheusSupport, included: false }, + { name: customPdfMetadata, included: false }, + ], + ENTERPRISE: [ + { name: selfHostedDeployment, included: true }, + { name: allPdfOperations, included: true }, + { name: secureLoginSupport, included: true }, + { name: communitySupport, included: true }, + { name: regularUpdates, included: true }, + { name: upToFiveUsersLowercase, included: false }, + { name: unlimitedUsers, included: false }, + { name: googleDriveIntegration, included: true }, + { name: externalDatabase, included: true }, + { name: editingTextInPdfs, included: true }, + { name: usersLimitedToSeats, included: true }, + { name: sso, included: true }, + { name: saml, included: true }, + { name: auditing, included: true }, + { name: usageTracking, included: true }, + { name: prometheusSupport, included: true }, + { name: customPdfMetadata, included: true }, + ], + }; + }, [t]); +}; + +export const usePlanHighlights = (): PlanHighlightsMap => { + const { t } = useTranslation(); + + return useMemo(() => { + const selfHostedOnInfrastructure = t( + "plan.highlights.selfHostedOnInfrastructure", + "Self-hosted on your infrastructure", + ); + const unlimitedUsers = t( + "plan.highlights.unlimitedUsers", + "Unlimited users", + ); + const advancedIntegrations = t( + "plan.highlights.advancedIntegrations", + "Advanced integrations", + ); + const ssoOAuth = t("plan.highlights.ssoOAuth", "SSO (OAuth2/OIDC)"); + const editingTextInPdfsCaps = t( + "plan.highlights.editingTextInPdfsCaps", + "Editing text in PDFs", + ); + const enterpriseFeatures = t( + "plan.highlights.enterpriseFeatures", + "Enterprise features (SAML, Auditing)", + ); + const usageTrackingPrometheus = t( + "plan.highlights.usageTrackingPrometheus", + "Usage tracking & Prometheus", + ); + const customPdfMetadata = t( + "plan.highlights.customPdfMetadata", + "Custom PDF metadata", + ); + const saveWithAnnualBilling = t( + "plan.highlights.saveWithAnnualBilling", + "Save with annual billing", + ); + + return { + FREE: [ + t("plan.highlights.upToFiveUsers", "Up to 5 users"), + t("plan.highlights.selfHosted", "Self-hosted"), + t("plan.highlights.allBasicFeatures", "All basic features"), + ], + SERVER_MONTHLY: [ + selfHostedOnInfrastructure, + unlimitedUsers, + advancedIntegrations, + ssoOAuth, + editingTextInPdfsCaps, + t("plan.highlights.cancelAnytime", "Cancel anytime"), + ], + SERVER_YEARLY: [ + selfHostedOnInfrastructure, + unlimitedUsers, + advancedIntegrations, + ssoOAuth, + editingTextInPdfsCaps, + saveWithAnnualBilling, + ], + ENTERPRISE_MONTHLY: [ + enterpriseFeatures, + usageTrackingPrometheus, + customPdfMetadata, + t("plan.highlights.perSeatLicensing", "Per-seat licensing"), + ], + ENTERPRISE_YEARLY: [ + enterpriseFeatures, + usageTrackingPrometheus, + customPdfMetadata, + saveWithAnnualBilling, + ], + }; + }, [t]); +}; diff --git a/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx b/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx index 5c6406639a..e050114bee 100644 --- a/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx +++ b/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx @@ -4,6 +4,8 @@ import React, { useState, useCallback, useEffect, + lazy, + Suspense, ReactNode, } from "react"; import { useTranslation } from "react-i18next"; @@ -13,7 +15,12 @@ import licenseService, { mapLicenseToTier, PlanTier, } from "@app/services/licenseService"; -import { StripeCheckout } from "@app/components/shared/stripeCheckout"; + +const StripeCheckout = lazy(() => + import("@app/components/shared/stripeCheckout").then((m) => ({ + default: m.StripeCheckout, + })), +); import { userManagementService } from "@app/services/userManagementService"; import { alert } from "@app/components/toast"; import { @@ -24,6 +31,10 @@ import { import { useLicense } from "@app/contexts/LicenseContext"; import { isSupabaseConfigured } from "@app/services/supabaseClient"; import { getPreferredCurrency } from "@app/utils/currencyDetection"; +import { + usePlanFeatures, + usePlanHighlights, +} from "@app/constants/planConstants"; export interface CheckoutOptions { minimumSeats?: number; // Override calculated seats for enterprise @@ -57,6 +68,8 @@ export const CheckoutProvider: React.FC = ({ }) => { const { t, i18n } = useTranslation(); const { refetchLicense } = useLicense(); + const planFeatures = usePlanFeatures(); + const planHighlights = usePlanHighlights(); const [isOpen, setIsOpen] = useState(false); const [isLoading, setIsLoading] = useState(false); const [selectedPlanGroup, setSelectedPlanGroup] = @@ -85,7 +98,11 @@ export const CheckoutProvider: React.FC = ({ try { setPlansLoading(true); - const response = await licenseService.getPlans(currency); + const response = await licenseService.getPlans( + planFeatures, + planHighlights, + currency, + ); setPlans(response.plans); setPlansLoaded(true); } catch (error) { @@ -95,7 +112,7 @@ export const CheckoutProvider: React.FC = ({ setPlansLoading(false); } }, - [plansLoading], + [plansLoading, planFeatures, planHighlights], ); const refetchPlans = useCallback(() => { @@ -423,16 +440,18 @@ export const CheckoutProvider: React.FC = ({ {/* Global Checkout Modal */} {selectedPlanGroup && ( - + + + )} ); diff --git a/frontend/editor/src/proprietary/hooks/usePlans.ts b/frontend/editor/src/proprietary/hooks/usePlans.ts index 8a9d0c13f2..45b7d8988e 100644 --- a/frontend/editor/src/proprietary/hooks/usePlans.ts +++ b/frontend/editor/src/proprietary/hooks/usePlans.ts @@ -1,8 +1,12 @@ -import { useState, useEffect } from "react"; +import { useState, useEffect, useCallback } from "react"; import licenseService, { PlanTier, PlansResponse, } from "@app/services/licenseService"; +import { + usePlanFeatures, + usePlanHighlights, +} from "@app/constants/planConstants"; export interface UsePlansReturn { plans: PlanTier[]; @@ -12,16 +16,22 @@ export interface UsePlansReturn { } export const usePlans = (currency: string = "gbp"): UsePlansReturn => { + const planFeatures = usePlanFeatures(); + const planHighlights = usePlanHighlights(); const [plans, setPlans] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); - const fetchPlans = async () => { + const fetchPlans = useCallback(async () => { try { setLoading(true); setError(null); - const data: PlansResponse = await licenseService.getPlans(currency); + const data: PlansResponse = await licenseService.getPlans( + planFeatures, + planHighlights, + currency, + ); setPlans(data.plans); } catch (err) { console.error("Error fetching plans:", err); @@ -29,11 +39,11 @@ export const usePlans = (currency: string = "gbp"): UsePlansReturn => { } finally { setLoading(false); } - }; + }, [currency, planFeatures, planHighlights]); useEffect(() => { fetchPlans(); - }, [currency]); + }, [fetchPlans]); return { plans, diff --git a/frontend/editor/src/proprietary/routes/AuthCallback.test.tsx b/frontend/editor/src/proprietary/routes/AuthCallback.test.tsx index fc05574fb1..4491358381 100644 --- a/frontend/editor/src/proprietary/routes/AuthCallback.test.tsx +++ b/frontend/editor/src/proprietary/routes/AuthCallback.test.tsx @@ -6,6 +6,7 @@ import { POST_LOGIN_REDIRECT_STORAGE_KEY, springAuth, } from "@app/auth/springAuthClient"; +import { expectConsole } from "@app/tests/failOnConsole"; // Mock springAuth; keep the real redirect-path helpers. vi.mock("@app/auth/springAuthClient", async () => { @@ -90,6 +91,7 @@ describe("AuthCallback", () => { }); it("should redirect to login when no access token in hash", async () => { + expectConsole.error(/\[AuthCallback\] No access_token in URL fragment/); // No hash or empty hash window.location.hash = ""; @@ -109,6 +111,7 @@ describe("AuthCallback", () => { }); it("should redirect to login when token validation fails", async () => { + expectConsole.error(/\[AuthCallback\] Failed to validate token/); const invalidToken = "invalid-oauth-token"; window.location.hash = `#access_token=${invalidToken}`; @@ -137,6 +140,7 @@ describe("AuthCallback", () => { }); it("should handle errors gracefully", async () => { + expectConsole.error(/\[AuthCallback\] Authentication failed/); const mockToken = "error-token"; window.location.hash = `#access_token=${mockToken}`; diff --git a/frontend/editor/src/proprietary/routes/AuthCallback.tsx b/frontend/editor/src/proprietary/routes/AuthCallback.tsx index 32f81b8411..fd2754ee88 100644 --- a/frontend/editor/src/proprietary/routes/AuthCallback.tsx +++ b/frontend/editor/src/proprietary/routes/AuthCallback.tsx @@ -18,42 +18,16 @@ export default function AuthCallback() { const navigate = useNavigate(); const processingRef = useRef(false); - // Log component lifecycle useEffect(() => { - const mountId = Math.random().toString(36).substring(7); - console.log(`[AuthCallback:${mountId}] 🔵 Component mounted`); - return () => { - console.log(`[AuthCallback:${mountId}] 🔴 Component unmounting`); - }; - }, []); + const startedAt = performance.now(); + const elapsed = () => `${(performance.now() - startedAt).toFixed(0)}ms`; - useEffect(() => { const handleCallback = async () => { - const startTime = performance.now(); - const executionId = Math.random().toString(36).substring(7); - - console.log( - `[AuthCallback:${executionId}] ════════════════════════════════════`, - ); - console.log( - `[AuthCallback:${executionId}] Starting authentication callback`, - ); - console.log(`[AuthCallback:${executionId}] URL: ${window.location.href}`); - console.log( - `[AuthCallback:${executionId}] Hash: ${window.location.hash}`, - ); - console.log( - `[AuthCallback:${executionId}] Document readyState: ${document.readyState}`, - ); - if ( typeof window !== "undefined" && window.sessionStorage.getItem("stirling_sso_auto_login_logged_out") === "1" ) { - console.warn( - `[AuthCallback:${executionId}] ⚠️ Logout block active, skipping token processing`, - ); navigate("/login", { replace: true, state: { error: "You have been signed out. Please sign in again." }, @@ -62,30 +36,16 @@ export default function AuthCallback() { } // Prevent double execution (React 18 Strict Mode + navigate dependency) - if (processingRef.current) { - console.warn( - `[AuthCallback:${executionId}] ⚠️ Already processing, skipping duplicate execution`, - ); - console.warn( - `[AuthCallback:${executionId}] This is expected in React Strict Mode (development)`, - ); - return; - } + if (processingRef.current) return; processingRef.current = true; try { - console.log( - `[AuthCallback:${executionId}] Step 1: Extracting token from URL fragment`, - ); - - // Extract JWT from URL fragment (#access_token=...) - const hash = window.location.hash.substring(1); // Remove '#' - const params = new URLSearchParams(hash); - const token = params.get("access_token"); + const hash = window.location.hash.substring(1); + const token = new URLSearchParams(hash).get("access_token"); if (!token) { console.error( - `[AuthCallback:${executionId}] ❌ No access_token in URL fragment`, + `[AuthCallback] No access_token in URL fragment (${elapsed()})`, ); navigate("/login", { replace: true, @@ -94,39 +54,13 @@ export default function AuthCallback() { return; } - console.log( - `[AuthCallback:${executionId}] ✓ Token extracted (length: ${token.length})`, - ); - console.log( - `[AuthCallback:${executionId}] Step 2: Storing JWT in localStorage`, - ); - - // Store JWT in localStorage localStorage.setItem("stirling_jwt", token); - console.log( - `[AuthCallback:${executionId}] ✓ JWT stored in localStorage`, - ); - - console.log( - `[AuthCallback:${executionId}] Step 3: Dispatching 'jwt-available' event`, - ); - // Dispatch custom event for other components to react to JWT availability window.dispatchEvent(new CustomEvent("jwt-available")); - console.log(`[AuthCallback:${executionId}] ✓ Event dispatched`); - console.log( - `[AuthCallback:${executionId}] Elapsed after jwt-available: ${(performance.now() - startTime).toFixed(2)}ms`, - ); - console.log( - `[AuthCallback:${executionId}] Step 4: Validating token with backend`, - ); - // Validate the token and load user info - // This calls /api/v1/auth/me with the JWT to get user details const { data, error } = await springAuth.getSession(); - if (error || !data.session) { console.error( - `[AuthCallback:${executionId}] ❌ Failed to validate token:`, + `[AuthCallback] Failed to validate token (${elapsed()}):`, error, ); localStorage.removeItem("stirling_jwt"); @@ -137,68 +71,21 @@ export default function AuthCallback() { return; } - console.log( - `[AuthCallback:${executionId}] ✓ Token validated, user: ${data.session.user.username}`, - ); - console.log( - `[AuthCallback:${executionId}] Step 5: Running platform-specific callback handlers`, - ); - await handleAuthCallbackSuccess(token); - console.log( - `[AuthCallback:${executionId}] ✓ Callback handlers complete`, - ); - console.log( - `[AuthCallback:${executionId}] Step 6: Waiting for context stabilization`, - ); - // Wait for all context providers to process jwt-available event // This prevents infinite render loop when coming from cross-domain SAML redirect await new Promise((resolve) => setTimeout(resolve, 100)); - console.log( - `[AuthCallback:${executionId}] Elapsed after stabilization wait: ${(performance.now() - startTime).toFixed(2)}ms`, - ); const target = consumePostLoginRedirectPath() ?? "/"; - console.log( - `[AuthCallback:${executionId}] Step 7: Navigating to ${target}`, + console.info( + `[AuthCallback] Authenticated ${data.session.user.username} in ${elapsed()}, navigating to ${target}`, ); navigate(target, { replace: true }); - - const duration = performance.now() - startTime; - console.log( - `[AuthCallback:${executionId}] ✓ Authentication complete (${duration.toFixed(2)}ms)`, - ); - console.log( - `[AuthCallback:${executionId}] ════════════════════════════════════`, - ); } catch (error) { - const duration = performance.now() - startTime; console.error( - `[AuthCallback:${executionId}] ════════════════════════════════════`, - ); - console.error( - `[AuthCallback:${executionId}] ❌ FATAL ERROR during authentication`, - ); - console.error(`[AuthCallback:${executionId}] Error:`, error); - console.error( - `[AuthCallback:${executionId}] Error name:`, - (error as Error)?.name, - ); - console.error( - `[AuthCallback:${executionId}] Error message:`, - (error as Error)?.message, - ); - console.error( - `[AuthCallback:${executionId}] Error stack:`, - (error as Error)?.stack, - ); - console.error( - `[AuthCallback:${executionId}] Duration before failure: ${duration.toFixed(2)}ms`, - ); - console.error( - `[AuthCallback:${executionId}] ════════════════════════════════════`, + `[AuthCallback] Authentication failed (${elapsed()}):`, + error, ); navigate("/login", { replace: true, diff --git a/frontend/editor/src/proprietary/routes/Login.test.tsx b/frontend/editor/src/proprietary/routes/Login.test.tsx index 0a4b79324b..6b9306a722 100644 --- a/frontend/editor/src/proprietary/routes/Login.test.tsx +++ b/frontend/editor/src/proprietary/routes/Login.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; -import { render, screen, waitFor } from "@testing-library/react"; +import { act, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { BrowserRouter, MemoryRouter } from "react-router-dom"; import { MantineProvider } from "@mantine/core"; @@ -176,7 +176,7 @@ describe("Login", () => { }); }); - it("should show loading state while auth is loading", () => { + it("should show loading state while auth is loading", async () => { vi.mocked(useAuth).mockReturnValue({ session: null, user: null, @@ -187,13 +187,15 @@ describe("Login", () => { refreshSession: vi.fn(), }); - render( - - - - - , - ); + await act(async () => { + render( + + + + + , + ); + }); // Component shouldn't redirect or show form while loading expect(mockNavigate).not.toHaveBeenCalled(); @@ -510,26 +512,30 @@ describe("Login", () => { expect(springAuth.signInWithPassword).not.toHaveBeenCalled(); }); - it("should display session expired message from URL param", () => { - render( - - - - - , - ); + it("should display session expired message from URL param", async () => { + await act(async () => { + render( + + + + + , + ); + }); expect(screen.getByText(/session.*expired/i)).toBeInTheDocument(); }); - it("should display account created success message", () => { - render( - - - - - , - ); + it("should display account created success message", async () => { + await act(async () => { + render( + + + + + , + ); + }); expect(screen.getByText(/account created/i)).toBeInTheDocument(); }); diff --git a/frontend/editor/src/proprietary/routes/Login.tsx b/frontend/editor/src/proprietary/routes/Login.tsx index 22d9c57e17..85c42fce86 100644 --- a/frontend/editor/src/proprietary/routes/Login.tsx +++ b/frontend/editor/src/proprietary/routes/Login.tsx @@ -287,8 +287,6 @@ export default function Login() { setPostLoginRedirectPath(returnPath); } - console.log(`[Login] Signing in with provider: ${provider}`); - // Redirect to Spring OAuth2 endpoint using the actual provider ID from backend // The backend returns the correct registration ID (e.g., 'authentik', 'oidc', 'keycloak') const { error } = await springAuth.signInWithOAuth({ @@ -520,8 +518,6 @@ export default function Login() { setError(null); clearLogoutBlock(); - console.log("[Login] Signing in with email:", email); - const { user, session, error } = await springAuth.signInWithPassword({ email: email.trim(), password: password, @@ -529,13 +525,11 @@ export default function Login() { }); if (error) { - console.error("[Login] Email sign in error:", error); setError(error.message); if (error.mfaRequired || error.code === "invalid_mfa_code") { setRequiresMfa(true); } } else if (user && session) { - console.log("[Login] Email sign in successful"); clearLogoutBlock(); setRequiresMfa(false); setMfaCode(""); diff --git a/frontend/editor/src/proprietary/services/licenseService.ts b/frontend/editor/src/proprietary/services/licenseService.ts index 600498b333..e87f304eb6 100644 --- a/frontend/editor/src/proprietary/services/licenseService.ts +++ b/frontend/editor/src/proprietary/services/licenseService.ts @@ -1,7 +1,10 @@ import apiClient from "@app/services/apiClient"; import { supabase, isSupabaseConfigured } from "@app/services/supabaseClient"; import { getCheckoutMode } from "@app/utils/protocolDetection"; -import { PLAN_FEATURES, PLAN_HIGHLIGHTS } from "@app/constants/planConstants"; +import type { + PlanFeaturesMap, + PlanHighlightsMap, +} from "@app/constants/planConstants"; import type { LicenseInfo, PlanFeature } from "@app/types/license"; export interface PlanTier { @@ -102,9 +105,18 @@ const SELF_HOSTED_LOOKUP_KEYS = [ const licenseService = { /** - * Get available plans with pricing for the specified currency + * Get available plans with pricing for the specified currency. + * + * Feature and highlight labels are passed in by the caller (a React hook / + * context that resolves them via `usePlanFeatures` / `usePlanHighlights`), + * which keeps this service free of React/i18n dependencies while still + * returning fully localized plan content. */ - async getPlans(currency: string = "usd"): Promise { + async getPlans( + planFeatures: PlanFeaturesMap, + planHighlights: PlanHighlightsMap, + currency: string = "usd", + ): Promise { try { // Check if Supabase is configured if (!isSupabaseConfigured || !supabase) { @@ -179,8 +191,8 @@ const licenseService = { currency: currencySymbol, period: "/month", popular: false, - features: PLAN_FEATURES.SERVER, - highlights: PLAN_HIGHLIGHTS.SERVER_MONTHLY, + features: planFeatures.SERVER, + highlights: planHighlights.SERVER_MONTHLY, }, { id: "selfhosted:server:yearly", @@ -190,8 +202,8 @@ const licenseService = { currency: currencySymbol, period: "/year", popular: true, - features: PLAN_FEATURES.SERVER, - highlights: PLAN_HIGHLIGHTS.SERVER_YEARLY, + features: planFeatures.SERVER, + highlights: planHighlights.SERVER_YEARLY, }, { id: "selfhosted:enterprise:monthly", @@ -203,8 +215,8 @@ const licenseService = { period: "/month", popular: false, requiresSeats: true, - features: PLAN_FEATURES.ENTERPRISE, - highlights: PLAN_HIGHLIGHTS.ENTERPRISE_MONTHLY, + features: planFeatures.ENTERPRISE, + highlights: planHighlights.ENTERPRISE_MONTHLY, }, { id: "selfhosted:enterprise:yearly", @@ -216,8 +228,8 @@ const licenseService = { period: "/year", popular: false, requiresSeats: true, - features: PLAN_FEATURES.ENTERPRISE, - highlights: PLAN_HIGHLIGHTS.ENTERPRISE_YEARLY, + features: planFeatures.ENTERPRISE, + highlights: planHighlights.ENTERPRISE_YEARLY, }, ]; @@ -240,8 +252,8 @@ const licenseService = { currency: currencySymbol, period: "", popular: false, - features: PLAN_FEATURES.FREE, - highlights: PLAN_HIGHLIGHTS.FREE, + features: planFeatures.FREE, + highlights: planHighlights.FREE, }; const allPlans = [freePlan, ...validPlans]; diff --git a/frontend/editor/src/saas/components/TrialExpiredBootstrap.tsx b/frontend/editor/src/saas/components/TrialExpiredBootstrap.tsx index 1b185684cb..40826f1904 100644 --- a/frontend/editor/src/saas/components/TrialExpiredBootstrap.tsx +++ b/frontend/editor/src/saas/components/TrialExpiredBootstrap.tsx @@ -1,7 +1,10 @@ -import { useEffect, useState } from "react"; +import { lazy, Suspense, useEffect, useState } from "react"; import { useAuth } from "@app/auth/UseSession"; import { TrialExpiredModal } from "@app/components/shared/TrialExpiredModal"; -import StripeCheckout from "@app/components/shared/StripeCheckoutSaas"; + +const StripeCheckout = lazy( + () => import("@app/components/shared/StripeCheckoutSaas"), +); /** * Bootstrap component that shows the trial expired modal when a user's trial has ended @@ -106,20 +109,22 @@ export default function TrialExpiredBootstrap() { onSubscribe={handleSubscribe} /> - {user && ( - - console.error("[TrialExpired] Checkout error:", error) - } - isTrialConversion={false} // Trial already ended, so this is not a conversion - /> + {user && checkoutOpened && ( + + + console.error("[TrialExpired] Checkout error:", error) + } + isTrialConversion={false} // Trial already ended, so this is not a conversion + /> + )} ); diff --git a/frontend/editor/src/saas/components/shared/StripeCheckoutSaas.tsx b/frontend/editor/src/saas/components/shared/StripeCheckoutSaas.tsx index fffefc91dd..43c96a9b33 100644 --- a/frontend/editor/src/saas/components/shared/StripeCheckoutSaas.tsx +++ b/frontend/editor/src/saas/components/shared/StripeCheckoutSaas.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useEffect, useMemo, useState } from "react"; import { Modal, Button, Text, Alert, Loader, Stack } from "@mantine/core"; import { useTranslation } from "react-i18next"; import { loadStripe } from "@stripe/stripe-js"; @@ -9,7 +9,7 @@ import { import { supabase } from "@app/auth/supabase"; import { Z_INDEX_OVER_SETTINGS_MODAL } from "@app/styles/zIndex"; -const stripePromise = loadStripe(import.meta.env.VITE_STRIPE_PUBLISHABLE_KEY); +const STRIPE_KEY = import.meta.env.VITE_STRIPE_PUBLISHABLE_KEY; export type PurchaseType = "subscription" | "credits"; export type CreditsPack = "xsmall" | "small" | "medium" | "large" | null; @@ -68,6 +68,11 @@ const StripeCheckout: React.FC = ({ }) => { const { t } = useTranslation(); const [state, setState] = useState({ status: "idle" }); + // Load Stripe.js lazily, only when this checkout component mounts. Loading + // at module scope pulled the Stripe script into every page that imports + // this file, which triggered the dev "HTTPS required" warning on every + // non-payment route. + const stripePromise = useMemo(() => loadStripe(STRIPE_KEY), []); const createCheckoutSession = async () => { try { diff --git a/frontend/editor/src/saas/components/shared/TrialStatusBanner.tsx b/frontend/editor/src/saas/components/shared/TrialStatusBanner.tsx index e6e7322ec2..7efec40eb4 100644 --- a/frontend/editor/src/saas/components/shared/TrialStatusBanner.tsx +++ b/frontend/editor/src/saas/components/shared/TrialStatusBanner.tsx @@ -1,11 +1,14 @@ -import { useEffect, useState, useCallback } from "react"; +import { lazy, Suspense, useEffect, useState, useCallback } from "react"; import { useBanner } from "@app/contexts/BannerContext"; import { useAuth } from "@app/auth/UseSession"; import { useTranslation } from "react-i18next"; import { InfoBanner } from "@app/components/shared/InfoBanner"; -import StripeCheckout from "@app/components/shared/StripeCheckoutSaas"; import { BASE_PATH } from "@app/constants/app"; +const StripeCheckout = lazy( + () => import("@app/components/shared/StripeCheckoutSaas"), +); + const SESSION_STORAGE_KEY = "trialBannerDismissed"; export function TrialStatusBanner() { @@ -113,18 +116,20 @@ export function TrialStatusBanner() { return ( <> - {trialStatus && ( - setCheckoutOpen(false)} - purchaseType="subscription" - planId="pro" - creditsPack={null} - planName="Pro" - onSuccess={handleCheckoutSuccess} - onError={(error) => console.error("Checkout error:", error)} - isTrialConversion={true} - /> + {trialStatus && checkoutOpen && ( + + setCheckoutOpen(false)} + purchaseType="subscription" + planId="pro" + creditsPack={null} + planName="Pro" + onSuccess={handleCheckoutSuccess} + onError={(error) => console.error("Checkout error:", error)} + isTrialConversion={true} + /> + )} ); diff --git a/frontend/editor/src/saas/components/shared/config/configSections/Plan.tsx b/frontend/editor/src/saas/components/shared/config/configSections/Plan.tsx index af7dcb2f83..8571c5db47 100644 --- a/frontend/editor/src/saas/components/shared/config/configSections/Plan.tsx +++ b/frontend/editor/src/saas/components/shared/config/configSections/Plan.tsx @@ -1,11 +1,15 @@ -import React, { useState, useCallback, useEffect } from "react"; +import React, { lazy, Suspense, useState, useCallback, useEffect } from "react"; import { Divider, Loader, Alert, Select, Group, Text } from "@mantine/core"; import { usePlans, PlanTier } from "@app/hooks/usePlans"; -import StripeCheckout, { +import type { PurchaseType, CreditsPack, PlanID, } from "@app/components/shared/StripeCheckoutSaas"; + +const StripeCheckout = lazy( + () => import("@app/components/shared/StripeCheckoutSaas"), +); import AvailablePlansSection from "@app/components/shared/config/configSections/plan/AvailablePlansSection"; import ApiPackagesSection from "@app/components/shared/config/configSections/plan/ApiPackagesSection"; import ActivePlanSection from "@app/components/shared/config/configSections/plan/ActivePlanSection"; @@ -204,29 +208,36 @@ const Plan: React.FC = () => { /> {/* Stripe Checkout Modal */} - pkg.id === selectedCreditsPack) - ?.name || "" - } - onSuccess={handlePaymentSuccess} - onError={handlePaymentError} - isTrialConversion={ - trialStatus?.isTrialing && purchaseType === "subscription" - } - /> + {checkoutOpen && + (selectedPlan !== null || selectedCreditsPack !== null) && ( + + pkg.id === selectedCreditsPack, + )?.name || "" + } + onSuccess={handlePaymentSuccess} + onError={handlePaymentError} + isTrialConversion={ + trialStatus?.isTrialing && purchaseType === "subscription" + } + /> + + )} ); }; diff --git a/frontend/editor/src/saas/components/shared/config/configSections/plan/ApiPackagesSection.tsx b/frontend/editor/src/saas/components/shared/config/configSections/plan/ApiPackagesSection.tsx index 09e238be04..f707d5d9a9 100644 --- a/frontend/editor/src/saas/components/shared/config/configSections/plan/ApiPackagesSection.tsx +++ b/frontend/editor/src/saas/components/shared/config/configSections/plan/ApiPackagesSection.tsx @@ -1,7 +1,7 @@ import React from "react"; import { Button, Card, Text, Stack, Flex, Slider } from "@mantine/core"; import { useTranslation } from "react-i18next"; -import { CreditsPack } from "@app/components/shared/StripeCheckoutSaas"; +import type { CreditsPack } from "@app/components/shared/StripeCheckoutSaas"; interface ApiPackage { id: string; diff --git a/frontend/editor/src/saas/services/apiClient.test.ts b/frontend/editor/src/saas/services/apiClient.test.ts index 01a9649387..56480bb7c7 100644 --- a/frontend/editor/src/saas/services/apiClient.test.ts +++ b/frontend/editor/src/saas/services/apiClient.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import type { Session, AuthError } from "@supabase/supabase-js"; import { supabase } from "@app/auth/supabase"; +import { expectConsole } from "@app/tests/failOnConsole"; // Mock supabase vi.mock("@app/auth/supabase", () => ({ @@ -177,6 +178,7 @@ describe("apiClient", () => { }); it("should handle refresh token failure", async () => { + expectConsole.error(/\[API Client\] Token refresh failed/); const oldToken = "old-token"; const oldSession = { diff --git a/frontend/editor/src/saas/setupTests.ts b/frontend/editor/src/saas/setupTests.ts index c461557ffa..2c5f53e271 100644 --- a/frontend/editor/src/saas/setupTests.ts +++ b/frontend/editor/src/saas/setupTests.ts @@ -1,5 +1,8 @@ import "@testing-library/jest-dom"; import { vi } from "vitest"; +import { installFailOnConsole } from "@app/tests/failOnConsole"; + +installFailOnConsole(); // Mock localStorage for tests const localStorageMock = (() => { diff --git a/testing/compose/docker-compose-saas.yml b/testing/compose/docker-compose-saas.yml new file mode 100644 index 0000000000..880518b423 --- /dev/null +++ b/testing/compose/docker-compose-saas.yml @@ -0,0 +1,104 @@ +services: + # --------------------------------------------------------------------- + # Postgres holding the stirling_pdf schema. Flyway migrates V1-V13 on + # backend startup against this DB. Exposed on host port 5433 so the + # cucumber harness can connect via psycopg from features/steps/payg_*. + # --------------------------------------------------------------------- + postgres-saas: + image: postgres:17-alpine + container_name: payg-cucumber-postgres + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - "5433:5432" + volumes: + # Bootstrap stirling_pdf schema + seed test team / api key. + # Runs once when the container is first created. + - ./payg/saas-init.sql:/docker-entrypoint-initdb.d/00-init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 3s + timeout: 5s + retries: 20 + networks: + - payg-cucumber-network + + # --------------------------------------------------------------------- + # Stirling-PDF backend with STIRLING_FLAVOR=saas. Authenticates via + # API key (SECURITY_CUSTOMGLOBALAPIKEY) — Supabase JWT enforcement is + # disabled for the test profile via SAAS_DB_PROJECT_REF=disabled so the + # OAuth resource-server filter no-ops. The PaygChargeInterceptor's + # resolveUser() ApiKey-token path is what the cucumber tests exercise. + # --------------------------------------------------------------------- + stirling-pdf-saas: + build: + context: ../.. + dockerfile: docker/embedded/Dockerfile + args: + STIRLING_FLAVOR: saas + container_name: payg-cucumber-stirling + depends_on: + postgres-saas: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: + memory: 4G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP'"] + interval: 5s + timeout: 10s + retries: 24 + ports: + - "8080:8080" + environment: + # Activate the saas profile + bring the saas subproject onto the classpath. + # `payg-cucumber` registers the test-only throw-500 controller in + # `app/saas/.../payg/test/PaygCucumberThrowController.java` — that bean is + # @Profile("payg-cucumber") so it never registers in production. + SPRING_PROFILES_ACTIVE: "saas,payg-cucumber" + STIRLING_FLAVOR: "saas" + ENABLE_SAAS: "true" + DISABLE_ADDITIONAL_FEATURES: "false" + + # Datasource — point Flyway + JPA at the test postgres. + SAAS_DB_URL: "jdbc:postgresql://postgres-saas:5432/postgres" + SAAS_DB_USERNAME: "postgres" + SAAS_DB_PASSWORD: "postgres" + SAAS_DB_PROJECT_REF: "disabled" + + # The saas Flyway migrations assume `users` and `teams` already exist + # (V2 ALTERs `users`; V5 references `teams(id)`) because in production + # those tables are provisioned by Supabase before Stirling starts. On + # a clean test postgres they don't exist, so we disable Flyway and let + # Hibernate's `ddl-auto=create-drop` build the full schema from the + # entity graph. The default-pricing-policy row that V12 seeds in + # production is re-seeded by testing/compose/payg/saas-seed.sql. + SPRING_FLYWAY_ENABLED: "false" + SPRING_JPA_HIBERNATE_DDL_AUTO: "create-drop" + + # Disable Supabase JWT enforcement for the test profile. The + # PaygChargeInterceptor resolves via ApiKey-token authority — that + # path is what we cover here. + SPRING_AUTOCONFIGURE_EXCLUDE: "org.springframework.boot.autoconfigure.security.oauth2.resource.servlet.OAuth2ResourceServerAutoConfiguration" + + # Test API key matching the user seeded by saas-init.sql. + SECURITY_CUSTOMGLOBALAPIKEY: "payg-cucumber-key" + SECURITY_ENABLELOGIN: "false" + + # Conservative defaults. The cucumber runs send small fixtures so + # the in-memory threshold never bites; the kill-switch is needed by + # one scenario but is toggled via a separate compose override. + PAYG_FILTER_ENABLED: "true" + PAYG_FILTER_RESPONSE_IN_MEMORY_THRESHOLD_BYTES: "10485760" + + SYSTEM_DEFAULTLOCALE: "en-US" + SYSTEM_MAXFILESIZE: "100" + networks: + - payg-cucumber-network + +networks: + payg-cucumber-network: + driver: bridge diff --git a/testing/compose/payg/saas-init.sql b/testing/compose/payg/saas-init.sql new file mode 100644 index 0000000000..1a40cdde66 --- /dev/null +++ b/testing/compose/payg/saas-init.sql @@ -0,0 +1,25 @@ +-- Bootstrap minimal stirling_pdf state for PAYG cucumber tests. +-- +-- Runs once when the postgres-saas container is first created (loaded via +-- /docker-entrypoint-initdb.d on the official postgres image). Flyway then +-- migrates V1-V13 on backend startup, which fills in the schema; this file +-- only seeds the data rows the cucumber scenarios reference by name. +-- +-- Idempotent — if the container is re-created, this whole file runs again +-- but every INSERT is guarded against duplicates. + +CREATE SCHEMA IF NOT EXISTS stirling_pdf; + +-- Note: the actual seed of teams / users / payg_team_extensions / wallet_policy +-- happens AFTER Flyway migrations have applied. We can't insert here because +-- the tables don't exist yet at container-init time. Instead we register a +-- helper function the backend invokes once at startup (or the cucumber +-- harness invokes via psql before running scenarios). +-- +-- For "make a start", the simplest approach is to keep this file as the +-- schema bootstrap, and put the seed inserts in a separate sidecar SQL that +-- the cucumber test.sh harness pipes through psql AFTER waiting for backend +-- health. See testing/compose/payg/saas-seed.sql below. + +-- Reserved for future container-init-time schema bootstrap if we ever need it. +SELECT 1; diff --git a/testing/compose/payg/saas-seed.sql b/testing/compose/payg/saas-seed.sql new file mode 100644 index 0000000000..612cbc9d1c --- /dev/null +++ b/testing/compose/payg/saas-seed.sql @@ -0,0 +1,118 @@ +-- Seed test team / payg policy / wallet_policy in PAYG_SHADOW mode for the +-- cucumber harness. Piped through psql AFTER Hibernate has built the schema +-- on backend startup (compose disables Flyway — see docker-compose-saas.yml +-- for the rationale: saas Flyway migrations assume `users` + `teams` exist +-- because Supabase normally provisions them). +-- +-- Idempotent — guarded against duplicate keys so re-running on the same +-- container is safe between scenarios. + +-- --------------------------------------------------------------------------- +-- 0. Default pricing policy + per-source step limits (V12 in production). +-- Required because PricingPolicyService.getEffectivePolicy() throws if +-- no row has is_default = TRUE. Explicit timestamps because Hibernate's +-- @CreationTimestamp is application-side; direct INSERTs bypass it. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.pricing_policy ( + version, effective_from, doc_pages_per_unit, doc_bytes_per_unit, + min_charge_units, file_unit_cap, is_default, notes, created_by, + created_at +) +SELECT + 'v1-cucumber', CURRENT_TIMESTAMP, 25, 5242880, + 1, 1000, TRUE, + 'Cucumber test default policy', 'system', + CURRENT_TIMESTAMP +WHERE NOT EXISTS ( + SELECT 1 FROM stirling_pdf.pricing_policy WHERE is_default = TRUE +); + +INSERT INTO stirling_pdf.pricing_policy_step_limit (policy_id, job_source, step_limit) +SELECT p.policy_id, src.job_source, src.step_limit +FROM stirling_pdf.pricing_policy p +CROSS JOIN ( + VALUES + ('WEB', 10), + ('API', 10), + ('PIPELINE', 20), + ('DESKTOP_APP', 10) +) AS src(job_source, step_limit) +WHERE p.is_default = TRUE + AND NOT EXISTS ( + SELECT 1 FROM stirling_pdf.pricing_policy_step_limit s + WHERE s.policy_id = p.policy_id AND s.job_source = src.job_source + ); + +-- --------------------------------------------------------------------------- +-- 1. Test team. The Stirling-PDF backend auto-creates `Default` and +-- `Internal` teams at boot; we add a third one specifically for the +-- PAYG scenarios so we can isolate state and assert per-team. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.teams (name) +SELECT 'payg-cucumber-team' +WHERE NOT EXISTS ( + SELECT 1 FROM stirling_pdf.teams WHERE name = 'payg-cucumber-team' +); + +-- --------------------------------------------------------------------------- +-- 2. payg_team_extensions sidecar — uses the default pricing policy. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.payg_team_extensions (team_id, pricing_policy_id, stripe_customer_id) +SELECT t.team_id, NULL, NULL +FROM stirling_pdf.teams t +WHERE t.name = 'payg-cucumber-team' + AND NOT EXISTS ( + SELECT 1 FROM stirling_pdf.payg_team_extensions ext WHERE ext.team_id = t.team_id + ); + +-- --------------------------------------------------------------------------- +-- 3. Bind the auto-created CUSTOM_API_USER to our cucumber team. The user +-- is created by the backend's SECURITY_CUSTOMGLOBALAPIKEY handling — we +-- don't seed our own user (would collide on the unique api_key). +-- Update their primary `team_id` so JobChargeService picks up +-- payg-cucumber-team as the owner team on requests. +-- --------------------------------------------------------------------------- +UPDATE stirling_pdf.users u +SET team_id = (SELECT team_id FROM stirling_pdf.teams WHERE name = 'payg-cucumber-team') +WHERE u.username = 'CUSTOM_API_USER'; + +-- --------------------------------------------------------------------------- +-- 4. Team membership row so /teams/* admin paths recognise the user. +-- All timestamp columns set explicitly because Hibernate-generated DDL +-- doesn't carry the Flyway-migration DEFAULT CURRENT_TIMESTAMP. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.team_memberships ( + team_id, user_id, role, invited_at, created_at, updated_at +) +SELECT t.team_id, u.user_id, 'LEADER', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP +FROM stirling_pdf.teams t, stirling_pdf.users u +WHERE t.name = 'payg-cucumber-team' + AND u.username = 'CUSTOM_API_USER' + AND NOT EXISTS ( + SELECT 1 FROM stirling_pdf.team_memberships m + WHERE m.team_id = t.team_id AND m.user_id = u.user_id + ); + +-- --------------------------------------------------------------------------- +-- 5. wallet_policy in PAYG_SHADOW mode. +-- auto_group_strategy is the dead column tracked for drop in PR-R11 +-- (PAYG_DESIGN.md §7.7) — until that ships, the NOT NULL constraint +-- means we have to populate it. 'AUTO' is the prod default. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.wallet_policy ( + team_id, engine, cap_period, warn_at_pct, degrade_at_pct, + degraded_feature_set, auto_group_strategy, notification_emails, + updated_at +) +SELECT t.team_id, 'PAYG_SHADOW', 'CALENDAR_MONTH', 80, 100, + 'MINIMAL', 'AUTO', '[]'::jsonb, CURRENT_TIMESTAMP +FROM stirling_pdf.teams t +WHERE t.name = 'payg-cucumber-team' + AND NOT EXISTS ( + SELECT 1 FROM stirling_pdf.wallet_policy wp WHERE wp.team_id = t.team_id + ); + +UPDATE stirling_pdf.wallet_policy +SET engine = 'PAYG_SHADOW', updated_at = CURRENT_TIMESTAMP +WHERE team_id = (SELECT team_id FROM stirling_pdf.teams WHERE name = 'payg-cucumber-team'); diff --git a/testing/cucumber/behave.ini b/testing/cucumber/behave.ini index 202b26576a..b398282b2a 100644 --- a/testing/cucumber/behave.ini +++ b/testing/cucumber/behave.ini @@ -5,4 +5,12 @@ # # python -m behave features/enterprise # -exclude_re = features/enterprise +# PAYG shadow-mode features live in features/payg/. They need the saas +# profile + Postgres + seeded test team (testing/compose/docker-compose-saas.yml). +# The default run excludes them; the saas-cucumber CI job invokes +# behave with `features/payg` explicitly. The PAYG harness in +# testing/test-payg.sh swaps a temp behave.ini that drops the payg +# exclusion + tags @manual to skip restart-hook scenarios. +# +exclude_re = features/(enterprise|payg) +tags = ~@manual diff --git a/testing/cucumber/features/payg/README.md b/testing/cucumber/features/payg/README.md new file mode 100644 index 0000000000..584ef446a0 --- /dev/null +++ b/testing/cucumber/features/payg/README.md @@ -0,0 +1,58 @@ +# PAYG cucumber scenarios + +End-to-end coverage for the PAYG shadow charging engine (PR #6519 / PR-S3 +in `notes/PAYG_DESIGN.md` §7.5). + +## Running locally + +```bash +./testing/test-payg.sh +``` + +That script: +1. Boots `testing/compose/docker-compose-saas.yml` (Stirling-PDF with + `STIRLING_FLAVOR=saas` + a Postgres holding the `stirling_pdf` schema) +2. Waits for backend health +3. Pipes `testing/compose/payg/saas-seed.sql` into the test postgres, + creating a `payg-cucumber-team` flipped to `wallet_policy.engine = + 'PAYG_SHADOW'` and a test user with API key `payg-cucumber-key` +4. Runs `python -m behave features/payg` +5. Tears the stack down + +## What's covered (automated, run by `docker-compose-tests-saas.yml` on CI) + +| Scenario | Validates | +|---|---| +| First tool call writes a CHARGED row | Filter + interceptor fire end-to-end; shadow row written | +| Lineage join — second call on output | `JobService.joinOrOpen` lineage matching; no new shadow row | +| First-step 5xx refunds + closes the process | `markFirstStepFailed` flips row to REFUNDED, job to CLOSED | +| 4xx leaves the row CHARGED | "customer paid for the attempt" semantics | +| ZIP-returning tool records per-PDF OUTPUT | `PaygOutputExtractor` unpacks + records signatures | +| Multi-file input writes a single shadow row | Multi-input group sizing | +| `X-Stirling-Automation` sets PIPELINE source | Header → `JobSource` detection | + +The 5xx scenario drives the refund path through `PaygCucumberThrowController` +— a `@Profile("payg-cucumber")` stub in `app/saas/.../payg/test/` that always +throws. The profile is activated by `docker-compose-saas.yml`'s +`SPRING_PROFILES_ACTIVE=saas,payg-cucumber` and is never set in production. + +## Manual-only scenarios + +One part of the shadow engine can't reasonably be driven from this suite +and is verified by hand each time its code path changes. The procedure +lives in `notes/PAYG_DESIGN.md` §7.5.2 "PAYG cucumber: manual-only +scenarios". + +- **Kill-switch (`PAYG_FILTER_ENABLED=false`).** Needs a docker-container + restart mid-suite; orchestrating that in behave is more harness fragility + than it's worth for a flag that's only flipped during incident response. + +## Fixtures + +The scenarios reuse `testing/cucumber/exampleFiles/`: +- `ghost1.pdf` — single-page reference PDF +- `tables.pdf` — multi-page input for split / ZIP scenarios + +If those filenames change in the main cucumber harness, update +`features/steps/payg_step_definitions.py` SINGLE_PAGE_PDF / THREE_PAGE_PDF +constants to match. diff --git a/testing/cucumber/features/payg/shadow_charges.feature b/testing/cucumber/features/payg/shadow_charges.feature new file mode 100644 index 0000000000..2141b66b0a --- /dev/null +++ b/testing/cucumber/features/payg/shadow_charges.feature @@ -0,0 +1,105 @@ +Feature: PAYG shadow-mode charging + # End-to-end coverage for the PAYG shadow charging engine via the filter + + # interceptor stack landed in PR #6519. Runs against the saas-profile + # docker-compose target (testing/compose/docker-compose-saas.yml). + # + # Each scenario sets up a test team in PAYG_SHADOW mode, hits a real tool + # endpoint, then asserts the shape of the rows written to payg_shadow_charge + # and processing_job / processing_job_step. + # + # Lives under features/payg so it's only loaded when the saas-cucumber job + # explicitly includes this directory (separate from the main behave run + # which boots the proprietary-flavor stack and has no PAYG tables). + + Background: + Given the SaaS stack is running with PAYG enabled + And team "payg-cucumber-team" exists with wallet_policy.engine = "PAYG_SHADOW" + And I am authenticated as a member of team "payg-cucumber-team" + + Scenario: First tool call writes a CHARGED shadow row + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF to "/api/v1/security/add-password" + Then the response status is 200 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest shadow charge row has status "CHARGED" + And the latest shadow charge row has payg_units >= 1 + And the latest shadow charge row's job is OPEN + And the latest job has 1 step recorded with status "OK" + + Scenario: Lineage join — second call on the same output joins the first process + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF to "/api/v1/security/add-password" + And I take the response body as "step1-output" + And I POST "step1-output" to "/api/v1/security/sanitize-pdf" + Then exactly 1 shadow charge row exists for team "payg-cucumber-team" + # The second call joined the first process — no new shadow row + And the latest job has step_count = 2 + And the latest job is OPEN + + Scenario: First-step 5xx refunds the shadow row + closes the process + # No reliably-5xx-ing real tool endpoint exists (every malformed input is + # caught as 4xx by GlobalExceptionHandler), so we drive the refund path + # through PaygCucumberThrowController — a @Profile("payg-cucumber") stub + # that always throws IllegalStateException → 500. The profile is active + # only inside docker-compose-saas.yml, never in production. + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF to "/api/v1/payg-cucumber/throw-500" + Then the response status is 500 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest shadow charge row has status "REFUNDED" + And the latest shadow charge row's refunded_at is not null + And the latest shadow charge row's refund_reason starts with "first-step-5xx" + And the latest job is CLOSED + And the latest job has 1 step recorded with status "FAILED" + + # ────────────────────────────────────────────────────────────────────────── + # NOTE: The kill-switch scenario (PAYG_FILTER_ENABLED=false) is NOT + # automated — it requires restarting the stirling-pdf-saas container with + # a different env var mid-run, which couples test setup to compose state + # more than it's worth for a flag that only flips during incident response. + # See notes/PAYG_DESIGN.md §7.5.2 "PAYG cucumber: manual-only scenarios". + # ────────────────────────────────────────────────────────────────────────── + + Scenario: 4xx leaves the shadow row CHARGED (customer pays for the attempt) + # /sanitize-pdf on an encrypted PDF without the password reliably 400s + # via GlobalExceptionHandler's PdfPasswordException → ProblemDetail path. + # We chain: first call encrypts a PDF (CHARGED), second call tries to + # sanitize WITHOUT the password and 400s. The 4xx assertion is on the + # SECOND call's behaviour. + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF to "/api/v1/security/add-password" + And I take the response body as "encrypted" + And I POST "encrypted" to "/api/v1/security/sanitize-pdf" + Then the response status is >= 400 and < 500 + # Note: shadow_charges count is 1 because the second call lineage-joins + # the first (its input matches the first's output). The 4xx therefore + # appears as a FAILED step on the existing process, not a new shadow row. + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest shadow charge row has status "CHARGED" + And the latest job has step_count = 2 + And the latest step's error_code matches the response status + + Scenario: ZIP-returning tool records OUTPUT signatures per inner PDF + # /split-pages with multiple page numbers returns a ZIP. Stirling sends + # application/octet-stream rather than application/zip — the extractor + # sniffs the PK\x03\x04 magic so the ZIP unpack path still fires. + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a 3-page PDF to "/api/v1/general/split-pages" with form fields: + | pageNumbers | 1,2 | + Then the response status is 200 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest job has at least 2 OUTPUT artifact hashes recorded + + Scenario: Multi-file input writes a single shadow row sized by the group + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST two single-page PDFs as a multi-file payload to "/api/v1/general/merge-pdfs" + Then the response status is 200 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest shadow charge row has payg_units >= 1 + + Scenario: PIPELINE header sets the job source + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF with header "X-Stirling-Automation: true" to "/api/v1/security/add-password" + Then the response status is 200 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest job's source is "PIPELINE" diff --git a/testing/cucumber/features/steps/payg_step_definitions.py b/testing/cucumber/features/steps/payg_step_definitions.py new file mode 100644 index 0000000000..5069f9a007 --- /dev/null +++ b/testing/cucumber/features/steps/payg_step_definitions.py @@ -0,0 +1,473 @@ +""" +Step definitions for PAYG shadow-mode end-to-end tests. + +Runs against the saas-profile stack defined in +testing/compose/docker-compose-saas.yml — the backend with STIRLING_FLAVOR=saas +plus a Postgres container holding the stirling_pdf schema. + +The test harness talks to the backend over HTTP and inspects the resulting +rows in `payg_shadow_charge`, `processing_job`, `processing_job_step`, and +`job_artifact_hash` via a direct psycopg connection. Direct DB inspection is +deliberate — we want to verify the *side effects* of the filter, not relay +them through another API layer that itself might be wrong. + +Auth model: the saas profile expects Supabase JWTs. For cucumber we +configure the stack with a test user whose API key is recognised via the +X-API-KEY header, which the PaygChargeInterceptor.resolveUser() path +handles natively. The companion docker-compose-saas.yml seeds the user + +team rows via saas-init.sql so each scenario starts from a known state. +""" + +import os +import time + +import psycopg +import requests +from behave import given, then, when + +BASE_URL = os.environ.get("PAYG_BASE_URL", "http://localhost:8080") +API_KEY = os.environ.get("PAYG_API_KEY", "payg-cucumber-key") + +DB_HOST = os.environ.get("PAYG_DB_HOST", "localhost") +DB_PORT = int(os.environ.get("PAYG_DB_PORT", "5433")) +DB_NAME = os.environ.get("PAYG_DB_NAME", "postgres") +DB_USER = os.environ.get("PAYG_DB_USER", "postgres") +DB_PASSWORD = os.environ.get("PAYG_DB_PASSWORD", "postgres") +DB_SCHEMA = os.environ.get("PAYG_DB_SCHEMA", "stirling_pdf") + +# Fixture paths — small PDFs that ship with the cucumber harness. +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "exampleFiles", +) +# Existing cucumber fixtures (verified page counts via pypdf): +# tables.pdf = 1 page → SINGLE_PAGE_PDF +# ghost1.pdf = 3 pages → THREE_PAGE_PDF +# images.pdf = 5 pages (available if a scenario needs more) +# If you add more PAYG scenarios that need a different shape, drop the +# new fixture in exampleFiles/ and reference it here. +SINGLE_PAGE_PDF = os.path.join(FIXTURE_DIR, "tables.pdf") +THREE_PAGE_PDF = os.path.join(FIXTURE_DIR, "ghost1.pdf") + + +# --------------------------------------------------------------------------- +# DB helpers +# --------------------------------------------------------------------------- + + +def _db(): + """Open a fresh connection for each step — keeps things simple.""" + return psycopg.connect( + host=DB_HOST, + port=DB_PORT, + dbname=DB_NAME, + user=DB_USER, + password=DB_PASSWORD, + autocommit=True, + ) + + +def _team_id_for(team_name, conn): + """Look up our test team's id (seeded by saas-init.sql).""" + with conn.cursor() as cur: + cur.execute( + f"SELECT team_id FROM {DB_SCHEMA}.teams WHERE name = %s LIMIT 1", + (team_name,), + ) + row = cur.fetchone() + assert row is not None, f"No team named '{team_name}' in {DB_SCHEMA}.teams" + return row[0] + + +def _shadow_rows_for(team_name): + with _db() as conn: + team_id = _team_id_for(team_name, conn) + with conn.cursor() as cur: + cur.execute( + f"""SELECT shadow_id, payg_units, status, refunded_at, refund_reason, job_id + FROM {DB_SCHEMA}.payg_shadow_charge + WHERE team_id = %s + ORDER BY occurred_at DESC""", + (team_id,), + ) + return cur.fetchall() + + +def _latest_job_for(team_name): + """Return the most recently opened job for the team as a dict.""" + with _db() as conn: + team_id = _team_id_for(team_name, conn) + with conn.cursor() as cur: + cur.execute( + f"""SELECT job_id, status, source, step_count, started_at, closed_at + FROM {DB_SCHEMA}.processing_job + WHERE owner_team_id = %s + ORDER BY started_at DESC + LIMIT 1""", + (team_id,), + ) + row = cur.fetchone() + assert row is not None, f"No processing_job rows for team '{team_name}'" + return { + "job_id": row[0], + "status": row[1], + "source": row[2], + "step_count": row[3], + "started_at": row[4], + "closed_at": row[5], + } + + +def _steps_for_job(job_id): + with _db() as conn: + with conn.cursor() as cur: + cur.execute( + f"""SELECT step_id, tool_id, status, error_code + FROM {DB_SCHEMA}.processing_job_step + WHERE job_id = %s + ORDER BY started_at ASC""", + (job_id,), + ) + return cur.fetchall() + + +def _output_artifact_count(job_id): + with _db() as conn: + with conn.cursor() as cur: + cur.execute( + f"""SELECT COUNT(*) FROM {DB_SCHEMA}.job_artifact_hash + WHERE job_id = %s AND kind = 'OUTPUT'""", + (job_id,), + ) + return cur.fetchone()[0] + + +# --------------------------------------------------------------------------- +# HTTP helpers +# --------------------------------------------------------------------------- + + +def _api_headers(extra=None): + headers = {"X-API-KEY": API_KEY} + if extra: + headers.update(extra) + return headers + + +def _wait_for_health(timeout_seconds=60): + """Block until /api/v1/info/status returns 2xx, or timeout.""" + deadline = time.time() + timeout_seconds + while time.time() < deadline: + try: + r = requests.get(f"{BASE_URL}/api/v1/info/status", timeout=5) + if 200 <= r.status_code < 300: + return + except requests.RequestException: + pass + time.sleep(2) + raise AssertionError( + f"SaaS stack did not become healthy within {timeout_seconds}s at {BASE_URL}" + ) + + +# --------------------------------------------------------------------------- +# GIVEN — environment + test fixtures +# --------------------------------------------------------------------------- + + +@given("the SaaS stack is running with PAYG enabled") +def step_saas_stack_running(context): + _wait_for_health() + + +@given('team "{team_name}" exists with wallet_policy.engine = "{engine}"') +def step_team_exists_with_engine(context, team_name, engine): + """Verify the seed migration created the team + flipped its engine.""" + with _db() as conn: + team_id = _team_id_for(team_name, conn) + with conn.cursor() as cur: + cur.execute( + f"SELECT engine FROM {DB_SCHEMA}.wallet_policy WHERE team_id = %s", + (team_id,), + ) + row = cur.fetchone() + assert row is not None, f"No wallet_policy row for team '{team_name}'" + assert row[0] == engine, ( + f"Team '{team_name}' engine is '{row[0]}', expected '{engine}'. " + "Check saas-init.sql seeded the row correctly." + ) + context.team_name = team_name + + +@given('I am authenticated as a member of team "{team_name}"') +def step_authenticated_as_team_member(context, team_name): + # The seeded API key is bound to a member of this team via saas-init.sql. + context.team_name = team_name + + +@given('there are no existing shadow charges for team "{team_name}"') +def step_clear_shadow_charges(context, team_name): + with _db() as conn: + team_id = _team_id_for(team_name, conn) + with conn.cursor() as cur: + cur.execute( + f"DELETE FROM {DB_SCHEMA}.payg_shadow_charge WHERE team_id = %s", + (team_id,), + ) + cur.execute( + f"""DELETE FROM {DB_SCHEMA}.job_artifact_hash + WHERE job_id IN ( + SELECT job_id FROM {DB_SCHEMA}.processing_job + WHERE owner_team_id = %s + )""", + (team_id,), + ) + cur.execute( + f"""DELETE FROM {DB_SCHEMA}.processing_job_step + WHERE job_id IN ( + SELECT job_id FROM {DB_SCHEMA}.processing_job + WHERE owner_team_id = %s + )""", + (team_id,), + ) + cur.execute( + f"DELETE FROM {DB_SCHEMA}.processing_job WHERE owner_team_id = %s", + (team_id,), + ) + + +# NOTE: The kill-switch scenario (PAYG_FILTER_ENABLED=false) is verified +# manually — see notes/PAYG_DESIGN.md §7.5 "PAYG cucumber: manual-only +# scenarios" for the procedure. No automated step def is provided because +# toggling the env var requires restarting the docker container, and +# orchestrating that mid-suite would couple test setup to compose state +# in ways that have historically been fragile. + + +# --------------------------------------------------------------------------- +# WHEN — invoke tool endpoints +# --------------------------------------------------------------------------- + + +@when('I POST a single-page PDF to "{endpoint}"') +def step_post_single_pdf(context, endpoint): + _post_pdf(context, endpoint, SINGLE_PAGE_PDF) + + +@when('I POST a 3-page PDF to "{endpoint}"') +def step_post_three_page_pdf(context, endpoint): + _post_pdf(context, endpoint, THREE_PAGE_PDF) + + +@when('I POST a single-page PDF to "{endpoint}" with form fields:') +def step_post_with_form_fields(context, endpoint): + """The Gherkin step takes a `table` of key/value pairs that become + multipart form fields. Use this when the default `password=...` field + from _post_pdf() isn't what we want for the scenario.""" + data = {row[0]: row[1] for row in context.table.rows} if context.table else {} + _post_pdf(context, endpoint, SINGLE_PAGE_PDF, form_data=data) + + +@when('I POST a 3-page PDF to "{endpoint}" with form fields:') +def step_post_three_page_with_form_fields(context, endpoint): + data = {row[0]: row[1] for row in context.table.rows} if context.table else {} + _post_pdf(context, endpoint, THREE_PAGE_PDF, form_data=data) + + +@when('I POST a single-page PDF with header "{header_name}: {header_value}" to "{endpoint}"') +def step_post_with_header(context, header_name, header_value, endpoint): + _post_pdf(context, endpoint, SINGLE_PAGE_PDF, extra_headers={header_name: header_value}) + + +@when('I POST two single-page PDFs as a multi-file payload to "{endpoint}"') +def step_post_two_pdfs(context, endpoint): + with open(SINGLE_PAGE_PDF, "rb") as a, open(SINGLE_PAGE_PDF, "rb") as b: + files = [ + ("fileInput", ("a.pdf", a.read(), "application/pdf")), + ("fileInput", ("b.pdf", b.read(), "application/pdf")), + ] + context.response = requests.post( + f"{BASE_URL}{endpoint}", + files=files, + headers=_api_headers(), + timeout=30, + ) + + +@when('I take the response body as "{name}"') +def step_capture_response_body(context, name): + assert context.response is not None, "No response on context" + if not hasattr(context, "captured_bodies"): + context.captured_bodies = {} + context.captured_bodies[name] = context.response.content + + +@when('I POST "{captured_name}" to "{endpoint}"') +def step_post_captured(context, captured_name, endpoint): + body = context.captured_bodies[captured_name] + files = {"fileInput": (f"{captured_name}.pdf", body, "application/pdf")} + # Most tools need at least one extra form field; for the sanitize endpoint + # the defaults are sufficient. + context.response = requests.post( + f"{BASE_URL}{endpoint}", + files=files, + headers=_api_headers(), + timeout=30, + ) + + +def _post_pdf(context, endpoint, fixture_path, extra_headers=None, form_data=None): + """Default form data carries `password=cucumber-test-password` because + /add-password (our most-used scenario endpoint) requires it. Pass + {@code form_data} to override (e.g. to send the wrong password to + /remove-password for the 4xx scenario).""" + with open(fixture_path, "rb") as f: + files = {"fileInput": (os.path.basename(fixture_path), f, "application/pdf")} + data = form_data if form_data is not None else {"password": "cucumber-test-password"} + context.response = requests.post( + f"{BASE_URL}{endpoint}", + files=files, + data=data, + headers=_api_headers(extra_headers), + timeout=60, + ) + + +# --------------------------------------------------------------------------- +# THEN — assertions +# --------------------------------------------------------------------------- + + +@then("the response status is {status:d}") +def step_assert_status(context, status): + assert context.response.status_code == status, _status_mismatch_message( + context.response, status + ) + + +def _status_mismatch_message(response, expected_status): + """Verbose diagnostic for unexpected statuses — dumps URL, headers, body + so we don't have to round-trip via container logs to figure out what came + back. Kept as a helper so future status-range / status-min steps can call + it too.""" + body_preview = response.text[:1000] if response.text else "" + headers_brief = { + k: v + for k, v in response.headers.items() + if k.lower() + in ("content-type", "content-length", "x-content-type-options", "location") + } + return ( + f"Expected {expected_status}, got {response.status_code}\n" + f" URL: {response.request.method} {response.request.url}\n" + f" Headers: {headers_brief}\n" + f" Body: {body_preview}" + ) + + +@then("the response status is >= {minimum:d}") +def step_assert_status_min(context, minimum): + assert context.response.status_code >= minimum, ( + f"Expected >= {minimum}, got {context.response.status_code}" + ) + + +@then("the response status is >= {minimum:d} and < {maximum:d}") +def step_assert_status_range(context, minimum, maximum): + code = context.response.status_code + assert minimum <= code < maximum, f"Expected [{minimum}, {maximum}), got {code}" + + +@then('the response Content-Type is "{expected}"') +def step_assert_content_type(context, expected): + actual = (context.response.headers.get("Content-Type") or "").split(";")[0].strip() + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then('exactly {n:d} shadow charge row exists for team "{team_name}"') +@then('exactly {n:d} shadow charge rows exist for team "{team_name}"') +def step_assert_shadow_count(context, n, team_name): + rows = _shadow_rows_for(team_name) + assert len(rows) == n, f"Expected {n} shadow rows for '{team_name}', found {len(rows)}: {rows}" + + +@then('the latest shadow charge row has status "{status}"') +def step_assert_latest_shadow_status(context, status): + rows = _shadow_rows_for(context.team_name) + assert rows, "No shadow rows for team" + assert rows[0][2] == status, f"Expected status '{status}', got '{rows[0][2]}'" + + +@then("the latest shadow charge row has payg_units >= {minimum:d}") +def step_assert_payg_units_min(context, minimum): + rows = _shadow_rows_for(context.team_name) + assert rows[0][1] >= minimum, f"Expected payg_units >= {minimum}, got {rows[0][1]}" + + +@then("the latest shadow charge row's refunded_at is not null") +def step_assert_refunded_at_set(context): + rows = _shadow_rows_for(context.team_name) + assert rows[0][3] is not None, "refunded_at is null on the latest shadow row" + + +@then('the latest shadow charge row\'s refund_reason starts with "{prefix}"') +def step_assert_refund_reason(context, prefix): + rows = _shadow_rows_for(context.team_name) + reason = rows[0][4] or "" + assert reason.startswith(prefix), f"refund_reason '{reason}' does not start with '{prefix}'" + + +@then("the latest shadow charge row's job is {status}") +def step_assert_latest_jobstatus_via_shadow(context, status): + job = _latest_job_for(context.team_name) + assert job["status"] == status, f"Latest job status is '{job['status']}', expected '{status}'" + + +@then("the latest job is {status}") +def step_assert_latest_job_status(context, status): + job = _latest_job_for(context.team_name) + assert job["status"] == status, f"Latest job status is '{job['status']}', expected '{status}'" + + +@then("the latest job has step_count = {expected:d}") +def step_assert_step_count(context, expected): + job = _latest_job_for(context.team_name) + assert job["step_count"] == expected, ( + f"Latest job step_count is {job['step_count']}, expected {expected}" + ) + + +@then('the latest job\'s source is "{source}"') +def step_assert_job_source(context, source): + job = _latest_job_for(context.team_name) + assert job["source"] == source, f"Latest job source is '{job['source']}', expected '{source}'" + + +@then('the latest job has {n:d} step recorded with status "{status}"') +@then('the latest job has {n:d} steps recorded with status "{status}"') +def step_assert_step_count_with_status(context, n, status): + job = _latest_job_for(context.team_name) + steps = _steps_for_job(job["job_id"]) + matching = [s for s in steps if s[2] == status] + assert len(matching) == n, ( + f"Expected {n} steps with status '{status}', got {len(matching)}: {steps}" + ) + + +@then("the latest step's error_code matches the response status") +def step_assert_step_error_code(context): + job = _latest_job_for(context.team_name) + steps = _steps_for_job(job["job_id"]) + assert steps, "No steps recorded" + latest_step = steps[-1] + expected = str(context.response.status_code) + assert latest_step[3] == expected, ( + f"Latest step error_code is '{latest_step[3]}', expected '{expected}'" + ) + + +@then("the latest job has at least {n:d} OUTPUT artifact hashes recorded") +def step_assert_output_artifact_count(context, n): + job = _latest_job_for(context.team_name) + count = _output_artifact_count(job["job_id"]) + assert count >= n, f"Latest job has {count} OUTPUT artifacts, expected >= {n}" diff --git a/testing/cucumber/requirements.in b/testing/cucumber/requirements.in index 8b1f01d1f8..ae968c9e34 100644 --- a/testing/cucumber/requirements.in +++ b/testing/cucumber/requirements.in @@ -5,3 +5,12 @@ pypdf reportlab PyCryptodome qrcode[pil] +# PAYG cucumber scenarios inspect shadow rows directly against the +# saas-profile Postgres container — see testing/cucumber/features/steps/payg_step_definitions.py. +# Loaded by the saas-cucumber CI job only; main behave run doesn't touch it. +psycopg[binary] +# psycopg declares `typing-extensions>=4.6` under a Python-version marker that +# pip-compile leaves unpinned, which trips `--require-hashes`. Listing it here +# forces pip-compile to lock it to a concrete version with a hash. Without this, +# both saas-cucumber and the main docker-compose-tests fail on `pip install`. +typing-extensions diff --git a/testing/cucumber/requirements.txt b/testing/cucumber/requirements.txt index 66d26c2c3e..dedcfe43b0 100644 --- a/testing/cucumber/requirements.txt +++ b/testing/cucumber/requirements.txt @@ -1,19 +1,19 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --generate-hashes --output-file='testing\cucumber\requirements.txt' --strip-extras 'testing\cucumber\requirements.in' +# pip-compile --generate-hashes --output-file=testing/cucumber/requirements.txt --strip-extras testing/cucumber/requirements.in # behave==1.3.3 \ --hash=sha256:2b8f4b64ed2ea756a5a2a73e23defc1c4631e9e724c499e46661778453ebaf51 \ --hash=sha256:89bdb62af8fb9f147ce245736a5de69f025e5edfb66f1fbe16c5007493f842c0 # via - # -r requirements.in + # -r testing/cucumber/requirements.in # behave-html-formatter behave-html-formatter==0.9.10 \ --hash=sha256:c5a9ad3edcac7be5766b14aacce46794885c749c4741fc93f8fc3a4bf2a891aa \ --hash=sha256:fff7ac2118463701423645ad5a12636845bfd6c8a2dd52097b524b4f290aa7c8 - # via -r requirements.in + # via -r testing/cucumber/requirements.in certifi==2026.2.25 \ --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 @@ -154,7 +154,9 @@ charset-normalizer==3.4.7 \ colorama==0.4.6 \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 - # via behave + # via + # behave + # qrcode cucumber-expressions==19.0.0 \ --hash=sha256:8eb5ae46dd03dd37fec1163ace1510529501d7d1868ff372c1ab2cd5aa4543a8 \ --hash=sha256:f452e6c73258c1677043ad67ad5f538c87284d6b502004720510fb6b7452d9c5 @@ -272,6 +274,67 @@ pillow==12.2.0 \ # via # qrcode # reportlab +psycopg==3.3.4 \ + --hash=sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a \ + --hash=sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc + # via -r testing/cucumber/requirements.in +psycopg-binary==3.3.4 \ + --hash=sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070 \ + --hash=sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c \ + --hash=sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc \ + --hash=sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e \ + --hash=sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68 \ + --hash=sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae \ + --hash=sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829 \ + --hash=sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097 \ + --hash=sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c \ + --hash=sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992 \ + --hash=sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97 \ + --hash=sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6 \ + --hash=sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a \ + --hash=sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d \ + --hash=sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228 \ + --hash=sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e \ + --hash=sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4 \ + --hash=sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41 \ + --hash=sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949 \ + --hash=sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9 \ + --hash=sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089 \ + --hash=sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf \ + --hash=sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da \ + --hash=sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578 \ + --hash=sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014 \ + --hash=sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e \ + --hash=sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e \ + --hash=sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31 \ + --hash=sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652 \ + --hash=sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b \ + --hash=sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839 \ + --hash=sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277 \ + --hash=sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7 \ + --hash=sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4 \ + --hash=sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e \ + --hash=sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70 \ + --hash=sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf \ + --hash=sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a \ + --hash=sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429 \ + --hash=sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8 \ + --hash=sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3 \ + --hash=sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13 \ + --hash=sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007 \ + --hash=sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc \ + --hash=sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38 \ + --hash=sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9 \ + --hash=sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95 \ + --hash=sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d \ + --hash=sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16 \ + --hash=sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d \ + --hash=sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260 \ + --hash=sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744 \ + --hash=sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28 \ + --hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \ + --hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7 + # via psycopg pycryptodome==3.23.0 \ --hash=sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4 \ --hash=sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c \ @@ -314,29 +377,37 @@ pycryptodome==3.23.0 \ --hash=sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa \ --hash=sha256:e3f2d0aaf8080bda0587d58fc9fe4766e012441e2eed4269a77de6aea981c8be \ --hash=sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7 - # via -r requirements.in + # via -r testing/cucumber/requirements.in pypdf==6.11.0 \ --hash=sha256:062b51c81b0910e6d2755e99e1c5547a0a23b7d0a32322af66240d8edcfabe87 \ --hash=sha256:769394d5756d5b304c9b6bef88b54b1816b328e7e6fc9254e625529a15ed4ab8 - # via -r requirements.in + # via -r testing/cucumber/requirements.in qrcode==8.2 \ --hash=sha256:16e64e0716c14960108e85d853062c9e8bba5ca8252c0b4d0231b9df4060ff4f \ --hash=sha256:35c3f2a4172b33136ab9f6b3ef1c00260dd2f66f858f24d88418a015f446506c - # via -r requirements.in + # via -r testing/cucumber/requirements.in reportlab==4.5.1 \ --hash=sha256:06fce8cb56c83307cfa4909cdf4e6a2ddbb44e5d6ef4d2edca896d7e9769f091 \ --hash=sha256:9fdf68f4de9171ec66acb4a5feed8f8ca2af43479e707a6fbb0daa75d88e5494 - # via -r requirements.in + # via -r testing/cucumber/requirements.in requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed - # via -r requirements.in + # via -r testing/cucumber/requirements.in six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via # behave # parse-type +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via -r testing/cucumber/requirements.in +tzdata==2026.2 \ + --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \ + --hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7 + # via psycopg urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 diff --git a/testing/test-payg.sh b/testing/test-payg.sh new file mode 100644 index 0000000000..d3683c4482 --- /dev/null +++ b/testing/test-payg.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# +# Cucumber harness for the PAYG shadow-mode scenarios. Brings up the +# saas-profile docker-compose, waits for backend health, pipes the seed SQL +# into the test postgres, then invokes Behave against features/payg/. +# +# Companion to testing/test.sh (which covers the proprietary-flavor stack +# and skips features/payg via behave.ini's exclude_re). Kept as a separate +# entrypoint so the saas variant can be reviewed + iterated on without +# touching the main cucumber harness. + +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMPOSE_FILE="$PROJECT_ROOT/testing/compose/docker-compose-saas.yml" +SEED_FILE="$PROJECT_ROOT/testing/compose/payg/saas-seed.sql" + +cd "$PROJECT_ROOT" + +# Swap behave.ini for the run. The project-default behave.ini excludes +# features/payg so the proprietary CI (which boots without the saas profile) +# can't try to run scenarios that need PAYG tables. behave's exclude_re takes +# priority over a path argument, so even `behave features/payg` would find +# zero features against the default config. The PAYG harness needs a config +# without the payg exclusion — restored on exit. +BEHAVE_INI="$PROJECT_ROOT/testing/cucumber/behave.ini" +BEHAVE_INI_BACKUP="$BEHAVE_INI.payg-backup" + +cleanup() { + # Always dump container logs before tearing the stack down — the workflow + # has a "Dump saas container logs on failure" step but it runs AFTER this + # cleanup, by which point the containers are gone. Cheap diagnostic on + # success, essential on failure. (Earlier ERR-trap conditional didn't + # fire reliably under `set -e` when behave was the script's last command.) + echo "===== STIRLING_BACKEND_LOG_DUMP_START =====" + docker compose -f "$COMPOSE_FILE" logs --tail 500 stirling-pdf-saas || true + echo "===== STIRLING_BACKEND_LOG_DUMP_END =====" + echo "==> Tearing down saas compose stack" + docker compose -f "$COMPOSE_FILE" down -v || true + if [ -f "$BEHAVE_INI_BACKUP" ]; then + mv "$BEHAVE_INI_BACKUP" "$BEHAVE_INI" + fi +} +trap cleanup EXIT + +cp "$BEHAVE_INI" "$BEHAVE_INI_BACKUP" +cat > "$BEHAVE_INI" <<'EOF' +# Temporary behave.ini written by testing/test-payg.sh for the PAYG harness +# run only. Restored on exit. The project default (in git) excludes +# features/payg so the proprietary-flavor CI doesn't try to run them. +[behave] +exclude_re = features/enterprise +EOF + +echo "==> Building + starting saas compose stack" +docker compose -f "$COMPOSE_FILE" up -d --build + +echo "==> Waiting for backend health (max 180s)" +deadline=$(( SECONDS + 180 )) +until curl -fsS http://localhost:8080/api/v1/info/status > /dev/null 2>&1; do + if [ $SECONDS -ge $deadline ]; then + echo "Backend did not become healthy in 180s" + docker compose -f "$COMPOSE_FILE" logs --tail 200 stirling-pdf-saas + exit 1 + fi + sleep 3 +done +echo "Backend healthy." + +echo "==> Seeding test team / user / wallet_policy (PAYG_SHADOW)" +docker compose -f "$COMPOSE_FILE" exec -T postgres-saas \ + psql -U postgres -d postgres < "$SEED_FILE" + +echo "==> Running PAYG cucumber scenarios" +cd "$PROJECT_ROOT/testing/cucumber" +PAYG_BASE_URL="${PAYG_BASE_URL:-http://localhost:8080}" \ +PAYG_API_KEY="${PAYG_API_KEY:-payg-cucumber-key}" \ +PAYG_DB_HOST="${PAYG_DB_HOST:-localhost}" \ +PAYG_DB_PORT="${PAYG_DB_PORT:-5433}" \ +PAYG_DB_USER="${PAYG_DB_USER:-postgres}" \ +PAYG_DB_PASSWORD="${PAYG_DB_PASSWORD:-postgres}" \ +PAYG_DB_NAME="${PAYG_DB_NAME:-postgres}" \ +PAYG_DB_SCHEMA="${PAYG_DB_SCHEMA:-stirling_pdf}" \ +python -m behave features/payg \ + -f behave_html_formatter:HTMLFormatter -o report-payg.html \ + -f pretty \ + --junit --junit-directory junit-payg + +echo "==> PAYG cucumber run complete"