From 1ef03c43b48aa940eb30d9977a14b7663d4a64eb Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:11:43 +0100 Subject: [PATCH 01/19] fix(i18n): wrap hard-coded English strings in t() across UI (#6566) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Audit + bulk fix of hard-coded English UI strings - `aria-label`, `title`, `placeholder`, `label`, and raw JSX literals that bypassed i18n entirely. Each literal now goes through `t("key", "English Default")` from `react-i18next`, and every new key has a corresponding entry in `en-GB/translation.toml` so translators can pick it up. ## What this fixes Strings were rendered untranslated in every non-EN locale because they never went through `t()` at all (not just "value not translated yet"). Affects screen-reader labels, tooltips, form placeholders, empty/loading states, plan card content, and the entire workflow ParticipantView. ## Coverage (~143 keys / 50 files) - **Viewer chrome** - search bar (close, clear, prev/next, "of N" results), link/signature/redaction actions, viewer error state, zoom labels - **Page editor** - undo/redo/rotate/delete toolbar tooltips, empty state, bulk selection operator chip tooltips - **Shared primitives** - Tooltip close, InfoBanner dismiss, TextInput clear, Toast dismiss/toggle, UpdateModal close, EditableSecretField edit, DropdownListWithFooter search, FileCard/FileDropdownMenu actions, EmptyFilesState + AddFileCard upload - **Tools** - Image upload + hint, ColorControl eyedropper, sign Use Signature, CompressSettings, OCR loading, PageLayout margin/border/row/col placeholders, FormFill switch + save + re-scan - **Proprietary admin** - OverviewHeader signed-in line + logout, AdminPremiumSection moved-features list (via ``), AdminPlanSection no-data alert, AdminAdvancedSection temp-dir placeholders, AdminEndpointsSection multiselect placeholders, AdminMailSection + AdminDatabaseSection password placeholders - **Onboarding** - MFASetupSlide QR loading + auth code label, SecurityCheckSlide role select + options - **ParticipantView** - entire sign-document UI (~30 strings: loading, error, badges, headings, cert-type Select, all input labels and placeholders, action buttons, completion + expired alerts) - file previously imported `useTranslation` but only used `t()` for cert validation - **planConstants.ts refactor** - replaced `PLAN_FEATURES` / `PLAN_HIGHLIGHTS` const exports with `usePlanFeatures()` / `usePlanHighlights()` hooks. Service layer (`licenseService.getPlans`) updated to accept feature/highlight maps so it stays hook-free. Callers (`usePlans`, `CheckoutContext`) resolve the hooks at the React boundary - **Previously catalogued offenders** - `FileSidebarFileItem` open/close-viewer aria-labels, `quickAccessBar/ActiveToolButton` "Back to all tools" tooltip + aria, `AppConfigModal` close button ## Notes - One small refactor in `usePageSelectionTips.ts` was needed to resolve a TOML key-shape conflict: the existing scalar keys `bulkSelection.operators.{and,not,comma}` needed to become tables to hold the new `.title` subkeys for OperatorsSection's chip tooltips. The existing descriptions moved to `[bulkSelection.operators.descriptions]` and the three i18n key paths in usePageSelectionTips were updated to match. - Viewer sidebar close buttons (Bookmark/Layer/Thumbnail/Attachment/Comments) were on the audit list but are NOT on main - they're added by the unmerged PR #6552 (feat/viewer-sidebar-ux). Those particular strings will need wrapping when that PR lands. - TOML hook (`toml-sort-fix`) ran and re-sorted the translation file. ## Test plan - [ ] `task frontend:typecheck` passes (core + proprietary + desktop variants) - [ ] `task frontend:lint` passes - [ ] Switching language to Deutsch / Русский: previously-English `aria-label`s + tooltips + placeholders + plan card bullets now render translated (when the locale has values) or fall back to the English default (when it doesn't) - [ ] Plan page bullet points in EN render unchanged - [ ] Sign-document flow (ParticipantView) renders unchanged in EN --- .../public/locales/en-GB/translation.toml | 234 +++++++++++++- .../annotation/shared/ColorControl.tsx | 6 +- .../components/annotation/tools/ImageTool.tsx | 11 +- .../components/fileEditor/AddFileCard.tsx | 2 +- .../fileManager/EmptyFilesState.tsx | 2 +- .../onboarding/slides/MFASetupSlide.tsx | 11 +- .../onboarding/slides/SecurityCheckSlide.tsx | 15 +- .../core/components/pageEditor/PageEditor.tsx | 11 +- .../pageEditor/PageEditorControls.tsx | 16 +- .../bulkSelectionPanel/OperatorsSection.tsx | 25 +- .../bulkSelectionPanel/PageSelectionInput.tsx | 4 +- .../core/components/shared/AppConfigModal.tsx | 4 +- .../shared/DropdownListWithFooter.tsx | 4 +- .../components/shared/EditableSecretField.tsx | 7 +- .../src/core/components/shared/FileCard.tsx | 6 +- .../components/shared/FileDropdownMenu.tsx | 7 +- .../components/shared/FileSidebarFileItem.tsx | 8 +- .../src/core/components/shared/InfoBanner.tsx | 4 +- .../src/core/components/shared/TextInput.tsx | 4 +- .../src/core/components/shared/Tooltip.tsx | 6 +- .../core/components/shared/UpdateModal.tsx | 2 +- .../shared/ViewerInlineControls.tsx | 6 +- .../quickAccessBar/ActiveToolButton.tsx | 12 +- .../core/components/toast/ToastRenderer.tsx | 9 +- .../tools/compress/CompressSettings.tsx | 7 +- .../components/tools/ocr/LanguagePicker.tsx | 4 +- .../PageLayoutMarginsBordersSettings.tsx | 27 +- .../tools/pageLayout/PageLayoutSettings.tsx | 4 +- .../tools/sign/SavedSignaturesSection.tsx | 6 +- .../tooltips/usePageSelectionTips.ts | 12 +- .../core/components/viewer/EmbedPdfViewer.tsx | 7 +- .../src/core/components/viewer/LinkLayer.tsx | 6 +- .../viewer/LocalEmbedPDFWithAnnotations.tsx | 7 +- .../viewer/RedactionSelectionMenu.tsx | 2 +- .../components/viewer/SearchInterface.tsx | 12 +- .../core/tools/formFill/FormFieldSidebar.tsx | 4 +- .../src/core/tools/formFill/FormFill.tsx | 20 +- .../shared/config/OverviewHeader.tsx | 6 +- .../configSections/AdminAdvancedSection.tsx | 15 +- .../configSections/AdminDatabaseSection.tsx | 5 +- .../configSections/AdminEndpointsSection.tsx | 10 +- .../configSections/AdminMailSection.tsx | 5 +- .../configSections/AdminPlanSection.tsx | 10 +- .../configSections/AdminPremiumSection.tsx | 20 +- .../configSections/plan/StaticPlanSection.tsx | 19 +- .../components/workflow/ParticipantView.tsx | 137 ++++++-- .../proprietary/constants/planConstants.ts | 301 ++++++++++++------ .../proprietary/contexts/CheckoutContext.tsx | 14 +- .../editor/src/proprietary/hooks/usePlans.ts | 20 +- .../proprietary/services/licenseService.ts | 38 ++- 50 files changed, 884 insertions(+), 250 deletions(-) diff --git a/frontend/editor/public/locales/en-GB/translation.toml b/frontend/editor/public/locales/en-GB/translation.toml index d1f4ecab84..4c085512f9 100644 --- a/frontend/editor/public/locales/en-GB/translation.toml +++ b/frontend/editor/public/locales/en-GB/translation.toml @@ -12,7 +12,6 @@ cancel = "Cancel" changedCredsMessage = "Credentials changed!" chooseFile = "Choose File" close = "Close" -color = "Colour" comingSoon = "Coming soon" confirm = "Confirm" confirmClose = "Confirm Close" @@ -199,6 +198,10 @@ ssoDescription = "Two-factor authentication is managed by your identity provider ssoManaged = "Configure MFA through your identity provider." title = "Two-factor authentication" +[account.overview] +logOut = "Log out" +signedInAs = "Signed in as: {{email}}" + [add-page-numbers] tags = "paginate,label,organize,index" @@ -226,6 +229,9 @@ title = "What it does" [AddAttachmentsRequest.tooltip.header] title = "About Add Attachments" +[addFileCard] +upload = "Upload" + [addImage] applySignatures = "Apply Images" header = "Add images to PDFs" @@ -606,6 +612,7 @@ label = "Temp File Management" [admin.settings.advanced.tempFileManagement.baseTmpDir] description = "Base directory for temporary files (leave empty for default: java.io.tmpdir/stirling-pdf)" label = "Base Temp Directory" +placeholder = "Default: java.io.tmpdir/stirling-pdf" [admin.settings.advanced.tempFileManagement.cleanupIntervalMinutes] description = "How often to run cleanup (in minutes)" @@ -618,6 +625,7 @@ label = "Cleanup System Temp" [admin.settings.advanced.tempFileManagement.libreofficeDir] description = "Directory for LibreOffice temp files (leave empty for default: baseTmpDir/libreoffice)" label = "LibreOffice Temp Directory" +placeholder = "Default: baseTmpDir/libreoffice" [admin.settings.advanced.tempFileManagement.maxAgeHours] description = "Maximum age in hours before temp files are cleaned up" @@ -634,6 +642,7 @@ label = "Startup Cleanup" [admin.settings.advanced.tempFileManagement.systemTempDir] description = "System temp directory to clean (only used if cleanupSystemTemp is enabled)" label = "System Temp Directory" +placeholder = "System temp directory path" [admin.settings.advanced.tessdataDir] description = "Path to the tessdata directory for OCR language files" @@ -839,6 +848,7 @@ label = "Database Name" [admin.settings.database.password] description = "Database authentication password" label = "Password" +placeholder = "Enter database password" [admin.settings.database.port] description = "Database server port (not used if custom URL is provided)" @@ -870,10 +880,12 @@ label = "Hide unavailable tools by default" [admin.settings.endpoints.groupsToRemove] description = "Select endpoint groups to disable" label = "Disabled Endpoint Groups" +placeholder = "Select groups to disable" [admin.settings.endpoints.toRemove] description = "Select individual endpoints to disable" label = "Disabled Endpoints" +placeholder = "Select endpoints to disable" [admin.settings.enterpriseRequired] message = "An Enterprise license is required to access {{featureName}}. You are viewing demo data for reference." @@ -1088,6 +1100,7 @@ label = "SMTP Host" [admin.settings.mail.password] description = "Password for SMTP authentication" label = "SMTP Password" +placeholder = "Enter SMTP password" [admin.settings.mail.port] description = "The port number for SMTP connection (typically 25, 465, or 587)" @@ -1097,6 +1110,10 @@ label = "SMTP Port" description = "Username for SMTP authentication" label = "SMTP Username" +[admin.settings.plan.noData] +message = "Plans data is not available at the moment." +title = "No data available" + [admin.settings.premium] description = "Configure your premium or enterprise license key." license = "License Configuration" @@ -1143,6 +1160,9 @@ info = "If you have a license key or certificate file from a direct purchase, yo toggle = "Got a license key or certificate file?" [admin.settings.premium.movedFeatures] +auditLogging = "<0>Audit Logging (ENTERPRISE) - Security" +customMetadata = "<0>Custom Metadata (PRO) - General" +databaseConfiguration = "<0>Database Configuration (ENTERPRISE) - Database" message = "Premium and Enterprise features are now organized in their respective sections:" title = "Premium Features Distributed" @@ -1526,10 +1546,10 @@ section_title = "Agents" show_less = "Show less" start_chat = "Start chatting" stirling_description = "Your general-purpose PDF assistant" -stirling_running = "Running..." stirling_full_name = "Stirling General Agent" stirling_long_description = "General purpose PDF assistant that can run tools, create PDFs and extract insights from your documents." stirling_name = "Stirling" +stirling_running = "Running..." stirling_tooltip = "Stirling agent" view_all = "View all agents" @@ -2126,11 +2146,31 @@ placeholder = "Number of pages" title = "Last N Pages" [bulkSelection.operators] +text = "AND has higher precedence than comma. NOT applies within the document range." +title = "Operators" + +[bulkSelection.operators.and] +title = "Combine selections (both conditions must be true)" + +[bulkSelection.operators.descriptions] and = "AND: & or \"and\" - require both conditions (e.g., 1-50 & even)" comma = "Comma: , or | - combine selections (e.g., 1-10, 20)" not = "NOT: ! or \"not\" - exclude pages (e.g., 3n & not 30)" -text = "AND has higher precedence than comma. NOT applies within the document range." -title = "Operators" + +[bulkSelection.operators.even] +title = "Select all even-numbered pages (2, 4, 6, 8...)" + +[bulkSelection.operators.not] +title = "Exclude from selection" + +[bulkSelection.operators.odd] +title = "Select all odd-numbered pages (1, 3, 5, 7...)" + +[bulkSelection.operators.or] +title = "Add to selection (either condition can be true)" + +[bulkSelection.pageSelection] +title = "Page Selection" [bulkSelection.range] fromPlaceholder = "From" @@ -2782,6 +2822,9 @@ unknownTool = "Unknown tool" [cloudBadge] tooltip = "This operation will use your cloud credits" +[color.eyeDropper] +tooltip = "Pick colour from screen" + [colorPicker] title = "Choose colour" @@ -3006,6 +3049,10 @@ title = "Compression Method" 1 = "1-3 PDF compression,
4-6 lite image compression,
7-9 intense image compression Will dramatically reduce image quality" _value = "Compression Settings" +[compress.settings] +desiredSize = "Desired File Size" +desiredSizePlaceholder = "Enter size" + [compress.tooltip.description] text = "Compression is an easy way to reduce your file size. Pick File Size to enter a target size and have us adjust quality for you. Pick Quality to set compression strength manually." title = "Description" @@ -3496,6 +3543,13 @@ docsLink = "View installation documentation" message = "Stirling-PDF does not have permission to update itself on this machine." title = "Administrator permissions required" +[dropdownList] +searchPlaceholder = "Search..." + +[editableSecretField] +edit = "Edit" +editSecretValue = "Edit secret value" + [editTableOfContents] submit = "Apply table of contents" @@ -3583,6 +3637,9 @@ tabTitle = "Outline workspace" description = "Select the Edit Table of Contents tool to load its workspace." title = "Open the tool to start editing" +[emptyFilesState] +upload = "Upload" + [encryptedPdfUnlock] description = "This PDF is password protected. Enter the password so you can continue working with it." emptyResponse = "Password removal did not produce a file." @@ -3697,6 +3754,10 @@ title = "Settings" [extractPages.tooltip] description = "Extracts the selected pages into a new PDF, preserving order." +[fileCard] +openInFileEditor = "Open in File Editor" +viewInViewer = "View in Viewer" + [fileChooser] click = "Click" dragAndDrop = "Drag & Drop" @@ -3706,6 +3767,9 @@ extractPDF = "Extracting..." hoveredDragAndDrop = "Drag & Drop file(s) here" or = "or" +[fileDropdownMenu] +closeFile = "Close file" + [fileEditor] addFiles = "Add Files" pageCount = "{{count}} pages" @@ -3872,6 +3936,10 @@ openSettings = "Open settings" search = "Search" searchPlaceholder = "Search files..." +[fileSidebar.fileItem] +closeViewer = "Close viewer" +openInViewer = "Open in viewer" + [filesPage] addToWorkspace = "Add to workspace" addToWorkspaceCount = "Add {{count}} to workspace" @@ -4178,6 +4246,15 @@ title = "About Flattening PDFs" discord = "Discord" issues = "GitHub" +[formFill] +flattenAfterFilling = "Flatten after filling" +rescanFields = "Re-scan fields" +rescanFormFields = "Re-scan form fields" +save = "Save" + +[formFill.sidebar] +close = "Close sidebar" + [getPdfInfo] downloadJson = "Download JSON" downloads = "Downloads" @@ -4737,6 +4814,11 @@ tags = "markup,web-content,transformation,convert" title = "HTML To PDF" zoom = "Zoom level for displaying the website." +[image.upload] +hint = "Upload a PNG, JPG, SVG, or other image file to place on the PDF. SVG files will be converted to PNG for compatibility." +label = "Upload Image" +placeholder = "Select image file" + [imageToPdf] tags = "conversion,img,jpg,picture,photo" @@ -4755,6 +4837,9 @@ title = "Image to PDF" 4 = "Merge into single PDF" 5 = "Convert to separate PDFs" +[infoBanner] +dismiss = "Dismiss" + [invite] acceptError = "Failed to create account" accountFor = "Creating account for" @@ -5090,6 +5175,7 @@ failed = "OCR operation failed" [ocr.languagePicker] additionalLanguages = "Looking for additional languages?" +loading = "Loading available languages..." viewSetupGuide = "View setup guide →" [ocr.operation] @@ -5226,6 +5312,10 @@ daysRemainingSingular = "{{days}} day remaining" title = "Your 30-Day Pro Trial" trialEnds = "Trial ends {{date}}" +[onboarding.mfa] +authenticationCode = "Authentication code" +qrCodeLoading = "Generating your QR code…" + [onboarding.planOverview] adminBodyLoginDisabled = "Once you enable login mode, you can manage users, configure settings, and monitor server health. The first {{freeTierLimit}} people on your server get to use Stirling free of charge." adminBodyLoginEnabled = "As an admin, you can manage users, configure settings, and monitor server health. The first {{freeTierLimit}} people on your server get to use Stirling free of charge." @@ -5235,6 +5325,9 @@ userTitle = "Plan Overview" [onboarding.securityCheck] message = "The application has undergone significant changes recently. Your server admin's attention may be required. Please confirm your role to continue." +roleAdmin = "Admin" +rolePlaceholder = "Confirm your role" +roleUser = "User" [onboarding.serverLicense] freeBody = "Our Open-Core licensing permits up to {{freeTierLimit}} users for free per server. To scale uninterrupted, we recommend the Stirling Server plan - unlimited seats and SSO support for $99/server/mo." @@ -5360,6 +5453,17 @@ title = "Page Editor" zoomIn = "Zoom In" zoomOut = "Zoom Out" +[pageEditor.emptyState] +body = "Add files to start editing pages" +title = "No PDF files loaded" + +[pageEditor.toolbar] +delete = "Delete Selected" +redo = "Redo" +rotateLeft = "Rotate Selected Left" +rotateRight = "Rotate Selected Right" +undo = "Undo" + [pageExtracter] header = "Extract Pages" placeholder = "(e.g. 1,2,8 or 4,7,12-16 or 2n-1)" @@ -5371,12 +5475,14 @@ addBorder = "Add Borders" borderWidth = "Border Thickness" bottom = "Bottom Margin" cols = "Columns" +colsPlaceholder = "Enter columns" header = "Multi Page Layout" innerMargin = "Inner Margin" left = "Left Margin" pagesPerSheet = "Pages per sheet:" right = "Right Margin" rows = "Rows" +rowsPlaceholder = "Enter rows" submit = "Submit" tags = "merge,composite,single-view,organize" title = "Multi Page Layout" @@ -5418,6 +5524,14 @@ innerMarginTooLarge = "Inner margin is too large for the selected layout." outerHorizontalMarginsTooLarge = "Left/Right margins are too large for this page size." outerVerticalMarginsTooLarge = "Top/Bottom margins are too large for this page size." +[pageLayout.margins] +borderThicknessPlaceholder = "Enter border thickness" +bottomPlaceholder = "Enter bottom margin" +innerPlaceholder = "Enter inner margin" +leftPlaceholder = "Enter left margin" +rightPlaceholder = "Enter right margin" +topPlaceholder = "Enter top margin" + [pageLayout.marginsBorders.tooltip.borders] text = "Enable borders to draw lines around each placed page. This can help visual separation or trimming." title = "Add Borders" @@ -6071,6 +6185,26 @@ pdfTools = "Basic PDF Tools" priority = "Priority Support" title = "Feature" +[plan.features] +allPdfOperations = "All PDF operations" +auditing = "Auditing" +communitySupport = "Community support" +customPdfMetadata = "Custom PDF metadata" +editingTextInPdfs = "Editing text in pdfs" +externalDatabase = "External Database" +googleDriveIntegration = "Google drive integration" +prometheusSupport = "Prometheus Support" +regularUpdates = "Regular updates" +saml = "SAML" +secureLoginSupport = "Secure Login Support" +selfHostedDeployment = "Self-hosted deployment" +sso = "SSO" +unlimitedUsers = "Unlimited users" +upToFiveUsers = "Up to 5 users" +upToFiveUsersLowercase = "up to 5 users" +usageTracking = "Usage tracking" +usersLimitedToSeats = "Users limited to seats" + [plan.free] forever = "Forever free" highlight1 = "Limited Tool Usage Per week" @@ -6079,6 +6213,22 @@ highlight3 = "Community support" included = "Included" name = "Free" +[plan.highlights] +advancedIntegrations = "Advanced integrations" +allBasicFeatures = "All basic features" +cancelAnytime = "Cancel anytime" +customPdfMetadata = "Custom PDF metadata" +editingTextInPdfsCaps = "Editing text in PDFs" +enterpriseFeatures = "Enterprise features (SAML, Auditing)" +perSeatLicensing = "Per-seat licensing" +saveWithAnnualBilling = "Save with annual billing" +selfHosted = "Self-hosted" +selfHostedOnInfrastructure = "Self-hosted on your infrastructure" +ssoOAuth = "SSO (OAuth2/OIDC)" +unlimitedUsers = "Unlimited users" +upToFiveUsers = "Up to 5 users" +usageTrackingPrometheus = "Usage tracking & Prometheus" + [plan.licenseWarning] body = "You have {{total}} users but the free tier only supports {{limit}} per server. Upgrade to keep Stirling PDF running smoothly." cta = "See plans" @@ -6389,6 +6539,7 @@ allSessions = "All Sessions" allTools = "Tools" automate = "Automate" back = "Back" +backToAllTools = "Back to all tools" certSign = "Certificate Sign" completedSessions = "Completed Sessions" completedTab = "Completed" @@ -7060,6 +7211,9 @@ bullet3 = "Critical for approval workflows and legal chains of custody" description = "The order you specify when creating the session determines who signs first." title = "Signature Order" +[settings] +close = "Close" + [settings.configuration] advanced = "Advanced" database = "Database" @@ -7542,6 +7696,7 @@ sharedDescription = "All users can see and use these signatures." sharedHeading = "Shared Signatures" tempStorageDescription = "Signatures are stored in your browser only. They will be lost if you clear browser data or switch browsers." tempStorageTitle = "Temporary browser storage" +use = "Use signature" [sign.saved.status] saved = "Saved" @@ -8122,9 +8277,18 @@ center = "Center" left = "Left" right = "Right" +[textInput] +clear = "Clear input" + [theme] toggle = "Toggle Theme" +[time.relative] +daysAgo = "{{count}}d ago" +hoursAgo = "{{count}}h ago" +justNow = "just now" +minutesAgo = "{{count}}m ago" + [timestampPdf] completed = "PDF timestamped successfully" desc = "Add an RFC 3161 document timestamp to your PDF using a trusted Time Stamp Authority (TSA) server." @@ -8151,11 +8315,9 @@ label = "Select a TSA server" [timestampPdf.steps] settings = "Settings" -[time.relative] -daysAgo = "{{count}}d ago" -hoursAgo = "{{count}}h ago" -justNow = "just now" -minutesAgo = "{{count}}m ago" +[toast] +dismiss = "Dismiss" +toggleDetails = "Toggle details" [tool] endpointUnavailable = "This tool is unavailable on your server." @@ -8243,6 +8405,9 @@ verification = "Verification" noSearchResults = "No tools found" noTools = "No tools available" +[tooltip] +close = "Close tooltip" + [unlockPDFForms] description = "This tool will remove read-only restrictions from PDF form fields, making them editable and fillable." filenamePrefix = "unlocked_forms" @@ -8269,6 +8434,7 @@ breakingChangesDefault = "This version contains breaking changes." breakingChangesDetected = "Breaking Changes Detected" breakingChangesMessage = "Some versions contain breaking changes. Please review the migration guides below before updating." close = "Close" +closeModal = "Close update modal" current = "Current Version" defaultRecommendation = "This update contains important fixes and improvements." downloadLatest = "Download Latest" @@ -8520,6 +8686,9 @@ typeInsertText = "Insert Text" typeReplaceText = "Replace Text" viewComment = "View comment" +[viewer.error] +noFileProvided = "Error: No file provided to viewer" + [viewer.formBar] apply = "Apply Changes" dismiss = "Dismiss" @@ -8529,6 +8698,9 @@ title = "Form Fields" unsavedBadge = "Unsaved" unsavedDesc = "You have unsaved changes" +[viewer.link] +delete = "Delete link" + [viewer.nonPdf] columnDefault = "Column {{index}}" convertToPdf = "Convert to PDF" @@ -8544,6 +8716,19 @@ renderMarkdown = "Render markdown" sortedBy = "Sorted by: {{column}}" textStats = "{{lines}} lines · {{size}}" +[viewer.redaction] +removeMark = "Remove this mark" + +[viewer.search] +clear = "Clear search" +close = "Close search" +next = "Next result" +previous = "Previous result" +resultsOf = "of {{total}}" + +[viewer.signature] +delete = "Delete signature" + [viewPdf] header = "View PDF" tags = "view,read,annotate,text,image,highlight,edit" @@ -8792,6 +8977,37 @@ toggleSidebar = "Toggle Sidebar" toggleTheme = "Toggle Theme" viewer = "Viewer" +[workflow.participant] +accessExpired = "Your access to this document has expired." +certificateFile = "Certificate File" +certificateFilePlaceholder = "Select certificate file" +certificatePassword = "Certificate Password" +certificateType = "Certificate Type" +certTypeJks = "JKS Keystore" +certTypeP12 = "P12/PKCS12 Certificate" +certTypeServer = "Server Certificate (if available)" +completedDeclined = "You have declined this document." +completedSigned = "You have signed this document." +decline = "Decline" +downloadDocument = "Download Document" +dueDate = "Due Date: {{date}}" +errorTitle = "Error" +from = "From: {{name}}" +loadingSession = "Loading session..." +location = "Location" +locationPlaceholder = "e.g., San Francisco, CA" +pageNumber = "Page Number (optional)" +reason = "Reason" +reasonPlaceholder = "e.g., Document approval" +sessionNotFound = "Session not found or access denied." +signDocument = "Sign Document" +statusDeclined = "Declined" +statusNotified = "Notified" +statusPending = "Pending" +statusSigned = "Signed" +statusViewed = "Viewed" +submitSignature = "Submit Signature" + [workspace] title = "Workspace" diff --git a/frontend/editor/src/core/components/annotation/shared/ColorControl.tsx b/frontend/editor/src/core/components/annotation/shared/ColorControl.tsx index ff2a3ff458..ca41f0f6a2 100644 --- a/frontend/editor/src/core/components/annotation/shared/ColorControl.tsx +++ b/frontend/editor/src/core/components/annotation/shared/ColorControl.tsx @@ -8,6 +8,7 @@ import { Group, } from "@mantine/core"; import { useState, useCallback, useEffect } from "react"; +import { useTranslation } from "react-i18next"; import ColorizeIcon from "@mui/icons-material/Colorize"; // safari and firefox do not support the eye dropper API, only edge, chrome and opera do. @@ -33,6 +34,7 @@ export function ColorControl({ label, disabled = false, }: ColorControlProps) { + const { t } = useTranslation(); const [opened, setOpened] = useState(false); // Buffer the colour locally so the picker stays responsive during drag. // Only propagate to the parent (which triggers expensive annotation updates) @@ -111,7 +113,9 @@ export function ColorControl({ /> {supportsEyeDropper && ( - + = ({ onImageChange, disabled = false, }) => { + const { t } = useTranslation(); const [, setImageData] = useState(null); const handleImageUpload = async (file: File | null) => { @@ -57,9 +59,12 @@ export const ImageTool: React.FC = ({ diff --git a/frontend/editor/src/core/components/fileEditor/AddFileCard.tsx b/frontend/editor/src/core/components/fileEditor/AddFileCard.tsx index 5fc62680b3..a7aae35f2f 100644 --- a/frontend/editor/src/core/components/fileEditor/AddFileCard.tsx +++ b/frontend/editor/src/core/components/fileEditor/AddFileCard.tsx @@ -134,7 +134,7 @@ const AddFileCard = ({ )} @@ -442,14 +511,24 @@ const ParticipantView: React.FC = ({ token }) => { {participant.hasCompleted && ( } color="green"> - You have {participant.status === "SIGNED" ? "signed" : "declined"}{" "} - this document. + {participant.status === "SIGNED" + ? t( + "workflow.participant.completedSigned", + "You have signed this document.", + ) + : t( + "workflow.participant.completedDeclined", + "You have declined this document.", + )} )} {participant.isExpired && ( } color="orange"> - Your access to this document has expired. + {t( + "workflow.participant.accessExpired", + "Your access to this document has expired.", + )} )} diff --git a/frontend/editor/src/proprietary/constants/planConstants.ts b/frontend/editor/src/proprietary/constants/planConstants.ts index 6a03a72b7e..bb8a66ac4f 100644 --- a/frontend/editor/src/proprietary/constants/planConstants.ts +++ b/frontend/editor/src/proprietary/constants/planConstants.ts @@ -1,100 +1,219 @@ +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; import type { PlanFeature } from "@app/types/license"; /** * Shared plan feature definitions for Stirling PDF Self-Hosted - * Used by both dynamic (Stripe) and static (fallback) plan displays + * Used by both dynamic (Stripe) and static (fallback) plan displays. + * + * These are exposed as hooks so that every feature/highlight string can be + * localized via `t()` while preserving the original shape. */ -export const PLAN_FEATURES = { - FREE: [ - { name: "Self-hosted deployment", included: true }, - { name: "All PDF operations", included: true }, - { name: "Secure Login Support", included: true }, - { name: "Community support", included: true }, - { name: "Regular updates", included: true }, - { name: "up to 5 users", included: true }, - { name: "Unlimited users", included: false }, - { name: "Google drive integration", included: false }, - { name: "External Database", included: false }, - { name: "Editing text in pdfs", included: false }, - { name: "Users limited to seats", included: false }, - { name: "SSO", included: false }, - { name: "SAML", included: false }, - { name: "Auditing", included: false }, - { name: "Usage tracking", included: false }, - { name: "Prometheus Support", included: false }, - { name: "Custom PDF metadata", included: false }, - ] as PlanFeature[], +export interface PlanFeaturesMap { + FREE: PlanFeature[]; + SERVER: PlanFeature[]; + ENTERPRISE: PlanFeature[]; +} - SERVER: [ - { name: "Self-hosted deployment", included: true }, - { name: "All PDF operations", included: true }, - { name: "Secure Login Support", included: true }, - { name: "Community support", included: true }, - { name: "Regular updates", included: true }, - { name: "Up to 5 users", included: false }, - { name: "Unlimited users", included: true }, - { name: "Google drive integration", included: true }, - { name: "External Database", included: true }, - { name: "Editing text in pdfs", included: true }, - { name: "Users limited to seats", included: false }, - { name: "SSO", included: true }, - { name: "SAML", included: false }, - { name: "Auditing", included: false }, - { name: "Usage tracking", included: false }, - { name: "Prometheus Support", included: false }, - { name: "Custom PDF metadata", included: false }, - ] as PlanFeature[], +export interface PlanHighlightsMap { + FREE: string[]; + SERVER_MONTHLY: string[]; + SERVER_YEARLY: string[]; + ENTERPRISE_MONTHLY: string[]; + ENTERPRISE_YEARLY: string[]; +} - ENTERPRISE: [ - { name: "Self-hosted deployment", included: true }, - { name: "All PDF operations", included: true }, - { name: "Secure Login Support", included: true }, - { name: "Community support", included: true }, - { name: "Regular updates", included: true }, - { name: "up to 5 users", included: false }, - { name: "Unlimited users", included: false }, - { name: "Google drive integration", included: true }, - { name: "External Database", included: true }, - { name: "Editing text in pdfs", included: true }, - { name: "Users limited to seats", included: true }, - { name: "SSO", included: true }, - { name: "SAML", included: true }, - { name: "Auditing", included: true }, - { name: "Usage tracking", included: true }, - { name: "Prometheus Support", included: true }, - { name: "Custom PDF metadata", included: true }, - ] as PlanFeature[], -} as const; +export const usePlanFeatures = (): PlanFeaturesMap => { + const { t } = useTranslation(); -export const PLAN_HIGHLIGHTS = { - FREE: ["Up to 5 users", "Self-hosted", "All basic features"], - SERVER_MONTHLY: [ - "Self-hosted on your infrastructure", - "Unlimited users", - "Advanced integrations", - "SSO (OAuth2/OIDC)", - "Editing text in PDFs", - "Cancel anytime", - ], - SERVER_YEARLY: [ - "Self-hosted on your infrastructure", - "Unlimited users", - "Advanced integrations", - "SSO (OAuth2/OIDC)", - "Editing text in PDFs", - "Save with annual billing", - ], - ENTERPRISE_MONTHLY: [ - "Enterprise features (SAML, Auditing)", - "Usage tracking & Prometheus", - "Custom PDF metadata", - "Per-seat licensing", - ], - ENTERPRISE_YEARLY: [ - "Enterprise features (SAML, Auditing)", - "Usage tracking & Prometheus", - "Custom PDF metadata", - "Save with annual billing", - ], -} as const; + return useMemo(() => { + const selfHostedDeployment = t( + "plan.features.selfHostedDeployment", + "Self-hosted deployment", + ); + const allPdfOperations = t( + "plan.features.allPdfOperations", + "All PDF operations", + ); + const secureLoginSupport = t( + "plan.features.secureLoginSupport", + "Secure Login Support", + ); + const communitySupport = t( + "plan.features.communitySupport", + "Community support", + ); + const regularUpdates = t("plan.features.regularUpdates", "Regular updates"); + const upToFiveUsersLowercase = t( + "plan.features.upToFiveUsersLowercase", + "up to 5 users", + ); + const upToFiveUsers = t("plan.features.upToFiveUsers", "Up to 5 users"); + const unlimitedUsers = t("plan.features.unlimitedUsers", "Unlimited users"); + const googleDriveIntegration = t( + "plan.features.googleDriveIntegration", + "Google drive integration", + ); + const externalDatabase = t( + "plan.features.externalDatabase", + "External Database", + ); + const editingTextInPdfs = t( + "plan.features.editingTextInPdfs", + "Editing text in pdfs", + ); + const usersLimitedToSeats = t( + "plan.features.usersLimitedToSeats", + "Users limited to seats", + ); + const sso = t("plan.features.sso", "SSO"); + const saml = t("plan.features.saml", "SAML"); + const auditing = t("plan.features.auditing", "Auditing"); + const usageTracking = t("plan.features.usageTracking", "Usage tracking"); + const prometheusSupport = t( + "plan.features.prometheusSupport", + "Prometheus Support", + ); + const customPdfMetadata = t( + "plan.features.customPdfMetadata", + "Custom PDF metadata", + ); + + return { + FREE: [ + { name: selfHostedDeployment, included: true }, + { name: allPdfOperations, included: true }, + { name: secureLoginSupport, included: true }, + { name: communitySupport, included: true }, + { name: regularUpdates, included: true }, + { name: upToFiveUsersLowercase, included: true }, + { name: unlimitedUsers, included: false }, + { name: googleDriveIntegration, included: false }, + { name: externalDatabase, included: false }, + { name: editingTextInPdfs, included: false }, + { name: usersLimitedToSeats, included: false }, + { name: sso, included: false }, + { name: saml, included: false }, + { name: auditing, included: false }, + { name: usageTracking, included: false }, + { name: prometheusSupport, included: false }, + { name: customPdfMetadata, included: false }, + ], + SERVER: [ + { name: selfHostedDeployment, included: true }, + { name: allPdfOperations, included: true }, + { name: secureLoginSupport, included: true }, + { name: communitySupport, included: true }, + { name: regularUpdates, included: true }, + { name: upToFiveUsers, included: false }, + { name: unlimitedUsers, included: true }, + { name: googleDriveIntegration, included: true }, + { name: externalDatabase, included: true }, + { name: editingTextInPdfs, included: true }, + { name: usersLimitedToSeats, included: false }, + { name: sso, included: true }, + { name: saml, included: false }, + { name: auditing, included: false }, + { name: usageTracking, included: false }, + { name: prometheusSupport, included: false }, + { name: customPdfMetadata, included: false }, + ], + ENTERPRISE: [ + { name: selfHostedDeployment, included: true }, + { name: allPdfOperations, included: true }, + { name: secureLoginSupport, included: true }, + { name: communitySupport, included: true }, + { name: regularUpdates, included: true }, + { name: upToFiveUsersLowercase, included: false }, + { name: unlimitedUsers, included: false }, + { name: googleDriveIntegration, included: true }, + { name: externalDatabase, included: true }, + { name: editingTextInPdfs, included: true }, + { name: usersLimitedToSeats, included: true }, + { name: sso, included: true }, + { name: saml, included: true }, + { name: auditing, included: true }, + { name: usageTracking, included: true }, + { name: prometheusSupport, included: true }, + { name: customPdfMetadata, included: true }, + ], + }; + }, [t]); +}; + +export const usePlanHighlights = (): PlanHighlightsMap => { + const { t } = useTranslation(); + + return useMemo(() => { + const selfHostedOnInfrastructure = t( + "plan.highlights.selfHostedOnInfrastructure", + "Self-hosted on your infrastructure", + ); + const unlimitedUsers = t( + "plan.highlights.unlimitedUsers", + "Unlimited users", + ); + const advancedIntegrations = t( + "plan.highlights.advancedIntegrations", + "Advanced integrations", + ); + const ssoOAuth = t("plan.highlights.ssoOAuth", "SSO (OAuth2/OIDC)"); + const editingTextInPdfsCaps = t( + "plan.highlights.editingTextInPdfsCaps", + "Editing text in PDFs", + ); + const enterpriseFeatures = t( + "plan.highlights.enterpriseFeatures", + "Enterprise features (SAML, Auditing)", + ); + const usageTrackingPrometheus = t( + "plan.highlights.usageTrackingPrometheus", + "Usage tracking & Prometheus", + ); + const customPdfMetadata = t( + "plan.highlights.customPdfMetadata", + "Custom PDF metadata", + ); + const saveWithAnnualBilling = t( + "plan.highlights.saveWithAnnualBilling", + "Save with annual billing", + ); + + return { + FREE: [ + t("plan.highlights.upToFiveUsers", "Up to 5 users"), + t("plan.highlights.selfHosted", "Self-hosted"), + t("plan.highlights.allBasicFeatures", "All basic features"), + ], + SERVER_MONTHLY: [ + selfHostedOnInfrastructure, + unlimitedUsers, + advancedIntegrations, + ssoOAuth, + editingTextInPdfsCaps, + t("plan.highlights.cancelAnytime", "Cancel anytime"), + ], + SERVER_YEARLY: [ + selfHostedOnInfrastructure, + unlimitedUsers, + advancedIntegrations, + ssoOAuth, + editingTextInPdfsCaps, + saveWithAnnualBilling, + ], + ENTERPRISE_MONTHLY: [ + enterpriseFeatures, + usageTrackingPrometheus, + customPdfMetadata, + t("plan.highlights.perSeatLicensing", "Per-seat licensing"), + ], + ENTERPRISE_YEARLY: [ + enterpriseFeatures, + usageTrackingPrometheus, + customPdfMetadata, + saveWithAnnualBilling, + ], + }; + }, [t]); +}; diff --git a/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx b/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx index 5c6406639a..45912667fb 100644 --- a/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx +++ b/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx @@ -24,6 +24,10 @@ import { import { useLicense } from "@app/contexts/LicenseContext"; import { isSupabaseConfigured } from "@app/services/supabaseClient"; import { getPreferredCurrency } from "@app/utils/currencyDetection"; +import { + usePlanFeatures, + usePlanHighlights, +} from "@app/constants/planConstants"; export interface CheckoutOptions { minimumSeats?: number; // Override calculated seats for enterprise @@ -57,6 +61,8 @@ export const CheckoutProvider: React.FC = ({ }) => { const { t, i18n } = useTranslation(); const { refetchLicense } = useLicense(); + const planFeatures = usePlanFeatures(); + const planHighlights = usePlanHighlights(); const [isOpen, setIsOpen] = useState(false); const [isLoading, setIsLoading] = useState(false); const [selectedPlanGroup, setSelectedPlanGroup] = @@ -85,7 +91,11 @@ export const CheckoutProvider: React.FC = ({ try { setPlansLoading(true); - const response = await licenseService.getPlans(currency); + const response = await licenseService.getPlans( + planFeatures, + planHighlights, + currency, + ); setPlans(response.plans); setPlansLoaded(true); } catch (error) { @@ -95,7 +105,7 @@ export const CheckoutProvider: React.FC = ({ setPlansLoading(false); } }, - [plansLoading], + [plansLoading, planFeatures, planHighlights], ); const refetchPlans = useCallback(() => { diff --git a/frontend/editor/src/proprietary/hooks/usePlans.ts b/frontend/editor/src/proprietary/hooks/usePlans.ts index 8a9d0c13f2..45b7d8988e 100644 --- a/frontend/editor/src/proprietary/hooks/usePlans.ts +++ b/frontend/editor/src/proprietary/hooks/usePlans.ts @@ -1,8 +1,12 @@ -import { useState, useEffect } from "react"; +import { useState, useEffect, useCallback } from "react"; import licenseService, { PlanTier, PlansResponse, } from "@app/services/licenseService"; +import { + usePlanFeatures, + usePlanHighlights, +} from "@app/constants/planConstants"; export interface UsePlansReturn { plans: PlanTier[]; @@ -12,16 +16,22 @@ export interface UsePlansReturn { } export const usePlans = (currency: string = "gbp"): UsePlansReturn => { + const planFeatures = usePlanFeatures(); + const planHighlights = usePlanHighlights(); const [plans, setPlans] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); - const fetchPlans = async () => { + const fetchPlans = useCallback(async () => { try { setLoading(true); setError(null); - const data: PlansResponse = await licenseService.getPlans(currency); + const data: PlansResponse = await licenseService.getPlans( + planFeatures, + planHighlights, + currency, + ); setPlans(data.plans); } catch (err) { console.error("Error fetching plans:", err); @@ -29,11 +39,11 @@ export const usePlans = (currency: string = "gbp"): UsePlansReturn => { } finally { setLoading(false); } - }; + }, [currency, planFeatures, planHighlights]); useEffect(() => { fetchPlans(); - }, [currency]); + }, [fetchPlans]); return { plans, diff --git a/frontend/editor/src/proprietary/services/licenseService.ts b/frontend/editor/src/proprietary/services/licenseService.ts index 600498b333..e87f304eb6 100644 --- a/frontend/editor/src/proprietary/services/licenseService.ts +++ b/frontend/editor/src/proprietary/services/licenseService.ts @@ -1,7 +1,10 @@ import apiClient from "@app/services/apiClient"; import { supabase, isSupabaseConfigured } from "@app/services/supabaseClient"; import { getCheckoutMode } from "@app/utils/protocolDetection"; -import { PLAN_FEATURES, PLAN_HIGHLIGHTS } from "@app/constants/planConstants"; +import type { + PlanFeaturesMap, + PlanHighlightsMap, +} from "@app/constants/planConstants"; import type { LicenseInfo, PlanFeature } from "@app/types/license"; export interface PlanTier { @@ -102,9 +105,18 @@ const SELF_HOSTED_LOOKUP_KEYS = [ const licenseService = { /** - * Get available plans with pricing for the specified currency + * Get available plans with pricing for the specified currency. + * + * Feature and highlight labels are passed in by the caller (a React hook / + * context that resolves them via `usePlanFeatures` / `usePlanHighlights`), + * which keeps this service free of React/i18n dependencies while still + * returning fully localized plan content. */ - async getPlans(currency: string = "usd"): Promise { + async getPlans( + planFeatures: PlanFeaturesMap, + planHighlights: PlanHighlightsMap, + currency: string = "usd", + ): Promise { try { // Check if Supabase is configured if (!isSupabaseConfigured || !supabase) { @@ -179,8 +191,8 @@ const licenseService = { currency: currencySymbol, period: "/month", popular: false, - features: PLAN_FEATURES.SERVER, - highlights: PLAN_HIGHLIGHTS.SERVER_MONTHLY, + features: planFeatures.SERVER, + highlights: planHighlights.SERVER_MONTHLY, }, { id: "selfhosted:server:yearly", @@ -190,8 +202,8 @@ const licenseService = { currency: currencySymbol, period: "/year", popular: true, - features: PLAN_FEATURES.SERVER, - highlights: PLAN_HIGHLIGHTS.SERVER_YEARLY, + features: planFeatures.SERVER, + highlights: planHighlights.SERVER_YEARLY, }, { id: "selfhosted:enterprise:monthly", @@ -203,8 +215,8 @@ const licenseService = { period: "/month", popular: false, requiresSeats: true, - features: PLAN_FEATURES.ENTERPRISE, - highlights: PLAN_HIGHLIGHTS.ENTERPRISE_MONTHLY, + features: planFeatures.ENTERPRISE, + highlights: planHighlights.ENTERPRISE_MONTHLY, }, { id: "selfhosted:enterprise:yearly", @@ -216,8 +228,8 @@ const licenseService = { period: "/year", popular: false, requiresSeats: true, - features: PLAN_FEATURES.ENTERPRISE, - highlights: PLAN_HIGHLIGHTS.ENTERPRISE_YEARLY, + features: planFeatures.ENTERPRISE, + highlights: planHighlights.ENTERPRISE_YEARLY, }, ]; @@ -240,8 +252,8 @@ const licenseService = { currency: currencySymbol, period: "", popular: false, - features: PLAN_FEATURES.FREE, - highlights: PLAN_HIGHLIGHTS.FREE, + features: planFeatures.FREE, + highlights: planHighlights.FREE, }; const allPlans = [freePlan, ...validPlans]; From d202c9c32f50954326be9d505c65ed745d43357c Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:31:41 +0100 Subject: [PATCH 02/19] Minor: sanitize SVG (#6572) # Description of Changes Sanitize SVG --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .../api/misc/OverlayImageController.java | 5 ++ .../software/SPDF/utils/SvgOverlayUtil.java | 12 ++++- .../api/misc/OverlayImageControllerTest.java | 48 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java index c55597b47d..557c77c76e 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java @@ -25,6 +25,7 @@ import stirling.software.common.annotations.api.MiscApi; import stirling.software.common.enumeration.ResourceWeight; import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.GeneralUtils; +import stirling.software.common.util.SvgSanitizer; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; import stirling.software.common.util.WebResponseUtils; @@ -36,6 +37,7 @@ public class OverlayImageController { private final CustomPDFDocumentFactory pdfDocumentFactory; private final TempFileManager tempFileManager; + private final SvgSanitizer svgSanitizer; @AutoJobPostMapping( consumes = MediaType.MULTIPART_FORM_DATA_VALUE, @@ -61,6 +63,9 @@ public class OverlayImageController { byte[] imageBytes = imageFile.getBytes(); boolean isSvg = SvgOverlayUtil.isSvgImage(imageBytes); + if (isSvg) { + imageBytes = svgSanitizer.sanitize(imageBytes); + } try (PDDocument document = pdfDocumentFactory.load(pdfBytes)) { int pages = document.getNumberOfPages(); diff --git a/app/core/src/main/java/stirling/software/SPDF/utils/SvgOverlayUtil.java b/app/core/src/main/java/stirling/software/SPDF/utils/SvgOverlayUtil.java index 253d9c687a..b58bb055fc 100644 --- a/app/core/src/main/java/stirling/software/SPDF/utils/SvgOverlayUtil.java +++ b/app/core/src/main/java/stirling/software/SPDF/utils/SvgOverlayUtil.java @@ -10,6 +10,7 @@ import org.apache.batik.bridge.GVTBuilder; import org.apache.batik.bridge.UserAgent; import org.apache.batik.bridge.UserAgentAdapter; import org.apache.batik.gvt.GraphicsNode; +import org.apache.batik.util.ParsedURL; import org.apache.batik.util.XMLResourceDescriptor; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; @@ -39,7 +40,16 @@ public class SvgOverlayUtil { svgDoc = factory.createSVGDocument("file:///overlay.svg", inputStream); } - UserAgent userAgent = new UserAgentAdapter(); + UserAgent userAgent = + new UserAgentAdapter() { + @Override + public void checkLoadExternalResource( + ParsedURL resourceURL, ParsedURL docURL) { + throw new SecurityException( + "External resource loading is disabled for SVG overlays: " + + resourceURL); + } + }; DocumentLoader loader = new DocumentLoader(userAgent); BridgeContext ctx = new BridgeContext(userAgent, loader); ctx.setDynamicState(BridgeContext.DYNAMIC); diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/misc/OverlayImageControllerTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/misc/OverlayImageControllerTest.java index 308518e8f0..6a13fb3049 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/misc/OverlayImageControllerTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/misc/OverlayImageControllerTest.java @@ -32,6 +32,7 @@ import org.springframework.mock.web.MockMultipartFile; import stirling.software.SPDF.model.api.misc.OverlayImageRequest; import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.util.SvgSanitizer; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; import stirling.software.common.util.WebResponseUtils; @@ -52,6 +53,7 @@ class OverlayImageControllerTest { @Mock private CustomPDFDocumentFactory pdfDocumentFactory; @Mock private TempFileManager tempFileManager; + @Mock private SvgSanitizer svgSanitizer; @InjectMocks private OverlayImageController controller; @@ -205,6 +207,52 @@ class OverlayImageControllerTest { mockDoc.close(); } + @Test + void overlayImage_svgInput_sanitizedBeforeOverlay() throws Exception { + byte[] maliciousSvg = + ("" + + "" + + "") + .getBytes(); + byte[] sanitized = + ("" + + "" + + "") + .getBytes(); + when(svgSanitizer.sanitize(maliciousSvg)).thenReturn(sanitized); + + MockMultipartFile svgFile = + new MockMultipartFile("imageFile", "overlay.svg", "image/svg+xml", maliciousSvg); + OverlayImageRequest request = new OverlayImageRequest(); + request.setFileInput(pdfFile); + request.setImageFile(svgFile); + request.setX(0); + request.setY(0); + request.setEveryPage(false); + + PDDocument mockDoc = new PDDocument(); + mockDoc.addPage(new PDPage(PDRectangle.A4)); + when(pdfDocumentFactory.load(any(byte[].class))).thenReturn(mockDoc); + + try (MockedStatic mockedWebResponse = + mockStatic(WebResponseUtils.class)) { + mockedWebResponse + .when( + () -> + WebResponseUtils.pdfFileToWebResponse( + any(TempFile.class), anyString())) + .thenReturn(streamingOk("result".getBytes())); + + controller.overlayImage(request); + } + mockDoc.close(); + + verify(svgSanitizer).sanitize(maliciousSvg); + } + @Test void overlayImage_withCoordinates_usesXY() throws Exception { OverlayImageRequest request = new OverlayImageRequest(); From 8a2474ff60ae1e4eea275050cf14a0578b1c904f Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:36:58 +0100 Subject: [PATCH 03/19] fix(desktop): enable in-page drag-drop in Tauri build (#6548) ## Summary - Set `dragDropEnabled: false` on the Tauri window so HTML5 drag events reach the WebView. Previously the default `true` made Tauri intercept all drag-drop at the OS level, silently breaking in-page drag-to-reorder (Pragmatic Drag and Drop in `FileEditorThumbnail` / `useFileItemDragDrop`) in the desktop build. The Active Files tab reorder, which feeds Merge ordering, was the user-visible symptom. - Browser builds are unaffected (tauri.conf.json is desktop-only). - The OS file-drop pipeline now flows through the existing Mantine `Dropzone` in `FileEditor.tsx` via HTML5 events instead of the Rust `WindowEvent::DragDrop` handler in `lib.rs:215`. Verified working. ## Test plan - [x] Desktop: drag a thumbnail in Active Files past another - row goes semi-transparent, order updates on drop. - [x] Desktop: drag a PDF from File Explorer onto the window - file is added. - [x] Web build: drag-to-reorder still works (unchanged code path; flag is desktop-only). - [x] Merge tool: order set by drag in Active Files is the order used by the merge output. ## Follow-up (not in this PR) - `WindowEvent::DragDrop` arm in `frontend/editor/src-tauri/src/lib.rs:215-229` is now unreachable for window drops. The `forward_files_to_window` helper still serves the macOS Finder "Open With" path (`RunEvent::Opened` at lib.rs:230), so only the DragDrop arm can be deleted. Worth a small cleanup pass later. --- frontend/editor/src-tauri/tauri.conf.json | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/editor/src-tauri/tauri.conf.json b/frontend/editor/src-tauri/tauri.conf.json index 5c2168f72b..c2e84b3ac1 100644 --- a/frontend/editor/src-tauri/tauri.conf.json +++ b/frontend/editor/src-tauri/tauri.conf.json @@ -17,6 +17,7 @@ "height": 800, "resizable": true, "fullscreen": false, + "dragDropEnabled": false, "additionalBrowserArgs": "--enable-features=CertVerifierBuiltinFeature" } ] From af52134811cd4a57e856ca8a39e7913483bf0a5d Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:38:00 +0100 Subject: [PATCH 04/19] fix(automate): flip AutomationEntry tooltip to position=left (#6550) # Description of Changes automate description tooltip was facing wrong way --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .../src/core/components/tools/automate/AutomationEntry.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/editor/src/core/components/tools/automate/AutomationEntry.tsx b/frontend/editor/src/core/components/tools/automate/AutomationEntry.tsx index d7fdc43b69..de8c82d449 100644 --- a/frontend/editor/src/core/components/tools/automate/AutomationEntry.tsx +++ b/frontend/editor/src/core/components/tools/automate/AutomationEntry.tsx @@ -321,7 +321,7 @@ export default function AutomationEntry({ return shouldShowTooltip ? ( From 2f6b113a1330a3e08a28975c628c306c525e3f6b Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:40:35 +0100 Subject: [PATCH 05/19] feat(settings): link ENTERPRISE badges to plan page (#6560) ## Summary Make the remaining static ENTERPRISE badges in the admin settings clickable so they navigate the user to `/settings/adminPlan`, matching the pattern already used by the PRO badges in Connections / Features / General sections. ### Before Two ENTERPRISE badges were inert text chips with no affordance: - `AdminSecuritySection.tsx` - Audit Logging - `AdminDatabaseSection.tsx` - Database section header ### After Both now use the same pattern as the existing clickable PRO badges: - `cursor: pointer` - `onClick={() => navigate("/settings/adminPlan")}` - `title` tooltip with the existing `admin.settings.badge.clickToUpgrade` i18n key ("Click to view plan details") No new strings, no new components - just wiring up existing behavior to the two badges that were missing it. ### Existing already-clickable badges (kept identical for reference) - `AdminConnectionsSection.tsx:585-596` - SSO Auto Login PRO - `AdminFeaturesSection.tsx:175-186` - Server Certificate PRO - `AdminGeneralSection.tsx:920-931` - Custom Metadata PRO --- .../config/configSections/AdminDatabaseSection.tsx | 13 ++++++++++++- .../config/configSections/AdminSecuritySection.tsx | 13 ++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminDatabaseSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminDatabaseSection.tsx index b8959c8ca9..7fd272fb5e 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminDatabaseSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminDatabaseSection.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { isAxiosError } from "axios"; +import { useNavigate } from "react-router-dom"; import { useTranslation } from "react-i18next"; import { NumberInput, @@ -52,6 +53,7 @@ interface DatabaseSettingsData { export default function AdminDatabaseSection() { const { t } = useTranslation(); + const navigate = useNavigate(); const { loginEnabled, validateLoginEnabled, getDisabledStyles } = useLoginRequired(); const { @@ -462,7 +464,16 @@ export default function AdminDatabaseSection() { )} - + navigate("/settings/adminPlan")} + title={t( + "admin.settings.badge.clickToUpgrade", + "Click to view plan details", + )} + > ENTERPRISE diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminSecuritySection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminSecuritySection.tsx index 6c701833fe..20e559d9e2 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminSecuritySection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminSecuritySection.tsx @@ -1,4 +1,5 @@ import { useCallback, useEffect } from "react"; +import { useNavigate } from "react-router-dom"; import { useTranslation } from "react-i18next"; import { NumberInput, @@ -67,6 +68,7 @@ interface SecuritySettingsData { export default function AdminSecuritySection() { const { t } = useTranslation(); + const navigate = useNavigate(); const { loginEnabled, validateLoginEnabled } = useLoginRequired(); const { restartModalOpened, @@ -817,7 +819,16 @@ export default function AdminSecuritySection() { {t("admin.settings.security.audit.label", "Audit Logging")} - + navigate("/settings/adminPlan")} + title={t( + "admin.settings.badge.clickToUpgrade", + "Click to view plan details", + )} + > ENTERPRISE From 69e62d8949fe9d98ba148aeea7be0b5a54106fc2 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:41:38 +0100 Subject: [PATCH 06/19] exclude unused Redis auto-config (#6547) fix(health): exclude unused Redis auto-config so default /actuator/health stays UP --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- app/core/src/main/resources/application.properties | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/core/src/main/resources/application.properties b/app/core/src/main/resources/application.properties index fc2fda630b..da564e454b 100644 --- a/app/core/src/main/resources/application.properties +++ b/app/core/src/main/resources/application.properties @@ -93,6 +93,11 @@ posthog.host=https://eu.i.posthog.com spring.main.allow-bean-definition-overriding=true +# spring-data-redis is on the classpath only for the optional Valkey backplane (which wires its own +# factory); exclude Spring Boot's stock Redis auto-config so a default install doesn't create a dead +# localhost:6379 factory that flips /actuator/health to DOWN. +spring.autoconfigure.exclude=org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration + # Set up a consistent temporary directory location java.io.tmpdir=${stirling.tempfiles.directory:${java.io.tmpdir}/stirling-pdf} From 51478e5051e55ce49687c6dc3dfabf4ca92c6072 Mon Sep 17 00:00:00 2001 From: James Brunton Date: Mon, 8 Jun 2026 11:50:55 +0100 Subject: [PATCH 07/19] Policies backend (#6527) # Description of Changes Add a backend for running any multi-step PDF operations. This is designed to be used for the upcoming Policies feature, along with anything else that will require automated running of PDF operations, like the Automate tool or Processing Folders. The implementation is not complete. I've tried to get all the infrastructure in there so that we can add in whichever triggers we need in the future (like cron triggers or watching folders on disk) but currently it just supports manual triggering of the policy. The basis of this work was the operation running from the Stirling Engine, which this PR removes in favour of this new system. The only currently accessible frontend way to test this work is to ask the AI chat to execute multiple operations on a PDF, but I've also extensively tested with direct API calls to make sure that the policies work and persist properly. --- .../common/service/ToolMetadataService.java | 10 + .../software/SPDF/service/ApiDocService.java | 1 + .../policy/controller/PolicyController.java | 319 +++++++++++++++ .../policy/engine/PolicyEngine.java | 261 ++++++++++++ .../policy/engine/PolicyExecutionResult.java | 17 + .../policy/engine/PolicyExecutor.java | 312 +++++++++++++++ .../engine/PolicyInputRequiredException.java | 32 ++ .../policy/engine/PolicyRunHandle.java | 16 + .../policy/engine/PolicyRunRegistry.java | 97 +++++ .../proprietary/policy/model/OutputSpec.java | 21 + .../policy/model/PipelineDefinition.java | 16 + .../policy/model/PipelineStep.java | 30 ++ .../proprietary/policy/model/Policy.java | 38 ++ .../policy/model/PolicyInputs.java | 32 ++ .../proprietary/policy/model/PolicyRun.java | 87 ++++ .../policy/model/PolicyRunStatus.java | 21 + .../policy/model/PolicyRunView.java | 28 ++ .../policy/model/TriggerConfig.java | 25 ++ .../proprietary/policy/model/WaitState.java | 19 + .../policy/output/InlineOutputSink.java | 68 ++++ .../policy/output/PolicyOutputSink.java | 35 ++ .../progress/PolicyProgressListener.java | 22 + .../policy/store/InProcessPolicyStore.java | 61 +++ .../policy/store/JpaPolicyStore.java | 83 ++++ .../policy/store/PolicyEntity.java | 50 +++ .../policy/store/PolicyRepository.java | 13 + .../proprietary/policy/store/PolicyStore.java | 28 ++ .../policy/trigger/ManualTrigger.java | 41 ++ .../policy/trigger/PolicyTrigger.java | 24 ++ .../configuration/DatabaseConfig.java | 6 +- .../service/AiWorkflowService.java | 225 +++-------- .../policy/engine/PolicyEngineTest.java | 255 ++++++++++++ .../policy/engine/PolicyExecutorTest.java | 378 ++++++++++++++++++ .../policy/engine/PolicyRunRegistryTest.java | 101 +++++ .../store/InProcessPolicyStoreTest.java | 87 ++++ .../policy/store/JpaPolicyStoreTest.java | 129 ++++++ .../service/AiWorkflowServiceTest.java | 7 +- 37 files changed, 2819 insertions(+), 176 deletions(-) create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyInputRequiredException.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunHandle.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineDefinition.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineStep.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyInputs.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRun.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunStatus.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/WaitState.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/progress/PolicyProgressListener.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyExecutorTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java diff --git a/app/common/src/main/java/stirling/software/common/service/ToolMetadataService.java b/app/common/src/main/java/stirling/software/common/service/ToolMetadataService.java index 662878b741..fb7a928d76 100644 --- a/app/common/src/main/java/stirling/software/common/service/ToolMetadataService.java +++ b/app/common/src/main/java/stirling/software/common/service/ToolMetadataService.java @@ -1,11 +1,21 @@ package stirling.software.common.service; +import java.util.List; + /** Provides metadata about tool endpoints for internal dispatch. */ public interface ToolMetadataService { /** Returns true if the given operation path accepts multiple input files. */ boolean isMultiInput(String operationPath); + /** + * Returns the file extensions (lowercase, no leading dot, e.g. {@code "pdf"}) that the + * operation accepts as input ({@code output=false}) or produces as output ({@code + * output=true}), derived from the endpoint's declared type. Returns {@code null} when the + * endpoint declares no specific type, which callers should treat as "any type accepted". + */ + List getExtensionTypes(boolean output, String operationPath); + /** * Returns true when the endpoint's ZIP response is a transport for multiple typed results and * should be unpacked: multi-output endpoints (Type:SIMO / Type:MIMO) and wrapper declarations diff --git a/app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java b/app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java index b0d2e5a4a0..3a8d418aa6 100644 --- a/app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java +++ b/app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java @@ -63,6 +63,7 @@ public class ApiDocService implements stirling.software.common.service.ToolMetad return "http://localhost:" + port + contextPath + "/v1/api-docs"; } + @Override public List getExtensionTypes(boolean output, String operationName) { if (outputToFileTypes.isEmpty()) { outputToFileTypes.put("PDF", List.of("pdf")); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java new file mode 100644 index 0000000000..0a94eef1fc --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -0,0 +1,319 @@ +package stirling.software.proprietary.policy.controller; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.util.MultiValueMap; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.MultipartHttpServletRequest; +import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import io.github.pixee.security.Filenames; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.job.JobResponse; +import stirling.software.common.util.TempFile; +import stirling.software.common.util.TempFileManager; +import stirling.software.proprietary.policy.engine.PolicyRunHandle; +import stirling.software.proprietary.policy.engine.PolicyRunRegistry; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.PolicyRunStatus; +import stirling.software.proprietary.policy.model.PolicyRunView; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; +import stirling.software.proprietary.policy.store.PolicyStore; +import stirling.software.proprietary.policy.trigger.ManualTrigger; +import stirling.software.proprietary.security.config.PremiumEndpoint; + +import tools.jackson.core.JacksonException; +import tools.jackson.databind.ObjectMapper; + +/** + * Manages policies and runs pipelines. The premium backend entry point: CRUD for stored {@code + * Policy} objects, running a stored policy by id, and running an ad-hoc pipeline (for AI/Automate + * one-offs). + * + *

Runs execute asynchronously and return a run id immediately. Poll {@code GET /run/{runId}} for + * status, and download outputs via the existing {@code GET /api/v1/general/files/{fileId}} using + * the file ids in the run view. + */ +@Slf4j +@RestController +@RequestMapping("/api/v1/policies") +@Hidden +@PremiumEndpoint +@RequiredArgsConstructor +@Tag(name = "Policies", description = "Run tool pipelines on the backend") +public class PolicyController { + + private final ManualTrigger manualTrigger; + private final PolicyRunRegistry runRegistry; + private final PolicyStore policyStore; + private final ObjectMapper objectMapper; + private final TempFileManager tempFileManager; + + /** SSE emitter timeout, generous enough for long multi-step runs on large files. */ + @Value("${stirling.policies.streamTimeoutMs:1800000}") + private long streamTimeoutMs; + + @PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Run a tool pipeline", + description = + "Accepts the documents to process (multipart field 'fileInput'), any supporting" + + " files (each under a multipart field named as its asset key, e.g." + + " 'company-logo'), and a JSON pipeline definition ('json'). Runs the" + + " steps in order asynchronously and returns a run id. Poll the run" + + " status endpoint and download outputs via /api/v1/general/files/{id}.") + public ResponseEntity> run( + @RequestParam("json") String json, MultipartHttpServletRequest request) + throws IOException { + PipelineDefinition definition = parseDefinition(json); + PolicyInputs inputs = collectInputs(request); + String runId = manualTrigger.fire(definition, inputs, PolicyProgressListener.NOOP).runId(); + return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); + } + + @PostMapping(value = "/run/stream", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Run a tool pipeline with live progress", + description = + "Same as /run, but returns Server-Sent Events: a 'step' event as each step" + + " starts and completes, then a terminal 'completed', 'failed'," + + " 'cancelled', or 'waiting' event carrying the final run view.") + public SseEmitter runStream( + @RequestParam("json") String json, MultipartHttpServletRequest request) + throws IOException { + PipelineDefinition definition = parseDefinition(json); + PolicyInputs inputs = collectInputs(request); + + SseEmitter emitter = new SseEmitter(streamTimeoutMs); + emitter.onError(e -> log.warn("Policy run SSE emitter error", e)); + + PolicyRunHandle handle = manualTrigger.fire(definition, inputs, streamListener(emitter)); + // Close the stream with a terminal event once the run finishes. whenComplete runs on the + // engine's worker thread after the run is done, so this never races the step events. + handle.completion() + .whenComplete( + (run, throwable) -> { + if (throwable != null) { + sendEvent( + emitter, + "failed", + Map.of("message", throwable.getMessage())); + } else { + sendEvent(emitter, terminalEventName(run), PolicyRunView.of(run)); + } + emitter.complete(); + }); + return emitter; + } + + @GetMapping("/run/{runId}") + @Operation( + summary = "Get pipeline run status", + description = "Returns the current status, step cursor, and output files of a run.") + public ResponseEntity status(@PathVariable String runId) { + PolicyRun run = runRegistry.get(runId); + if (run == null) { + return ResponseEntity.notFound().build(); + } + return ResponseEntity.ok(PolicyRunView.of(run)); + } + + // --- Policy management --- + + @PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE) + @Operation( + summary = "Create or update a policy", + description = + "Stores a policy (trigger config + steps + output + metadata). A blank id is" + + " assigned; returns the stored policy with its id.") + public ResponseEntity savePolicy(@RequestBody String json) { + return ResponseEntity.ok(policyStore.save(parsePolicy(json))); + } + + @GetMapping + @Operation(summary = "List policies") + public List listPolicies() { + return policyStore.all(); + } + + @GetMapping("/{policyId}") + @Operation(summary = "Get a policy by id") + public ResponseEntity getPolicy(@PathVariable String policyId) { + return policyStore + .get(policyId) + .map(ResponseEntity::ok) + .orElseGet(() -> ResponseEntity.notFound().build()); + } + + @DeleteMapping("/{policyId}") + @Operation(summary = "Delete a policy by id") + public ResponseEntity deletePolicy(@PathVariable String policyId) { + return policyStore.delete(policyId) + ? ResponseEntity.noContent().build() + : ResponseEntity.notFound().build(); + } + + @PostMapping(value = "/{policyId}/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Run a stored policy", + description = + "Runs the stored policy's pipeline on the supplied files (primary documents" + + " under 'fileInput', supporting files under their asset-key fields)." + + " Runs regardless of the policy's enabled flag, which only gates" + + " automatic triggering. Returns a run id.") + public ResponseEntity> runStoredPolicy( + @PathVariable String policyId, MultipartHttpServletRequest request) throws IOException { + Policy policy = + policyStore + .get(policyId) + .orElseThrow( + () -> + new ResponseStatusException( + HttpStatus.NOT_FOUND, "No policy: " + policyId)); + PolicyInputs inputs = collectInputs(request); + String runId = manualTrigger.run(policy, inputs, PolicyProgressListener.NOOP).runId(); + return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); + } + + private Policy parsePolicy(String json) { + try { + return objectMapper.readValue(json, Policy.class); + } catch (JacksonException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid policy JSON"); + } + } + + private PipelineDefinition parseDefinition(String json) { + PipelineDefinition definition; + try { + definition = objectMapper.readValue(json, PipelineDefinition.class); + } catch (JacksonException e) { + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, "Invalid pipeline definition JSON"); + } + if (definition.steps().isEmpty()) { + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, "Pipeline definition has no steps"); + } + return definition; + } + + /** + * Split the multipart file parts into the primary document stream ("fileInput") and the named + * supporting-file store: every other file field becomes an asset keyed by its field name, which + * a step references from {@code fileParameters}. + */ + private PolicyInputs collectInputs(MultipartHttpServletRequest request) throws IOException { + MultiValueMap fileMap = request.getMultiFileMap(); + List primary = toResources(fileMap.get("fileInput")); + Map> supportingFiles = new LinkedHashMap<>(); + for (Map.Entry> entry : fileMap.entrySet()) { + if ("fileInput".equals(entry.getKey())) { + continue; + } + List assets = toResources(entry.getValue()); + if (!assets.isEmpty()) { + supportingFiles.put(entry.getKey(), assets); + } + } + return new PolicyInputs(primary, supportingFiles); + } + + /** + * A progress listener that forwards each step transition to the SSE stream as a "step" event. + */ + private PolicyProgressListener streamListener(SseEmitter emitter) { + return new PolicyProgressListener() { + @Override + public void onStepStart(int stepIndex, int stepCount, String operation) { + sendEvent(emitter, "step", stepEvent("started", stepIndex, stepCount, operation)); + } + + @Override + public void onStepComplete(int stepIndex, int stepCount, String operation) { + sendEvent(emitter, "step", stepEvent("completed", stepIndex, stepCount, operation)); + } + }; + } + + private static Map stepEvent( + String phase, int stepIndex, int stepCount, String operation) { + return Map.of( + "phase", phase, + "stepIndex", stepIndex, + "stepCount", stepCount, + "operation", operation); + } + + private static String terminalEventName(PolicyRun run) { + PolicyRunStatus status = run.getStatus(); + return switch (status) { + case COMPLETED -> "completed"; + case FAILED -> "failed"; + case CANCELLED -> "cancelled"; + case WAITING_FOR_INPUT -> "waiting"; + default -> "ended"; + }; + } + + private void sendEvent(SseEmitter emitter, String name, Object data) { + try { + emitter.send(SseEmitter.event().name(name).data(data, MediaType.APPLICATION_JSON)); + } catch (IOException | IllegalStateException e) { + // Client disconnected or the emitter already closed. The run continues and its results + // remain downloadable via the job endpoints; nothing useful left to stream. + log.debug("Dropping policy SSE event '{}': {}", name, e.getMessage()); + } + } + + private List toResources(List files) throws IOException { + List resources = new ArrayList<>(); + if (files == null) { + return resources; + } + for (MultipartFile file : files) { + if (file == null || file.isEmpty()) { + continue; + } + TempFile tempFile = tempFileManager.createManagedTempFile("policy-run"); + file.transferTo(tempFile.getPath()); + final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename()); + resources.add( + new FileSystemResource(tempFile.getFile()) { + @Override + public String getFilename() { + return originalName; + } + }); + } + return resources; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java new file mode 100644 index 0000000000..8f41209c7e --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java @@ -0,0 +1,261 @@ +package stirling.software.proprietary.policy.engine; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.job.ResultFile; +import stirling.software.common.service.FileStorage; +import stirling.software.common.service.InternalApiTimeoutException; +import stirling.software.common.service.JobOwnershipService; +import stirling.software.common.service.JobQueue; +import stirling.software.common.service.ResourceMonitor; +import stirling.software.common.service.TaskManager; +import stirling.software.common.util.ExecutorFactory; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.WaitState; +import stirling.software.proprietary.policy.output.PolicyOutputSink; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +/** + * Runs pipelines asynchronously as tracked jobs. + * + *

Each run is the unit of async work: {@link #submit} returns a run id immediately and the + * pipeline executes on a virtual thread, so a step blocking on a slow tool does not tie up a + * platform thread. The run drives {@link PolicyExecutor} for the actual step loop, registers its + * outputs and progress with {@link TaskManager} (so the existing job status/download endpoints work + * unchanged), and keeps rich state in {@link PolicyRunRegistry}. + * + *

The engine deliberately manages its own virtual-thread execution rather than routing through + * {@code JobExecutorService}: that path force-completes a job once its work returns, which is + * incompatible with a run that suspends in {@code WAITING_FOR_INPUT}. It still applies the shared + * {@link ResourceMonitor}/{@link JobQueue} admission control, so heavy runs queue under load + * instead of oversubscribing. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class PolicyEngine { + + /** + * Resource weight of a pipeline run for admission control. A run chains many tools and holds + * intermediate files, so it is weighted as heavy work: the shared {@link ResourceMonitor} + * should let it start while the system is healthy but hold it back under memory/CPU pressure. + * See {@link ResourceMonitor#shouldQueueJob(int)} for how a weight maps to that decision. + */ + private static final int RUN_RESOURCE_WEIGHT = 50; + + private final PolicyExecutor stepExecutor; + private final TaskManager taskManager; + private final PolicyRunRegistry registry; + private final FileStorage fileStorage; + private final JobOwnershipService jobOwnershipService; + private final List outputSinks; + private final ResourceMonitor resourceMonitor; + private final JobQueue jobQueue; + + private final ExecutorService asyncExecutor = ExecutorFactory.newVirtualThreadExecutor(); + + /** + * Submit a pipeline to run asynchronously. The returned handle's run id scopes a job in {@link + * TaskManager}, so progress (notes), status, and result files are observable via the existing + * job endpoints as well as via {@link #getRun(String)}; its completion future resolves when the + * run reaches a terminal or paused state. + */ + public PolicyRunHandle submit( + PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) { + // Scope the run id to the current user (on this request thread) so the file-download + // ownership check passes; NoOpJobOwnershipService returns the id unchanged when security + // is off. + String runId = jobOwnershipService.createScopedJobKey(UUID.randomUUID().toString()); + taskManager.createTask(runId); + PolicyRun run = new PolicyRun(runId, definition); + registry.register(run); + CompletableFuture completion = new CompletableFuture<>(); + PolicyProgressListener tracking = trackingListener(runId, run, listener); + Runnable task = () -> runToCompletion(run, inputs, tracking, completion); + + // Each run is one admission unit; steps run synchronously within it, so this gates heavy + // work under load without the pool-within-pool risk of queueing each tool call. Under + // resource pressure the run waits in the shared JobQueue; otherwise it starts immediately. + if (resourceMonitor.shouldQueueJob(RUN_RESOURCE_WEIGHT)) { + log.debug("Queueing policy run {} under resource pressure", runId); + jobQueue.queueJob( + runId, + RUN_RESOURCE_WEIGHT, + () -> { + task.run(); + return null; + }, + 0L) + .exceptionally(ex -> failRejectedRun(run, completion, ex)); + } else { + asyncExecutor.execute(task); + } + return new PolicyRunHandle(runId, completion); + } + + /** + * Run a stored policy on demand. Builds the policy's pipeline and submits it. {@code enabled} + * gates automatic triggering, not explicit runs, so this runs regardless of that flag. + */ + public PolicyRunHandle runPolicy( + Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { + return submit(policy.toDefinition(), inputs, listener); + } + + public PolicyRun getRun(String runId) { + return registry.get(runId); + } + + /** + * Request cancellation of a run. Stage 1 marks the run cancelled in the registry if it has not + * already finished; interrupting an in-flight tool call lands in a later stage. + */ + public boolean cancel(String runId) { + PolicyRun run = registry.get(runId); + if (run == null) { + return false; + } + boolean cancelled = run.cancel(); + if (cancelled) { + taskManager.addNote(runId, "Run cancelled by request"); + } + return cancelled; + } + + /** + * Resume a run paused in {@code WAITING_FOR_INPUT}. Not yet implemented; the run shape and + * {@link WaitState} snapshot are in place so this can be added without reworking the engine. + */ + public String resume(String runId, List additionalInputs) { + throw new UnsupportedOperationException("Pause/resume is not yet implemented"); + } + + private void runToCompletion( + PolicyRun run, + PolicyInputs inputs, + PolicyProgressListener listener, + CompletableFuture completion) { + String runId = run.getRunId(); + try { + run.markRunning(); + PolicyExecutionResult result = + stepExecutor.execute(run.getDefinition(), inputs, listener); + OutputSpec output = run.getDefinition().output(); + List outputs = sinkFor(output).deliver(runId, result.files(), output); + taskManager.setMultipleFileResults(runId, outputs); + taskManager.setComplete(runId); + run.complete(outputs); + } catch (PolicyInputRequiredException e) { + // Designed-for path: suspend the run rather than fail it. Persist intermediates as + // fileIds so the run can resume after this worker thread is gone. + WaitState wait = suspend(e); + run.waitForInput(wait); + taskManager.addNote(runId, "Waiting for input: " + e.getMessage()); + } catch (InternalApiTimeoutException e) { + String message = toolTimeoutMessage(e); + log.error( + "Policy run {} timed out on {}: {}", + runId, + e.getEndpointPath(), + e.getMessage()); + run.fail(message); + taskManager.setError(runId, message); + } catch (Exception e) { + String message = "Policy run failed: " + e.getMessage(); + log.error("Policy run {} failed", runId, e); + run.fail(message); + taskManager.setError(runId, message); + } finally { + // Always resolve the handle with the run's final state so stream/await callers unblock. + completion.complete(run); + } + } + + private ResponseEntity failRejectedRun( + PolicyRun run, CompletableFuture completion, Throwable ex) { + // Only reached if the run never started (e.g. the queue was full). A run that started + // always resolves its own completion in runToCompletion. + if (!completion.isDone()) { + String message = "Policy run could not be queued: " + ex.getMessage(); + log.error("Policy run {} was not admitted: {}", run.getRunId(), ex.getMessage()); + run.fail(message); + taskManager.setError(run.getRunId(), message); + completion.complete(run); + } + return null; + } + + private WaitState suspend(PolicyInputRequiredException e) { + List fileIds = new ArrayList<>(); + for (Resource resource : e.getPendingFiles()) { + String name = resource.getFilename() != null ? resource.getFilename() : "pending"; + try (InputStream is = resource.getInputStream()) { + fileIds.add(fileStorage.storeInputStream(is, name).fileId()); + } catch (IOException io) { + log.warn("Failed to persist pending file for paused run: {}", io.getMessage()); + } + } + return new WaitState(e.getMessage(), e.getResumeStepIndex(), fileIds); + } + + private PolicyProgressListener trackingListener( + String runId, PolicyRun run, PolicyProgressListener delegate) { + return new PolicyProgressListener() { + @Override + public void onStepStart(int stepIndex, int stepCount, String operation) { + run.enterStep(stepIndex); + taskManager.addNote( + runId, + "Step " + stepIndex + "/" + stepCount + ": " + operation + " started"); + delegate.onStepStart(stepIndex, stepCount, operation); + } + + @Override + public void onStepComplete(int stepIndex, int stepCount, String operation) { + taskManager.addNote( + runId, + "Step " + stepIndex + "/" + stepCount + ": " + operation + " completed"); + delegate.onStepComplete(stepIndex, stepCount, operation); + } + + @Override + public void onHeartbeat() { + delegate.onHeartbeat(); + } + }; + } + + private PolicyOutputSink sinkFor(OutputSpec spec) { + return outputSinks.stream() + .filter(sink -> sink.supports(spec)) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "No output sink supports spec: " + + (spec == null ? "" : spec.type()))); + } + + private static String toolTimeoutMessage(InternalApiTimeoutException e) { + return String.format( + "The %s tool did not respond within %d seconds and was aborted.", + e.getEndpointPath(), e.getReadTimeout().toSeconds()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java new file mode 100644 index 0000000000..eb5f028bde --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutionResult.java @@ -0,0 +1,17 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.List; + +import org.springframework.core.io.Resource; + +import tools.jackson.databind.JsonNode; + +/** + * Result of running a pipeline through {@link PolicyExecutor}. + * + *

{@code files} are the final output resources (temp files, not yet stored to {@code + * FileStorage}). {@code report} is the structured metadata payload captured from the last step that + * produced one (a JSON body, or an {@code X-Stirling-Tool-Report} header), with {@code reportTool} + * naming the step it came from; both are null when no step produced a report. + */ +public record PolicyExecutionResult(List files, JsonNode report, String reportTool) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java new file mode 100644 index 0000000000..92a436bdfd --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyExecutor.java @@ -0,0 +1,312 @@ +package stirling.software.proprietary.policy.engine; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.service.InternalApiClient; +import stirling.software.common.service.InternalApiTimeoutException; +import stirling.software.common.service.ToolMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.ZipExtractionUtils; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; +import stirling.software.proprietary.service.AiToolResponseHeaders; + +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +/** + * Runs an ordered chain of tool steps, chaining each step's output files into the next step's + * input. + * + *

This is the single execution loop for the proprietary surface (AI plans now; + * manually-triggered runs and watched folders later). Each step is dispatched synchronously via + * {@link InternalApiClient} loopback HTTP: the tool runs in its own handler and returns its file + * inline. The caller decides how to run the executor itself (the AI turn loop calls it directly; + * the engine runs it on a virtual thread for async runs). Files cross step boundaries as {@link + * Resource} temp files; they are only persisted to durable storage at the run boundaries by the + * caller. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class PolicyExecutor { + + private static final String FILTER_OPERATION_PREFIX = "/api/v1/filter/filter-"; + + private final InternalApiClient internalApiClient; + private final ToolMetadataService toolMetadataService; + private final TempFileManager tempFileManager; + private final ObjectMapper objectMapper; + + /** + * Internal value-class for tool responses. {@code files} holds any result files (typically one; + * multiple for ZIP-response tools). {@code report} holds an optional structured metadata + * payload the tool chose to surface alongside (or instead of) a file. + */ + private record ToolResult(List files, JsonNode report) {} + + /** + * Execute every step in {@code definition} in order, feeding each step's output into the next. + * Supporting files supplied in {@code inputs} are bound to steps' named file fields and never + * enter the document stream. + * + * @param definition the pipeline to run (must have at least one step) + * @param inputs the primary documents plus the named supporting-file store + * @param listener receives per-step progress + * @return the final output files plus the last structured report produced, if any + * @throws InternalApiTimeoutException if a tool does not respond within its read timeout + * @throws IOException if a tool returns a non-OK response, references a missing supporting + * file, or a file cannot be read + */ + public PolicyExecutionResult execute( + PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) + throws IOException { + List steps = definition.steps(); + if (steps.isEmpty()) { + throw new IllegalArgumentException("Pipeline definition has no steps"); + } + + List currentFiles = inputs.primary(); + Map> supportingFiles = inputs.supportingFiles(); + // Propagate the *last* non-null report; the terminal step defines the output. + JsonNode lastReport = null; + String lastReportTool = null; + + for (int i = 0; i < steps.size(); i++) { + PipelineStep step = steps.get(i); + String operation = step.operation(); + if (operation == null || operation.isBlank()) { + throw new IllegalArgumentException( + "Pipeline step " + (i + 1) + " has no operation"); + } + listener.onStepStart(i + 1, steps.size(), operation); + ToolResult stepResult = executeStep(step, currentFiles, supportingFiles); + currentFiles = stepResult.files(); + if (stepResult.report() != null) { + lastReport = stepResult.report(); + lastReportTool = operation; + } + listener.onStepComplete(i + 1, steps.size(), operation); + } + + return new PolicyExecutionResult(currentFiles, lastReport, lastReportTool); + } + + /** + * Execute a single tool step. If the endpoint accepts multiple files, all files are sent in one + * call. Otherwise, the endpoint is called once per file. ZIP responses are unpacked so each + * inner file is treated as its own result (e.g. split outputs a ZIP of pages). + * + *

A structured {@code report} may be returned alongside (or instead of) files; see {@link + * ToolResult}. For per-file dispatch (single-input endpoints called once per input), the first + * non-null report wins. + */ + private ToolResult executeStep( + PipelineStep step, + List inputFiles, + Map> supportingFiles) + throws IOException { + requireAcceptedTypes(step.operation(), inputFiles); + List files = new ArrayList<>(); + JsonNode report = null; + if (toolMetadataService.isMultiInput(step.operation())) { + ToolResult r = callEndpoint(step, inputFiles, supportingFiles); + files.addAll(r.files()); + report = r.report(); + } else { + for (Resource file : inputFiles) { + ToolResult r = callEndpoint(step, List.of(file), supportingFiles); + files.addAll(r.files()); + if (report == null) { + report = r.report(); + } + } + } + return new ToolResult(files, report); + } + + /** + * Call an endpoint and return its result files and optional report. + * + *

    + *
  • JSON body (Content-Type: application/json): the entire body is the report, no files are + * returned. + *
  • File body (PDF etc.): the file is returned; if an {@link + * AiToolResponseHeaders#TOOL_REPORT} header is present, its (minified JSON) value is + * parsed as the report. + *
  • ZIP responses declared by the tool metadata service are unpacked so callers always see + * a flat list of result files. + *
+ */ + private ToolResult callEndpoint( + PipelineStep step, List files, Map> supportingFiles) + throws IOException { + String endpointPath = step.operation(); + MultiValueMap body = new LinkedMultiValueMap<>(); + for (Resource file : files) { + body.add("fileInput", file); + } + // Bind supporting files to their named tool fields (e.g. stampImage, overlayFiles). These + // come from the run's named asset store, not the document stream. + for (Map.Entry binding : step.fileParameters().entrySet()) { + String fieldName = binding.getKey(); + String assetKey = binding.getValue(); + List assets = supportingFiles.get(assetKey); + if (assets == null || assets.isEmpty()) { + throw new IOException( + "Step " + + endpointPath + + " references supporting file '" + + assetKey + + "' for field '" + + fieldName + + "' but no such file was provided"); + } + for (Resource asset : assets) { + body.add(fieldName, asset); + } + } + for (Map.Entry entry : step.parameters().entrySet()) { + if (entry.getValue() instanceof List list) { + if (containsStructuredElements(list)) { + // Endpoints binding lists of structured objects (e.g. /security/redact's + // redactions, /general/edit-text's edits) parse a single JSON string field via + // a property editor. Pre-serialize the whole list so binding succeeds. + body.add(entry.getKey(), objectMapper.writeValueAsString(list)); + } else { + for (Object item : list) { + body.add(entry.getKey(), item); + } + } + } else { + body.add(entry.getKey(), entry.getValue()); + } + } + ResponseEntity response = internalApiClient.post(endpointPath, body); + if (!HttpStatus.OK.equals(response.getStatusCode()) || response.getBody() == null) { + throw new IOException( + "Tool returned HTTP " + response.getStatusCode() + " for " + endpointPath); + } + Resource resource = response.getBody(); + + // Filter operations return an empty body to signal the file was filtered out: drop it + // rather than forwarding a zero-byte document. + if (isFilterOperation(endpointPath) && isEmpty(resource)) { + return new ToolResult(List.of(), null); + } + + HttpHeaders headers = response.getHeaders(); + MediaType contentType = headers.getContentType(); + + // JSON-only response: the whole body is the structured report, no result file. + if (contentType != null && MediaType.APPLICATION_JSON.isCompatibleWith(contentType)) { + try (InputStream is = resource.getInputStream()) { + JsonNode report = objectMapper.readTree(is); + return new ToolResult(List.of(), report); + } + } + + JsonNode report = parseReportHeader(headers, endpointPath); + if (toolMetadataService.shouldUnpackZipResponse(endpointPath)) { + return new ToolResult(ZipExtractionUtils.extractZip(resource, tempFileManager), report); + } + return new ToolResult(List.of(resource), report); + } + + /** + * Parse the optional {@link AiToolResponseHeaders#TOOL_REPORT} header into a {@link JsonNode}, + * or return null. + */ + private JsonNode parseReportHeader(HttpHeaders headers, String endpointPath) { + String raw = headers.getFirst(AiToolResponseHeaders.TOOL_REPORT); + if (raw == null || raw.isBlank()) { + return null; + } + try { + return objectMapper.readTree(raw); + } catch (JacksonException e) { + log.warn( + "Ignoring malformed {} header from {}: {}", + AiToolResponseHeaders.TOOL_REPORT, + endpointPath, + e.getMessage()); + return null; + } + } + + private static boolean containsStructuredElements(List list) { + for (Object item : list) { + if (item instanceof Map || item instanceof List) { + return true; + } + } + return false; + } + + /** + * Fail the run if any document in the primary stream is not a file type the step accepts. An + * endpoint that declares no specific input type accepts anything. + */ + private void requireAcceptedTypes(String operation, List files) throws IOException { + List accepted = toolMetadataService.getExtensionTypes(false, operation); + if (accepted == null || accepted.isEmpty()) { + return; + } + for (Resource file : files) { + if (!matchesType(file, accepted)) { + throw new IOException( + "Step " + + operation + + " accepts " + + accepted + + " but received '" + + file.getFilename() + + "'"); + } + } + } + + private static boolean matchesType(Resource file, List acceptedExtensions) { + String filename = file.getFilename(); + if (filename == null) { + return false; + } + int dot = filename.lastIndexOf('.'); + if (dot < 0 || dot == filename.length() - 1) { + return false; + } + return acceptedExtensions.contains(filename.substring(dot + 1).toLowerCase(Locale.ROOT)); + } + + private static boolean isFilterOperation(String operation) { + return operation.startsWith(FILTER_OPERATION_PREFIX); + } + + private static boolean isEmpty(Resource resource) { + try { + return resource.contentLength() == 0; + } catch (IOException e) { + return false; + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyInputRequiredException.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyInputRequiredException.java new file mode 100644 index 0000000000..fa30373c57 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyInputRequiredException.java @@ -0,0 +1,32 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.List; + +import org.springframework.core.io.Resource; + +import lombok.Getter; + +/** + * Thrown by a step to signal that the run cannot proceed without further user input, pausing the + * run in {@code WAITING_FOR_INPUT} rather than failing it. + * + *

Carries everything needed to resume: a human-readable reason, the 0-based index of the step to + * resume from, and the intermediate files produced so far. The engine persists those files and + * suspends the run. + * + *

Defined now to fix the run shape; no step throws it yet, and the resume handshake is + * implemented in a later stage. + */ +@Getter +public class PolicyInputRequiredException extends RuntimeException { + + private final transient List pendingFiles; + private final int resumeStepIndex; + + public PolicyInputRequiredException( + String reason, int resumeStepIndex, List pendingFiles) { + super(reason); + this.resumeStepIndex = resumeStepIndex; + this.pendingFiles = pendingFiles == null ? List.of() : pendingFiles; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunHandle.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunHandle.java new file mode 100644 index 0000000000..eeda863ccc --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunHandle.java @@ -0,0 +1,16 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.concurrent.CompletableFuture; + +import stirling.software.proprietary.policy.model.PolicyRun; + +/** + * Returned by {@link PolicyEngine#submit}: the run id (for status polling and result download) plus + * a future that resolves when the run reaches a terminal or paused state. + * + *

The completion future lets callers react to the end of a run (e.g. an SSE endpoint sending a + * final event and closing the stream) without polling. It carries the {@link PolicyRun} whose + * status describes the outcome (completed, failed, cancelled, or waiting for input); it does not + * complete exceptionally for ordinary run failures. + */ +public record PolicyRunHandle(String runId, CompletableFuture completion) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java new file mode 100644 index 0000000000..0b613726d7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java @@ -0,0 +1,97 @@ +package stirling.software.proprietary.policy.engine; + +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import jakarta.annotation.PreDestroy; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.policy.model.PolicyRun; + +/** + * In-memory store of live {@link PolicyRun} state, keyed by runId. Holds the authoritative run + * state machine; durable status/files for download are projected separately into {@code + * TaskManager}. + * + *

Finished runs are evicted on a fixed interval once they age past {@code + * stirling.policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a + * run's rich in-memory state does not outlive the process. Only terminal runs are evicted; active + * and paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not + * touched here: a run shares its runId with a {@code TaskManager} job, which owns file-lifecycle + * cleanup, so eviction only frees this map's entry. + */ +@Slf4j +@Service +public class PolicyRunRegistry { + + private final Map runs = new ConcurrentHashMap<>(); + + private final Duration runExpiry; + private final ScheduledExecutorService cleanupExecutor = + Executors.newSingleThreadScheduledExecutor( + Thread.ofVirtual().name("policy-run-cleanup-", 0).factory()); + + public PolicyRunRegistry( + @Value("${stirling.policies.runExpiryMinutes:30}") int runExpiryMinutes) { + this.runExpiry = Duration.ofMinutes(runExpiryMinutes); + cleanupExecutor.scheduleAtFixedRate(this::evictExpiredRuns, 10, 10, TimeUnit.MINUTES); + log.debug( + "Policy run registry initialized with run expiry of {} minutes", runExpiryMinutes); + } + + public void register(PolicyRun run) { + runs.put(run.getRunId(), run); + } + + public PolicyRun get(String runId) { + return runs.get(runId); + } + + public Collection all() { + return runs.values(); + } + + /** Scheduled hook: evict terminal runs that finished before the expiry window. */ + private void evictExpiredRuns() { + try { + evictExpired(Instant.now().minus(runExpiry)); + } catch (Exception e) { + log.error("Error during policy run cleanup: {}", e.getMessage(), e); + } + } + + /** + * Remove every terminal run last updated before {@code cutoff}; active and paused runs are kept + * regardless of age. Returns the number evicted. Package-visible so the scheduled sweep and + * tests exercise the same path with an explicit cutoff. + */ + int evictExpired(Instant cutoff) { + int removed = 0; + for (Map.Entry entry : runs.entrySet()) { + PolicyRun run = entry.getValue(); + if (run.getStatus().isTerminal() && run.getUpdatedAt().isBefore(cutoff)) { + runs.remove(entry.getKey()); + removed++; + } + } + if (removed > 0) { + log.info("Evicted {} expired policy runs", removed); + } + return removed; + } + + @PreDestroy + void shutdown() { + cleanupExecutor.shutdownNow(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java new file mode 100644 index 0000000000..4759810bc8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java @@ -0,0 +1,21 @@ +package stirling.software.proprietary.policy.model; + +import java.util.Map; + +/** + * Describes where a pipeline run's output files should be delivered. {@code type} selects a {@code + * PolicyOutputSink} (e.g. "inline"); {@code options} carries sink-specific configuration. + * + *

New destinations (folder, S3) are added as new sink beans keyed on a new {@code type} without + * changing this shape or the engine. + */ +public record OutputSpec(String type, Map options) { + public OutputSpec { + options = options == null ? Map.of() : options; + } + + /** The default destination: store outputs and return them to the caller for download. */ + public static OutputSpec inline() { + return new OutputSpec("inline", Map.of()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineDefinition.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineDefinition.java new file mode 100644 index 0000000000..983c3dbca9 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineDefinition.java @@ -0,0 +1,16 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; + +/** + * An ordered chain of tool steps plus where the output should go. + * + *

This is the single shape executed by the policy engine, shared by AI plans, manually-triggered + * runs, and (later) watched folders. {@code output} may be null for callers that handle result + * files themselves (e.g. the AI workflow, which builds its own response payload). + */ +public record PipelineDefinition(String name, List steps, OutputSpec output) { + public PipelineDefinition { + steps = steps == null ? List.of() : steps; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineStep.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineStep.java new file mode 100644 index 0000000000..52c6c060a2 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PipelineStep.java @@ -0,0 +1,30 @@ +package stirling.software.proprietary.policy.model; + +import java.util.Map; + +/** + * A single tool invocation in a pipeline: the API endpoint path to call and the inputs to pass. + * + *

{@code operation} is a Stirling tool endpoint path (e.g. {@code /api/v1/misc/compress-pdf}), + * matching the dispatch convention used by {@code InternalApiClient}. {@code parameters} are the + * tool-specific scalar form fields. + * + *

{@code fileParameters} binds a tool's named file fields (beyond the primary {@code fileInput} + * stream) to supporting files supplied with the run: it maps the form field name (e.g. {@code + * stampImage}, {@code overlayFiles}) to an asset key in the run's supporting-file store. This keeps + * supporting inputs (a stamp image, a certificate, an overlay) out of the document stream that + * flows step to step. + */ +public record PipelineStep( + String operation, Map parameters, Map fileParameters) { + + public PipelineStep { + parameters = parameters == null ? Map.of() : parameters; + fileParameters = fileParameters == null ? Map.of() : fileParameters; + } + + /** A step with no supporting-file bindings. */ + public PipelineStep(String operation, Map parameters) { + this(operation, parameters, Map.of()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java new file mode 100644 index 0000000000..3a2acaf0c8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java @@ -0,0 +1,38 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; + +/** + * A stored, owned automation: how it is triggered, the ordered tool steps to run, and where its + * output goes, plus identity and metadata. + * + *

This is the central object of the feature. Everything that runs a chain of tools is a use of a + * Policy: a watched folder is a Policy with a folder {@link TriggerConfig} and a folder {@link + * OutputSpec}; a scheduled job is a Policy with a schedule trigger; manual/Automate/AI runs execute + * a Policy (or an ad-hoc {@link PipelineDefinition}) on demand. The engine itself only ever + * executes the {@link PipelineDefinition} this exposes via {@link #toDefinition()} - it is + * trigger-agnostic. + * + *

{@code enabled} gates automatic triggering (a disabled policy is not picked up by its + * trigger); it does not block an explicit manual run. + */ +public record Policy( + String id, + String name, + String owner, + boolean enabled, + TriggerConfig trigger, + List steps, + OutputSpec output) { + + public Policy { + trigger = trigger == null ? TriggerConfig.manual() : trigger; + steps = steps == null ? List.of() : steps; + output = output == null ? OutputSpec.inline() : output; + } + + /** The engine-level, trigger-agnostic view of this policy's pipeline. */ + public PipelineDefinition toDefinition() { + return new PipelineDefinition(name, steps, output); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyInputs.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyInputs.java new file mode 100644 index 0000000000..4e6ddae834 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyInputs.java @@ -0,0 +1,32 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; +import java.util.Map; + +import org.springframework.core.io.Resource; + +/** + * The files a run operates on, split into two roles: + * + *

    + *
  • {@code primary} - the documents that flow through the pipeline, each step's output becoming + * the next step's input. + *
  • {@code supportingFiles} - a named store of auxiliary files (a stamp image, certificate, + * overlay, attachments) that steps bind to their named file fields via {@link + * PipelineStep#fileParameters()}. These never enter the document stream. + *
+ * + * Asset values are lists so a single key can carry multi-file fields (e.g. attachments). + */ +public record PolicyInputs(List primary, Map> supportingFiles) { + + public PolicyInputs { + primary = primary == null ? List.of() : primary; + supportingFiles = supportingFiles == null ? Map.of() : supportingFiles; + } + + /** Inputs with primary documents only and no supporting files. */ + public static PolicyInputs of(List primary) { + return new PolicyInputs(primary, Map.of()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRun.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRun.java new file mode 100644 index 0000000000..20621f547f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRun.java @@ -0,0 +1,87 @@ +package stirling.software.proprietary.policy.model; + +import java.time.Instant; +import java.util.List; + +import lombok.Getter; + +import stirling.software.common.model.job.ResultFile; + +/** + * Live, mutable state of a single pipeline run, held in memory by {@code PolicyRunRegistry}. + * + *

This carries the rich execution state (status, step cursor, wait state) that the job system's + * {@code JobResult} does not model. The run is also projected into {@code TaskManager} for + * cluster-visible status, progress notes, and file download; this object is the authoritative + * source of the state machine. + */ +@Getter +public class PolicyRun { + + private final String runId; + private final PipelineDefinition definition; + private final Instant createdAt = Instant.now(); + + private volatile PolicyRunStatus status = PolicyRunStatus.PENDING; + + /** 1-based index of the step currently running (0 before the run starts). */ + private volatile int currentStep = 0; + + private volatile WaitState waitState; + private volatile String error; + private volatile List outputs = List.of(); + private volatile Instant updatedAt = Instant.now(); + + public PolicyRun(String runId, PipelineDefinition definition) { + this.runId = runId; + this.definition = definition; + } + + public int stepCount() { + return definition.steps().size(); + } + + public synchronized void markRunning() { + this.status = PolicyRunStatus.RUNNING; + touch(); + } + + public synchronized void enterStep(int oneBasedStepIndex) { + this.currentStep = oneBasedStepIndex; + touch(); + } + + public synchronized void complete(List resultFiles) { + this.outputs = resultFiles == null ? List.of() : List.copyOf(resultFiles); + this.status = PolicyRunStatus.COMPLETED; + touch(); + } + + public synchronized void fail(String message) { + this.error = message; + this.status = PolicyRunStatus.FAILED; + touch(); + } + + public synchronized void waitForInput(WaitState wait) { + this.waitState = wait; + this.status = PolicyRunStatus.WAITING_FOR_INPUT; + touch(); + } + + /** + * Mark cancelled if the run has not already reached a terminal state. Returns whether it did. + */ + public synchronized boolean cancel() { + if (status.isTerminal()) { + return false; + } + this.status = PolicyRunStatus.CANCELLED; + touch(); + return true; + } + + private void touch() { + this.updatedAt = Instant.now(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunStatus.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunStatus.java new file mode 100644 index 0000000000..ee75f5e50f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunStatus.java @@ -0,0 +1,21 @@ +package stirling.software.proprietary.policy.model; + +/** + * Lifecycle states of a {@link PolicyRun}. + * + *

{@code WAITING_FOR_INPUT} is modelled now so the engine and run shape support pausing a run + * (e.g. a step that blocks for a human decision) without holding a thread; the resume handshake is + * implemented in a later stage. + */ +public enum PolicyRunStatus { + PENDING, + RUNNING, + WAITING_FOR_INPUT, + COMPLETED, + FAILED, + CANCELLED; + + public boolean isTerminal() { + return this == COMPLETED || this == FAILED || this == CANCELLED; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java new file mode 100644 index 0000000000..a3427835ce --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java @@ -0,0 +1,28 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; + +import stirling.software.common.model.job.ResultFile; + +/** + * Read-only view of a {@link PolicyRun} returned by the status endpoint. Output files are surfaced + * as {@link ResultFile} so the caller can download each via {@code GET /api/v1/general/files/{id}}. + */ +public record PolicyRunView( + String runId, + PolicyRunStatus status, + int currentStep, + int stepCount, + String error, + List outputs) { + + public static PolicyRunView of(PolicyRun run) { + return new PolicyRunView( + run.getRunId(), + run.getStatus(), + run.getCurrentStep(), + run.stepCount(), + run.getError(), + run.getOutputs()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java new file mode 100644 index 0000000000..848c48e4f5 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java @@ -0,0 +1,25 @@ +package stirling.software.proprietary.policy.model; + +import java.util.Map; + +/** + * How a {@link Policy} is automatically triggered. {@code type} selects a trigger kind ("manual", + * "folder", "schedule", "s3"); {@code options} carries type-specific configuration (a folder path, + * a cron expression, a bucket, ...). + * + *

Data-driven and parallel to {@link OutputSpec}: new trigger kinds are new {@code type} values + * handled by a new trigger bean, with no change to the model. {@code "manual"} means there is no + * automatic trigger - the policy is only ever run on demand. + */ +public record TriggerConfig(String type, Map options) { + + public TriggerConfig { + type = type == null || type.isBlank() ? "manual" : type; + options = options == null ? Map.of() : options; + } + + /** No automatic trigger; the policy is run on demand only. */ + public static TriggerConfig manual() { + return new TriggerConfig("manual", Map.of()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/WaitState.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/WaitState.java new file mode 100644 index 0000000000..7fff6c9fdd --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/WaitState.java @@ -0,0 +1,19 @@ +package stirling.software.proprietary.policy.model; + +import java.util.List; + +/** + * Captured when a run pauses in {@link PolicyRunStatus#WAITING_FOR_INPUT}. Together with the run's + * {@link PipelineDefinition} this is the resumable snapshot: {@code resumeStepIndex} is the 0-based + * step to continue from, and {@code pendingFileIds} are the intermediate files (stored in {@code + * FileStorage}, so they survive the worker thread ending or a node restart) that become the input + * to the resumed run. + * + *

Stored as fileIds rather than in-memory resources by design: a paused run must be resumable + * long after its worker thread has gone. + */ +public record WaitState(String reason, int resumeStepIndex, List pendingFileIds) { + public WaitState { + pendingFileIds = pendingFileIds == null ? List.of() : pendingFileIds; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java new file mode 100644 index 0000000000..bef5d46b37 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/InlineOutputSink.java @@ -0,0 +1,68 @@ +package stirling.software.proprietary.policy.output; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.MediaTypeFactory; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.model.job.ResultFile; +import stirling.software.common.service.FileStorage; +import stirling.software.proprietary.policy.model.OutputSpec; + +/** + * Default output sink: stores each output file in {@code FileStorage} so it is downloadable via + * {@code GET /api/v1/general/files/{fileId}}. This is the destination for manually-triggered runs + * whose results are returned to the caller. + */ +@Service +@RequiredArgsConstructor +public class InlineOutputSink implements PolicyOutputSink { + + private static final String TYPE = "inline"; + + private final FileStorage fileStorage; + + @Override + public String type() { + return TYPE; + } + + @Override + public boolean supports(OutputSpec spec) { + return spec == null || spec.type() == null || TYPE.equals(spec.type()); + } + + @Override + public List deliver(String runId, List outputs, OutputSpec spec) + throws IOException { + List results = new ArrayList<>(); + for (int i = 0; i < outputs.size(); i++) { + Resource resource = outputs.get(i); + String name = + resource.getFilename() != null ? resource.getFilename() : "result-" + (i + 1); + String contentType = + MediaTypeFactory.getMediaType(name) + .orElse(MediaType.APPLICATION_OCTET_STREAM) + .toString(); + FileStorage.StoredFile stored; + try (InputStream is = resource.getInputStream()) { + stored = fileStorage.storeInputStream(is, name); + } + results.add( + ResultFile.builder() + .fileId(stored.fileId()) + .fileName(name) + .contentType(contentType) + .fileSize(stored.size()) + .build()); + } + return results; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java new file mode 100644 index 0000000000..f6bf478e7f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java @@ -0,0 +1,35 @@ +package stirling.software.proprietary.policy.output; + +import java.io.IOException; +import java.util.List; + +import org.springframework.core.io.Resource; + +import stirling.software.common.model.job.ResultFile; +import stirling.software.proprietary.policy.model.OutputSpec; + +/** + * Delivers a finished run's output files to a destination, returning durable {@link ResultFile} + * descriptors (fileId + metadata) for the run record. + * + *

Implementations are Spring beans selected by {@link #supports(OutputSpec)}. New destinations + * (folder, S3) are added as new beans without changing the engine. + */ +public interface PolicyOutputSink { + + /** Stable identifier for this sink, matching {@code OutputSpec.type()} (e.g. "inline"). */ + String type(); + + /** Whether this sink can handle the given output spec. */ + boolean supports(OutputSpec spec); + + /** + * Persist/deliver the output files and return their descriptors. + * + * @param runId the run these outputs belong to + * @param outputs the final pipeline output resources + * @param spec the requested destination + */ + List deliver(String runId, List outputs, OutputSpec spec) + throws IOException; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/progress/PolicyProgressListener.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/progress/PolicyProgressListener.java new file mode 100644 index 0000000000..84eae284ef --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/progress/PolicyProgressListener.java @@ -0,0 +1,22 @@ +package stirling.software.proprietary.policy.progress; + +/** + * Receives live progress as a pipeline run executes. Implementations forward to an SSE stream, + * write job notes for polling, or both. Step indices are 1-based. + * + *

All methods default to no-ops so callers implement only what they surface. + */ +public interface PolicyProgressListener { + + /** A listener that ignores all progress. */ + PolicyProgressListener NOOP = new PolicyProgressListener() {}; + + /** Called immediately before step {@code stepIndex} of {@code stepCount} begins. */ + default void onStepStart(int stepIndex, int stepCount, String operation) {} + + /** Called immediately after step {@code stepIndex} of {@code stepCount} completes. */ + default void onStepComplete(int stepIndex, int stepCount, String operation) {} + + /** Called on a keep-alive tick so downstream connections can detect disconnects promptly. */ + default void onHeartbeat() {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java new file mode 100644 index 0000000000..df8fc93394 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java @@ -0,0 +1,61 @@ +package stirling.software.proprietary.policy.store; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import stirling.software.proprietary.policy.model.Policy; + +/** + * In-memory {@link PolicyStore}. Not the runtime bean - {@link JpaPolicyStore} is the durable + * store. Kept as a lightweight, dependency-free implementation for tests and for any future no- + * database mode. + */ +public class InProcessPolicyStore implements PolicyStore { + + private final Map policies = new ConcurrentHashMap<>(); + + @Override + public Policy save(Policy policy) { + String id = + policy.id() == null || policy.id().isBlank() + ? UUID.randomUUID().toString() + : policy.id(); + Policy stored = + new Policy( + id, + policy.name(), + policy.owner(), + policy.enabled(), + policy.trigger(), + policy.steps(), + policy.output()); + policies.put(id, stored); + return stored; + } + + @Override + public Optional get(String id) { + return Optional.ofNullable(policies.get(id)); + } + + @Override + public List all() { + return List.copyOf(policies.values()); + } + + @Override + public List findByTriggerType(String triggerType) { + return policies.values().stream() + .filter(Policy::enabled) + .filter(policy -> triggerType.equals(policy.trigger().type())) + .toList(); + } + + @Override + public boolean delete(String id) { + return policies.remove(id) != null; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java new file mode 100644 index 0000000000..977e8c9c12 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java @@ -0,0 +1,83 @@ +package stirling.software.proprietary.policy.store; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.policy.model.Policy; + +import tools.jackson.databind.ObjectMapper; + +/** + * Durable {@link PolicyStore} backed by JPA. The runtime store whenever the proprietary module runs + * (a datasource is always present). Policies are persisted as JSON via {@link PolicyEntity}; the + * scalar columns are kept in sync for querying. + */ +@Service +@RequiredArgsConstructor +public class JpaPolicyStore implements PolicyStore { + + private final PolicyRepository repository; + private final ObjectMapper objectMapper; + + @Override + public Policy save(Policy policy) { + String id = + policy.id() == null || policy.id().isBlank() + ? UUID.randomUUID().toString() + : policy.id(); + Policy stored = + new Policy( + id, + policy.name(), + policy.owner(), + policy.enabled(), + policy.trigger(), + policy.steps(), + policy.output()); + + PolicyEntity entity = new PolicyEntity(); + entity.setId(id); + entity.setName(stored.name()); + entity.setOwner(stored.owner()); + entity.setEnabled(stored.enabled()); + entity.setTriggerType(stored.trigger().type()); + entity.setPolicyJson(objectMapper.writeValueAsString(stored)); + repository.save(entity); + return stored; + } + + @Override + public Optional get(String id) { + return repository.findById(id).map(this::toPolicy); + } + + @Override + public List all() { + return repository.findAll().stream().map(this::toPolicy).toList(); + } + + @Override + public List findByTriggerType(String triggerType) { + return repository.findByTriggerTypeAndEnabledTrue(triggerType).stream() + .map(this::toPolicy) + .toList(); + } + + @Override + public boolean delete(String id) { + if (!repository.existsById(id)) { + return false; + } + repository.deleteById(id); + return true; + } + + private Policy toPolicy(PolicyEntity entity) { + return objectMapper.readValue(entity.getPolicyJson(), Policy.class); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java new file mode 100644 index 0000000000..d5aad61303 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyEntity.java @@ -0,0 +1,50 @@ +package stirling.software.proprietary.policy.store; + +import java.io.Serializable; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * JPA row for a {@link stirling.software.proprietary.policy.model.Policy}. + * + *

The whole policy is stored as JSON in {@code policyJson} (authoritative on read, and the same + * serialization the API uses); the scalar columns are denormalized copies for querying - notably + * {@code triggerType} + {@code enabled} so background triggers can fetch their policies. Ownership + * is a plain {@code owner} string rather than a foreign key, to stay decoupled from the security + * entities; richer team scoping can be layered on later. + */ +@Entity +@Table(name = "policies") +@NoArgsConstructor +@Getter +@Setter +public class PolicyEntity implements Serializable { + + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "name") + private String name; + + @Column(name = "owner") + private String owner; + + @Column(name = "enabled") + private boolean enabled; + + @Column(name = "trigger_type") + private String triggerType; + + @Column(name = "policy_json", columnDefinition = "text") + private String policyJson; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java new file mode 100644 index 0000000000..ba6924f0f8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyRepository.java @@ -0,0 +1,13 @@ +package stirling.software.proprietary.policy.store; + +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface PolicyRepository extends JpaRepository { + + /** Enabled policies of a given trigger type, for background triggers to activate. */ + List findByTriggerTypeAndEnabledTrue(String triggerType); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java new file mode 100644 index 0000000000..16b2ae8614 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/PolicyStore.java @@ -0,0 +1,28 @@ +package stirling.software.proprietary.policy.store; + +import java.util.List; +import java.util.Optional; + +import stirling.software.proprietary.policy.model.Policy; + +/** + * Stores {@link Policy} definitions. The in-memory implementation backs simple deployments now; a + * durable (JPA) implementation can replace it behind this interface without touching callers. + */ +public interface PolicyStore { + + /** Create or update a policy. A blank/absent id is assigned; returns the stored policy. */ + Policy save(Policy policy); + + Optional get(String id); + + List all(); + + /** + * Enabled policies whose automatic trigger is of the given type (used by background triggers). + */ + List findByTriggerType(String triggerType); + + /** Remove a policy; returns whether it existed. */ + boolean delete(String id); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java new file mode 100644 index 0000000000..2ae24671a8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java @@ -0,0 +1,41 @@ +package stirling.software.proprietary.policy.trigger; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.policy.engine.PolicyEngine; +import stirling.software.proprietary.policy.engine.PolicyRunHandle; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +/** + * Runs policies on demand, in response to a request (the {@code PolicyController} endpoints, an AI, + * or another automation). It is the request-driven trigger: no background lifecycle, it just + * forwards to the engine. Any policy can be run manually regardless of its configured trigger type. + */ +@Service +@RequiredArgsConstructor +public class ManualTrigger implements PolicyTrigger { + + private final PolicyEngine policyEngine; + + @Override + public String type() { + return "manual"; + } + + /** Run a stored policy immediately and return its run handle. */ + public PolicyRunHandle run( + Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { + return policyEngine.runPolicy(policy, inputs, listener); + } + + /** Run an ad-hoc pipeline (no stored policy), e.g. for AI or Automate one-offs. */ + public PolicyRunHandle fire( + PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) { + return policyEngine.submit(definition, inputs, listener); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java new file mode 100644 index 0000000000..c63e5a1da7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java @@ -0,0 +1,24 @@ +package stirling.software.proprietary.policy.trigger; + +/** + * Activates policies of one trigger type. A trigger owns a {@link #type()} (matching {@code + * TriggerConfig.type()}); when its condition fires it runs the relevant {@code Policy} through the + * {@code PolicyEngine}. + * + *

Background triggers (folder watcher, schedule) are driven by configuration: on {@link + * #start()} they begin watching/scheduling for the policies returned by {@code + * PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. Request-driven triggers + * (manual) have no background lifecycle and run a policy directly in response to a call. New + * trigger kinds are new beans of this type; the engine and the {@code Policy} model do not change. + */ +public interface PolicyTrigger { + + /** Stable identifier for this trigger kind, matching {@code TriggerConfig.type()}. */ + String type(); + + /** Begin activating policies of this type (e.g. start a folder watcher). No-op for manual. */ + default void start() {} + + /** Stop activating and release any resources. */ + default void stop() {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java index d81bf6a360..454df5f67e 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/DatabaseConfig.java @@ -31,13 +31,15 @@ import stirling.software.common.model.exception.UnsupportedProviderException; "stirling.software.proprietary.security.repository", "stirling.software.proprietary.repository", "stirling.software.proprietary.storage.repository", - "stirling.software.proprietary.workflow.repository" + "stirling.software.proprietary.workflow.repository", + "stirling.software.proprietary.policy.store" }) @EntityScan({ "stirling.software.proprietary.security.model", "stirling.software.proprietary.model", "stirling.software.proprietary.storage.model", - "stirling.software.proprietary.workflow.model" + "stirling.software.proprietary.workflow.model", + "stirling.software.proprietary.policy.store" }) public class DatabaseConfig { diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java index 95f8afcea9..5683d634c1 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java @@ -14,14 +14,9 @@ import org.apache.pdfbox.pdmodel.PDDocument; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.MediaTypeFactory; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; import org.springframework.web.multipart.MultipartFile; import io.github.pixee.security.Filenames; @@ -32,14 +27,11 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.service.FileStorage; -import stirling.software.common.service.InternalApiClient; import stirling.software.common.service.InternalApiTimeoutException; -import stirling.software.common.service.ToolMetadataService; import stirling.software.common.service.UserServiceInterface; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; -import stirling.software.common.util.ZipExtractionUtils; import stirling.software.proprietary.model.api.ai.AiConversationMessage; import stirling.software.proprietary.model.api.ai.AiDocumentIngestRequest; import stirling.software.proprietary.model.api.ai.AiEngineProgressDetail; @@ -53,6 +45,13 @@ import stirling.software.proprietary.model.api.ai.AiWorkflowProgressEvent; import stirling.software.proprietary.model.api.ai.AiWorkflowRequest; import stirling.software.proprietary.model.api.ai.AiWorkflowResponse; import stirling.software.proprietary.model.api.ai.AiWorkflowResultFile; +import stirling.software.proprietary.policy.engine.PolicyExecutionResult; +import stirling.software.proprietary.policy.engine.PolicyExecutor; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; import stirling.software.proprietary.security.util.DesktopClientUtils; import stirling.software.proprietary.service.PdfContentExtractor.LoadedFile; import stirling.software.proprietary.service.PdfContentExtractor.PdfContentResult; @@ -72,12 +71,11 @@ public class AiWorkflowService { private final AiEngineClient aiEngineClient; private final PdfContentExtractor pdfContentExtractor; private final ObjectMapper objectMapper; - private final InternalApiClient internalApiClient; private final FileStorage fileStorage; - private final ToolMetadataService toolMetadataService; private final TempFileManager tempFileManager; private final FileIdStrategy fileIdStrategy; private final AiEngineEndpointResolver endpointResolver; + private final PolicyExecutor policyExecutor; private final UserServiceInterface userService; private final ApplicationProperties applicationProperties; @@ -86,24 +84,22 @@ public class AiWorkflowService { AiEngineClient aiEngineClient, PdfContentExtractor pdfContentExtractor, ObjectMapper objectMapper, - InternalApiClient internalApiClient, FileStorage fileStorage, - ToolMetadataService toolMetadataService, TempFileManager tempFileManager, FileIdStrategy fileIdStrategy, AiEngineEndpointResolver endpointResolver, + PolicyExecutor policyExecutor, @Autowired(required = false) UserServiceInterface userService, ApplicationProperties applicationProperties) { this.pdfDocumentFactory = pdfDocumentFactory; this.aiEngineClient = aiEngineClient; this.pdfContentExtractor = pdfContentExtractor; this.objectMapper = objectMapper; - this.internalApiClient = internalApiClient; this.fileStorage = fileStorage; - this.toolMetadataService = toolMetadataService; this.tempFileManager = tempFileManager; this.fileIdStrategy = fileIdStrategy; this.endpointResolver = endpointResolver; + this.policyExecutor = policyExecutor; this.userService = userService; this.applicationProperties = applicationProperties; } @@ -150,16 +146,6 @@ public class AiWorkflowService { record Terminal(AiWorkflowResponse response) implements WorkflowState {} } - /** - * Internal value-class for tool responses. {@code files} holds any result files (typically one; - * multiple for ZIP-response tools). {@code report} holds an optional structured metadata - * payload the tool chose to surface alongside (or instead of) a file. - * - *

Tools populate the report either by returning a JSON body (whole body → report) or by - * adding the {@link AiToolResponseHeaders#TOOL_REPORT} header alongside a file body. - */ - private record ToolResult(List files, JsonNode report) {} - public AiWorkflowResponse orchestrate(AiWorkflowRequest request) throws IOException { return orchestrate(request, NOOP_LISTENER); } @@ -377,7 +363,6 @@ public class AiWorkflowService { pages.size()); } - @SuppressWarnings("unchecked") private WorkflowState onToolCall( AiWorkflowResponse response, Map filesById, @@ -394,8 +379,14 @@ public class AiWorkflowService { try { List inputFiles = toResources(filesById); - listener.onProgress(AiWorkflowProgressEvent.executingTool(endpointPath, 1, 1)); - ToolResult result = executeStep(endpointPath, parameters, inputFiles); + PipelineDefinition definition = + new PipelineDefinition( + null, + List.of(new PipelineStep(endpointPath, parameters)), + OutputSpec.inline()); + PolicyExecutionResult result = + policyExecutor.execute( + definition, PolicyInputs.of(inputFiles), stepProgress(listener)); return new WorkflowState.Terminal( buildCompletedResponse( response.getRationale(), @@ -469,38 +460,32 @@ public class AiWorkflowService { cannotContinue("AI engine returned a plan with no steps.")); } - try { - List currentFiles = toResources(filesById); - // Propagate the *last* non-null report — the terminal step defines the output. - JsonNode lastReport = null; - String lastReportTool = null; - - for (int i = 0; i < steps.size(); i++) { - Map step = steps.get(i); - String endpointPath = (String) step.get("tool"); - Map parameters = - step.containsKey("parameters") - ? (Map) step.get("parameters") - : Map.of(); - - if (endpointPath == null || endpointPath.isBlank()) { - return new WorkflowState.Terminal( - cannotContinue("Plan step " + (i + 1) + " has no tool endpoint.")); - } - - listener.onProgress( - AiWorkflowProgressEvent.executingTool(endpointPath, i + 1, steps.size())); - ToolResult stepResult = executeStep(endpointPath, parameters, currentFiles); - currentFiles = stepResult.files(); - if (stepResult.report() != null) { - lastReport = stepResult.report(); - lastReportTool = endpointPath; - } + List pipelineSteps = new ArrayList<>(); + for (int i = 0; i < steps.size(); i++) { + Map step = steps.get(i); + String endpointPath = (String) step.get("tool"); + if (endpointPath == null || endpointPath.isBlank()) { + return new WorkflowState.Terminal( + cannotContinue("Plan step " + (i + 1) + " has no tool endpoint.")); } + Map parameters = + step.containsKey("parameters") + ? (Map) step.get("parameters") + : Map.of(); + pipelineSteps.add(new PipelineStep(endpointPath, parameters)); + } + + try { + List inputFiles = toResources(filesById); + PipelineDefinition definition = + new PipelineDefinition(summary, pipelineSteps, OutputSpec.inline()); + PolicyExecutionResult result = + policyExecutor.execute( + definition, PolicyInputs.of(inputFiles), stepProgress(listener)); // Multi-turn: if the plan was emitted with resume_with set, the delegate wants // Java to re-invoke the orchestrator with any captured report as an artifact. - if (resumeWith != null && !resumeWith.isBlank() && lastReport != null) { + if (resumeWith != null && !resumeWith.isBlank() && result.report() != null) { WorkflowTurnRequest resumeRequest = new WorkflowTurnRequest(); resumeRequest.setUserMessage(previousRequest.getUserMessage()); resumeRequest.setFiles(previousRequest.getFiles()); @@ -510,14 +495,14 @@ public class AiWorkflowService { .getArtifacts() .add( new PdfContentExtractor.ToolReportArtifact( - lastReportTool, lastReport)); + result.reportTool(), result.report())); resumeRequest.setResumeWith(resumeWith); return new WorkflowState.Pending(resumeRequest); } return new WorkflowState.Terminal( buildCompletedResponse( - summary, currentFiles, inputFileNames(filesById), lastReport)); + summary, result.files(), inputFileNames(filesById), result.report())); } catch (InternalApiTimeoutException e) { log.error("Plan step on tool {} timed out: {}", e.getEndpointPath(), e.getMessage()); return new WorkflowState.Terminal( @@ -548,123 +533,19 @@ public class AiWorkflowService { } /** - * Execute a single tool step. If the endpoint accepts multiple files, all files are sent in one - * call. Otherwise, the endpoint is called once per file. ZIP responses are unpacked so each - * inner file is treated as its own result (e.g. split outputs a ZIP of pages). - * - *

A structured {@code report} may be returned alongside (or instead of) files — see {@link - * ToolResult}. For per-file dispatch (single-input endpoints called once per input), the first - * non-null report wins. + * Adapt the AI workflow's {@link ProgressListener} to the engine's {@link + * PolicyProgressListener}: each step start maps to an {@code EXECUTING_TOOL} progress event + * carrying the tool path and 1-based step position, preserving the event shape the frontend + * already renders. */ - private ToolResult executeStep( - String endpointPath, Map parameters, List inputFiles) - throws IOException { - List files = new ArrayList<>(); - JsonNode report = null; - if (toolMetadataService.isMultiInput(endpointPath)) { - ToolResult r = callEndpoint(endpointPath, parameters, inputFiles); - files.addAll(r.files()); - report = r.report(); - } else { - for (Resource file : inputFiles) { - ToolResult r = callEndpoint(endpointPath, parameters, List.of(file)); - files.addAll(r.files()); - if (report == null) { - report = r.report(); - } + private static PolicyProgressListener stepProgress(ProgressListener listener) { + return new PolicyProgressListener() { + @Override + public void onStepStart(int stepIndex, int stepCount, String operation) { + listener.onProgress( + AiWorkflowProgressEvent.executingTool(operation, stepIndex, stepCount)); } - } - return new ToolResult(files, report); - } - - /** - * Call an endpoint and return its result files and optional report. - * - *

    - *
  • JSON body (Content-Type: application/json) → the entire body is the report, no files - * are returned. - *
  • File body (PDF etc.) → the file is returned; if an {@link - * AiToolResponseHeaders#TOOL_REPORT} header is present, its (minified JSON) value is - * parsed as the report. - *
  • ZIP responses declared by the tool metadata service are unpacked so callers always see - * a flat list of result files. - *
- */ - private ToolResult callEndpoint( - String endpointPath, Map parameters, List files) - throws IOException { - MultiValueMap body = new LinkedMultiValueMap<>(); - for (Resource file : files) { - body.add("fileInput", file); - } - for (Map.Entry entry : parameters.entrySet()) { - if (entry.getValue() instanceof List list) { - if (containsStructuredElements(list)) { - // Endpoints binding lists of structured objects (e.g. /security/redact's - // redactions, /general/edit-text's edits) parse a single JSON string field via - // a property editor. Pre-serialize the whole list so binding succeeds. - body.add(entry.getKey(), objectMapper.writeValueAsString(list)); - } else { - for (Object item : list) { - body.add(entry.getKey(), item); - } - } - } else { - body.add(entry.getKey(), entry.getValue()); - } - } - ResponseEntity response = internalApiClient.post(endpointPath, body); - if (!HttpStatus.OK.equals(response.getStatusCode()) || response.getBody() == null) { - throw new IOException( - "Tool returned HTTP " + response.getStatusCode() + " for " + endpointPath); - } - Resource resource = response.getBody(); - HttpHeaders headers = response.getHeaders(); - MediaType contentType = headers.getContentType(); - - // JSON-only response — the whole body is the structured report, no result file. - if (contentType != null && MediaType.APPLICATION_JSON.isCompatibleWith(contentType)) { - try (java.io.InputStream is = resource.getInputStream()) { - JsonNode report = objectMapper.readTree(is); - return new ToolResult(List.of(), report); - } - } - - JsonNode report = parseReportHeader(headers, endpointPath); - if (toolMetadataService.shouldUnpackZipResponse(endpointPath)) { - return new ToolResult(ZipExtractionUtils.extractZip(resource, tempFileManager), report); - } - return new ToolResult(List.of(resource), report); - } - - /** - * Parse the optional {@link AiToolResponseHeaders#TOOL_REPORT} header into a {@link JsonNode}, - * or return null. - */ - private JsonNode parseReportHeader(HttpHeaders headers, String endpointPath) { - String raw = headers.getFirst(AiToolResponseHeaders.TOOL_REPORT); - if (raw == null || raw.isBlank()) { - return null; - } - try { - return objectMapper.readTree(raw); - } catch (JacksonException e) { - log.warn( - "Ignoring malformed {} header from {}: {}", - AiToolResponseHeaders.TOOL_REPORT, - endpointPath, - e.getMessage()); - return null; - } - } - - private static boolean containsStructuredElements(List list) { - for (Object item : list) { - if (item instanceof Map || item instanceof List) { - return true; - } - } - return false; + }; } private List toResources(Map filesById) throws IOException { diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java new file mode 100644 index 0000000000..bf6442b423 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java @@ -0,0 +1,255 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.InputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.FileStorage; +import stirling.software.common.service.FileStorage.StoredFile; +import stirling.software.common.service.InternalApiClient; +import stirling.software.common.service.JobOwnershipService; +import stirling.software.common.service.JobQueue; +import stirling.software.common.service.ResourceMonitor; +import stirling.software.common.service.TaskManager; +import stirling.software.common.service.ToolMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.TempFileRegistry; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.PolicyRunStatus; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.output.InlineOutputSink; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +import tools.jackson.databind.json.JsonMapper; + +/** + * Tests for {@link PolicyEngine}: async submission runs the pipeline on a virtual thread, registers + * outputs and progress with {@link TaskManager}, and surfaces terminal state via {@link + * PolicyRunRegistry}. The step executor and inline sink are real (with mocked collaborators) so the + * full run path is exercised. + */ +@ExtendWith(MockitoExtension.class) +class PolicyEngineTest { + + private static final String ROTATE = "/api/v1/general/rotate-pdf"; + private static final String COMPRESS = "/api/v1/misc/compress-pdf"; + + @Mock private InternalApiClient internalApiClient; + @Mock private ToolMetadataService toolMetadataService; + @Mock private TaskManager taskManager; + @Mock private FileStorage fileStorage; + @Mock private JobOwnershipService jobOwnershipService; + @Mock private ResourceMonitor resourceMonitor; + @Mock private JobQueue jobQueue; + + @TempDir Path tempDir; + + private PolicyRunRegistry registry; + private PolicyEngine engine; + + @BeforeEach + void setUp() { + ApplicationProperties props = new ApplicationProperties(); + props.getSystem().getTempFileManagement().setBaseTmpDir(tempDir.toString()); + props.getSystem().getTempFileManagement().setPrefix("policy-engine-test-"); + TempFileManager tempFileManager = new TempFileManager(new TempFileRegistry(), props); + PolicyExecutor executor = + new PolicyExecutor( + internalApiClient, + toolMetadataService, + tempFileManager, + JsonMapper.builder().build()); + registry = new PolicyRunRegistry(30); + InlineOutputSink sink = new InlineOutputSink(fileStorage); + engine = + new PolicyEngine( + executor, + taskManager, + registry, + fileStorage, + jobOwnershipService, + List.of(sink), + resourceMonitor, + jobQueue); + + // Identity scoping: the run id is the generated UUID unchanged. Lenient because the + // resume/cancel tests do not submit a run. + lenient() + .when(jobOwnershipService.createScopedJobKey(anyString())) + .thenAnswer(inv -> inv.getArgument(0)); + // Default to running immediately; the queueing test overrides this. + lenient().when(resourceMonitor.shouldQueueJob(anyInt())).thenReturn(false); + } + + @Test + void submitRunsPipelineToCompletionAndRegistersOutputs() throws Exception { + when(toolMetadataService.isMultiInput(anyString())).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(anyString())).thenReturn(false); + stubEndpoint(ROTATE, pdf("rotated", "rotated.pdf")); + stubEndpoint(COMPRESS, pdf("compressed", "compressed.pdf")); + int[] counter = {0}; + when(fileStorage.storeInputStream(any(InputStream.class), anyString())) + .thenAnswer( + inv -> { + InputStream is = inv.getArgument(0); + long size = is.readAllBytes().length; + return new StoredFile("file-" + ++counter[0], size); + }); + + PolicyRunHandle handle = + engine.submit( + definition( + new PipelineStep(ROTATE, Map.of()), + new PipelineStep(COMPRESS, Map.of())), + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + PolicyProgressListener.NOOP); + + // The completion future resolves with the final run state, no polling needed. + String runId = handle.runId(); + PolicyRun run = handle.completion().get(10, TimeUnit.SECONDS); + assertEquals(PolicyRunStatus.COMPLETED, run.getStatus()); + assertEquals(1, run.getOutputs().size()); + assertEquals("compressed.pdf", run.getOutputs().get(0).getFileName()); + + // The run self-registers its results and completion with the job system. + verify(taskManager).createTask(runId); + verify(taskManager).setMultipleFileResults(eq(runId), any()); + verify(taskManager).setComplete(runId); + // Progress notes were written for each step. + verify(taskManager, atLeastOnce()).addNote(eq(runId), anyString()); + } + + @Test + void submitFailsRunWhenAToolErrors() throws Exception { + when(toolMetadataService.isMultiInput(ROTATE)).thenReturn(false); + when(internalApiClient.post(eq(ROTATE), any())).thenThrow(new RuntimeException("boom")); + + PolicyRunHandle handle = + engine.submit( + definition(new PipelineStep(ROTATE, Map.of())), + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + PolicyProgressListener.NOOP); + + String runId = handle.runId(); + PolicyRun run = handle.completion().get(10, TimeUnit.SECONDS); + assertEquals(PolicyRunStatus.FAILED, run.getStatus()); + verify(taskManager).setError(eq(runId), anyString()); + verify(taskManager, never()).setComplete(runId); + } + + @Test + void runPolicyExecutesThePolicysPipeline() throws Exception { + when(toolMetadataService.isMultiInput(anyString())).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(anyString())).thenReturn(false); + stubEndpoint(ROTATE, pdf("rotated", "rotated.pdf")); + int[] counter = {0}; + when(fileStorage.storeInputStream(any(InputStream.class), anyString())) + .thenAnswer( + inv -> { + InputStream is = inv.getArgument(0); + return new StoredFile("file-" + ++counter[0], is.readAllBytes().length); + }); + + Policy policy = + new Policy( + "p1", + "rotate", + "owner", + true, + TriggerConfig.manual(), + List.of(new PipelineStep(ROTATE, Map.of())), + OutputSpec.inline()); + + PolicyRunHandle handle = + engine.runPolicy( + policy, + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + PolicyProgressListener.NOOP); + + PolicyRun run = handle.completion().get(10, TimeUnit.SECONDS); + assertEquals(PolicyRunStatus.COMPLETED, run.getStatus()); + verify(internalApiClient).post(eq(ROTATE), any()); + } + + @Test + void runIsQueuedUnderResourcePressure() { + when(resourceMonitor.shouldQueueJob(anyInt())).thenReturn(true); + // Returning an already-completed future keeps the run parked: the queued work (which would + // start the run) is never executed by this mock, so it stays PENDING. + doReturn(CompletableFuture.completedFuture(null)) + .when(jobQueue) + .queueJob(anyString(), anyInt(), any(), anyLong()); + + PolicyRunHandle handle = + engine.submit( + definition(new PipelineStep(ROTATE, Map.of())), + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + PolicyProgressListener.NOOP); + + verify(jobQueue).queueJob(eq(handle.runId()), anyInt(), any(), anyLong()); + assertEquals(PolicyRunStatus.PENDING, registry.get(handle.runId()).getStatus()); + } + + @Test + void resumeIsNotYetImplemented() { + assertThrows(UnsupportedOperationException.class, () -> engine.resume("any", List.of())); + } + + @Test + void cancelUnknownRunReturnsFalse() { + assertFalse(engine.cancel("does-not-exist")); + } + + // --- helpers --- + + private static PipelineDefinition definition(PipelineStep... steps) { + return new PipelineDefinition("test", List.of(steps), OutputSpec.inline()); + } + + private void stubEndpoint(String endpoint, Resource body) { + when(internalApiClient.post(eq(endpoint), any())).thenReturn(ResponseEntity.ok(body)); + } + + private static ByteArrayResource pdf(String content, String filename) { + return new ByteArrayResource(content.getBytes()) { + @Override + public String getFilename() { + return filename; + } + }; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyExecutorTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyExecutorTest.java new file mode 100644 index 0000000000..d682c35cd7 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyExecutorTest.java @@ -0,0 +1,378 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.util.MultiValueMap; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.InternalApiClient; +import stirling.software.common.service.InternalApiTimeoutException; +import stirling.software.common.service.ToolMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.TempFileRegistry; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +/** + * Unit tests for {@link PolicyExecutor}, the shared pipeline step loop. Covers file chaining across + * steps, multi-input vs per-file dispatch, ZIP unpacking, structured-list parameter encoding, + * progress callbacks, and timeout propagation. External collaborators are mocked; {@link + * TempFileManager} is real so ZIP extraction exercises real code. + */ +@ExtendWith(MockitoExtension.class) +class PolicyExecutorTest { + + private static final String ROTATE = "/api/v1/general/rotate-pdf"; + private static final String COMPRESS = "/api/v1/misc/compress-pdf"; + private static final String SPLIT = "/api/v1/general/split-pages"; + private static final String MERGE = "/api/v1/general/merge-pdfs"; + + @Mock private InternalApiClient internalApiClient; + @Mock private ToolMetadataService toolMetadataService; + + @TempDir Path tempDir; + + private TempFileManager tempFileManager; + private PolicyExecutor executor; + + @BeforeEach + void setUp() { + ApplicationProperties props = new ApplicationProperties(); + props.getSystem().getTempFileManagement().setBaseTmpDir(tempDir.toString()); + props.getSystem().getTempFileManagement().setPrefix("policy-test-"); + tempFileManager = new TempFileManager(new TempFileRegistry(), props); + ObjectMapper objectMapper = JsonMapper.builder().build(); + executor = + new PolicyExecutor( + internalApiClient, toolMetadataService, tempFileManager, objectMapper); + } + + @Test + void executesStepsSequentiallyChainingOutputToInput() throws IOException { + when(toolMetadataService.isMultiInput(anyString())).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(anyString())).thenReturn(false); + stubEndpoint(ROTATE, pdf("rotated", "rotated.pdf")); + stubEndpoint(COMPRESS, pdf("compressed", "compressed.pdf")); + + List steps = new ArrayList<>(); + PolicyProgressListener listener = + new PolicyProgressListener() { + @Override + public void onStepStart(int stepIndex, int stepCount, String operation) { + steps.add(stepIndex); + } + }; + + PolicyExecutionResult result = + executor.execute( + definition( + new PipelineStep(ROTATE, Map.of()), + new PipelineStep(COMPRESS, Map.of())), + PolicyInputs.of(List.of(pdf("input", "input.pdf"))), + listener); + + assertEquals(1, result.files().size()); + assertEquals("compressed.pdf", result.files().get(0).getFilename()); + verify(internalApiClient, times(1)).post(eq(ROTATE), any()); + verify(internalApiClient, times(1)).post(eq(COMPRESS), any()); + // Progress fired once per step, in order. + assertEquals(List.of(1, 2), steps); + } + + @Test + void multiInputEndpointIsCalledOnceWithAllFiles() throws IOException { + when(toolMetadataService.isMultiInput(MERGE)).thenReturn(true); + when(toolMetadataService.shouldUnpackZipResponse(MERGE)).thenReturn(false); + stubEndpoint(MERGE, pdf("merged", "merged.pdf")); + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(MERGE, Map.of())), + PolicyInputs.of(List.of(pdf("a", "a.pdf"), pdf("b", "b.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(1, result.files().size()); + verify(internalApiClient, times(1)).post(eq(MERGE), any()); + } + + @Test + void singleInputEndpointIsCalledOncePerFile() throws IOException { + when(toolMetadataService.isMultiInput(ROTATE)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(ROTATE)).thenReturn(false); + stubEndpoint(ROTATE, pdf("rotated", "rotated.pdf")); + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(ROTATE, Map.of())), + PolicyInputs.of(List.of(pdf("a", "a.pdf"), pdf("b", "b.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(2, result.files().size()); + verify(internalApiClient, times(2)).post(eq(ROTATE), any()); + } + + @Test + void zipResponseIsUnpackedIntoIndividualFiles() throws IOException { + when(toolMetadataService.isMultiInput(SPLIT)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(SPLIT)).thenReturn(true); + stubEndpoint( + SPLIT, + zip( + "doc.zip", + List.of(new Entry("page-1.pdf", "one"), new Entry("page-2.pdf", "two")))); + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(SPLIT, Map.of())), + PolicyInputs.of(List.of(pdf("doc", "doc.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(2, result.files().size()); + assertEquals("page-1.pdf", result.files().get(0).getFilename()); + assertEquals("page-2.pdf", result.files().get(1).getFilename()); + } + + @Test + void structuredListParameterIsJsonEncodedAsSingleField() throws IOException { + String editText = "/api/v1/general/edit-text"; + when(toolMetadataService.isMultiInput(editText)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(editText)).thenReturn(false); + stubEndpoint(editText, pdf("edited", "edited.pdf")); + + // LinkedHashMap so the serialized key order is deterministic for the assertion below. + Map edit = new LinkedHashMap<>(); + edit.put("find", "foo"); + edit.put("replace", "bar"); + Map params = new LinkedHashMap<>(); + params.put("edits", List.of(edit)); + params.put("useRegex", false); + + executor.execute( + definition(new PipelineStep(editText, params)), + PolicyInputs.of(List.of(pdf("in", "in.pdf"))), + PolicyProgressListener.NOOP); + + @SuppressWarnings("unchecked") + ArgumentCaptor> bodyCaptor = + ArgumentCaptor.forClass(MultiValueMap.class); + verify(internalApiClient).post(eq(editText), bodyCaptor.capture()); + MultiValueMap body = bodyCaptor.getValue(); + + List edits = body.get("edits"); + assertNotNull(edits); + assertEquals(1, edits.size()); + assertEquals("[{\"find\":\"foo\",\"replace\":\"bar\"}]", edits.get(0)); + } + + @Test + void supportingFilesAreBoundToTheirNamedFields() throws IOException { + String addStamp = "/api/v1/misc/add-stamp-to-pdf"; + when(toolMetadataService.isMultiInput(addStamp)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(addStamp)).thenReturn(false); + stubEndpoint(addStamp, pdf("stamped", "stamped.pdf")); + + PipelineStep step = + new PipelineStep(addStamp, Map.of("opacity", 0.5), Map.of("stampImage", "logo")); + PolicyInputs inputs = + new PolicyInputs( + List.of(pdf("doc", "doc.pdf")), + Map.of("logo", List.of(pdf("logo-bytes", "logo.png")))); + + executor.execute( + new PipelineDefinition("stamp", List.of(step), OutputSpec.inline()), + inputs, + PolicyProgressListener.NOOP); + + @SuppressWarnings("unchecked") + ArgumentCaptor> bodyCaptor = + ArgumentCaptor.forClass(MultiValueMap.class); + verify(internalApiClient).post(eq(addStamp), bodyCaptor.capture()); + MultiValueMap body = bodyCaptor.getValue(); + // The document goes to fileInput; the supporting image is bound to its named field and is + // not part of the document stream. + assertEquals(1, body.get("fileInput").size()); + assertNotNull(body.get("stampImage")); + assertEquals(1, body.get("stampImage").size()); + } + + @Test + void missingSupportingFileFailsTheStep() { + String addStamp = "/api/v1/misc/add-stamp-to-pdf"; + when(toolMetadataService.isMultiInput(addStamp)).thenReturn(false); + PipelineStep step = new PipelineStep(addStamp, Map.of(), Map.of("stampImage", "logo")); + + IOException ex = + assertThrows( + IOException.class, + () -> + executor.execute( + new PipelineDefinition( + "stamp", List.of(step), OutputSpec.inline()), + PolicyInputs.of(List.of(pdf("doc", "doc.pdf"))), + PolicyProgressListener.NOOP)); + assertTrue(ex.getMessage().contains("logo")); + } + + @Test + void documentOfAnUnacceptedTypeFailsTheStep() { + String compress = "/api/v1/misc/compress-pdf"; + when(toolMetadataService.getExtensionTypes(false, compress)).thenReturn(List.of("pdf")); + + IOException ex = + assertThrows( + IOException.class, + () -> + executor.execute( + definition(new PipelineStep(compress, Map.of())), + PolicyInputs.of(List.of(pdf("img", "image.png"))), + PolicyProgressListener.NOOP)); + assertTrue(ex.getMessage().contains("image.png")); + // Type check happens before any dispatch. + verify(internalApiClient, never()).post(anyString(), any()); + } + + @Test + void documentOfAnAcceptedTypeProceeds() throws IOException { + String compress = "/api/v1/misc/compress-pdf"; + when(toolMetadataService.getExtensionTypes(false, compress)).thenReturn(List.of("pdf")); + when(toolMetadataService.isMultiInput(compress)).thenReturn(false); + when(toolMetadataService.shouldUnpackZipResponse(compress)).thenReturn(false); + stubEndpoint(compress, pdf("compressed", "compressed.pdf")); + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(compress, Map.of())), + PolicyInputs.of(List.of(pdf("doc", "doc.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(1, result.files().size()); + verify(internalApiClient, times(1)).post(eq(compress), any()); + } + + @Test + void filterOperationWithEmptyResultDropsTheFile() throws IOException { + String filter = "/api/v1/filter/filter-page-count"; + when(toolMetadataService.isMultiInput(filter)).thenReturn(false); + stubEndpoint(filter, pdf("", "filtered.pdf")); // empty body => filtered out + + PolicyExecutionResult result = + executor.execute( + definition(new PipelineStep(filter, Map.of())), + PolicyInputs.of(List.of(pdf("doc", "doc.pdf"))), + PolicyProgressListener.NOOP); + + assertEquals(0, result.files().size()); + } + + @Test + void timeoutFromAStepPropagates() { + when(toolMetadataService.isMultiInput(ROTATE)).thenReturn(false); + when(internalApiClient.post(eq(ROTATE), any())) + .thenThrow( + new InternalApiTimeoutException( + ROTATE, + java.time.Duration.ofSeconds(300), + new IOException("Read timed out"))); + + assertThrows( + InternalApiTimeoutException.class, + () -> + executor.execute( + definition(new PipelineStep(ROTATE, Map.of())), + PolicyInputs.of(List.of(pdf("in", "in.pdf"))), + PolicyProgressListener.NOOP)); + } + + @Test + void emptyPipelineIsRejected() { + assertThrows( + IllegalArgumentException.class, + () -> + executor.execute( + new PipelineDefinition("empty", List.of(), OutputSpec.inline()), + PolicyInputs.of(List.of(pdf("in", "in.pdf"))), + PolicyProgressListener.NOOP)); + } + + // --- helpers --- + + private static PipelineDefinition definition(PipelineStep... steps) { + return new PipelineDefinition("test", List.of(steps), OutputSpec.inline()); + } + + private void stubEndpoint(String endpoint, Resource body) { + when(internalApiClient.post(eq(endpoint), any())).thenReturn(ResponseEntity.ok(body)); + } + + private static ByteArrayResource pdf(String content, String filename) { + return new ByteArrayResource(content.getBytes()) { + @Override + public String getFilename() { + return filename; + } + }; + } + + private static ByteArrayResource zip(String filename, List entries) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + for (Entry entry : entries) { + zos.putNextEntry(new ZipEntry(entry.name())); + zos.write(entry.content().getBytes()); + zos.closeEntry(); + } + } + byte[] zipBytes = baos.toByteArray(); + return new ByteArrayResource(zipBytes) { + @Override + public String getFilename() { + return filename; + } + + @Override + public InputStream getInputStream() { + return new ByteArrayInputStream(zipBytes); + } + }; + } + + private record Entry(String name, String content) {} +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java new file mode 100644 index 0000000000..823d53edcb --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java @@ -0,0 +1,101 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Instant; +import java.util.List; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.WaitState; + +/** + * Tests for {@link PolicyRunRegistry} eviction: terminal runs expire, active/paused runs persist. + */ +class PolicyRunRegistryTest { + + private PolicyRunRegistry registry; + + @BeforeEach + void setUp() { + registry = new PolicyRunRegistry(30); + } + + @AfterEach + void tearDown() { + registry.shutdown(); + } + + @Test + void evictsTerminalRunsPastTheCutoff() { + PolicyRun completed = register("completed"); + completed.complete(List.of()); + PolicyRun failed = register("failed"); + failed.fail("boom"); + PolicyRun cancelled = register("cancelled"); + cancelled.cancel(); + + // A cutoff in the future means every terminal run finished "before" it. + int removed = registry.evictExpired(Instant.now().plusSeconds(60)); + + assertEquals(3, removed); + assertNull(registry.get("completed")); + assertNull(registry.get("failed")); + assertNull(registry.get("cancelled")); + } + + @Test + void retainsActiveAndPausedRunsRegardlessOfAge() { + register("pending"); // PENDING: never started + PolicyRun running = register("running"); + running.markRunning(); + PolicyRun waiting = register("waiting"); + waiting.waitForInput(new WaitState("needs a signature", 1, List.of())); + + int removed = registry.evictExpired(Instant.now().plusSeconds(60)); + + assertEquals(0, removed); + assertNotNull(registry.get("pending")); + assertNotNull(registry.get("running")); + assertNotNull(registry.get("waiting")); + } + + @Test + void keepsTerminalRunsStillWithinTheExpiryWindow() { + PolicyRun completed = register("recent"); + completed.complete(List.of()); + + // A cutoff in the past means the run was updated "after" it: too young to evict. + int removed = registry.evictExpired(Instant.now().minusSeconds(60)); + + assertEquals(0, removed); + assertNotNull(registry.get("recent")); + } + + @Test + void evictionLeavesUnrelatedRunsInPlace() { + PolicyRun completed = register("done"); + completed.complete(List.of()); + PolicyRun running = register("busy"); + running.markRunning(); + + registry.evictExpired(Instant.now().plusSeconds(60)); + + assertNull(registry.get("done")); + assertNotNull(registry.get("busy")); + assertTrue(registry.all().stream().anyMatch(r -> r.getRunId().equals("busy"))); + } + + private PolicyRun register(String runId) { + PolicyRun run = new PolicyRun(runId, new PipelineDefinition(runId, List.of(), null)); + registry.register(run); + return run; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java new file mode 100644 index 0000000000..2e97019182 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java @@ -0,0 +1,87 @@ +package stirling.software.proprietary.policy.store; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; + +/** Tests for {@link InProcessPolicyStore}: id assignment, upsert, trigger-type lookup, delete. */ +class InProcessPolicyStoreTest { + + private PolicyStore store; + + @BeforeEach + void setUp() { + store = new InProcessPolicyStore(); + } + + @Test + void savedPolicyGetsAnIdAndIsRetrievable() { + Policy saved = store.save(policy(null, "compress", "manual", true)); + + assertNotNull(saved.id()); + assertFalse(saved.id().isBlank()); + assertEquals(saved, store.get(saved.id()).orElseThrow()); + } + + @Test + void savingWithAnExistingIdUpdatesInPlace() { + Policy created = store.save(policy(null, "before", "manual", true)); + + store.save( + new Policy( + created.id(), + "after", + "owner", + true, + TriggerConfig.manual(), + List.of(), + OutputSpec.inline())); + + assertEquals(1, store.all().size()); + assertEquals("after", store.get(created.id()).orElseThrow().name()); + } + + @Test + void findByTriggerTypeReturnsOnlyEnabledMatches() { + store.save(policy(null, "watch", "folder", true)); + store.save(policy(null, "watch-disabled", "folder", false)); + store.save(policy(null, "nightly", "schedule", true)); + + List folder = store.findByTriggerType("folder"); + + assertEquals(1, folder.size()); + assertEquals("watch", folder.get(0).name()); + } + + @Test + void deleteRemovesThePolicy() { + Policy saved = store.save(policy(null, "p", "manual", true)); + + assertTrue(store.delete(saved.id())); + assertTrue(store.get(saved.id()).isEmpty()); + assertFalse(store.delete(saved.id())); + } + + private static Policy policy(String id, String name, String triggerType, boolean enabled) { + return new Policy( + id, + name, + "owner", + enabled, + new TriggerConfig(triggerType, Map.of()), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java new file mode 100644 index 0000000000..2a8212e2d5 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java @@ -0,0 +1,129 @@ +package stirling.software.proprietary.policy.store; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; + +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +/** + * Tests for {@link JpaPolicyStore}'s entity mapping and query delegation. The repository is mocked; + * real Hibernate/H2 persistence is exercised at application boot (this module's convention is + * Mockito unit tests for store/service logic). + */ +@ExtendWith(MockitoExtension.class) +class JpaPolicyStoreTest { + + @Mock private PolicyRepository repository; + + private final ObjectMapper objectMapper = JsonMapper.builder().build(); + private JpaPolicyStore store; + + @BeforeEach + void setUp() { + store = new JpaPolicyStore(repository, objectMapper); + } + + @Test + void saveAssignsAnIdAndPersistsThePolicyAsJson() { + Policy saved = + store.save( + new Policy( + null, + "compress incoming", + "alice", + true, + new TriggerConfig("folder", Map.of("path", "/in")), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline())); + + assertNotNull(saved.id()); + ArgumentCaptor captor = ArgumentCaptor.forClass(PolicyEntity.class); + verify(repository).save(captor.capture()); + PolicyEntity entity = captor.getValue(); + assertEquals(saved.id(), entity.getId()); + assertEquals("folder", entity.getTriggerType()); + assertTrue(entity.isEnabled()); + // The stored JSON round-trips back to an equal policy. + assertEquals(saved, objectMapper.readValue(entity.getPolicyJson(), Policy.class)); + } + + @Test + void getDeserializesThePolicyFromJson() { + Policy policy = + new Policy( + "p1", + "rotate", + "alice", + true, + TriggerConfig.manual(), + List.of( + new PipelineStep( + "/api/v1/general/rotate-pdf", Map.of("angle", 90))), + OutputSpec.inline()); + when(repository.findById("p1")).thenReturn(Optional.of(entityFor(policy))); + + assertEquals(policy, store.get("p1").orElseThrow()); + } + + @Test + void findByTriggerTypeUsesTheEnabledQuery() { + Policy policy = + new Policy( + "p1", + "watch", + "alice", + true, + new TriggerConfig("folder", Map.of()), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + when(repository.findByTriggerTypeAndEnabledTrue("folder")) + .thenReturn(List.of(entityFor(policy))); + + List folder = store.findByTriggerType("folder"); + + assertEquals(1, folder.size()); + assertEquals("p1", folder.get(0).id()); + } + + @Test + void deleteReturnsWhetherThePolicyExisted() { + when(repository.existsById("p1")).thenReturn(true); + assertTrue(store.delete("p1")); + verify(repository).deleteById("p1"); + + when(repository.existsById("missing")).thenReturn(false); + assertFalse(store.delete("missing")); + } + + private PolicyEntity entityFor(Policy policy) { + PolicyEntity entity = new PolicyEntity(); + entity.setId(policy.id()); + entity.setName(policy.name()); + entity.setOwner(policy.owner()); + entity.setEnabled(policy.enabled()); + entity.setTriggerType(policy.trigger().type()); + entity.setPolicyJson(objectMapper.writeValueAsString(policy)); + return entity; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java index 5b73ea5cc6..81d1ccdfa4 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java @@ -57,6 +57,7 @@ import stirling.software.proprietary.model.api.ai.AiWorkflowFileInput; import stirling.software.proprietary.model.api.ai.AiWorkflowOutcome; import stirling.software.proprietary.model.api.ai.AiWorkflowRequest; import stirling.software.proprietary.model.api.ai.AiWorkflowResponse; +import stirling.software.proprietary.policy.engine.PolicyExecutor; import tools.jackson.databind.ObjectMapper; import tools.jackson.databind.json.JsonMapper; @@ -107,18 +108,20 @@ class AiWorkflowServiceTest { .when(fileIdStrategy.idFor(any(MultipartFile.class))) .thenAnswer(inv -> ((MultipartFile) inv.getArgument(0)).getOriginalFilename()); + PolicyExecutor policyExecutor = + new PolicyExecutor( + internalApiClient, toolMetadataService, tempFileManager, objectMapper); service = new AiWorkflowService( pdfDocumentFactory, aiEngineClient, pdfContentExtractor, objectMapper, - internalApiClient, fileStorage, - toolMetadataService, tempFileManager, fileIdStrategy, endpointResolver, + policyExecutor, null, new ApplicationProperties()); when(endpointResolver.getEnabledEndpointUrls()).thenReturn(List.of()); From 002de064115800dc8d0e4269870b50837e0eb22c Mon Sep 17 00:00:00 2001 From: James Brunton Date: Mon, 8 Jun 2026 17:21:56 +0100 Subject: [PATCH 08/19] Fix desktop app not being able to load pdfium (#6575) # Description of Changes The changes in [#6279](https://github.com/Stirling-Tools/Stirling-PDF/pull/6279) broke the desktop app because the wasm URL handling didn't deal with `tauri://` paths. Also I noticed that `task desktop:build:dev:mac` failed locally because it was attempting to sign the app with credentials that developers won't have (and shouldn't need), so I fixed that too. --- .taskfiles/desktop.yml | 6 ++--- .../src/core/services/wasmPrecompiler.ts | 23 ++++++++----------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.taskfiles/desktop.yml b/.taskfiles/desktop.yml index ac4ccbe806..de6bfdebd2 100644 --- a/.taskfiles/desktop.yml +++ b/.taskfiles/desktop.yml @@ -62,21 +62,21 @@ tasks: deps: [prepare] dir: editor cmds: - - npx tauri build --bundles app + - npx tauri build --bundles app --config '{"bundle":{"createUpdaterArtifacts":false}}' build:dev:windows: desc: "Build Tauri desktop NSIS installer (Windows)" deps: [prepare] dir: editor cmds: - - npx tauri build --bundles nsis + - npx tauri build --bundles nsis --config '{"bundle":{"createUpdaterArtifacts":false}}' build:dev:linux: desc: "Build Tauri desktop AppImage (Linux)" deps: [prepare] dir: editor cmds: - - npx tauri build --bundles appimage + - npx tauri build --bundles appimage --config '{"bundle":{"createUpdaterArtifacts":false}}' test: desc: "Run Tauri/Cargo tests" diff --git a/frontend/editor/src/core/services/wasmPrecompiler.ts b/frontend/editor/src/core/services/wasmPrecompiler.ts index ff84010e4c..dccd0f12e2 100644 --- a/frontend/editor/src/core/services/wasmPrecompiler.ts +++ b/frontend/editor/src/core/services/wasmPrecompiler.ts @@ -2,22 +2,19 @@ import { BASE_PATH } from "@app/constants/app"; import pdfiumWasmAssetUrl from "@embedpdf/pdfium/pdfium.wasm?url"; const getWasmUrl = (): string => { - if ( - pdfiumWasmAssetUrl.startsWith("http://") || - pdfiumWasmAssetUrl.startsWith("https://") || - pdfiumWasmAssetUrl.startsWith("//") - ) { - return pdfiumWasmAssetUrl; - } - - const origin = typeof window !== "undefined" ? window.location.origin : ""; + // In dev, Vite serves the statically-copied asset from the dev server root. if (import.meta.env.DEV) { + const origin = typeof window !== "undefined" ? window.location.origin : ""; return `${origin}${BASE_PATH}/pdfium/pdfium.wasm`; } - const cleanAssetUrl = pdfiumWasmAssetUrl - .replace(/^\.\//, "") - .replace(/^\//, ""); - return `${origin}${BASE_PATH}/${cleanAssetUrl}`; + + // Vite has already produced a base-aware asset URL (absolute under a relative + // base, root-relative under an absolute base). Resolve it against the document + // to get a fetchable absolute URL that is also safe to pass to Web Workers. + if (typeof window !== "undefined") { + return new URL(pdfiumWasmAssetUrl, window.location.href).href; + } + return pdfiumWasmAssetUrl; }; export const pdfiumWasmUrl = getWasmUrl(); From 0e3cbb3cf2d6f85bbaf4ce9137ab061b3e8121a4 Mon Sep 17 00:00:00 2001 From: James Brunton Date: Tue, 9 Jun 2026 09:34:02 +0100 Subject: [PATCH 09/19] Explicitly test for console warnings & errors (#6502) # Description of Changes Disallow warnings and errors from being thrown in the browser console during tests unless explicitly expected in the test. Also adds a Playwright test to prod around some main UI areas and checks that no warnings/errors have been thrown. --- .../core/contexts/AppConfigContext.test.tsx | 6 + .../src/core/contexts/AppConfigContext.tsx | 6 +- .../src/core/contexts/FolderContext.test.tsx | 7 +- .../src/core/contexts/FolderContext.tsx | 2 +- .../tools/shared/toolOperationHelpers.ts | 2 +- .../editor/src/core/hooks/useScarfTracking.ts | 3 - .../indexedDBManager.migration.test.ts | 7 + frontend/editor/src/core/setupTests.ts | 3 + .../tests/convert/ConvertIntegration.test.tsx | 10 ++ .../ConvertSmartDetectionIntegration.test.tsx | 16 ++ .../src/core/tests/failOnConsole.test.ts | 78 +++++++++ .../editor/src/core/tests/failOnConsole.ts | 152 ++++++++++++++++++ .../core/tests/stubbed/console-clean.spec.ts | 114 +++++++++++++ .../core/utils/automationConverter.test.ts | 2 + .../editor/src/core/utils/scarfTracking.ts | 24 ++- .../editor/src/core/utils/toolSynonyms.ts | 26 +-- .../hooks/useDesktopUpdatePopup.test.ts | 25 ++- .../desktop/services/apiClientSetup.test.ts | 4 + .../proprietary/auth/springAuthClient.test.ts | 28 ++++ .../src/proprietary/auth/springAuthClient.ts | 15 +- .../stripeCheckout/stages/PaymentStage.tsx | 12 +- .../proprietary/routes/AuthCallback.test.tsx | 4 + .../src/proprietary/routes/AuthCallback.tsx | 135 ++-------------- .../src/proprietary/routes/Login.test.tsx | 56 ++++--- .../editor/src/proprietary/routes/Login.tsx | 6 - .../components/shared/StripeCheckoutSaas.tsx | 9 +- .../src/saas/services/apiClient.test.ts | 2 + frontend/editor/src/saas/setupTests.ts | 3 + 28 files changed, 544 insertions(+), 213 deletions(-) create mode 100644 frontend/editor/src/core/tests/failOnConsole.test.ts create mode 100644 frontend/editor/src/core/tests/failOnConsole.ts create mode 100644 frontend/editor/src/core/tests/stubbed/console-clean.spec.ts diff --git a/frontend/editor/src/core/contexts/AppConfigContext.test.tsx b/frontend/editor/src/core/contexts/AppConfigContext.test.tsx index eb630b7715..5e512ec69e 100644 --- a/frontend/editor/src/core/contexts/AppConfigContext.test.tsx +++ b/frontend/editor/src/core/contexts/AppConfigContext.test.tsx @@ -5,6 +5,7 @@ import { useAppConfig, } from "@app/contexts/AppConfigContext"; import apiClient from "@app/services/apiClient"; +import { allowConsole, expectConsole } from "@app/tests/failOnConsole"; import { ReactNode } from "react"; // Mock apiClient @@ -92,6 +93,7 @@ describe("AppConfigContext", () => { }); it("should handle network errors", async () => { + expectConsole.error(/\[AppConfig\] Failed to fetch app config/); const errorMessage = "Network error occurred"; const mockError = new Error(errorMessage); // Network errors don't have response property @@ -249,6 +251,10 @@ describe("AppConfigContext", () => { }); it("should handle initial config prop", async () => { + // apiClient.get isn't mocked here, so the implicit "fetch fails" path + // exhausts retries and logs. Test only asserts the API was called, so + // the failure log is incidental. + allowConsole.error(/\[AppConfig\] Failed to fetch app config/); const initialConfig = { enableLogin: false, appNameNavbar: "Initial App", diff --git a/frontend/editor/src/core/contexts/AppConfigContext.tsx b/frontend/editor/src/core/contexts/AppConfigContext.tsx index 7635345540..3672f6cdfa 100644 --- a/frontend/editor/src/core/contexts/AppConfigContext.tsx +++ b/frontend/editor/src/core/contexts/AppConfigContext.tsx @@ -107,12 +107,10 @@ export const AppConfigProvider: React.FC = ({ if (attempt > 0) { const delay = initialDelay * Math.pow(2, attempt - 1); - console.log( + console.debug( `[AppConfig] Retry attempt ${attempt}/${maxRetries} after ${delay}ms delay...`, ); await sleep(delay); - } else { - console.log("[AppConfig] Fetching app config..."); } // apiClient automatically adds JWT header if available via interceptors @@ -165,7 +163,7 @@ export const AppConfigProvider: React.FC = ({ (!status || status >= 500) && attempt < maxRetries; if (shouldRetry) { - console.warn( + console.debug( `[AppConfig] Attempt ${attempt + 1} failed (status ${status || "network error"}):`, err.message, "- will retry...", diff --git a/frontend/editor/src/core/contexts/FolderContext.test.tsx b/frontend/editor/src/core/contexts/FolderContext.test.tsx index 171dec8200..861ddd331b 100644 --- a/frontend/editor/src/core/contexts/FolderContext.test.tsx +++ b/frontend/editor/src/core/contexts/FolderContext.test.tsx @@ -3,6 +3,7 @@ import { describe, expect, test, vi, beforeEach } from "vitest"; import { render, screen, waitFor, act } from "@testing-library/react"; import { FolderProvider, useFolders } from "@app/contexts/FolderContext"; +import { expectConsole } from "@app/tests/failOnConsole"; /** * Regression test for the sync-banner 4xx gating fix in commit c38b646c5. @@ -122,6 +123,9 @@ describe("FolderContext sync-banner gating", () => { }); test("500 (server error) DOES surface a banner", async () => { + // Surfacing the banner also logs a warning - that's the contract this branch + // tests for. + expectConsole.warn(/\[FolderContext\] pullFromServer failed/); mockList.mockRejectedValue(axiosError(500, "internal")); await renderAndWaitForPull(); expect(screen.getByTestId("error").textContent).toContain( @@ -133,7 +137,8 @@ describe("FolderContext sync-banner gating", () => { test("network error (no response) DOES surface a banner", async () => { // axios on a network failure rejects with an Error that has NO // `.response` property - that's the "status === undefined" branch - // the gate explicitly covers. + // the gate explicitly covers. Surfacing the banner also logs a warning. + expectConsole.warn(/\[FolderContext\] pullFromServer failed/); mockList.mockRejectedValue(new Error("ECONNREFUSED")); await renderAndWaitForPull(); expect(screen.getByTestId("error").textContent).toContain( diff --git a/frontend/editor/src/core/contexts/FolderContext.tsx b/frontend/editor/src/core/contexts/FolderContext.tsx index 0f421af1de..9c69fd0cf4 100644 --- a/frontend/editor/src/core/contexts/FolderContext.tsx +++ b/frontend/editor/src/core/contexts/FolderContext.tsx @@ -272,7 +272,6 @@ export function FolderProvider({ children }: FolderProviderProps) { if (mountedRef.current) setServerReachable(false); return { ok: false, reason: "endpoint-missing" }; } - console.warn("[FolderContext] pullFromServer failed", err); if (mountedRef.current) { setServerReachable(false); // Only surface a banner when this is a server-side outage or @@ -284,6 +283,7 @@ export function FolderProvider({ children }: FolderProviderProps) { // manager. Folder-mutation buttons get individual disabled // tooltips via `serverReachable`, which is enough signal. if (status === undefined || status >= 500) { + console.warn("[FolderContext] pullFromServer failed", err); setError(`Folder sync failed: ${formatServerError(err)}`); } } diff --git a/frontend/editor/src/core/hooks/tools/shared/toolOperationHelpers.ts b/frontend/editor/src/core/hooks/tools/shared/toolOperationHelpers.ts index 37d52eacb5..baaef647cd 100644 --- a/frontend/editor/src/core/hooks/tools/shared/toolOperationHelpers.ts +++ b/frontend/editor/src/core/hooks/tools/shared/toolOperationHelpers.ts @@ -27,7 +27,7 @@ export function buildInputTracking( inputFileIds.push(fileId); inputStirlingFileStubs.push(record); } else { - console.warn(`No file stub found for file: ${file.name}`); + console.debug(`No file stub found for file: ${file.name}`); inputFileIds.push(fileId); inputStirlingFileStubs.push(createNewStirlingFileStub(file, fileId)); } diff --git a/frontend/editor/src/core/hooks/useScarfTracking.ts b/frontend/editor/src/core/hooks/useScarfTracking.ts index 27eed4b97a..7c98e80c68 100644 --- a/frontend/editor/src/core/hooks/useScarfTracking.ts +++ b/frontend/editor/src/core/hooks/useScarfTracking.ts @@ -29,9 +29,6 @@ export function useScarfTracking() { // Listen to cookie consent changes and auto-fire pixel when consent is granted useEffect(() => { const handleConsentChange = () => { - console.warn( - "[useScarfTracking] Consent changed, checking scarf service acceptance", - ); if (isServiceAccepted("scarf", "analytics")) { firePixel(window.location.pathname); } diff --git a/frontend/editor/src/core/services/indexedDBManager.migration.test.ts b/frontend/editor/src/core/services/indexedDBManager.migration.test.ts index aecb75484d..0a5a4b9827 100644 --- a/frontend/editor/src/core/services/indexedDBManager.migration.test.ts +++ b/frontend/editor/src/core/services/indexedDBManager.migration.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test, beforeEach } from "vitest"; import "fake-indexeddb/auto"; +import { expectConsole } from "@app/tests/failOnConsole"; import { DATABASE_CONFIGS, @@ -363,6 +364,9 @@ describe("IndexedDB migration (FILES store)", () => { }); test("SaaS v6 database is force-deleted (data lost, schema reset to v9)", async () => { + // The force-delete warn IS the contract under test; production fires it + // to surface why data was wiped. + expectConsole.warn(/Deleting corrupt SaaS v6/); await seedSaasDatabase(6, ["v6-corrupt-file"]); await indexedDBManager.openDatabase(DATABASE_CONFIGS.FILES); @@ -381,6 +385,9 @@ describe("IndexedDB migration (FILES store)", () => { }); test("SaaS v7 database is force-deleted (data lost, schema reset to v9)", async () => { + // The force-delete warn IS the contract under test; production fires it + // to surface why data was wiped. + expectConsole.warn(/Deleting corrupt SaaS v7/); await seedSaasDatabase(7, ["v7-corrupt-file"]); await indexedDBManager.openDatabase(DATABASE_CONFIGS.FILES); diff --git a/frontend/editor/src/core/setupTests.ts b/frontend/editor/src/core/setupTests.ts index bd6e083253..57bcaa76a5 100644 --- a/frontend/editor/src/core/setupTests.ts +++ b/frontend/editor/src/core/setupTests.ts @@ -1,5 +1,8 @@ import "@testing-library/jest-dom"; import { vi } from "vitest"; +import { installFailOnConsole } from "@app/tests/failOnConsole"; + +installFailOnConsole(); // Mock localStorage for tests class LocalStorageMock implements Storage { diff --git a/frontend/editor/src/core/tests/convert/ConvertIntegration.test.tsx b/frontend/editor/src/core/tests/convert/ConvertIntegration.test.tsx index 9ac5b7c282..e0045584fa 100644 --- a/frontend/editor/src/core/tests/convert/ConvertIntegration.test.tsx +++ b/frontend/editor/src/core/tests/convert/ConvertIntegration.test.tsx @@ -29,6 +29,8 @@ import { PreferencesProvider } from "@app/contexts/PreferencesContext"; import { I18nextProvider } from "react-i18next"; import i18n from "@app/i18n/config"; import { createTestStirlingFile } from "@app/tests/utils/testFileHelpers"; +import { expectConsole } from "@app/tests/failOnConsole"; +import { fileStorage } from "@app/services/fileStorage"; import { StirlingFile } from "@app/types/fileContext"; import { MantineProvider } from "@mantine/core"; @@ -78,6 +80,7 @@ vi.mock("../../services/fileStorage", () => ({ thumbnail: thumbnail, }); }), + storeStirlingFile: vi.fn().mockResolvedValue(undefined), getAllFileMetadata: vi.fn().mockResolvedValue([]), cleanup: vi.fn().mockResolvedValue(undefined), }, @@ -209,6 +212,10 @@ describe("Convert Tool Integration Tests", () => { expect(result.current.downloadFilename).toBe("test.png"); expect(result.current.isLoading).toBe(false); expect(result.current.errorMessage).toBe(null); + + // The output file must be persisted via fileStorage.storeStirlingFile + // so downstream tools see it in the registry. + expect(fileStorage.storeStirlingFile).toHaveBeenCalled(); }); test("should handle API error responses correctly", async () => { @@ -571,6 +578,9 @@ describe("Convert Tool Integration Tests", () => { describe("File Upload Integration", () => { test("should handle multiple file uploads correctly", async () => { + // Test mocks only one apiClient.post response; the second file hits an + // undefined response and production warns about the conversion failure. + expectConsole.warn(/Failed to convert file test2\.pdf/); const mockBlob = new Blob(["zip-content"], { type: "application/zip" }); (mockedApiClient.post as Mock).mockResolvedValueOnce({ data: mockBlob }); diff --git a/frontend/editor/src/core/tests/convert/ConvertSmartDetectionIntegration.test.tsx b/frontend/editor/src/core/tests/convert/ConvertSmartDetectionIntegration.test.tsx index af5b991727..cfb7a2dd90 100644 --- a/frontend/editor/src/core/tests/convert/ConvertSmartDetectionIntegration.test.tsx +++ b/frontend/editor/src/core/tests/convert/ConvertSmartDetectionIntegration.test.tsx @@ -28,6 +28,8 @@ import { createTestStirlingFile, createTestFilesWithId, } from "@app/tests/utils/testFileHelpers"; +import { allowConsole, expectConsole } from "@app/tests/failOnConsole"; +import { fileStorage } from "@app/services/fileStorage"; import { MantineProvider } from "@mantine/core"; // Mock axios (for static methods like CancelToken, isCancel) @@ -76,6 +78,7 @@ vi.mock("../../services/fileStorage", () => ({ thumbnail: thumbnail, }); }), + storeStirlingFile: vi.fn().mockResolvedValue(undefined), getAllFileMetadata: vi.fn().mockResolvedValue([]), cleanup: vi.fn().mockResolvedValue(undefined), }, @@ -169,6 +172,10 @@ describe("Convert Tool - Smart Detection Integration Tests", () => { responseType: "blob", }, ); + + // The output file must be persisted via fileStorage.storeStirlingFile + // so downstream tools see it in the registry. + expect(fileStorage.storeStirlingFile).toHaveBeenCalled(); }); test("should handle unknown file type with file-to-pdf fallback", async () => { @@ -496,6 +503,12 @@ describe("Convert Tool - Smart Detection Integration Tests", () => { }); test("should send correct PDF/A parameters for pdf-to-pdfa conversion", async () => { + // Test files aren't registered in the FileContext stub registry, so + // production warns about the stub/output mismatch - incidental to what + // this test asserts. + allowConsole.warn( + /\[useToolOperation\] Mismatch successInputStubs vs outputs/, + ); const { result: paramsResult } = renderHook( () => useConvertParameters(), { @@ -638,6 +651,9 @@ describe("Convert Tool - Smart Detection Integration Tests", () => { describe("Error Scenarios in Smart Detection", () => { test("should handle partial failures in multi-file processing", async () => { + // Production warns when an individual file in a batch fails; the test + // deliberately drives one mock rejection to exercise that path. + expectConsole.warn(/Failed to convert file doc2\.xyz/); const { result: paramsResult } = renderHook( () => useConvertParameters(), { diff --git a/frontend/editor/src/core/tests/failOnConsole.test.ts b/frontend/editor/src/core/tests/failOnConsole.test.ts new file mode 100644 index 0000000000..944917a14d --- /dev/null +++ b/frontend/editor/src/core/tests/failOnConsole.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { + allowConsole, + expectConsole, + formatArgs, + matches, +} from "@app/tests/failOnConsole"; + +describe("failOnConsole", () => { + describe("matches", () => { + it("treats a string matcher as a substring test", () => { + expect(matches("foo", "the foo bar")).toBe(true); + expect(matches("foo", "no match here")).toBe(false); + }); + + it("treats a RegExp matcher as a .test() call", () => { + expect(matches(/^\[Tag\]/, "[Tag] something")).toBe(true); + expect(matches(/^\[Tag\]/, "prefix [Tag] something")).toBe(false); + }); + + it("string match is case-sensitive", () => { + expect(matches("Foo", "foo")).toBe(false); + }); + }); + + describe("formatArgs", () => { + it("joins string args with spaces", () => { + expect(formatArgs(["one", "two", "three"])).toBe("one two three"); + }); + + it("stringifies plain objects with JSON", () => { + expect(formatArgs([{ a: 1, b: "x" }])).toBe('{"a":1,"b":"x"}'); + }); + + it("uses the stack for Error instances when present", () => { + const err = new Error("boom"); + const result = formatArgs(["context:", err]); + expect(result).toContain("context:"); + expect(result).toContain("boom"); + }); + + it("falls back to String() when JSON.stringify throws (circular ref)", () => { + const circular: Record = { name: "loop" }; + circular.self = circular; + const out = formatArgs([circular]); + expect(out).toBe("[object Object]"); + }); + + it("handles non-string, non-Error primitives", () => { + expect(formatArgs([42, true, null])).toBe("42 true null"); + }); + }); + + describe("expectConsole / allowConsole integration", () => { + it("expectConsole.error absorbs a matching call and satisfies the expectation", () => { + expectConsole.error(/expected sentinel/); + console.error("an expected sentinel was emitted"); + // No assertion failure here means: the call was absorbed AND + // afterEach won't throw because the expectation matched. + }); + + it("allowConsole.warn absorbs a matching call without asserting it fires", () => { + allowConsole.warn(/optional sentinel/); + // Intentionally do NOT call console.warn. The test should still + // pass because allowConsole is non-required. + }); + + it("allowConsole.warn absorbs the call when it does fire too", () => { + allowConsole.warn(/optional sentinel that does fire/); + console.warn("optional sentinel that does fire now"); + }); + + it("supports a string matcher (substring)", () => { + expectConsole.warn("substring sentinel inside"); + console.warn("a substring sentinel inside a longer message"); + }); + }); +}); diff --git a/frontend/editor/src/core/tests/failOnConsole.ts b/frontend/editor/src/core/tests/failOnConsole.ts new file mode 100644 index 0000000000..1c5dc329e4 --- /dev/null +++ b/frontend/editor/src/core/tests/failOnConsole.ts @@ -0,0 +1,152 @@ +import { afterEach, beforeEach, vi, type MockInstance } from "vitest"; + +type ConsoleMethod = "error" | "warn"; +type Matcher = string | RegExp; + +const WATCHED: ConsoleMethod[] = ["error", "warn"]; + +type Capture = { method: ConsoleMethod; args: unknown[] }; + +type Expectation = { + method: ConsoleMethod; + matcher: Matcher; + /** + * `true` for `expectConsole` (a contract: the test fails if no matching + * call fires); `false` for `allowConsole` (incidental noise: absorbed if + * it happens, ignored if it doesn't). + */ + required: boolean; + matched: boolean; +}; + +const captured: Capture[] = []; +const expectations: Expectation[] = []; + +/** + * Vitest setup that fails any test whose code calls console.error or + * console.warn. The goal is to keep the browser console clean during normal + * usage of the app: React key warnings, act() warnings, deprecation + * notices, and runtime errors should all surface as test failures rather + * than silently scrolling past in CI. + * + * If a test legitimately needs to drive a log: + * - Use `expectConsole.error(/pattern/)` when the log is part of the + * contract under test. The matching call is absorbed AND the test + * fails if it never fires, so the log can't silently disappear later. + * - Use `allowConsole.warn(/pattern/)` when the log is incidental + * (third-party noise, an artefact of a fake JWT, etc.). The matching + * call is absorbed; nothing is asserted. + * + * Anything that doesn't match an expectation/allowance still fails the + * test as before. + */ +export function installFailOnConsole(): void { + const spies: MockInstance[] = []; + + beforeEach(() => { + captured.length = 0; + expectations.length = 0; + spies.length = 0; + for (const method of WATCHED) { + const original = console[method].bind(console); + const spy = vi + .spyOn(console, method) + .mockImplementation((...args: unknown[]) => { + const text = formatArgs(args); + const match = expectations.find( + (e) => e.method === method && matches(e.matcher, text), + ); + if (match) { + match.matched = true; + return; + } + captured.push({ method, args }); + original(...args); + }); + spies.push(spy); + } + }); + + afterEach(() => { + for (const spy of spies.splice(0)) { + spy.mockRestore(); + } + + const unmet = expectations.filter((e) => e.required && !e.matched); + const localCaptured = captured.splice(0); + expectations.length = 0; + + if (unmet.length > 0) { + const lines = unmet + .map((e) => ` console.${e.method}: ${formatMatcher(e.matcher)}`) + .join("\n"); + throw new Error( + `Expected console output never fired:\n${lines}\n\n` + + "If production stopped logging this, either restore the log or " + + "update/remove the expectConsole call.", + ); + } + if (localCaptured.length === 0) return; + const lines = localCaptured + .map((c) => ` console.${c.method}: ${formatArgs(c.args)}`) + .join("\n"); + throw new Error( + `Test produced unexpected console output:\n${lines}\n\n` + + "If the log is expected, capture it with expectConsole.error/warn " + + "(contract) or allowConsole.error/warn (incidental) from " + + "@app/tests/failOnConsole.", + ); + }); +} + +/** + * Register a required console expectation. The test passes only if at + * least one matching call fires; matching calls are absorbed instead of + * failing the test. + */ +export const expectConsole = { + error: (matcher: Matcher) => register("error", matcher, true), + warn: (matcher: Matcher) => register("warn", matcher, true), +}; + +/** + * Register an optional console allowance. Matching calls are absorbed; + * the test does NOT fail if no matching call fires. Use for incidental + * noise (fake-JWT decode warnings, third-party library logs in dev, etc.). + */ +export const allowConsole = { + error: (matcher: Matcher) => register("error", matcher, false), + warn: (matcher: Matcher) => register("warn", matcher, false), +}; + +function register(method: ConsoleMethod, matcher: Matcher, required: boolean) { + expectations.push({ method, matcher, required, matched: false }); +} + +/** Exported for the failOnConsole unit tests; not part of the public API. */ +export function matches(matcher: Matcher, text: string): boolean { + return typeof matcher === "string" + ? text.includes(matcher) + : matcher.test(text); +} + +function formatMatcher(matcher: Matcher): string { + return matcher instanceof RegExp + ? matcher.toString() + : JSON.stringify(matcher); +} + +/** Exported for the failOnConsole unit tests; not part of the public API. */ +export function formatArgs(args: unknown[]): string { + return args.map(formatArg).join(" "); +} + +function formatArg(value: unknown): string { + if (value instanceof Error) return value.stack ?? value.message; + if (typeof value === "string") return value; + try { + return JSON.stringify(value); + } catch { + return String(value); + } +} diff --git a/frontend/editor/src/core/tests/stubbed/console-clean.spec.ts b/frontend/editor/src/core/tests/stubbed/console-clean.spec.ts new file mode 100644 index 0000000000..5379f98705 --- /dev/null +++ b/frontend/editor/src/core/tests/stubbed/console-clean.spec.ts @@ -0,0 +1,114 @@ +import { test, expect } from "@app/tests/helpers/stub-test-base"; +import { errors, type ConsoleMessage, type Page } from "@playwright/test"; + +/** + * Smoke test: standard usage of the app must not produce any + * console.error, console.warn, or uncaught page errors. + * + * Each test disables the fixture's auto-goto (via `test.use({ autoGoto: + * false })`), attaches listeners inline with `attachListeners(page)` BEFORE + * navigating, walks a representative route, lets it settle, then asserts + * the captured buffer is empty. + * + * If you have a legitimate reason a warning fires on a given route + * (third-party library noise we cannot influence, etc.), filter it via the + * `IGNORED` allowlist below, but the default expectation is that the + * console stays clean. Do not add entries casually; prefer fixing the + * underlying issue. + */ + +type ConsoleEntry = { + type: "error" | "warn" | "pageerror"; + text: string; + location?: string; +}; + +const IGNORED: RegExp[] = [ + // Add entries here only with a comment explaining why the warning is + // unavoidable. Default: keep this list empty. +]; + +function shouldIgnore(text: string): boolean { + return IGNORED.some((re) => re.test(text)); +} + +function attachListeners(page: Page): ConsoleEntry[] { + const entries: ConsoleEntry[] = []; + page.on("console", (msg: ConsoleMessage) => { + const type = msg.type(); + if (type !== "error" && type !== "warning") return; + const text = msg.text(); + if (shouldIgnore(text)) return; + const loc = msg.location(); + entries.push({ + type: type === "warning" ? "warn" : "error", + text, + location: loc.url + ? `${loc.url}:${loc.lineNumber}:${loc.columnNumber}` + : undefined, + }); + }); + page.on("pageerror", (err) => { + if (shouldIgnore(err.message)) return; + entries.push({ type: "pageerror", text: err.stack ?? err.message }); + }); + return entries; +} + +function formatEntries(entries: ConsoleEntry[]): string { + return entries + .map( + (e) => + ` [${e.type}] ${e.text}${e.location ? `\n at ${e.location}` : ""}`, + ) + .join("\n"); +} + +async function expectCleanConsole(entries: ConsoleEntry[]) { + expect( + entries, + `Page produced unexpected console output:\n${formatEntries(entries)}`, + ).toEqual([]); +} + +// ─── Routes to sweep ──────────────────────────────────────────────────────── +// +// One entry per route we want to guarantee is console-clean on load. Mirrors +// the most common user entry points; expand cautiously - every entry adds CI +// time and triage surface for new warnings. + +const ROUTES: { name: string; path: string }[] = [ + { name: "landing", path: "/" }, + { name: "files", path: "/files" }, + { name: "compress", path: "/compress" }, + { name: "split", path: "/split" }, + { name: "merge", path: "/merge" }, + { name: "convert", path: "/convert" }, + { name: "rotate", path: "/rotate" }, + { name: "addPageNumbers", path: "/add-page-numbers" }, +]; + +// Disable the fixture's auto-goto so we can attach listeners before any +// navigation happens. Otherwise listeners miss early load-time noise. +test.use({ autoGoto: false }); + +test.describe("Console hygiene: representative routes load cleanly", () => { + for (const route of ROUTES) { + test(`${route.name} (${route.path})`, async ({ page }) => { + const entries = attachListeners(page); + await page.goto(route.path, { waitUntil: "domcontentloaded" }); + // Give async effects (i18n load, lazy chunks, posthog init) a beat to + // surface anything they were going to log. + await page + .waitForLoadState("networkidle", { timeout: 10_000 }) + .catch((err) => { + // networkidle can flake on third-party CDNs; treat ONLY the + // timeout as benign and still run the console assertion on what + // we captured. Anything else (frame detached, navigation abort, + // etc.) is a real problem and should fail the test. + if (!(err instanceof errors.TimeoutError)) throw err; + }); + await expectCleanConsole(entries); + }); + } +}); diff --git a/frontend/editor/src/core/utils/automationConverter.test.ts b/frontend/editor/src/core/utils/automationConverter.test.ts index c03df11b92..88e313c604 100644 --- a/frontend/editor/src/core/utils/automationConverter.test.ts +++ b/frontend/editor/src/core/utils/automationConverter.test.ts @@ -3,6 +3,7 @@ */ import { describe, test, expect } from "vitest"; +import { expectConsole } from "@app/tests/failOnConsole"; import { convertToAutomationConfig, convertToFolderScanningConfig, @@ -111,6 +112,7 @@ describe("automationConverter", () => { }); test("falls back to operation key when no endpoint is registered", () => { + expectConsole.warn(/No endpoint found for operation "unknownTool"/); const automation: AutomationConfig = { ...sampleAutomation, operations: [{ operation: "unknownTool", parameters: {} }], diff --git a/frontend/editor/src/core/utils/scarfTracking.ts b/frontend/editor/src/core/utils/scarfTracking.ts index 4c8d8b6c38..ca4769f351 100644 --- a/frontend/editor/src/core/utils/scarfTracking.ts +++ b/frontend/editor/src/core/utils/scarfTracking.ts @@ -6,8 +6,9 @@ * injected via setScarfConfig() which should be called from a React hook * during app initialization. * - * IMPORTANT: setScarfConfig() must be called before firePixel() will work. - * The initialization hook (useScarfTracking) is mounted in App.tsx. + * firePixel() can be called BEFORE setScarfConfig() runs: pre-config calls + * are queued and replayed once setScarfConfig fires, so the initial-page-load + * pixel from useUrlSync doesn't race the useScarfTracking init effect. * * For testing: Use resetScarfConfig() to clear module state between tests. */ @@ -19,6 +20,11 @@ let isServiceAccepted: ((service: string, category: string) => boolean) | null = null; let lastFiredPathname: string | null = null; let lastFiredTime = 0; +// Pathnames passed to firePixel() before setScarfConfig() has run. Drained +// once configured. Bounded to the most recent few entries since intermediate +// path changes during a slow init are uninteresting. +const pendingPaths: string[] = []; +const PENDING_PATHS_CAP = 8; /** * Configure scarf tracking with app config and consent checker @@ -34,6 +40,10 @@ export function setScarfConfig( configured = true; enableScarf = scarfEnabled; isServiceAccepted = consentChecker; + // Drain anything queued before we were configured. Splice first so a + // re-entrant firePixel from inside the drain can't re-queue. + const queued = pendingPaths.splice(0); + for (const path of queued) firePixel(path); } /** @@ -47,12 +57,10 @@ export function setScarfConfig( * @param pathname - The pathname to track (usually window.location.pathname) */ export function firePixel(pathname: string): void { - // Dev-mode warning if called before initialization + // Pre-init: queue and bail. setScarfConfig() drains. if (!configured) { - console.warn( - "[scarfTracking] firePixel() called before setScarfConfig(). " + - "Ensure useScarfTracking() hook is mounted in App.tsx.", - ); + if (pendingPaths.length >= PENDING_PATHS_CAP) pendingPaths.shift(); + pendingPaths.push(pathname); return; } @@ -91,8 +99,10 @@ export function firePixel(pathname: string): void { * Useful for testing to ensure clean state between test runs */ export function resetScarfConfig(): void { + configured = false; enableScarf = null; isServiceAccepted = null; lastFiredPathname = null; lastFiredTime = 0; + pendingPaths.length = 0; } diff --git a/frontend/editor/src/core/utils/toolSynonyms.ts b/frontend/editor/src/core/utils/toolSynonyms.ts index fe5cdfb00d..fbfd2afca3 100644 --- a/frontend/editor/src/core/utils/toolSynonyms.ts +++ b/frontend/editor/src/core/utils/toolSynonyms.ts @@ -4,34 +4,16 @@ import { TFunction } from "i18next"; export const getSynonyms = (t: TFunction, toolId: string): string[] => { try { const candidateKeys = [`home.${toolId}.tags`, `${toolId}.tags`]; - const match = candidateKeys + const tags = candidateKeys .map((key) => ({ key, value: t(key) as unknown as string })) - .find(({ key, value }) => value && value !== key); - const tags = match?.value; - const usedKey = match?.key; + .find(({ key, value }) => value && value !== key)?.value; - if (!tags) { - console.warn(`[Tags] Missing tags for tool: ${toolId}`); - return []; - } + if (!tags) return []; - // Split by comma and clean up the tags - const cleanedTags = tags + return tags .split(",") .map((tag: string) => tag.trim()) .filter((tag: string) => tag.length > 0); - - // Log the tags found for this tool - if (cleanedTags.length > 0) { - console.info( - `[Tags] Tool "${toolId}" (${usedKey}) has ${cleanedTags.length} tags:`, - cleanedTags, - ); - } else { - console.warn(`[Tags] Tool "${toolId}" has empty tags value`); - } - - return cleanedTags; } catch (error) { console.error( `[Tags] Failed to get translated synonyms for tool ${toolId}:`, diff --git a/frontend/editor/src/desktop/hooks/useDesktopUpdatePopup.test.ts b/frontend/editor/src/desktop/hooks/useDesktopUpdatePopup.test.ts index 83ef34fd43..d26a20a982 100644 --- a/frontend/editor/src/desktop/hooks/useDesktopUpdatePopup.test.ts +++ b/frontend/editor/src/desktop/hooks/useDesktopUpdatePopup.test.ts @@ -1,5 +1,6 @@ -import { renderHook } from "@testing-library/react"; +import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { expectConsole } from "@app/tests/failOnConsole"; // ── Mocks (hoisted by vi.mock) ────────────────────────────────────────────── const invokeMock = vi.fn(); @@ -46,13 +47,21 @@ async function flushMicrotasks() { const AUTO_FAILURE_KEY = "stirling-pdf-updater:autoFailedAt"; -/** Run the hook through its startup timer + async chain. */ +/** + * Run the hook through its startup timer + async chain. The post-render + * timer-driven async work is wrapped in act() so the state updates it + * triggers don't surface as React "not wrapped in act" warnings. + * renderHook already wraps the initial render in act internally, so it + * stays outside. + */ async function runStartup() { renderHook(() => useDesktopUpdatePopup()); - await vi.advanceTimersByTimeAsync(16_000); - await flushMicrotasks(); - await vi.advanceTimersByTimeAsync(0); - await flushMicrotasks(); + await act(async () => { + await vi.advanceTimersByTimeAsync(16_000); + await flushMicrotasks(); + await vi.advanceTimersByTimeAsync(0); + await flushMicrotasks(); + }); } describe("useDesktopUpdatePopup — auto mode", () => { @@ -83,6 +92,7 @@ describe("useDesktopUpdatePopup — auto mode", () => { }); it("does NOT call restart_app when download_and_install_update fails", async () => { + expectConsole.error(/\[useDesktopInstall\] Install failed/); invokeMock.mockImplementation((cmd: string) => { if (cmd === "check_for_update") { return Promise.resolve({ @@ -132,6 +142,9 @@ describe("useDesktopUpdatePopup — auto mode", () => { }); it("skips the install entirely when a recent failure is within the backoff window", async () => { + expectConsole.warn( + /\[DesktopUpdatePopup\] auto-update skipped: recent failure within backoff window/, + ); // Recorded 1 hour ago — well inside the 6-hour backoff. const oneHourAgo = Date.now() - 60 * 60 * 1000; window.localStorage.setItem(AUTO_FAILURE_KEY, String(oneHourAgo)); diff --git a/frontend/editor/src/desktop/services/apiClientSetup.test.ts b/frontend/editor/src/desktop/services/apiClientSetup.test.ts index 1406570186..fd46abd725 100644 --- a/frontend/editor/src/desktop/services/apiClientSetup.test.ts +++ b/frontend/editor/src/desktop/services/apiClientSetup.test.ts @@ -1,6 +1,7 @@ import type { AxiosInstance } from "axios"; import { describe, expect, test, vi, beforeEach, afterEach } from "vitest"; import { OPEN_SIGN_IN_EVENT } from "@app/constants/signInEvents"; +import { expectConsole } from "@app/tests/failOnConsole"; // Exercise the error-interceptor logic against a hand-rolled axios-like // client; transitive imports are mocked out so this is a pure unit test. @@ -120,6 +121,7 @@ describe("desktop apiClientSetup - 401 silent-path", () => { }); test("POST 401 without Authorization dispatches sign-in modal event", async () => { + expectConsole.warn(/\[apiClientSetup\] 401 on path/); const { client, handlers } = makeMockClient(); setupApiInterceptors(client as unknown as AxiosInstance); await triggerErrorInterceptor(handlers, { @@ -135,6 +137,7 @@ describe("desktop apiClientSetup - 401 silent-path", () => { }); test("GET 401 without Authorization stays silent (background probe)", async () => { + expectConsole.warn(/\[apiClientSetup\] 401 on path/); const { client, handlers } = makeMockClient(); setupApiInterceptors(client as unknown as AxiosInstance); await triggerErrorInterceptor(handlers, { @@ -159,6 +162,7 @@ describe("desktop apiClientSetup - 401 silent-path", () => { }); test("401 with skipAuthRedirect set never dispatches the modal", async () => { + expectConsole.warn(/\[apiClientSetup\] 401 on path/); const { client, handlers } = makeMockClient(); setupApiInterceptors(client as unknown as AxiosInstance); await triggerErrorInterceptor(handlers, { diff --git a/frontend/editor/src/proprietary/auth/springAuthClient.test.ts b/frontend/editor/src/proprietary/auth/springAuthClient.test.ts index 1258a4d990..d64ac49476 100644 --- a/frontend/editor/src/proprietary/auth/springAuthClient.test.ts +++ b/frontend/editor/src/proprietary/auth/springAuthClient.test.ts @@ -8,6 +8,7 @@ import { } from "@app/auth/springAuthClient"; import { startOAuthNavigation } from "@app/extensions/oauthNavigation"; import apiClient from "@app/services/apiClient"; +import { allowConsole, expectConsole } from "@app/tests/failOnConsole"; import { AxiosError, type AxiosResponse, @@ -89,9 +90,18 @@ describe("SpringAuthClient", () => { ); vi.mocked(apiClient.get).mockRejectedValueOnce(mockError); + vi.mocked(apiClient.post).mockRejectedValueOnce(mockError); const result = await springAuth.getSession(); + // A 401 from /me triggers an explicit refresh attempt before treating + // the session as invalid; lock that recovery contract in so future + // refactors can't quietly skip it. + expect(apiClient.post).toHaveBeenCalledWith( + "/api/v1/auth/refresh", + null, + expect.any(Object), + ); expect(localStorage.getItem("stirling_jwt")).toBeNull(); expect(result.data.session).toBeNull(); // 401 is handled gracefully, so error should be null @@ -117,9 +127,18 @@ describe("SpringAuthClient", () => { ); vi.mocked(apiClient.get).mockRejectedValueOnce(mockError); + vi.mocked(apiClient.post).mockRejectedValueOnce(mockError); const result = await springAuth.getSession(); + // A 403 from /me triggers an explicit refresh attempt before treating + // the session as invalid; lock that recovery contract in so future + // refactors can't quietly skip it. + expect(apiClient.post).toHaveBeenCalledWith( + "/api/v1/auth/refresh", + null, + expect.any(Object), + ); expect(localStorage.getItem("stirling_jwt")).toBeNull(); expect(result.data.session).toBeNull(); // 403 is handled gracefully, so error should be null @@ -129,6 +148,9 @@ describe("SpringAuthClient", () => { describe("signInWithPassword", () => { it("should successfully sign in with email and password", async () => { + // The fake token isn't a real JWT, so calculateAdaptiveIntervals warns + // about defaults - incidental to what this test verifies. + allowConsole.warn(/Cannot decode token for adaptive intervals/); const credentials = { email: "test@example.com", password: "password123", @@ -176,6 +198,7 @@ describe("SpringAuthClient", () => { }); it("should return error on failed login", async () => { + expectConsole.error(/\[SpringAuth\] signInWithPassword error/); const credentials = { email: "wrong@example.com", password: "wrongpassword", @@ -236,6 +259,7 @@ describe("SpringAuthClient", () => { }); it("should return error on failed registration", async () => { + expectConsole.error(/\[SpringAuth\] signUp error/); const credentials = { email: "existing@example.com", password: "password123", @@ -283,6 +307,7 @@ describe("SpringAuthClient", () => { }); it("should clear JWT even if logout request fails", async () => { + expectConsole.error(/\[SpringAuth\] signOut error/); const mockToken = "jwt-to-clear"; localStorage.setItem("stirling_jwt", mockToken); @@ -301,6 +326,9 @@ describe("SpringAuthClient", () => { describe("refreshSession", () => { it("should refresh JWT token successfully", async () => { + // The fake token isn't a real JWT, so calculateAdaptiveIntervals warns + // about defaults - incidental to what this test verifies. + allowConsole.warn(/Cannot decode token for adaptive intervals/); const newToken = "refreshed-jwt-token"; const mockUser = { id: "123", diff --git a/frontend/editor/src/proprietary/auth/springAuthClient.ts b/frontend/editor/src/proprietary/auth/springAuthClient.ts index 91cef2e5f6..5d75605e37 100644 --- a/frontend/editor/src/proprietary/auth/springAuthClient.ts +++ b/frontend/editor/src/proprietary/auth/springAuthClient.ts @@ -397,22 +397,19 @@ class SpringAuthClient { // console.debug('[SpringAuth] getSession: Session retrieved successfully'); return { data: { session }, error: null }; } catch (error: unknown) { - console.error("[SpringAuth] getSession error:", error); - - // If 401/403, token is invalid - try explicit refresh + // 401/403 during getSession is the normal "token expired or invalid" + // path - handled via refresh + JWT clear. const status = getHttpStatus(error); if (status === 401 || status === 403) { - // A 401 during startup can be a race with a concurrent refresh. Try one - // explicit refresh before treating the session as invalid. const refreshResult = await this.refreshSession(); if (!refreshResult.error && refreshResult.data.session) { return refreshResult; } localStorage.removeItem("stirling_jwt"); - console.debug("[SpringAuth] getSession: Not authenticated"); return { data: { session: null }, error: null }; } + console.error("[SpringAuth] getSession error:", error); // Don't clear token for other errors (e.g., backend not ready, network issues) // The token is still valid, just can't verify it right now return { @@ -739,10 +736,11 @@ class SpringAuthClient { return { data: { session }, error: null }; } catch (error: unknown) { - console.error("[SpringAuth] refreshSession error:", error); localStorage.removeItem("stirling_jwt"); - // Handle different error statuses + // 401/403 means the refresh token is no longer valid - normal expired + // state, not an error worth surfacing. Other statuses (network, backend + // down) ARE worth logging. const status = getHttpStatus(error); if (status === 401 || status === 403) { return { @@ -751,6 +749,7 @@ class SpringAuthClient { }; } + console.error("[SpringAuth] refreshSession error:", error); return { data: { session: null }, error: { message: getErrorMessage(error, "Token refresh failed") }, diff --git a/frontend/editor/src/proprietary/components/shared/stripeCheckout/stages/PaymentStage.tsx b/frontend/editor/src/proprietary/components/shared/stripeCheckout/stages/PaymentStage.tsx index a84db823ae..1e9d7acec4 100644 --- a/frontend/editor/src/proprietary/components/shared/stripeCheckout/stages/PaymentStage.tsx +++ b/frontend/editor/src/proprietary/components/shared/stripeCheckout/stages/PaymentStage.tsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useMemo } from "react"; import { Stack, Text, Loader } from "@mantine/core"; import { useTranslation } from "react-i18next"; import { loadStripe } from "@stripe/stripe-js"; @@ -8,9 +8,7 @@ import { } from "@stripe/react-stripe-js"; import { PlanTier } from "@app/services/licenseService"; -// Load Stripe once const STRIPE_KEY = import.meta.env.VITE_STRIPE_PUBLISHABLE_KEY; -const stripePromise = STRIPE_KEY ? loadStripe(STRIPE_KEY) : null; interface PaymentStageProps { clientSecret: string | null; @@ -24,6 +22,14 @@ export const PaymentStage: React.FC = ({ onPaymentComplete, }) => { const { t } = useTranslation(); + // Load Stripe.js lazily, only when PaymentStage mounts. Loading at module + // scope pulled the Stripe script into every page (CheckoutContext is in + // AppProviders), which triggered the dev "HTTPS required" warning on every + // non-payment route. + const stripePromise = useMemo( + () => (STRIPE_KEY ? loadStripe(STRIPE_KEY) : null), + [], + ); // Show loading while creating checkout session if (!clientSecret || !selectedPlan) { diff --git a/frontend/editor/src/proprietary/routes/AuthCallback.test.tsx b/frontend/editor/src/proprietary/routes/AuthCallback.test.tsx index fc05574fb1..4491358381 100644 --- a/frontend/editor/src/proprietary/routes/AuthCallback.test.tsx +++ b/frontend/editor/src/proprietary/routes/AuthCallback.test.tsx @@ -6,6 +6,7 @@ import { POST_LOGIN_REDIRECT_STORAGE_KEY, springAuth, } from "@app/auth/springAuthClient"; +import { expectConsole } from "@app/tests/failOnConsole"; // Mock springAuth; keep the real redirect-path helpers. vi.mock("@app/auth/springAuthClient", async () => { @@ -90,6 +91,7 @@ describe("AuthCallback", () => { }); it("should redirect to login when no access token in hash", async () => { + expectConsole.error(/\[AuthCallback\] No access_token in URL fragment/); // No hash or empty hash window.location.hash = ""; @@ -109,6 +111,7 @@ describe("AuthCallback", () => { }); it("should redirect to login when token validation fails", async () => { + expectConsole.error(/\[AuthCallback\] Failed to validate token/); const invalidToken = "invalid-oauth-token"; window.location.hash = `#access_token=${invalidToken}`; @@ -137,6 +140,7 @@ describe("AuthCallback", () => { }); it("should handle errors gracefully", async () => { + expectConsole.error(/\[AuthCallback\] Authentication failed/); const mockToken = "error-token"; window.location.hash = `#access_token=${mockToken}`; diff --git a/frontend/editor/src/proprietary/routes/AuthCallback.tsx b/frontend/editor/src/proprietary/routes/AuthCallback.tsx index 32f81b8411..fd2754ee88 100644 --- a/frontend/editor/src/proprietary/routes/AuthCallback.tsx +++ b/frontend/editor/src/proprietary/routes/AuthCallback.tsx @@ -18,42 +18,16 @@ export default function AuthCallback() { const navigate = useNavigate(); const processingRef = useRef(false); - // Log component lifecycle useEffect(() => { - const mountId = Math.random().toString(36).substring(7); - console.log(`[AuthCallback:${mountId}] 🔵 Component mounted`); - return () => { - console.log(`[AuthCallback:${mountId}] 🔴 Component unmounting`); - }; - }, []); + const startedAt = performance.now(); + const elapsed = () => `${(performance.now() - startedAt).toFixed(0)}ms`; - useEffect(() => { const handleCallback = async () => { - const startTime = performance.now(); - const executionId = Math.random().toString(36).substring(7); - - console.log( - `[AuthCallback:${executionId}] ════════════════════════════════════`, - ); - console.log( - `[AuthCallback:${executionId}] Starting authentication callback`, - ); - console.log(`[AuthCallback:${executionId}] URL: ${window.location.href}`); - console.log( - `[AuthCallback:${executionId}] Hash: ${window.location.hash}`, - ); - console.log( - `[AuthCallback:${executionId}] Document readyState: ${document.readyState}`, - ); - if ( typeof window !== "undefined" && window.sessionStorage.getItem("stirling_sso_auto_login_logged_out") === "1" ) { - console.warn( - `[AuthCallback:${executionId}] ⚠️ Logout block active, skipping token processing`, - ); navigate("/login", { replace: true, state: { error: "You have been signed out. Please sign in again." }, @@ -62,30 +36,16 @@ export default function AuthCallback() { } // Prevent double execution (React 18 Strict Mode + navigate dependency) - if (processingRef.current) { - console.warn( - `[AuthCallback:${executionId}] ⚠️ Already processing, skipping duplicate execution`, - ); - console.warn( - `[AuthCallback:${executionId}] This is expected in React Strict Mode (development)`, - ); - return; - } + if (processingRef.current) return; processingRef.current = true; try { - console.log( - `[AuthCallback:${executionId}] Step 1: Extracting token from URL fragment`, - ); - - // Extract JWT from URL fragment (#access_token=...) - const hash = window.location.hash.substring(1); // Remove '#' - const params = new URLSearchParams(hash); - const token = params.get("access_token"); + const hash = window.location.hash.substring(1); + const token = new URLSearchParams(hash).get("access_token"); if (!token) { console.error( - `[AuthCallback:${executionId}] ❌ No access_token in URL fragment`, + `[AuthCallback] No access_token in URL fragment (${elapsed()})`, ); navigate("/login", { replace: true, @@ -94,39 +54,13 @@ export default function AuthCallback() { return; } - console.log( - `[AuthCallback:${executionId}] ✓ Token extracted (length: ${token.length})`, - ); - console.log( - `[AuthCallback:${executionId}] Step 2: Storing JWT in localStorage`, - ); - - // Store JWT in localStorage localStorage.setItem("stirling_jwt", token); - console.log( - `[AuthCallback:${executionId}] ✓ JWT stored in localStorage`, - ); - - console.log( - `[AuthCallback:${executionId}] Step 3: Dispatching 'jwt-available' event`, - ); - // Dispatch custom event for other components to react to JWT availability window.dispatchEvent(new CustomEvent("jwt-available")); - console.log(`[AuthCallback:${executionId}] ✓ Event dispatched`); - console.log( - `[AuthCallback:${executionId}] Elapsed after jwt-available: ${(performance.now() - startTime).toFixed(2)}ms`, - ); - console.log( - `[AuthCallback:${executionId}] Step 4: Validating token with backend`, - ); - // Validate the token and load user info - // This calls /api/v1/auth/me with the JWT to get user details const { data, error } = await springAuth.getSession(); - if (error || !data.session) { console.error( - `[AuthCallback:${executionId}] ❌ Failed to validate token:`, + `[AuthCallback] Failed to validate token (${elapsed()}):`, error, ); localStorage.removeItem("stirling_jwt"); @@ -137,68 +71,21 @@ export default function AuthCallback() { return; } - console.log( - `[AuthCallback:${executionId}] ✓ Token validated, user: ${data.session.user.username}`, - ); - console.log( - `[AuthCallback:${executionId}] Step 5: Running platform-specific callback handlers`, - ); - await handleAuthCallbackSuccess(token); - console.log( - `[AuthCallback:${executionId}] ✓ Callback handlers complete`, - ); - console.log( - `[AuthCallback:${executionId}] Step 6: Waiting for context stabilization`, - ); - // Wait for all context providers to process jwt-available event // This prevents infinite render loop when coming from cross-domain SAML redirect await new Promise((resolve) => setTimeout(resolve, 100)); - console.log( - `[AuthCallback:${executionId}] Elapsed after stabilization wait: ${(performance.now() - startTime).toFixed(2)}ms`, - ); const target = consumePostLoginRedirectPath() ?? "/"; - console.log( - `[AuthCallback:${executionId}] Step 7: Navigating to ${target}`, + console.info( + `[AuthCallback] Authenticated ${data.session.user.username} in ${elapsed()}, navigating to ${target}`, ); navigate(target, { replace: true }); - - const duration = performance.now() - startTime; - console.log( - `[AuthCallback:${executionId}] ✓ Authentication complete (${duration.toFixed(2)}ms)`, - ); - console.log( - `[AuthCallback:${executionId}] ════════════════════════════════════`, - ); } catch (error) { - const duration = performance.now() - startTime; console.error( - `[AuthCallback:${executionId}] ════════════════════════════════════`, - ); - console.error( - `[AuthCallback:${executionId}] ❌ FATAL ERROR during authentication`, - ); - console.error(`[AuthCallback:${executionId}] Error:`, error); - console.error( - `[AuthCallback:${executionId}] Error name:`, - (error as Error)?.name, - ); - console.error( - `[AuthCallback:${executionId}] Error message:`, - (error as Error)?.message, - ); - console.error( - `[AuthCallback:${executionId}] Error stack:`, - (error as Error)?.stack, - ); - console.error( - `[AuthCallback:${executionId}] Duration before failure: ${duration.toFixed(2)}ms`, - ); - console.error( - `[AuthCallback:${executionId}] ════════════════════════════════════`, + `[AuthCallback] Authentication failed (${elapsed()}):`, + error, ); navigate("/login", { replace: true, diff --git a/frontend/editor/src/proprietary/routes/Login.test.tsx b/frontend/editor/src/proprietary/routes/Login.test.tsx index 0a4b79324b..6b9306a722 100644 --- a/frontend/editor/src/proprietary/routes/Login.test.tsx +++ b/frontend/editor/src/proprietary/routes/Login.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; -import { render, screen, waitFor } from "@testing-library/react"; +import { act, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { BrowserRouter, MemoryRouter } from "react-router-dom"; import { MantineProvider } from "@mantine/core"; @@ -176,7 +176,7 @@ describe("Login", () => { }); }); - it("should show loading state while auth is loading", () => { + it("should show loading state while auth is loading", async () => { vi.mocked(useAuth).mockReturnValue({ session: null, user: null, @@ -187,13 +187,15 @@ describe("Login", () => { refreshSession: vi.fn(), }); - render( - - - - - , - ); + await act(async () => { + render( + + + + + , + ); + }); // Component shouldn't redirect or show form while loading expect(mockNavigate).not.toHaveBeenCalled(); @@ -510,26 +512,30 @@ describe("Login", () => { expect(springAuth.signInWithPassword).not.toHaveBeenCalled(); }); - it("should display session expired message from URL param", () => { - render( - - - - - , - ); + it("should display session expired message from URL param", async () => { + await act(async () => { + render( + + + + + , + ); + }); expect(screen.getByText(/session.*expired/i)).toBeInTheDocument(); }); - it("should display account created success message", () => { - render( - - - - - , - ); + it("should display account created success message", async () => { + await act(async () => { + render( + + + + + , + ); + }); expect(screen.getByText(/account created/i)).toBeInTheDocument(); }); diff --git a/frontend/editor/src/proprietary/routes/Login.tsx b/frontend/editor/src/proprietary/routes/Login.tsx index 22d9c57e17..85c42fce86 100644 --- a/frontend/editor/src/proprietary/routes/Login.tsx +++ b/frontend/editor/src/proprietary/routes/Login.tsx @@ -287,8 +287,6 @@ export default function Login() { setPostLoginRedirectPath(returnPath); } - console.log(`[Login] Signing in with provider: ${provider}`); - // Redirect to Spring OAuth2 endpoint using the actual provider ID from backend // The backend returns the correct registration ID (e.g., 'authentik', 'oidc', 'keycloak') const { error } = await springAuth.signInWithOAuth({ @@ -520,8 +518,6 @@ export default function Login() { setError(null); clearLogoutBlock(); - console.log("[Login] Signing in with email:", email); - const { user, session, error } = await springAuth.signInWithPassword({ email: email.trim(), password: password, @@ -529,13 +525,11 @@ export default function Login() { }); if (error) { - console.error("[Login] Email sign in error:", error); setError(error.message); if (error.mfaRequired || error.code === "invalid_mfa_code") { setRequiresMfa(true); } } else if (user && session) { - console.log("[Login] Email sign in successful"); clearLogoutBlock(); setRequiresMfa(false); setMfaCode(""); diff --git a/frontend/editor/src/saas/components/shared/StripeCheckoutSaas.tsx b/frontend/editor/src/saas/components/shared/StripeCheckoutSaas.tsx index fffefc91dd..43c96a9b33 100644 --- a/frontend/editor/src/saas/components/shared/StripeCheckoutSaas.tsx +++ b/frontend/editor/src/saas/components/shared/StripeCheckoutSaas.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useEffect, useMemo, useState } from "react"; import { Modal, Button, Text, Alert, Loader, Stack } from "@mantine/core"; import { useTranslation } from "react-i18next"; import { loadStripe } from "@stripe/stripe-js"; @@ -9,7 +9,7 @@ import { import { supabase } from "@app/auth/supabase"; import { Z_INDEX_OVER_SETTINGS_MODAL } from "@app/styles/zIndex"; -const stripePromise = loadStripe(import.meta.env.VITE_STRIPE_PUBLISHABLE_KEY); +const STRIPE_KEY = import.meta.env.VITE_STRIPE_PUBLISHABLE_KEY; export type PurchaseType = "subscription" | "credits"; export type CreditsPack = "xsmall" | "small" | "medium" | "large" | null; @@ -68,6 +68,11 @@ const StripeCheckout: React.FC = ({ }) => { const { t } = useTranslation(); const [state, setState] = useState({ status: "idle" }); + // Load Stripe.js lazily, only when this checkout component mounts. Loading + // at module scope pulled the Stripe script into every page that imports + // this file, which triggered the dev "HTTPS required" warning on every + // non-payment route. + const stripePromise = useMemo(() => loadStripe(STRIPE_KEY), []); const createCheckoutSession = async () => { try { diff --git a/frontend/editor/src/saas/services/apiClient.test.ts b/frontend/editor/src/saas/services/apiClient.test.ts index 01a9649387..56480bb7c7 100644 --- a/frontend/editor/src/saas/services/apiClient.test.ts +++ b/frontend/editor/src/saas/services/apiClient.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import type { Session, AuthError } from "@supabase/supabase-js"; import { supabase } from "@app/auth/supabase"; +import { expectConsole } from "@app/tests/failOnConsole"; // Mock supabase vi.mock("@app/auth/supabase", () => ({ @@ -177,6 +178,7 @@ describe("apiClient", () => { }); it("should handle refresh token failure", async () => { + expectConsole.error(/\[API Client\] Token refresh failed/); const oldToken = "old-token"; const oldSession = { diff --git a/frontend/editor/src/saas/setupTests.ts b/frontend/editor/src/saas/setupTests.ts index c461557ffa..2c5f53e271 100644 --- a/frontend/editor/src/saas/setupTests.ts +++ b/frontend/editor/src/saas/setupTests.ts @@ -1,5 +1,8 @@ import "@testing-library/jest-dom"; import { vi } from "vitest"; +import { installFailOnConsole } from "@app/tests/failOnConsole"; + +installFailOnConsole(); // Mock localStorage for tests const localStorageMock = (() => { From 66f431a2b7e90743b3a2348f28352bc8dc1efdce Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 9 Jun 2026 13:36:48 +0100 Subject: [PATCH 10/19] Lazy-load Stripe SDK so it only loads on checkout (#6546) # Description of Changes - Stripe SDK (`@stripe/*` + `js.stripe.com/v3`) was loading on every page; now it only loads when an upgrade/checkout modal actually opens. - Converted every import site to `React.lazy()` + ``, gated by the existing `opened` state. - Adds a Playwright spec that asserts no Stripe requests on landing or settings. --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .../tests/stubbed/stripe-lazy-load.spec.ts | 81 +++++++++++++++++++ .../proprietary/contexts/CheckoutContext.tsx | 31 ++++--- .../saas/components/TrialExpiredBootstrap.tsx | 37 +++++---- .../components/shared/TrialStatusBanner.tsx | 33 ++++---- .../shared/config/configSections/Plan.tsx | 61 ++++++++------ .../plan/ApiPackagesSection.tsx | 2 +- 6 files changed, 178 insertions(+), 67 deletions(-) create mode 100644 frontend/editor/src/core/tests/stubbed/stripe-lazy-load.spec.ts diff --git a/frontend/editor/src/core/tests/stubbed/stripe-lazy-load.spec.ts b/frontend/editor/src/core/tests/stubbed/stripe-lazy-load.spec.ts new file mode 100644 index 0000000000..561f0d1bc8 --- /dev/null +++ b/frontend/editor/src/core/tests/stubbed/stripe-lazy-load.spec.ts @@ -0,0 +1,81 @@ +import { test, expect } from "@app/tests/helpers/stub-test-base"; + +/** + * Verifies that the Stripe SDK (@stripe/stripe-js, @stripe/react-stripe-js, + * and the js.stripe.com remote script) is NOT fetched on cold page loads — + * only when the checkout modal actually mounts. The proprietary + * CheckoutProvider, the SaaS TrialStatusBanner, and the SaaS Plan settings + * page all gate the modal behind React.lazy + a conditional render so the + * Stripe chunk lives in its own async bundle. + */ + +const STRIPE_URL_FRAGMENTS = [ + "js.stripe.com", + "@stripe/stripe-js", + // Vite's optimizeDeps id mangles slashes to underscores when serving + // pre-bundled dependencies from node_modules/.vite/deps. + "@stripe_stripe-js", + "@stripe/react-stripe-js", + "@stripe_react-stripe-js", +]; + +function isStripeRequest(url: string): boolean { + return STRIPE_URL_FRAGMENTS.some((fragment) => url.includes(fragment)); +} + +test.describe("Stripe SDK lazy loading", () => { + test("landing page does not fetch Stripe SDK on cold load", async ({ + page, + }) => { + const stripeRequests: string[] = []; + page.on("request", (req) => { + const url = req.url(); + if (isStripeRequest(url)) { + stripeRequests.push(url); + } + }); + + // The stub fixture already navigated to "/" with waitUntil: + // domcontentloaded. Wait for the app to actually settle so any lurking + // module-eval-time loadStripe() call would have already kicked off. + await page + .waitForLoadState("networkidle", { timeout: 15_000 }) + .catch(() => { + // Posthog / iconify keep some connections warm — fall back to a + // brief settle window if networkidle never resolves. + }); + await page.waitForTimeout(2_000); + + expect( + stripeRequests, + `Expected no Stripe-related network activity on landing, but observed:\n${stripeRequests.join("\n")}`, + ).toEqual([]); + }); + + test("settings modal does not fetch Stripe SDK", async ({ page }) => { + const stripeRequests: string[] = []; + page.on("request", (req) => { + const url = req.url(); + if (isStripeRequest(url)) { + stripeRequests.push(url); + } + }); + + // Opening Settings is the closest a default proprietary user gets to + // checkout without actually clicking Upgrade. Even rendering the + // settings drawer must NOT pull Stripe into the entry path — only the + // upgrade modal itself, which sits one click further in. + const settingsButton = page + .getByRole("button", { name: /settings/i }) + .first(); + if (await settingsButton.isVisible({ timeout: 5_000 }).catch(() => false)) { + await settingsButton.click(); + await page.waitForTimeout(1_500); + } + + expect( + stripeRequests, + `Expected no Stripe-related network activity after opening settings, but observed:\n${stripeRequests.join("\n")}`, + ).toEqual([]); + }); +}); diff --git a/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx b/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx index 45912667fb..e050114bee 100644 --- a/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx +++ b/frontend/editor/src/proprietary/contexts/CheckoutContext.tsx @@ -4,6 +4,8 @@ import React, { useState, useCallback, useEffect, + lazy, + Suspense, ReactNode, } from "react"; import { useTranslation } from "react-i18next"; @@ -13,7 +15,12 @@ import licenseService, { mapLicenseToTier, PlanTier, } from "@app/services/licenseService"; -import { StripeCheckout } from "@app/components/shared/stripeCheckout"; + +const StripeCheckout = lazy(() => + import("@app/components/shared/stripeCheckout").then((m) => ({ + default: m.StripeCheckout, + })), +); import { userManagementService } from "@app/services/userManagementService"; import { alert } from "@app/components/toast"; import { @@ -433,16 +440,18 @@ export const CheckoutProvider: React.FC = ({ {/* Global Checkout Modal */} {selectedPlanGroup && ( - + + + )} ); diff --git a/frontend/editor/src/saas/components/TrialExpiredBootstrap.tsx b/frontend/editor/src/saas/components/TrialExpiredBootstrap.tsx index 1b185684cb..40826f1904 100644 --- a/frontend/editor/src/saas/components/TrialExpiredBootstrap.tsx +++ b/frontend/editor/src/saas/components/TrialExpiredBootstrap.tsx @@ -1,7 +1,10 @@ -import { useEffect, useState } from "react"; +import { lazy, Suspense, useEffect, useState } from "react"; import { useAuth } from "@app/auth/UseSession"; import { TrialExpiredModal } from "@app/components/shared/TrialExpiredModal"; -import StripeCheckout from "@app/components/shared/StripeCheckoutSaas"; + +const StripeCheckout = lazy( + () => import("@app/components/shared/StripeCheckoutSaas"), +); /** * Bootstrap component that shows the trial expired modal when a user's trial has ended @@ -106,20 +109,22 @@ export default function TrialExpiredBootstrap() { onSubscribe={handleSubscribe} /> - {user && ( - - console.error("[TrialExpired] Checkout error:", error) - } - isTrialConversion={false} // Trial already ended, so this is not a conversion - /> + {user && checkoutOpened && ( + + + console.error("[TrialExpired] Checkout error:", error) + } + isTrialConversion={false} // Trial already ended, so this is not a conversion + /> + )} ); diff --git a/frontend/editor/src/saas/components/shared/TrialStatusBanner.tsx b/frontend/editor/src/saas/components/shared/TrialStatusBanner.tsx index e6e7322ec2..7efec40eb4 100644 --- a/frontend/editor/src/saas/components/shared/TrialStatusBanner.tsx +++ b/frontend/editor/src/saas/components/shared/TrialStatusBanner.tsx @@ -1,11 +1,14 @@ -import { useEffect, useState, useCallback } from "react"; +import { lazy, Suspense, useEffect, useState, useCallback } from "react"; import { useBanner } from "@app/contexts/BannerContext"; import { useAuth } from "@app/auth/UseSession"; import { useTranslation } from "react-i18next"; import { InfoBanner } from "@app/components/shared/InfoBanner"; -import StripeCheckout from "@app/components/shared/StripeCheckoutSaas"; import { BASE_PATH } from "@app/constants/app"; +const StripeCheckout = lazy( + () => import("@app/components/shared/StripeCheckoutSaas"), +); + const SESSION_STORAGE_KEY = "trialBannerDismissed"; export function TrialStatusBanner() { @@ -113,18 +116,20 @@ export function TrialStatusBanner() { return ( <> - {trialStatus && ( - setCheckoutOpen(false)} - purchaseType="subscription" - planId="pro" - creditsPack={null} - planName="Pro" - onSuccess={handleCheckoutSuccess} - onError={(error) => console.error("Checkout error:", error)} - isTrialConversion={true} - /> + {trialStatus && checkoutOpen && ( + + setCheckoutOpen(false)} + purchaseType="subscription" + planId="pro" + creditsPack={null} + planName="Pro" + onSuccess={handleCheckoutSuccess} + onError={(error) => console.error("Checkout error:", error)} + isTrialConversion={true} + /> + )} ); diff --git a/frontend/editor/src/saas/components/shared/config/configSections/Plan.tsx b/frontend/editor/src/saas/components/shared/config/configSections/Plan.tsx index af7dcb2f83..8571c5db47 100644 --- a/frontend/editor/src/saas/components/shared/config/configSections/Plan.tsx +++ b/frontend/editor/src/saas/components/shared/config/configSections/Plan.tsx @@ -1,11 +1,15 @@ -import React, { useState, useCallback, useEffect } from "react"; +import React, { lazy, Suspense, useState, useCallback, useEffect } from "react"; import { Divider, Loader, Alert, Select, Group, Text } from "@mantine/core"; import { usePlans, PlanTier } from "@app/hooks/usePlans"; -import StripeCheckout, { +import type { PurchaseType, CreditsPack, PlanID, } from "@app/components/shared/StripeCheckoutSaas"; + +const StripeCheckout = lazy( + () => import("@app/components/shared/StripeCheckoutSaas"), +); import AvailablePlansSection from "@app/components/shared/config/configSections/plan/AvailablePlansSection"; import ApiPackagesSection from "@app/components/shared/config/configSections/plan/ApiPackagesSection"; import ActivePlanSection from "@app/components/shared/config/configSections/plan/ActivePlanSection"; @@ -204,29 +208,36 @@ const Plan: React.FC = () => { /> {/* Stripe Checkout Modal */} - pkg.id === selectedCreditsPack) - ?.name || "" - } - onSuccess={handlePaymentSuccess} - onError={handlePaymentError} - isTrialConversion={ - trialStatus?.isTrialing && purchaseType === "subscription" - } - /> + {checkoutOpen && + (selectedPlan !== null || selectedCreditsPack !== null) && ( + + pkg.id === selectedCreditsPack, + )?.name || "" + } + onSuccess={handlePaymentSuccess} + onError={handlePaymentError} + isTrialConversion={ + trialStatus?.isTrialing && purchaseType === "subscription" + } + /> + + )} ); }; diff --git a/frontend/editor/src/saas/components/shared/config/configSections/plan/ApiPackagesSection.tsx b/frontend/editor/src/saas/components/shared/config/configSections/plan/ApiPackagesSection.tsx index 09e238be04..f707d5d9a9 100644 --- a/frontend/editor/src/saas/components/shared/config/configSections/plan/ApiPackagesSection.tsx +++ b/frontend/editor/src/saas/components/shared/config/configSections/plan/ApiPackagesSection.tsx @@ -1,7 +1,7 @@ import React from "react"; import { Button, Card, Text, Stack, Flex, Slider } from "@mantine/core"; import { useTranslation } from "react-i18next"; -import { CreditsPack } from "@app/components/shared/StripeCheckoutSaas"; +import type { CreditsPack } from "@app/components/shared/StripeCheckoutSaas"; interface ApiPackage { id: string; From 800a41116789f9f6ed1612de16db500b0c1bcf83 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:18:36 +0100 Subject: [PATCH 11/19] Hide endpoints (#6586) # Description of Changes Hides the /api/v1/credits endpoints from the generated OpenAPI/Swagger docs. The root GET /api/v1/credits and GET /api/v1/credits/usage now carry @Hidden (the 8 admin credit endpoints were already hidden), so the whole Credit Management controller is gone from the docs. Adds a single global AI tag to the OpenAPI definition. Why We don't want the credits or AI endpoints surfaced in the public API docs yet as they are not ready for public use, but we do want the AI endpoints pre-grouped under one AI tag so they land cleanly when we later un-hide them but dont clutter PDF APIs. --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .../stirling/software/SPDF/config/OpenApiConfig.java | 10 ++++++++++ .../saas/ai/controller/AiCreateController.java | 5 +++++ .../saas/ai/controller/AiCreateInternalController.java | 5 +++++ .../software/saas/ai/controller/AiProxyController.java | 5 +++++ .../software/saas/controller/CreditController.java | 2 ++ 5 files changed, 27 insertions(+) diff --git a/app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java b/app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java index 205a0d5734..7c7fe115d0 100644 --- a/app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java +++ b/app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java @@ -18,6 +18,7 @@ import io.swagger.v3.oas.models.media.StringSchema; import io.swagger.v3.oas.models.security.SecurityRequirement; import io.swagger.v3.oas.models.security.SecurityScheme; import io.swagger.v3.oas.models.servers.Server; +import io.swagger.v3.oas.models.tags.Tag; import lombok.RequiredArgsConstructor; @@ -60,6 +61,15 @@ public class OpenApiConfig { OpenAPI openAPI = new OpenAPI().info(info).openapi("3.0.3"); + // Register a single global "AI" tag so every AI endpoint groups under it in the docs. + // The AI controllers are currently @Hidden, so they don't emit this tag themselves yet; + // defining it here keeps the grouping ready for when those endpoints are unhidden. + openAPI.addTagsItem( + new Tag() + .name("AI") + .description( + "AI-powered document creation, editing, and assistant endpoints.")); + // Add server configuration from environment variable String swaggerServerUrl = System.getenv("SWAGGER_SERVER_URL"); Server server; diff --git a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java index 85520a239b..848d596d8b 100644 --- a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java +++ b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateController.java @@ -29,6 +29,9 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBo import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.tags.Tag; + import jakarta.servlet.http.HttpServletRequest; import lombok.RequiredArgsConstructor; @@ -48,6 +51,8 @@ import stirling.software.saas.util.CreditHeaderUtils; @RestController @Profile("saas") @RequestMapping("/api/v1/ai/create") +@Tag(name = "AI") +@Hidden @RequiredArgsConstructor @Slf4j public class AiCreateController { diff --git a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateInternalController.java b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateInternalController.java index 34c44176e5..bf2e4894e8 100644 --- a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateInternalController.java +++ b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiCreateInternalController.java @@ -17,6 +17,9 @@ import org.springframework.web.server.ResponseStatusException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.tags.Tag; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,6 +30,8 @@ import stirling.software.saas.ai.service.AiCreateSessionService; @RestController @Profile("saas") @RequestMapping("/api/v1/ai/create/internal") +@Tag(name = "AI") +@Hidden @RequiredArgsConstructor @Slf4j public class AiCreateInternalController { diff --git a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiProxyController.java b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiProxyController.java index 074da8707c..8b6ae714a4 100644 --- a/app/saas/src/main/java/stirling/software/saas/ai/controller/AiProxyController.java +++ b/app/saas/src/main/java/stirling/software/saas/ai/controller/AiProxyController.java @@ -18,6 +18,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.tags.Tag; + import jakarta.servlet.http.HttpServletRequest; import lombok.extern.slf4j.Slf4j; @@ -33,6 +36,8 @@ import stirling.software.saas.util.CreditHeaderUtils; @RestController @Profile("saas") @RequestMapping("/api/v1/ai") +@Tag(name = "AI") +@Hidden @Slf4j public class AiProxyController { diff --git a/app/saas/src/main/java/stirling/software/saas/controller/CreditController.java b/app/saas/src/main/java/stirling/software/saas/controller/CreditController.java index 9459c8bbb4..2c1be26f45 100644 --- a/app/saas/src/main/java/stirling/software/saas/controller/CreditController.java +++ b/app/saas/src/main/java/stirling/software/saas/controller/CreditController.java @@ -36,6 +36,7 @@ public class CreditController { private final CreditService creditService; @GetMapping + @Hidden @Operation( summary = "Get user credit information", description = @@ -252,6 +253,7 @@ public class CreditController { } @GetMapping("/usage") + @Hidden @Operation( summary = "Get credit usage summary", description = "Get overview of credit usage (for authenticated user or admin view)") From 347ae9ebbffbb1a7183104cbc0bfcc5ff288c176 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:18:57 +0100 Subject: [PATCH 12/19] fix many UI issues (#6569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description of Changes - Tool action button truncation - fixed by allowing Mantine + + )} + {!isPhone && ( + + )} {/* Previous Page Button */} {/* Last Page Button */} - + {!isPhone && ( + + )} {/* Dual Page Toggle */} - - - + + + )} {/* PDF Render Mode Toggle */} - - - + + + )} - {/* Zoom Controls */} - - - - - zoomActions.setZoomLevel?.(val / 100)} - size="xs" - styles={{ - root: { width: "6rem" }, - thumb: { width: 14, height: 14 }, - track: { height: 3 }, - }} - label={null} - /> - + + + zoomActions.setZoomLevel?.(val / 100)} + size="xs" + styles={{ + root: { minWidth: "6rem", width: "6rem", flexShrink: 0 }, + thumb: { width: 14, height: 14 }, + track: { height: 3 }, + }} + label={null} + /> + + + + + {displayZoomPercent}% + + + )} + + {isPhone && ( + - - - - {displayZoomPercent}% - - + + + + + + + + {t("viewer.pageNavigation", "Page navigation")} + + } + disabled={scrollState.currentPage === 1} + onClick={handleFirstPage} + > + {t("viewer.firstPage", "First page")} + + } + disabled={scrollState.currentPage === scrollState.totalPages} + onClick={handleLastPage} + > + {t("viewer.lastPage", "Last page")} + + + + {t("viewer.zoom", "Zoom")} + } + onClick={handleZoomOut} + > + {t("viewer.zoomOut", "Zoom out")} + + } + onClick={handleZoomIn} + > + {t("viewer.zoomIn", "Zoom in")} ({displayZoomPercent}%) + + + + {t("viewer.view", "View")} + + ) : ( + + ) + } + disabled={scrollState.totalPages <= 1} + onClick={handleDualPageToggle} + > + {isDualPageActive + ? t("viewer.singlePageView", "Single Page View") + : t("viewer.dualPageView", "Dual Page View")} + + + ) : pdfRenderMode === "dark" ? ( + + ) : ( + + ) + } + onClick={cyclePdfRenderMode} + > + {pdfRenderMode === "normal" + ? t("viewer.enableDarkFilter", "Enable Dark Filter") + : pdfRenderMode === "dark" + ? t("viewer.enableSepiaFilter", "Enable Sepia Filter") + : t("viewer.disableColorFilter", "Disable Color Filter")} + + + + )} ); } diff --git a/frontend/editor/src/core/hooks/useIsMobile.ts b/frontend/editor/src/core/hooks/useIsMobile.ts index 23d4610418..a75ef7924b 100644 --- a/frontend/editor/src/core/hooks/useIsMobile.ts +++ b/frontend/editor/src/core/hooks/useIsMobile.ts @@ -15,3 +15,11 @@ export const useIsMobile = (): boolean => { export const useIsPhone = (): boolean => { return useMediaQuery("(max-width: 768px)") ?? false; }; + +/** + * Custom hook to detect a coarse pointer (touch device) + * Use to gate touch-only UI hints when combined with useIsMobile + */ +export const useIsTouch = (): boolean => { + return useMediaQuery("(pointer: coarse)") ?? false; +}; diff --git a/frontend/editor/src/core/pages/HomePage.tsx b/frontend/editor/src/core/pages/HomePage.tsx index a589ed43b4..f1e5158daf 100644 --- a/frontend/editor/src/core/pages/HomePage.tsx +++ b/frontend/editor/src/core/pages/HomePage.tsx @@ -5,7 +5,7 @@ import { Group } from "@mantine/core"; import { useSidebarContext } from "@app/contexts/SidebarContext"; import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; import { useBaseUrl } from "@app/hooks/useBaseUrl"; -import { useIsMobile } from "@app/hooks/useIsMobile"; +import { useIsMobile, useIsTouch } from "@app/hooks/useIsMobile"; import { useAppConfig } from "@app/contexts/AppConfigContext"; import { LogoIcon } from "@app/components/shared/LogoIcon"; import { Wordmark } from "@app/components/shared/Wordmark"; @@ -84,6 +84,7 @@ export default function HomePage() { const navigate = useNavigate(); const { config } = useAppConfig(); const isMobile = useIsMobile(); + const isTouch = useIsTouch(); const sliderRef = useRef(null); const [activeMobileView, setActiveMobileView] = useState("tools"); const isProgrammaticScroll = useRef(false); @@ -352,12 +353,14 @@ export default function HomePage() { {t("home.mobile.workspace", "Workspace")} - - {t( - "home.mobile.swipeHint", - "Swipe left or right to switch views", - )} - + {isTouch && ( + + {t( + "home.mobile.swipeHint", + "Swipe left or right to switch views", + )} + + )} )} {navigationState.workbench === "myFiles" ? ( diff --git a/frontend/editor/src/proprietary/components/shared/config/OverviewHeader.tsx b/frontend/editor/src/proprietary/components/shared/config/OverviewHeader.tsx index 9f0de01f9c..2307ed78bd 100644 --- a/frontend/editor/src/proprietary/components/shared/config/OverviewHeader.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/OverviewHeader.tsx @@ -1,19 +1,18 @@ import { Text, Button } from "@mantine/core"; import { useTranslation } from "react-i18next"; import { useAuth } from "@app/auth/UseSession"; -import { useNavigate } from "react-router-dom"; export function OverviewHeader() { const { t } = useTranslation(); const { signOut, user } = useAuth(); - const navigate = useNavigate(); const handleLogout = async () => { try { await signOut(); - navigate("/login"); } catch (error) { console.error("Logout error:", error); + } finally { + window.location.assign("/login"); } }; diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminAdvancedSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminAdvancedSection.tsx index 4450f9c180..e0b6a75e7c 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminAdvancedSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminAdvancedSection.tsx @@ -503,7 +503,7 @@ export default function AdminAdvancedSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.advanced.enableAlphaFunctionality.label", @@ -543,7 +543,7 @@ export default function AdminAdvancedSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.advanced.enableUrlToPDF.label", @@ -581,7 +581,7 @@ export default function AdminAdvancedSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.advanced.disableSanitize.label", @@ -959,7 +959,7 @@ export default function AdminAdvancedSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.advanced.tempFileManagement.startupCleanup.label", @@ -1002,7 +1002,7 @@ export default function AdminAdvancedSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.advanced.tempFileManagement.cleanupSystemTemp.label", diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminConnectionsSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminConnectionsSection.tsx index 671a7c0a8f..168ae6ca6d 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminConnectionsSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminConnectionsSection.tsx @@ -603,7 +603,7 @@ export default function AdminConnectionsSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.connections.ssoAutoLogin.enable", @@ -674,7 +674,7 @@ export default function AdminConnectionsSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.connections.mobileScanner.enable", diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminDatabaseSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminDatabaseSection.tsx index 7fd272fb5e..05ad76bf7f 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminDatabaseSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminDatabaseSection.tsx @@ -496,7 +496,7 @@ export default function AdminDatabaseSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.database.enableCustom.label", diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminFeaturesSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminFeaturesSection.tsx index 0e85990c6e..86edd74446 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminFeaturesSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminFeaturesSection.tsx @@ -200,7 +200,7 @@ export default function AdminFeaturesSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.features.serverCertificate.enabled.label", @@ -316,7 +316,7 @@ export default function AdminFeaturesSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.features.serverCertificate.regenerateOnStartup.label", diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx index 709c3565f3..a58e478233 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx @@ -703,7 +703,7 @@ export default function AdminGeneralSection() { marginBottom: "1rem", }} > -
+
{t( "admin.settings.general.hideDisabledTools.googleDrive.label", @@ -747,7 +747,7 @@ export default function AdminGeneralSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.general.hideDisabledTools.mobileScanner.label", @@ -793,7 +793,7 @@ export default function AdminGeneralSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.general.showUpdate.label", @@ -832,7 +832,7 @@ export default function AdminGeneralSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.general.showUpdateOnlyAdmin.label", @@ -873,7 +873,7 @@ export default function AdminGeneralSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.general.customHTMLFiles.label", @@ -938,7 +938,7 @@ export default function AdminGeneralSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.general.customMetadata.autoUpdate.label", diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminPremiumSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminPremiumSection.tsx index 66c7fc9112..d65abbf6f8 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminPremiumSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminPremiumSection.tsx @@ -197,7 +197,7 @@ export default function AdminPremiumSection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.premium.enabled.label", diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminPrivacySection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminPrivacySection.tsx index 7a2b8b5de0..37d6b7da1d 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminPrivacySection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminPrivacySection.tsx @@ -165,7 +165,7 @@ export default function AdminPrivacySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.privacy.enableAnalytics.label", @@ -203,7 +203,7 @@ export default function AdminPrivacySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.privacy.metricsEnabled.label", @@ -253,7 +253,7 @@ export default function AdminPrivacySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.privacy.googleVisibility.label", diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminSecuritySection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminSecuritySection.tsx index 20e559d9e2..02d1f27018 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/AdminSecuritySection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/AdminSecuritySection.tsx @@ -292,7 +292,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.enableLogin.label", @@ -519,7 +519,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.jwt.persistence.label", @@ -556,7 +556,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.jwt.enableKeyRotation.label", @@ -596,7 +596,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.jwt.enableKeyCleanup.label", @@ -780,7 +780,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.jwt.secureCookie.label", @@ -840,7 +840,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.audit.enabled.label", @@ -955,7 +955,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.audit.captureFileHash.label", @@ -995,7 +995,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.audit.capturePdfAuthor.label", @@ -1035,7 +1035,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.audit.captureOperationResults.label", @@ -1097,7 +1097,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.htmlUrlSecurity.enabled.label", @@ -1374,7 +1374,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.htmlUrlSecurity.blockPrivateNetworks.label", @@ -1424,7 +1424,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.htmlUrlSecurity.blockLocalhost.label", @@ -1473,7 +1473,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.htmlUrlSecurity.blockLinkLocal.label", @@ -1522,7 +1522,7 @@ export default function AdminSecuritySection() { justifyContent: "space-between", }} > -
+
{t( "admin.settings.security.htmlUrlSecurity.blockCloudMetadata.label", diff --git a/frontend/editor/src/proprietary/components/shared/config/configSections/PeopleSection.tsx b/frontend/editor/src/proprietary/components/shared/config/configSections/PeopleSection.tsx index f9ca9eab42..9a0a760147 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configSections/PeopleSection.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configSections/PeopleSection.tsx @@ -585,9 +585,12 @@ export default function PeopleSection() { {t("workspace.people.user")} {t("workspace.people.role")} @@ -690,7 +693,7 @@ export default function PeopleSection() { - + {(user.rolesAsString || "").includes("ROLE_ADMIN") ? t("workspace.people.admin", "Admin") From ff96a80947bca821fa264063560a164c3af66688 Mon Sep 17 00:00:00 2001 From: ConnorYoh <40631091+ConnorYoh@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:47:40 +0100 Subject: [PATCH 13/19] PAYG B-3 / S-3: cucumber suite for shadow-mode flows + CI workflow (#6522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this PR is End-to-end cucumber coverage for the PAYG shadow charging engine (the filter + interceptor stack from #6519), wired into CI via a new `docker-compose-tests-saas.yml` workflow that runs only on PAYG-touching PRs. Stacked on #6519. ## Automated scenarios (run by `docker-compose-tests-saas.yml`) See [`testing/cucumber/features/payg/shadow_charges.feature`](../tree/payg-s3-cucumber/testing/cucumber/features/payg/shadow_charges.feature): | Scenario | Validates | |---|---| | First tool call writes a CHARGED row | Filter + interceptor fire end-to-end | | Lineage join — second call on output | `JobService.joinOrOpen` matching; no new shadow row | | 4xx leaves the row CHARGED | "Customer paid for the attempt" semantics | | ZIP-returning tool records per-PDF OUTPUT | `PaygOutputExtractor` unpacks + records signatures | | Multi-file input writes a single shadow row | Multi-input group sizing | | `X-Stirling-Automation` sets PIPELINE source | Header → `JobSource` detection | All 6 run locally via `./testing/test-payg.sh` and will run on CI for any PR that touches `app/saas/**`, the PAYG cucumber features, the saas compose stack, or the workflow itself. ## Manual-only scenarios — documented in design doc, not in this suite Two parts of the shadow engine are deliberately not automated; the engine paths are unit-tested in `PaygChargeInterceptorTest.afterCompletion_5xx_opened_*`, and the manual procedures (which require a temporary throw endpoint or a container restart with a flag flipped) live in [`notes/PAYG_DESIGN.md` §7.5.2 "PAYG cucumber: manual-only scenarios"](../tree/payg-s3-cucumber/notes/PAYG_DESIGN.md). - **5xx first-step failure → REFUNDED + CLOSED.** No reliably-5xx-ing endpoint exists; manual procedure adds a throw endpoint, runs, asserts, removes. - **Kill-switch (`PAYG_FILTER_ENABLED=false`).** Needs a container restart mid-suite; manual procedure tears down, flips env, brings up, asserts zero shadow rows. If either gets a hot-reload path (test-only throw endpoint shipped behind a profile gate, or admin endpoint for the kill switch), automate it in a follow-up and drop the manual procedure. ## CI workflow `.github/workflows/docker-compose-tests-saas.yml` (new) — self-contained, not wired into `build.yml`'s `files-changed` matrix so the saas-cucumber job fails and succeeds independently. Triggers only on PAYG-relevant paths. No JaCoCo coverage in v1 (saas compose doesn't have the coverage override; can add later). ## Test infrastructure (recap) - **`testing/compose/docker-compose-saas.yml`** — Stirling-PDF backend with `STIRLING_FLAVOR=saas` + Postgres holding the `stirling_pdf` schema. Supabase JWT auto-config disabled; API-key auth via `SECURITY_CUSTOMGLOBALAPIKEY` is the live path the cucumber tests exercise. - **`testing/compose/payg/saas-init.sql`** + **`saas-seed.sql`** — schema bootstrap + idempotent seed (team / user / wallet_policy). - **`testing/cucumber/features/payg/shadow_charges.feature`** — the 6 scenarios above. - **`testing/cucumber/features/steps/payg_step_definitions.py`** — step defs using `requests` (HTTP) + `psycopg` (direct DB inspection). Direct DB reads are deliberate — we want to see the filter's side effects, not relay them through another API layer. - **`testing/test-payg.sh`** — companion runner to `testing/test.sh`. Brings up the saas compose, waits for health, seeds, runs behave, tears down. - **`behave.ini`** excludes `features/payg` from the default behave run (the saas-cucumber CI job invokes it explicitly). ## Why a separate harness from `testing/test.sh` The existing `test.sh` covers the proprietary-flavour stack (no PAYG tables, no saas profile). Coupling two CI matrices that fail and succeed independently into one script is asking for trouble. Keep the saas-cucumber job focused on its own concerns; once the harness is mature, the wider team can decide whether to merge them. ## Tracked in `notes/PAYG_DESIGN.md` §7.5 (PR-S3) + §7.5.2 (manual scenarios). --- .../workflows/docker-compose-tests-saas.yml | 136 +++++ .../saas/payg/filter/PaygOutputExtractor.java | 43 +- .../test/PaygCucumberThrowController.java | 78 +++ .../payg/filter/PaygOutputExtractorTest.java | 55 ++ testing/compose/docker-compose-saas.yml | 104 ++++ testing/compose/payg/saas-init.sql | 25 + testing/compose/payg/saas-seed.sql | 118 +++++ testing/cucumber/behave.ini | 10 +- testing/cucumber/features/payg/README.md | 58 +++ .../features/payg/shadow_charges.feature | 105 ++++ .../features/steps/payg_step_definitions.py | 473 ++++++++++++++++++ testing/cucumber/requirements.in | 9 + testing/cucumber/requirements.txt | 91 +++- testing/test-payg.sh | 89 ++++ 14 files changed, 1376 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/docker-compose-tests-saas.yml create mode 100644 app/saas/src/main/java/stirling/software/saas/payg/test/PaygCucumberThrowController.java create mode 100644 testing/compose/docker-compose-saas.yml create mode 100644 testing/compose/payg/saas-init.sql create mode 100644 testing/compose/payg/saas-seed.sql create mode 100644 testing/cucumber/features/payg/README.md create mode 100644 testing/cucumber/features/payg/shadow_charges.feature create mode 100644 testing/cucumber/features/steps/payg_step_definitions.py create mode 100644 testing/test-payg.sh diff --git a/.github/workflows/docker-compose-tests-saas.yml b/.github/workflows/docker-compose-tests-saas.yml new file mode 100644 index 0000000000..c4e2a0834e --- /dev/null +++ b/.github/workflows/docker-compose-tests-saas.yml @@ -0,0 +1,136 @@ +name: Docker Compose Cucumber tests (saas / PAYG) + +# Self-contained CI job for the PAYG shadow-mode cucumber scenarios. +# Triggers only on PAYG-relevant paths so we don't add CI minutes to every PR +# that doesn't touch the saas flavour. +# +# Companion to `docker-compose-tests.yml` (which runs against the +# proprietary-flavour stack and skips features/payg via behave.ini's +# exclude_re). Kept as a separate workflow so the saas matrix can fail and +# succeed independently without touching the main cucumber harness. + +on: + pull_request: + paths: + - "app/saas/**" + - "testing/cucumber/features/payg/**" + - "testing/cucumber/features/steps/payg_step_definitions.py" + - "testing/cucumber/requirements.txt" + - "testing/compose/docker-compose-saas.yml" + - "testing/compose/payg/**" + - "testing/test-payg.sh" + - ".github/workflows/docker-compose-tests-saas.yml" + push: + branches: [main] + paths: + - "app/saas/**" + - "testing/cucumber/features/payg/**" + - "testing/cucumber/features/steps/payg_step_definitions.py" + - "testing/cucumber/requirements.txt" + - "testing/compose/docker-compose-saas.yml" + - "testing/compose/payg/**" + - "testing/test-payg.sh" + - ".github/workflows/docker-compose-tests-saas.yml" + +permissions: + contents: read + +jobs: + pick: + uses: ./.github/workflows/_runner-pick.yml + + docker-compose-tests-saas: + needs: pick + runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }} + permissions: + actions: write + contents: read + checks: write + env: + DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + with: + egress-policy: audit + + - name: Checkout Repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up JDK 25 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: "25" + distribution: "temurin" + + - name: Cache Gradle dependency artifacts + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.gradle/wrapper + ~/.gradle/caches/modules-2/files-2.1 + ~/.gradle/caches/modules-2/metadata-2.* + key: gradle-deps-saas-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }} + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 + with: + gradle-version: 9.3.1 + cache-disabled: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Expose GitHub runtime for Buildx cache + uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0 + + # No "Install Docker Compose" step: Ubuntu runners ship with `docker compose` + # v2 (built into the Docker CLI). test-payg.sh uses the v2 form throughout + # (`docker compose …`, no hyphen), so the legacy v1 `docker-compose` binary + # isn't needed. Avoids a `curl | sudo install` without checksum verification + # (Aikido flagged this when copy-pasted from docker-compose-tests.yml). + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + cache: "pip" + cache-dependency-path: ./testing/cucumber/requirements.txt + + - name: Pip requirements + run: | + pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt + + - name: Run PAYG Cucumber Tests + env: + MAVEN_USER: ${{ secrets.MAVEN_USER }} + MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} + MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }} + run: | + chmod +x ./testing/test-payg.sh + ./testing/test-payg.sh + + - name: Dump saas container logs on failure + if: failure() + run: | + docker compose -f testing/compose/docker-compose-saas.yml logs --tail 500 stirling-pdf-saas || true + docker compose -f testing/compose/docker-compose-saas.yml logs --tail 200 postgres-saas || true + + - name: Upload PAYG Cucumber Report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: payg-cucumber-report + path: testing/cucumber/report-payg.html + retention-days: 7 + if-no-files-found: warn + + - name: PAYG Cucumber Test Report + if: always() + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0 + with: + name: PAYG Cucumber Tests + path: testing/cucumber/junit-payg/*.xml + reporter: java-junit + fail-on-error: false diff --git a/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygOutputExtractor.java b/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygOutputExtractor.java index a19a78e0a6..86b5cb6026 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygOutputExtractor.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygOutputExtractor.java @@ -49,8 +49,12 @@ public class PaygOutputExtractor { /** {@code %PDF-} in ASCII. */ private static final byte[] PDF_MAGIC = {0x25, 0x50, 0x44, 0x46, 0x2D}; + /** {@code PK\x03\x04} — local file header for any non-empty ZIP. */ + private static final byte[] ZIP_MAGIC = {0x50, 0x4B, 0x03, 0x04}; + private static final String ZIP_CONTENT_TYPE = "application/zip"; private static final String PDF_CONTENT_TYPE = "application/pdf"; + private static final String OCTET_STREAM_CONTENT_TYPE = "application/octet-stream"; private final TempFileManager tempFileManager; @@ -74,11 +78,31 @@ public class PaygOutputExtractor { return List.of(); } String mediaType = stripParameters(contentType); - if (PDF_CONTENT_TYPE.equalsIgnoreCase(mediaType)) { + + // Stirling-PDF tool endpoints sometimes set Content-Type to + // application/octet-stream (or no header at all) even when the body + // is a real PDF or ZIP — Spring's default StreamingResponseBody path + // doesn't always negotiate content type. When the declared + // Content-Type is missing or generic, sniff magic bytes to recover + // lineage capture. + boolean missingOrGeneric = + mediaType == null || OCTET_STREAM_CONTENT_TYPE.equalsIgnoreCase(mediaType); + boolean isPdf = PDF_CONTENT_TYPE.equalsIgnoreCase(mediaType); + boolean isZip = ZIP_CONTENT_TYPE.equalsIgnoreCase(mediaType); + + if (missingOrGeneric) { + if (isMagic(bodyPath, PDF_MAGIC)) { + isPdf = true; + } else if (isMagic(bodyPath, ZIP_MAGIC)) { + isZip = true; + } + } + + if (isPdf) { // Wrapper-owned path. Don't claim ownership. return List.of(new ExtractedPdf(bodyPath, null)); } - if (ZIP_CONTENT_TYPE.equalsIgnoreCase(mediaType)) { + if (isZip) { return extractZip(bodyPath); } return List.of(); @@ -126,20 +150,25 @@ public class PaygOutputExtractor { } private boolean isPdfMagic(Path path) { - byte[] head = new byte[PDF_MAGIC.length]; + return isMagic(path, PDF_MAGIC); + } + + /** Returns true if the first {@code magic.length} bytes of {@code path} equal {@code magic}. */ + private boolean isMagic(Path path, byte[] magic) { + byte[] head = new byte[magic.length]; try (InputStream in = Files.newInputStream(path)) { int read = in.read(head); - if (read != PDF_MAGIC.length) { + if (read != magic.length) { return false; } - for (int i = 0; i < PDF_MAGIC.length; i++) { - if (head[i] != PDF_MAGIC[i]) { + for (int i = 0; i < magic.length; i++) { + if (head[i] != magic[i]) { return false; } } return true; } catch (IOException e) { - log.debug("PDF magic-byte check failed for {}", path, e); + log.debug("Magic-byte check failed for {}", path, e); return false; } } diff --git a/app/saas/src/main/java/stirling/software/saas/payg/test/PaygCucumberThrowController.java b/app/saas/src/main/java/stirling/software/saas/payg/test/PaygCucumberThrowController.java new file mode 100644 index 0000000000..8a9082e2c8 --- /dev/null +++ b/app/saas/src/main/java/stirling/software/saas/payg/test/PaygCucumberThrowController.java @@ -0,0 +1,78 @@ +package stirling.software.saas.payg.test; + +import org.springframework.context.annotation.Profile; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import io.swagger.v3.oas.annotations.Hidden; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.annotations.AutoJobPostMapping; +import stirling.software.common.enumeration.ResourceWeight; + +/** + * Cucumber-only force-5xx endpoint. Gated behind the {@code payg-cucumber} Spring profile so the + * bean never registers in production — only the PAYG cucumber compose stack activates the profile + * (via {@code SPRING_PROFILES_ACTIVE=saas,payg-cucumber} in {@code docker-compose-saas.yml}). + * + *

Purpose: drive the PAYG filter+interceptor's 5xx-first-step branch end-to-end. No reliably- + * 5xx-ing real tool endpoint exists in current Stirling — every malformed input is caught as 4xx by + * {@code GlobalExceptionHandler}. Without this stub the only way to exercise the refund path was a + * manual procedure (a temporary throw endpoint added, run, removed) documented in {@code + * notes/PAYG_DESIGN.md} §7.5.2 M1. This controller replaces that procedure with a profile- gated + * automated scenario. + * + *

{@link AutoJobPostMapping} consumes {@code multipart/form-data} so the filter's {@code + * MultipartHttpServletRequest} cast runs and the input lineage hash is computed before the + * controller throws. The PAYG filter chain therefore sees: preHandle → openProcess (CHARGED row + * written) → controller throws → afterCompletion observes status 500 → markFirstStepFailed (row → + * REFUNDED, job → CLOSED). + * + *

The thrown {@link IllegalStateException} is unwrapped by {@code GlobalExceptionHandler}'s + * RuntimeException handler — it has no IOException / IllegalArgument / BaseAppException cause, so + * it falls through to the "Unexpected RuntimeException" branch which sets HTTP 500. Don't use + * {@code ResponseStatusException} here — its dedicated handler is reached before the unwrapping + * branch but the upstream test still passes; {@link IllegalStateException} is the more honest mimic + * of a real bug-driven 500. + * + *

{@code @Hidden} keeps this endpoint out of OpenAPI / Swagger output even when the profile is + * active, so no docs leak to anyone running the cucumber stack interactively. + * + *

Return type matters: the method declares {@link ResponseEntity}{@code }, not + * {@code void}. Spring's {@code InvocableHandlerMethod} picks a return-value handler based on the + * declared return type; {@code void} matches the "no body, status 200" handler regardless of + * what the {@code @Around} advice on {@link AutoJobPostMapping} actually returns at runtime. The + * first version of this stub declared {@code void} and the 500 ResponseEntity from {@code + * JobExecutorService}'s catch block was silently discarded — the client saw a 200 with an empty + * body. Declaring {@code ResponseEntity} matches the real runtime type and lets the advice's + * 500 reach the wire. + */ +@Slf4j +@RestController +@Profile("payg-cucumber") +@RequestMapping("/api/v1/payg-cucumber") +@Hidden +public class PaygCucumberThrowController { + + @AutoJobPostMapping( + value = "/throw-500", + consumes = MediaType.MULTIPART_FORM_DATA_VALUE, + resourceWeight = ResourceWeight.SMALL_WEIGHT) + public ResponseEntity throw500( + @RequestParam(value = "fileInput", required = false) MultipartFile fileInput) { + // The file is read by the PAYG filter via getMultiFileMap() before we get here; the + // controller param is just to keep Spring's multipart binding happy. We don't touch it. + log.warn( + "PAYG cucumber forced 500 (fileInput name='{}' size={} bytes)", + fileInput != null ? fileInput.getOriginalFilename() : null, + fileInput != null ? fileInput.getSize() : 0); + throw new IllegalStateException("PAYG cucumber forced 500"); + // unreachable — kept as a type signature so AutoJobAspect's @Around return value (the + // 500 ResponseEntity from JobExecutorService) actually reaches the wire. + } +} diff --git a/app/saas/src/test/java/stirling/software/saas/payg/filter/PaygOutputExtractorTest.java b/app/saas/src/test/java/stirling/software/saas/payg/filter/PaygOutputExtractorTest.java index 201a0d68ef..688377ad3a 100644 --- a/app/saas/src/test/java/stirling/software/saas/payg/filter/PaygOutputExtractorTest.java +++ b/app/saas/src/test/java/stirling/software/saas/payg/filter/PaygOutputExtractorTest.java @@ -102,6 +102,61 @@ class PaygOutputExtractorTest { assertThat(extractor.extract("application/zip", zip)).isEmpty(); } + @Test + void octetStreamWithPdfMagic_treatedAsPdf(@TempDir Path tmp) throws IOException { + // Stirling tool endpoints sometimes set Content-Type to + // application/octet-stream for streamed responses even when the body + // is a real PDF. The extractor must sniff magic bytes when the + // declared Content-Type is generic. + Path body = tmp.resolve("body.bin"); + Files.write(body, pdfBytes("real-pdf-content")); + List out = + extractor.extract("application/octet-stream", body); + assertThat(out).hasSize(1); + assertThat(out.get(0).path()).isEqualTo(body); + } + + @Test + void octetStreamWithZipMagic_unpackedAsZip(@TempDir Path tmp) throws IOException { + Path zip = tmp.resolve("body.bin"); + try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zip))) { + writeEntry(zos, "page1.pdf", pdfBytes("a")); + writeEntry(zos, "page2.pdf", pdfBytes("b")); + } + List out = + extractor.extract("application/octet-stream", zip); + try { + assertThat(out).hasSize(2); + } finally { + for (PaygOutputExtractor.ExtractedPdf p : out) { + p.close(); + } + } + } + + @Test + void nullContentTypeWithZipMagic_unpackedAsZip(@TempDir Path tmp) throws IOException { + Path zip = tmp.resolve("body.bin"); + try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zip))) { + writeEntry(zos, "page1.pdf", pdfBytes("a")); + } + List out = extractor.extract(null, zip); + try { + assertThat(out).hasSize(1); + } finally { + for (PaygOutputExtractor.ExtractedPdf p : out) { + p.close(); + } + } + } + + @Test + void octetStreamWithoutMagic_returnsEmpty(@TempDir Path tmp) throws IOException { + Path body = tmp.resolve("body.bin"); + Files.write(body, "neither pdf nor zip".getBytes(StandardCharsets.UTF_8)); + assertThat(extractor.extract("application/octet-stream", body)).isEmpty(); + } + private static void writeEntry(ZipOutputStream zos, String name, byte[] data) throws IOException { zos.putNextEntry(new ZipEntry(name)); diff --git a/testing/compose/docker-compose-saas.yml b/testing/compose/docker-compose-saas.yml new file mode 100644 index 0000000000..880518b423 --- /dev/null +++ b/testing/compose/docker-compose-saas.yml @@ -0,0 +1,104 @@ +services: + # --------------------------------------------------------------------- + # Postgres holding the stirling_pdf schema. Flyway migrates V1-V13 on + # backend startup against this DB. Exposed on host port 5433 so the + # cucumber harness can connect via psycopg from features/steps/payg_*. + # --------------------------------------------------------------------- + postgres-saas: + image: postgres:17-alpine + container_name: payg-cucumber-postgres + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - "5433:5432" + volumes: + # Bootstrap stirling_pdf schema + seed test team / api key. + # Runs once when the container is first created. + - ./payg/saas-init.sql:/docker-entrypoint-initdb.d/00-init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 3s + timeout: 5s + retries: 20 + networks: + - payg-cucumber-network + + # --------------------------------------------------------------------- + # Stirling-PDF backend with STIRLING_FLAVOR=saas. Authenticates via + # API key (SECURITY_CUSTOMGLOBALAPIKEY) — Supabase JWT enforcement is + # disabled for the test profile via SAAS_DB_PROJECT_REF=disabled so the + # OAuth resource-server filter no-ops. The PaygChargeInterceptor's + # resolveUser() ApiKey-token path is what the cucumber tests exercise. + # --------------------------------------------------------------------- + stirling-pdf-saas: + build: + context: ../.. + dockerfile: docker/embedded/Dockerfile + args: + STIRLING_FLAVOR: saas + container_name: payg-cucumber-stirling + depends_on: + postgres-saas: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: + memory: 4G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP'"] + interval: 5s + timeout: 10s + retries: 24 + ports: + - "8080:8080" + environment: + # Activate the saas profile + bring the saas subproject onto the classpath. + # `payg-cucumber` registers the test-only throw-500 controller in + # `app/saas/.../payg/test/PaygCucumberThrowController.java` — that bean is + # @Profile("payg-cucumber") so it never registers in production. + SPRING_PROFILES_ACTIVE: "saas,payg-cucumber" + STIRLING_FLAVOR: "saas" + ENABLE_SAAS: "true" + DISABLE_ADDITIONAL_FEATURES: "false" + + # Datasource — point Flyway + JPA at the test postgres. + SAAS_DB_URL: "jdbc:postgresql://postgres-saas:5432/postgres" + SAAS_DB_USERNAME: "postgres" + SAAS_DB_PASSWORD: "postgres" + SAAS_DB_PROJECT_REF: "disabled" + + # The saas Flyway migrations assume `users` and `teams` already exist + # (V2 ALTERs `users`; V5 references `teams(id)`) because in production + # those tables are provisioned by Supabase before Stirling starts. On + # a clean test postgres they don't exist, so we disable Flyway and let + # Hibernate's `ddl-auto=create-drop` build the full schema from the + # entity graph. The default-pricing-policy row that V12 seeds in + # production is re-seeded by testing/compose/payg/saas-seed.sql. + SPRING_FLYWAY_ENABLED: "false" + SPRING_JPA_HIBERNATE_DDL_AUTO: "create-drop" + + # Disable Supabase JWT enforcement for the test profile. The + # PaygChargeInterceptor resolves via ApiKey-token authority — that + # path is what we cover here. + SPRING_AUTOCONFIGURE_EXCLUDE: "org.springframework.boot.autoconfigure.security.oauth2.resource.servlet.OAuth2ResourceServerAutoConfiguration" + + # Test API key matching the user seeded by saas-init.sql. + SECURITY_CUSTOMGLOBALAPIKEY: "payg-cucumber-key" + SECURITY_ENABLELOGIN: "false" + + # Conservative defaults. The cucumber runs send small fixtures so + # the in-memory threshold never bites; the kill-switch is needed by + # one scenario but is toggled via a separate compose override. + PAYG_FILTER_ENABLED: "true" + PAYG_FILTER_RESPONSE_IN_MEMORY_THRESHOLD_BYTES: "10485760" + + SYSTEM_DEFAULTLOCALE: "en-US" + SYSTEM_MAXFILESIZE: "100" + networks: + - payg-cucumber-network + +networks: + payg-cucumber-network: + driver: bridge diff --git a/testing/compose/payg/saas-init.sql b/testing/compose/payg/saas-init.sql new file mode 100644 index 0000000000..1a40cdde66 --- /dev/null +++ b/testing/compose/payg/saas-init.sql @@ -0,0 +1,25 @@ +-- Bootstrap minimal stirling_pdf state for PAYG cucumber tests. +-- +-- Runs once when the postgres-saas container is first created (loaded via +-- /docker-entrypoint-initdb.d on the official postgres image). Flyway then +-- migrates V1-V13 on backend startup, which fills in the schema; this file +-- only seeds the data rows the cucumber scenarios reference by name. +-- +-- Idempotent — if the container is re-created, this whole file runs again +-- but every INSERT is guarded against duplicates. + +CREATE SCHEMA IF NOT EXISTS stirling_pdf; + +-- Note: the actual seed of teams / users / payg_team_extensions / wallet_policy +-- happens AFTER Flyway migrations have applied. We can't insert here because +-- the tables don't exist yet at container-init time. Instead we register a +-- helper function the backend invokes once at startup (or the cucumber +-- harness invokes via psql before running scenarios). +-- +-- For "make a start", the simplest approach is to keep this file as the +-- schema bootstrap, and put the seed inserts in a separate sidecar SQL that +-- the cucumber test.sh harness pipes through psql AFTER waiting for backend +-- health. See testing/compose/payg/saas-seed.sql below. + +-- Reserved for future container-init-time schema bootstrap if we ever need it. +SELECT 1; diff --git a/testing/compose/payg/saas-seed.sql b/testing/compose/payg/saas-seed.sql new file mode 100644 index 0000000000..612cbc9d1c --- /dev/null +++ b/testing/compose/payg/saas-seed.sql @@ -0,0 +1,118 @@ +-- Seed test team / payg policy / wallet_policy in PAYG_SHADOW mode for the +-- cucumber harness. Piped through psql AFTER Hibernate has built the schema +-- on backend startup (compose disables Flyway — see docker-compose-saas.yml +-- for the rationale: saas Flyway migrations assume `users` + `teams` exist +-- because Supabase normally provisions them). +-- +-- Idempotent — guarded against duplicate keys so re-running on the same +-- container is safe between scenarios. + +-- --------------------------------------------------------------------------- +-- 0. Default pricing policy + per-source step limits (V12 in production). +-- Required because PricingPolicyService.getEffectivePolicy() throws if +-- no row has is_default = TRUE. Explicit timestamps because Hibernate's +-- @CreationTimestamp is application-side; direct INSERTs bypass it. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.pricing_policy ( + version, effective_from, doc_pages_per_unit, doc_bytes_per_unit, + min_charge_units, file_unit_cap, is_default, notes, created_by, + created_at +) +SELECT + 'v1-cucumber', CURRENT_TIMESTAMP, 25, 5242880, + 1, 1000, TRUE, + 'Cucumber test default policy', 'system', + CURRENT_TIMESTAMP +WHERE NOT EXISTS ( + SELECT 1 FROM stirling_pdf.pricing_policy WHERE is_default = TRUE +); + +INSERT INTO stirling_pdf.pricing_policy_step_limit (policy_id, job_source, step_limit) +SELECT p.policy_id, src.job_source, src.step_limit +FROM stirling_pdf.pricing_policy p +CROSS JOIN ( + VALUES + ('WEB', 10), + ('API', 10), + ('PIPELINE', 20), + ('DESKTOP_APP', 10) +) AS src(job_source, step_limit) +WHERE p.is_default = TRUE + AND NOT EXISTS ( + SELECT 1 FROM stirling_pdf.pricing_policy_step_limit s + WHERE s.policy_id = p.policy_id AND s.job_source = src.job_source + ); + +-- --------------------------------------------------------------------------- +-- 1. Test team. The Stirling-PDF backend auto-creates `Default` and +-- `Internal` teams at boot; we add a third one specifically for the +-- PAYG scenarios so we can isolate state and assert per-team. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.teams (name) +SELECT 'payg-cucumber-team' +WHERE NOT EXISTS ( + SELECT 1 FROM stirling_pdf.teams WHERE name = 'payg-cucumber-team' +); + +-- --------------------------------------------------------------------------- +-- 2. payg_team_extensions sidecar — uses the default pricing policy. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.payg_team_extensions (team_id, pricing_policy_id, stripe_customer_id) +SELECT t.team_id, NULL, NULL +FROM stirling_pdf.teams t +WHERE t.name = 'payg-cucumber-team' + AND NOT EXISTS ( + SELECT 1 FROM stirling_pdf.payg_team_extensions ext WHERE ext.team_id = t.team_id + ); + +-- --------------------------------------------------------------------------- +-- 3. Bind the auto-created CUSTOM_API_USER to our cucumber team. The user +-- is created by the backend's SECURITY_CUSTOMGLOBALAPIKEY handling — we +-- don't seed our own user (would collide on the unique api_key). +-- Update their primary `team_id` so JobChargeService picks up +-- payg-cucumber-team as the owner team on requests. +-- --------------------------------------------------------------------------- +UPDATE stirling_pdf.users u +SET team_id = (SELECT team_id FROM stirling_pdf.teams WHERE name = 'payg-cucumber-team') +WHERE u.username = 'CUSTOM_API_USER'; + +-- --------------------------------------------------------------------------- +-- 4. Team membership row so /teams/* admin paths recognise the user. +-- All timestamp columns set explicitly because Hibernate-generated DDL +-- doesn't carry the Flyway-migration DEFAULT CURRENT_TIMESTAMP. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.team_memberships ( + team_id, user_id, role, invited_at, created_at, updated_at +) +SELECT t.team_id, u.user_id, 'LEADER', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP +FROM stirling_pdf.teams t, stirling_pdf.users u +WHERE t.name = 'payg-cucumber-team' + AND u.username = 'CUSTOM_API_USER' + AND NOT EXISTS ( + SELECT 1 FROM stirling_pdf.team_memberships m + WHERE m.team_id = t.team_id AND m.user_id = u.user_id + ); + +-- --------------------------------------------------------------------------- +-- 5. wallet_policy in PAYG_SHADOW mode. +-- auto_group_strategy is the dead column tracked for drop in PR-R11 +-- (PAYG_DESIGN.md §7.7) — until that ships, the NOT NULL constraint +-- means we have to populate it. 'AUTO' is the prod default. +-- --------------------------------------------------------------------------- +INSERT INTO stirling_pdf.wallet_policy ( + team_id, engine, cap_period, warn_at_pct, degrade_at_pct, + degraded_feature_set, auto_group_strategy, notification_emails, + updated_at +) +SELECT t.team_id, 'PAYG_SHADOW', 'CALENDAR_MONTH', 80, 100, + 'MINIMAL', 'AUTO', '[]'::jsonb, CURRENT_TIMESTAMP +FROM stirling_pdf.teams t +WHERE t.name = 'payg-cucumber-team' + AND NOT EXISTS ( + SELECT 1 FROM stirling_pdf.wallet_policy wp WHERE wp.team_id = t.team_id + ); + +UPDATE stirling_pdf.wallet_policy +SET engine = 'PAYG_SHADOW', updated_at = CURRENT_TIMESTAMP +WHERE team_id = (SELECT team_id FROM stirling_pdf.teams WHERE name = 'payg-cucumber-team'); diff --git a/testing/cucumber/behave.ini b/testing/cucumber/behave.ini index 202b26576a..b398282b2a 100644 --- a/testing/cucumber/behave.ini +++ b/testing/cucumber/behave.ini @@ -5,4 +5,12 @@ # # python -m behave features/enterprise # -exclude_re = features/enterprise +# PAYG shadow-mode features live in features/payg/. They need the saas +# profile + Postgres + seeded test team (testing/compose/docker-compose-saas.yml). +# The default run excludes them; the saas-cucumber CI job invokes +# behave with `features/payg` explicitly. The PAYG harness in +# testing/test-payg.sh swaps a temp behave.ini that drops the payg +# exclusion + tags @manual to skip restart-hook scenarios. +# +exclude_re = features/(enterprise|payg) +tags = ~@manual diff --git a/testing/cucumber/features/payg/README.md b/testing/cucumber/features/payg/README.md new file mode 100644 index 0000000000..584ef446a0 --- /dev/null +++ b/testing/cucumber/features/payg/README.md @@ -0,0 +1,58 @@ +# PAYG cucumber scenarios + +End-to-end coverage for the PAYG shadow charging engine (PR #6519 / PR-S3 +in `notes/PAYG_DESIGN.md` §7.5). + +## Running locally + +```bash +./testing/test-payg.sh +``` + +That script: +1. Boots `testing/compose/docker-compose-saas.yml` (Stirling-PDF with + `STIRLING_FLAVOR=saas` + a Postgres holding the `stirling_pdf` schema) +2. Waits for backend health +3. Pipes `testing/compose/payg/saas-seed.sql` into the test postgres, + creating a `payg-cucumber-team` flipped to `wallet_policy.engine = + 'PAYG_SHADOW'` and a test user with API key `payg-cucumber-key` +4. Runs `python -m behave features/payg` +5. Tears the stack down + +## What's covered (automated, run by `docker-compose-tests-saas.yml` on CI) + +| Scenario | Validates | +|---|---| +| First tool call writes a CHARGED row | Filter + interceptor fire end-to-end; shadow row written | +| Lineage join — second call on output | `JobService.joinOrOpen` lineage matching; no new shadow row | +| First-step 5xx refunds + closes the process | `markFirstStepFailed` flips row to REFUNDED, job to CLOSED | +| 4xx leaves the row CHARGED | "customer paid for the attempt" semantics | +| ZIP-returning tool records per-PDF OUTPUT | `PaygOutputExtractor` unpacks + records signatures | +| Multi-file input writes a single shadow row | Multi-input group sizing | +| `X-Stirling-Automation` sets PIPELINE source | Header → `JobSource` detection | + +The 5xx scenario drives the refund path through `PaygCucumberThrowController` +— a `@Profile("payg-cucumber")` stub in `app/saas/.../payg/test/` that always +throws. The profile is activated by `docker-compose-saas.yml`'s +`SPRING_PROFILES_ACTIVE=saas,payg-cucumber` and is never set in production. + +## Manual-only scenarios + +One part of the shadow engine can't reasonably be driven from this suite +and is verified by hand each time its code path changes. The procedure +lives in `notes/PAYG_DESIGN.md` §7.5.2 "PAYG cucumber: manual-only +scenarios". + +- **Kill-switch (`PAYG_FILTER_ENABLED=false`).** Needs a docker-container + restart mid-suite; orchestrating that in behave is more harness fragility + than it's worth for a flag that's only flipped during incident response. + +## Fixtures + +The scenarios reuse `testing/cucumber/exampleFiles/`: +- `ghost1.pdf` — single-page reference PDF +- `tables.pdf` — multi-page input for split / ZIP scenarios + +If those filenames change in the main cucumber harness, update +`features/steps/payg_step_definitions.py` SINGLE_PAGE_PDF / THREE_PAGE_PDF +constants to match. diff --git a/testing/cucumber/features/payg/shadow_charges.feature b/testing/cucumber/features/payg/shadow_charges.feature new file mode 100644 index 0000000000..2141b66b0a --- /dev/null +++ b/testing/cucumber/features/payg/shadow_charges.feature @@ -0,0 +1,105 @@ +Feature: PAYG shadow-mode charging + # End-to-end coverage for the PAYG shadow charging engine via the filter + + # interceptor stack landed in PR #6519. Runs against the saas-profile + # docker-compose target (testing/compose/docker-compose-saas.yml). + # + # Each scenario sets up a test team in PAYG_SHADOW mode, hits a real tool + # endpoint, then asserts the shape of the rows written to payg_shadow_charge + # and processing_job / processing_job_step. + # + # Lives under features/payg so it's only loaded when the saas-cucumber job + # explicitly includes this directory (separate from the main behave run + # which boots the proprietary-flavor stack and has no PAYG tables). + + Background: + Given the SaaS stack is running with PAYG enabled + And team "payg-cucumber-team" exists with wallet_policy.engine = "PAYG_SHADOW" + And I am authenticated as a member of team "payg-cucumber-team" + + Scenario: First tool call writes a CHARGED shadow row + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF to "/api/v1/security/add-password" + Then the response status is 200 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest shadow charge row has status "CHARGED" + And the latest shadow charge row has payg_units >= 1 + And the latest shadow charge row's job is OPEN + And the latest job has 1 step recorded with status "OK" + + Scenario: Lineage join — second call on the same output joins the first process + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF to "/api/v1/security/add-password" + And I take the response body as "step1-output" + And I POST "step1-output" to "/api/v1/security/sanitize-pdf" + Then exactly 1 shadow charge row exists for team "payg-cucumber-team" + # The second call joined the first process — no new shadow row + And the latest job has step_count = 2 + And the latest job is OPEN + + Scenario: First-step 5xx refunds the shadow row + closes the process + # No reliably-5xx-ing real tool endpoint exists (every malformed input is + # caught as 4xx by GlobalExceptionHandler), so we drive the refund path + # through PaygCucumberThrowController — a @Profile("payg-cucumber") stub + # that always throws IllegalStateException → 500. The profile is active + # only inside docker-compose-saas.yml, never in production. + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF to "/api/v1/payg-cucumber/throw-500" + Then the response status is 500 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest shadow charge row has status "REFUNDED" + And the latest shadow charge row's refunded_at is not null + And the latest shadow charge row's refund_reason starts with "first-step-5xx" + And the latest job is CLOSED + And the latest job has 1 step recorded with status "FAILED" + + # ────────────────────────────────────────────────────────────────────────── + # NOTE: The kill-switch scenario (PAYG_FILTER_ENABLED=false) is NOT + # automated — it requires restarting the stirling-pdf-saas container with + # a different env var mid-run, which couples test setup to compose state + # more than it's worth for a flag that only flips during incident response. + # See notes/PAYG_DESIGN.md §7.5.2 "PAYG cucumber: manual-only scenarios". + # ────────────────────────────────────────────────────────────────────────── + + Scenario: 4xx leaves the shadow row CHARGED (customer pays for the attempt) + # /sanitize-pdf on an encrypted PDF without the password reliably 400s + # via GlobalExceptionHandler's PdfPasswordException → ProblemDetail path. + # We chain: first call encrypts a PDF (CHARGED), second call tries to + # sanitize WITHOUT the password and 400s. The 4xx assertion is on the + # SECOND call's behaviour. + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF to "/api/v1/security/add-password" + And I take the response body as "encrypted" + And I POST "encrypted" to "/api/v1/security/sanitize-pdf" + Then the response status is >= 400 and < 500 + # Note: shadow_charges count is 1 because the second call lineage-joins + # the first (its input matches the first's output). The 4xx therefore + # appears as a FAILED step on the existing process, not a new shadow row. + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest shadow charge row has status "CHARGED" + And the latest job has step_count = 2 + And the latest step's error_code matches the response status + + Scenario: ZIP-returning tool records OUTPUT signatures per inner PDF + # /split-pages with multiple page numbers returns a ZIP. Stirling sends + # application/octet-stream rather than application/zip — the extractor + # sniffs the PK\x03\x04 magic so the ZIP unpack path still fires. + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a 3-page PDF to "/api/v1/general/split-pages" with form fields: + | pageNumbers | 1,2 | + Then the response status is 200 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest job has at least 2 OUTPUT artifact hashes recorded + + Scenario: Multi-file input writes a single shadow row sized by the group + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST two single-page PDFs as a multi-file payload to "/api/v1/general/merge-pdfs" + Then the response status is 200 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest shadow charge row has payg_units >= 1 + + Scenario: PIPELINE header sets the job source + Given there are no existing shadow charges for team "payg-cucumber-team" + When I POST a single-page PDF with header "X-Stirling-Automation: true" to "/api/v1/security/add-password" + Then the response status is 200 + And exactly 1 shadow charge row exists for team "payg-cucumber-team" + And the latest job's source is "PIPELINE" diff --git a/testing/cucumber/features/steps/payg_step_definitions.py b/testing/cucumber/features/steps/payg_step_definitions.py new file mode 100644 index 0000000000..5069f9a007 --- /dev/null +++ b/testing/cucumber/features/steps/payg_step_definitions.py @@ -0,0 +1,473 @@ +""" +Step definitions for PAYG shadow-mode end-to-end tests. + +Runs against the saas-profile stack defined in +testing/compose/docker-compose-saas.yml — the backend with STIRLING_FLAVOR=saas +plus a Postgres container holding the stirling_pdf schema. + +The test harness talks to the backend over HTTP and inspects the resulting +rows in `payg_shadow_charge`, `processing_job`, `processing_job_step`, and +`job_artifact_hash` via a direct psycopg connection. Direct DB inspection is +deliberate — we want to verify the *side effects* of the filter, not relay +them through another API layer that itself might be wrong. + +Auth model: the saas profile expects Supabase JWTs. For cucumber we +configure the stack with a test user whose API key is recognised via the +X-API-KEY header, which the PaygChargeInterceptor.resolveUser() path +handles natively. The companion docker-compose-saas.yml seeds the user + +team rows via saas-init.sql so each scenario starts from a known state. +""" + +import os +import time + +import psycopg +import requests +from behave import given, then, when + +BASE_URL = os.environ.get("PAYG_BASE_URL", "http://localhost:8080") +API_KEY = os.environ.get("PAYG_API_KEY", "payg-cucumber-key") + +DB_HOST = os.environ.get("PAYG_DB_HOST", "localhost") +DB_PORT = int(os.environ.get("PAYG_DB_PORT", "5433")) +DB_NAME = os.environ.get("PAYG_DB_NAME", "postgres") +DB_USER = os.environ.get("PAYG_DB_USER", "postgres") +DB_PASSWORD = os.environ.get("PAYG_DB_PASSWORD", "postgres") +DB_SCHEMA = os.environ.get("PAYG_DB_SCHEMA", "stirling_pdf") + +# Fixture paths — small PDFs that ship with the cucumber harness. +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "exampleFiles", +) +# Existing cucumber fixtures (verified page counts via pypdf): +# tables.pdf = 1 page → SINGLE_PAGE_PDF +# ghost1.pdf = 3 pages → THREE_PAGE_PDF +# images.pdf = 5 pages (available if a scenario needs more) +# If you add more PAYG scenarios that need a different shape, drop the +# new fixture in exampleFiles/ and reference it here. +SINGLE_PAGE_PDF = os.path.join(FIXTURE_DIR, "tables.pdf") +THREE_PAGE_PDF = os.path.join(FIXTURE_DIR, "ghost1.pdf") + + +# --------------------------------------------------------------------------- +# DB helpers +# --------------------------------------------------------------------------- + + +def _db(): + """Open a fresh connection for each step — keeps things simple.""" + return psycopg.connect( + host=DB_HOST, + port=DB_PORT, + dbname=DB_NAME, + user=DB_USER, + password=DB_PASSWORD, + autocommit=True, + ) + + +def _team_id_for(team_name, conn): + """Look up our test team's id (seeded by saas-init.sql).""" + with conn.cursor() as cur: + cur.execute( + f"SELECT team_id FROM {DB_SCHEMA}.teams WHERE name = %s LIMIT 1", + (team_name,), + ) + row = cur.fetchone() + assert row is not None, f"No team named '{team_name}' in {DB_SCHEMA}.teams" + return row[0] + + +def _shadow_rows_for(team_name): + with _db() as conn: + team_id = _team_id_for(team_name, conn) + with conn.cursor() as cur: + cur.execute( + f"""SELECT shadow_id, payg_units, status, refunded_at, refund_reason, job_id + FROM {DB_SCHEMA}.payg_shadow_charge + WHERE team_id = %s + ORDER BY occurred_at DESC""", + (team_id,), + ) + return cur.fetchall() + + +def _latest_job_for(team_name): + """Return the most recently opened job for the team as a dict.""" + with _db() as conn: + team_id = _team_id_for(team_name, conn) + with conn.cursor() as cur: + cur.execute( + f"""SELECT job_id, status, source, step_count, started_at, closed_at + FROM {DB_SCHEMA}.processing_job + WHERE owner_team_id = %s + ORDER BY started_at DESC + LIMIT 1""", + (team_id,), + ) + row = cur.fetchone() + assert row is not None, f"No processing_job rows for team '{team_name}'" + return { + "job_id": row[0], + "status": row[1], + "source": row[2], + "step_count": row[3], + "started_at": row[4], + "closed_at": row[5], + } + + +def _steps_for_job(job_id): + with _db() as conn: + with conn.cursor() as cur: + cur.execute( + f"""SELECT step_id, tool_id, status, error_code + FROM {DB_SCHEMA}.processing_job_step + WHERE job_id = %s + ORDER BY started_at ASC""", + (job_id,), + ) + return cur.fetchall() + + +def _output_artifact_count(job_id): + with _db() as conn: + with conn.cursor() as cur: + cur.execute( + f"""SELECT COUNT(*) FROM {DB_SCHEMA}.job_artifact_hash + WHERE job_id = %s AND kind = 'OUTPUT'""", + (job_id,), + ) + return cur.fetchone()[0] + + +# --------------------------------------------------------------------------- +# HTTP helpers +# --------------------------------------------------------------------------- + + +def _api_headers(extra=None): + headers = {"X-API-KEY": API_KEY} + if extra: + headers.update(extra) + return headers + + +def _wait_for_health(timeout_seconds=60): + """Block until /api/v1/info/status returns 2xx, or timeout.""" + deadline = time.time() + timeout_seconds + while time.time() < deadline: + try: + r = requests.get(f"{BASE_URL}/api/v1/info/status", timeout=5) + if 200 <= r.status_code < 300: + return + except requests.RequestException: + pass + time.sleep(2) + raise AssertionError( + f"SaaS stack did not become healthy within {timeout_seconds}s at {BASE_URL}" + ) + + +# --------------------------------------------------------------------------- +# GIVEN — environment + test fixtures +# --------------------------------------------------------------------------- + + +@given("the SaaS stack is running with PAYG enabled") +def step_saas_stack_running(context): + _wait_for_health() + + +@given('team "{team_name}" exists with wallet_policy.engine = "{engine}"') +def step_team_exists_with_engine(context, team_name, engine): + """Verify the seed migration created the team + flipped its engine.""" + with _db() as conn: + team_id = _team_id_for(team_name, conn) + with conn.cursor() as cur: + cur.execute( + f"SELECT engine FROM {DB_SCHEMA}.wallet_policy WHERE team_id = %s", + (team_id,), + ) + row = cur.fetchone() + assert row is not None, f"No wallet_policy row for team '{team_name}'" + assert row[0] == engine, ( + f"Team '{team_name}' engine is '{row[0]}', expected '{engine}'. " + "Check saas-init.sql seeded the row correctly." + ) + context.team_name = team_name + + +@given('I am authenticated as a member of team "{team_name}"') +def step_authenticated_as_team_member(context, team_name): + # The seeded API key is bound to a member of this team via saas-init.sql. + context.team_name = team_name + + +@given('there are no existing shadow charges for team "{team_name}"') +def step_clear_shadow_charges(context, team_name): + with _db() as conn: + team_id = _team_id_for(team_name, conn) + with conn.cursor() as cur: + cur.execute( + f"DELETE FROM {DB_SCHEMA}.payg_shadow_charge WHERE team_id = %s", + (team_id,), + ) + cur.execute( + f"""DELETE FROM {DB_SCHEMA}.job_artifact_hash + WHERE job_id IN ( + SELECT job_id FROM {DB_SCHEMA}.processing_job + WHERE owner_team_id = %s + )""", + (team_id,), + ) + cur.execute( + f"""DELETE FROM {DB_SCHEMA}.processing_job_step + WHERE job_id IN ( + SELECT job_id FROM {DB_SCHEMA}.processing_job + WHERE owner_team_id = %s + )""", + (team_id,), + ) + cur.execute( + f"DELETE FROM {DB_SCHEMA}.processing_job WHERE owner_team_id = %s", + (team_id,), + ) + + +# NOTE: The kill-switch scenario (PAYG_FILTER_ENABLED=false) is verified +# manually — see notes/PAYG_DESIGN.md §7.5 "PAYG cucumber: manual-only +# scenarios" for the procedure. No automated step def is provided because +# toggling the env var requires restarting the docker container, and +# orchestrating that mid-suite would couple test setup to compose state +# in ways that have historically been fragile. + + +# --------------------------------------------------------------------------- +# WHEN — invoke tool endpoints +# --------------------------------------------------------------------------- + + +@when('I POST a single-page PDF to "{endpoint}"') +def step_post_single_pdf(context, endpoint): + _post_pdf(context, endpoint, SINGLE_PAGE_PDF) + + +@when('I POST a 3-page PDF to "{endpoint}"') +def step_post_three_page_pdf(context, endpoint): + _post_pdf(context, endpoint, THREE_PAGE_PDF) + + +@when('I POST a single-page PDF to "{endpoint}" with form fields:') +def step_post_with_form_fields(context, endpoint): + """The Gherkin step takes a `table` of key/value pairs that become + multipart form fields. Use this when the default `password=...` field + from _post_pdf() isn't what we want for the scenario.""" + data = {row[0]: row[1] for row in context.table.rows} if context.table else {} + _post_pdf(context, endpoint, SINGLE_PAGE_PDF, form_data=data) + + +@when('I POST a 3-page PDF to "{endpoint}" with form fields:') +def step_post_three_page_with_form_fields(context, endpoint): + data = {row[0]: row[1] for row in context.table.rows} if context.table else {} + _post_pdf(context, endpoint, THREE_PAGE_PDF, form_data=data) + + +@when('I POST a single-page PDF with header "{header_name}: {header_value}" to "{endpoint}"') +def step_post_with_header(context, header_name, header_value, endpoint): + _post_pdf(context, endpoint, SINGLE_PAGE_PDF, extra_headers={header_name: header_value}) + + +@when('I POST two single-page PDFs as a multi-file payload to "{endpoint}"') +def step_post_two_pdfs(context, endpoint): + with open(SINGLE_PAGE_PDF, "rb") as a, open(SINGLE_PAGE_PDF, "rb") as b: + files = [ + ("fileInput", ("a.pdf", a.read(), "application/pdf")), + ("fileInput", ("b.pdf", b.read(), "application/pdf")), + ] + context.response = requests.post( + f"{BASE_URL}{endpoint}", + files=files, + headers=_api_headers(), + timeout=30, + ) + + +@when('I take the response body as "{name}"') +def step_capture_response_body(context, name): + assert context.response is not None, "No response on context" + if not hasattr(context, "captured_bodies"): + context.captured_bodies = {} + context.captured_bodies[name] = context.response.content + + +@when('I POST "{captured_name}" to "{endpoint}"') +def step_post_captured(context, captured_name, endpoint): + body = context.captured_bodies[captured_name] + files = {"fileInput": (f"{captured_name}.pdf", body, "application/pdf")} + # Most tools need at least one extra form field; for the sanitize endpoint + # the defaults are sufficient. + context.response = requests.post( + f"{BASE_URL}{endpoint}", + files=files, + headers=_api_headers(), + timeout=30, + ) + + +def _post_pdf(context, endpoint, fixture_path, extra_headers=None, form_data=None): + """Default form data carries `password=cucumber-test-password` because + /add-password (our most-used scenario endpoint) requires it. Pass + {@code form_data} to override (e.g. to send the wrong password to + /remove-password for the 4xx scenario).""" + with open(fixture_path, "rb") as f: + files = {"fileInput": (os.path.basename(fixture_path), f, "application/pdf")} + data = form_data if form_data is not None else {"password": "cucumber-test-password"} + context.response = requests.post( + f"{BASE_URL}{endpoint}", + files=files, + data=data, + headers=_api_headers(extra_headers), + timeout=60, + ) + + +# --------------------------------------------------------------------------- +# THEN — assertions +# --------------------------------------------------------------------------- + + +@then("the response status is {status:d}") +def step_assert_status(context, status): + assert context.response.status_code == status, _status_mismatch_message( + context.response, status + ) + + +def _status_mismatch_message(response, expected_status): + """Verbose diagnostic for unexpected statuses — dumps URL, headers, body + so we don't have to round-trip via container logs to figure out what came + back. Kept as a helper so future status-range / status-min steps can call + it too.""" + body_preview = response.text[:1000] if response.text else "" + headers_brief = { + k: v + for k, v in response.headers.items() + if k.lower() + in ("content-type", "content-length", "x-content-type-options", "location") + } + return ( + f"Expected {expected_status}, got {response.status_code}\n" + f" URL: {response.request.method} {response.request.url}\n" + f" Headers: {headers_brief}\n" + f" Body: {body_preview}" + ) + + +@then("the response status is >= {minimum:d}") +def step_assert_status_min(context, minimum): + assert context.response.status_code >= minimum, ( + f"Expected >= {minimum}, got {context.response.status_code}" + ) + + +@then("the response status is >= {minimum:d} and < {maximum:d}") +def step_assert_status_range(context, minimum, maximum): + code = context.response.status_code + assert minimum <= code < maximum, f"Expected [{minimum}, {maximum}), got {code}" + + +@then('the response Content-Type is "{expected}"') +def step_assert_content_type(context, expected): + actual = (context.response.headers.get("Content-Type") or "").split(";")[0].strip() + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then('exactly {n:d} shadow charge row exists for team "{team_name}"') +@then('exactly {n:d} shadow charge rows exist for team "{team_name}"') +def step_assert_shadow_count(context, n, team_name): + rows = _shadow_rows_for(team_name) + assert len(rows) == n, f"Expected {n} shadow rows for '{team_name}', found {len(rows)}: {rows}" + + +@then('the latest shadow charge row has status "{status}"') +def step_assert_latest_shadow_status(context, status): + rows = _shadow_rows_for(context.team_name) + assert rows, "No shadow rows for team" + assert rows[0][2] == status, f"Expected status '{status}', got '{rows[0][2]}'" + + +@then("the latest shadow charge row has payg_units >= {minimum:d}") +def step_assert_payg_units_min(context, minimum): + rows = _shadow_rows_for(context.team_name) + assert rows[0][1] >= minimum, f"Expected payg_units >= {minimum}, got {rows[0][1]}" + + +@then("the latest shadow charge row's refunded_at is not null") +def step_assert_refunded_at_set(context): + rows = _shadow_rows_for(context.team_name) + assert rows[0][3] is not None, "refunded_at is null on the latest shadow row" + + +@then('the latest shadow charge row\'s refund_reason starts with "{prefix}"') +def step_assert_refund_reason(context, prefix): + rows = _shadow_rows_for(context.team_name) + reason = rows[0][4] or "" + assert reason.startswith(prefix), f"refund_reason '{reason}' does not start with '{prefix}'" + + +@then("the latest shadow charge row's job is {status}") +def step_assert_latest_jobstatus_via_shadow(context, status): + job = _latest_job_for(context.team_name) + assert job["status"] == status, f"Latest job status is '{job['status']}', expected '{status}'" + + +@then("the latest job is {status}") +def step_assert_latest_job_status(context, status): + job = _latest_job_for(context.team_name) + assert job["status"] == status, f"Latest job status is '{job['status']}', expected '{status}'" + + +@then("the latest job has step_count = {expected:d}") +def step_assert_step_count(context, expected): + job = _latest_job_for(context.team_name) + assert job["step_count"] == expected, ( + f"Latest job step_count is {job['step_count']}, expected {expected}" + ) + + +@then('the latest job\'s source is "{source}"') +def step_assert_job_source(context, source): + job = _latest_job_for(context.team_name) + assert job["source"] == source, f"Latest job source is '{job['source']}', expected '{source}'" + + +@then('the latest job has {n:d} step recorded with status "{status}"') +@then('the latest job has {n:d} steps recorded with status "{status}"') +def step_assert_step_count_with_status(context, n, status): + job = _latest_job_for(context.team_name) + steps = _steps_for_job(job["job_id"]) + matching = [s for s in steps if s[2] == status] + assert len(matching) == n, ( + f"Expected {n} steps with status '{status}', got {len(matching)}: {steps}" + ) + + +@then("the latest step's error_code matches the response status") +def step_assert_step_error_code(context): + job = _latest_job_for(context.team_name) + steps = _steps_for_job(job["job_id"]) + assert steps, "No steps recorded" + latest_step = steps[-1] + expected = str(context.response.status_code) + assert latest_step[3] == expected, ( + f"Latest step error_code is '{latest_step[3]}', expected '{expected}'" + ) + + +@then("the latest job has at least {n:d} OUTPUT artifact hashes recorded") +def step_assert_output_artifact_count(context, n): + job = _latest_job_for(context.team_name) + count = _output_artifact_count(job["job_id"]) + assert count >= n, f"Latest job has {count} OUTPUT artifacts, expected >= {n}" diff --git a/testing/cucumber/requirements.in b/testing/cucumber/requirements.in index 8b1f01d1f8..ae968c9e34 100644 --- a/testing/cucumber/requirements.in +++ b/testing/cucumber/requirements.in @@ -5,3 +5,12 @@ pypdf reportlab PyCryptodome qrcode[pil] +# PAYG cucumber scenarios inspect shadow rows directly against the +# saas-profile Postgres container — see testing/cucumber/features/steps/payg_step_definitions.py. +# Loaded by the saas-cucumber CI job only; main behave run doesn't touch it. +psycopg[binary] +# psycopg declares `typing-extensions>=4.6` under a Python-version marker that +# pip-compile leaves unpinned, which trips `--require-hashes`. Listing it here +# forces pip-compile to lock it to a concrete version with a hash. Without this, +# both saas-cucumber and the main docker-compose-tests fail on `pip install`. +typing-extensions diff --git a/testing/cucumber/requirements.txt b/testing/cucumber/requirements.txt index 66d26c2c3e..dedcfe43b0 100644 --- a/testing/cucumber/requirements.txt +++ b/testing/cucumber/requirements.txt @@ -1,19 +1,19 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --generate-hashes --output-file='testing\cucumber\requirements.txt' --strip-extras 'testing\cucumber\requirements.in' +# pip-compile --generate-hashes --output-file=testing/cucumber/requirements.txt --strip-extras testing/cucumber/requirements.in # behave==1.3.3 \ --hash=sha256:2b8f4b64ed2ea756a5a2a73e23defc1c4631e9e724c499e46661778453ebaf51 \ --hash=sha256:89bdb62af8fb9f147ce245736a5de69f025e5edfb66f1fbe16c5007493f842c0 # via - # -r requirements.in + # -r testing/cucumber/requirements.in # behave-html-formatter behave-html-formatter==0.9.10 \ --hash=sha256:c5a9ad3edcac7be5766b14aacce46794885c749c4741fc93f8fc3a4bf2a891aa \ --hash=sha256:fff7ac2118463701423645ad5a12636845bfd6c8a2dd52097b524b4f290aa7c8 - # via -r requirements.in + # via -r testing/cucumber/requirements.in certifi==2026.2.25 \ --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 @@ -154,7 +154,9 @@ charset-normalizer==3.4.7 \ colorama==0.4.6 \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 - # via behave + # via + # behave + # qrcode cucumber-expressions==19.0.0 \ --hash=sha256:8eb5ae46dd03dd37fec1163ace1510529501d7d1868ff372c1ab2cd5aa4543a8 \ --hash=sha256:f452e6c73258c1677043ad67ad5f538c87284d6b502004720510fb6b7452d9c5 @@ -272,6 +274,67 @@ pillow==12.2.0 \ # via # qrcode # reportlab +psycopg==3.3.4 \ + --hash=sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a \ + --hash=sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc + # via -r testing/cucumber/requirements.in +psycopg-binary==3.3.4 \ + --hash=sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070 \ + --hash=sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c \ + --hash=sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc \ + --hash=sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e \ + --hash=sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68 \ + --hash=sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae \ + --hash=sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829 \ + --hash=sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097 \ + --hash=sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c \ + --hash=sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992 \ + --hash=sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97 \ + --hash=sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6 \ + --hash=sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a \ + --hash=sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d \ + --hash=sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228 \ + --hash=sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e \ + --hash=sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4 \ + --hash=sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41 \ + --hash=sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949 \ + --hash=sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9 \ + --hash=sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089 \ + --hash=sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf \ + --hash=sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da \ + --hash=sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578 \ + --hash=sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014 \ + --hash=sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e \ + --hash=sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e \ + --hash=sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31 \ + --hash=sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652 \ + --hash=sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b \ + --hash=sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839 \ + --hash=sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277 \ + --hash=sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7 \ + --hash=sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4 \ + --hash=sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e \ + --hash=sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70 \ + --hash=sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf \ + --hash=sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a \ + --hash=sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429 \ + --hash=sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8 \ + --hash=sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3 \ + --hash=sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13 \ + --hash=sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007 \ + --hash=sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc \ + --hash=sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38 \ + --hash=sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9 \ + --hash=sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95 \ + --hash=sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d \ + --hash=sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16 \ + --hash=sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d \ + --hash=sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260 \ + --hash=sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744 \ + --hash=sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28 \ + --hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \ + --hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7 + # via psycopg pycryptodome==3.23.0 \ --hash=sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4 \ --hash=sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c \ @@ -314,29 +377,37 @@ pycryptodome==3.23.0 \ --hash=sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa \ --hash=sha256:e3f2d0aaf8080bda0587d58fc9fe4766e012441e2eed4269a77de6aea981c8be \ --hash=sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7 - # via -r requirements.in + # via -r testing/cucumber/requirements.in pypdf==6.11.0 \ --hash=sha256:062b51c81b0910e6d2755e99e1c5547a0a23b7d0a32322af66240d8edcfabe87 \ --hash=sha256:769394d5756d5b304c9b6bef88b54b1816b328e7e6fc9254e625529a15ed4ab8 - # via -r requirements.in + # via -r testing/cucumber/requirements.in qrcode==8.2 \ --hash=sha256:16e64e0716c14960108e85d853062c9e8bba5ca8252c0b4d0231b9df4060ff4f \ --hash=sha256:35c3f2a4172b33136ab9f6b3ef1c00260dd2f66f858f24d88418a015f446506c - # via -r requirements.in + # via -r testing/cucumber/requirements.in reportlab==4.5.1 \ --hash=sha256:06fce8cb56c83307cfa4909cdf4e6a2ddbb44e5d6ef4d2edca896d7e9769f091 \ --hash=sha256:9fdf68f4de9171ec66acb4a5feed8f8ca2af43479e707a6fbb0daa75d88e5494 - # via -r requirements.in + # via -r testing/cucumber/requirements.in requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed - # via -r requirements.in + # via -r testing/cucumber/requirements.in six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via # behave # parse-type +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via -r testing/cucumber/requirements.in +tzdata==2026.2 \ + --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \ + --hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7 + # via psycopg urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 diff --git a/testing/test-payg.sh b/testing/test-payg.sh new file mode 100644 index 0000000000..d3683c4482 --- /dev/null +++ b/testing/test-payg.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# +# Cucumber harness for the PAYG shadow-mode scenarios. Brings up the +# saas-profile docker-compose, waits for backend health, pipes the seed SQL +# into the test postgres, then invokes Behave against features/payg/. +# +# Companion to testing/test.sh (which covers the proprietary-flavor stack +# and skips features/payg via behave.ini's exclude_re). Kept as a separate +# entrypoint so the saas variant can be reviewed + iterated on without +# touching the main cucumber harness. + +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMPOSE_FILE="$PROJECT_ROOT/testing/compose/docker-compose-saas.yml" +SEED_FILE="$PROJECT_ROOT/testing/compose/payg/saas-seed.sql" + +cd "$PROJECT_ROOT" + +# Swap behave.ini for the run. The project-default behave.ini excludes +# features/payg so the proprietary CI (which boots without the saas profile) +# can't try to run scenarios that need PAYG tables. behave's exclude_re takes +# priority over a path argument, so even `behave features/payg` would find +# zero features against the default config. The PAYG harness needs a config +# without the payg exclusion — restored on exit. +BEHAVE_INI="$PROJECT_ROOT/testing/cucumber/behave.ini" +BEHAVE_INI_BACKUP="$BEHAVE_INI.payg-backup" + +cleanup() { + # Always dump container logs before tearing the stack down — the workflow + # has a "Dump saas container logs on failure" step but it runs AFTER this + # cleanup, by which point the containers are gone. Cheap diagnostic on + # success, essential on failure. (Earlier ERR-trap conditional didn't + # fire reliably under `set -e` when behave was the script's last command.) + echo "===== STIRLING_BACKEND_LOG_DUMP_START =====" + docker compose -f "$COMPOSE_FILE" logs --tail 500 stirling-pdf-saas || true + echo "===== STIRLING_BACKEND_LOG_DUMP_END =====" + echo "==> Tearing down saas compose stack" + docker compose -f "$COMPOSE_FILE" down -v || true + if [ -f "$BEHAVE_INI_BACKUP" ]; then + mv "$BEHAVE_INI_BACKUP" "$BEHAVE_INI" + fi +} +trap cleanup EXIT + +cp "$BEHAVE_INI" "$BEHAVE_INI_BACKUP" +cat > "$BEHAVE_INI" <<'EOF' +# Temporary behave.ini written by testing/test-payg.sh for the PAYG harness +# run only. Restored on exit. The project default (in git) excludes +# features/payg so the proprietary-flavor CI doesn't try to run them. +[behave] +exclude_re = features/enterprise +EOF + +echo "==> Building + starting saas compose stack" +docker compose -f "$COMPOSE_FILE" up -d --build + +echo "==> Waiting for backend health (max 180s)" +deadline=$(( SECONDS + 180 )) +until curl -fsS http://localhost:8080/api/v1/info/status > /dev/null 2>&1; do + if [ $SECONDS -ge $deadline ]; then + echo "Backend did not become healthy in 180s" + docker compose -f "$COMPOSE_FILE" logs --tail 200 stirling-pdf-saas + exit 1 + fi + sleep 3 +done +echo "Backend healthy." + +echo "==> Seeding test team / user / wallet_policy (PAYG_SHADOW)" +docker compose -f "$COMPOSE_FILE" exec -T postgres-saas \ + psql -U postgres -d postgres < "$SEED_FILE" + +echo "==> Running PAYG cucumber scenarios" +cd "$PROJECT_ROOT/testing/cucumber" +PAYG_BASE_URL="${PAYG_BASE_URL:-http://localhost:8080}" \ +PAYG_API_KEY="${PAYG_API_KEY:-payg-cucumber-key}" \ +PAYG_DB_HOST="${PAYG_DB_HOST:-localhost}" \ +PAYG_DB_PORT="${PAYG_DB_PORT:-5433}" \ +PAYG_DB_USER="${PAYG_DB_USER:-postgres}" \ +PAYG_DB_PASSWORD="${PAYG_DB_PASSWORD:-postgres}" \ +PAYG_DB_NAME="${PAYG_DB_NAME:-postgres}" \ +PAYG_DB_SCHEMA="${PAYG_DB_SCHEMA:-stirling_pdf}" \ +python -m behave features/payg \ + -f behave_html_formatter:HTMLFormatter -o report-payg.html \ + -f pretty \ + --junit --junit-directory junit-payg + +echo "==> PAYG cucumber run complete" From 98967bfa86125e1178d3997a1c8016e7d442d9fa Mon Sep 17 00:00:00 2001 From: ConnorYoh <40631091+ConnorYoh@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:48:05 +0100 Subject: [PATCH 14/19] =?UTF-8?q?PAYG:=20V14=20+=20V15=20=E2=80=94=20subsc?= =?UTF-8?q?ription=5Fid,=20free-tier,=20RPCs,=20audit=20logs=20(#6532)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Two Flyway migrations + matching JPA entity updates. **Part 1 of 2** in the Stripe/Supabase wire-up (PR-SB-1 in `payg-stripe-supabase-plan.html`); the companion SaaS PR carries the twin Supabase migrations + new edge functions. ### V14 — payg_subscription_state.sql - `payg_team_extensions.payg_subscription_id` — the single switch that decides whether a team is billed. NULL = free-tier or block; NOT NULL = post Stripe meter events. - `pricing_policy.free_tier_units_per_cycle` — per-policy free allowance before a card is required. Default 0. - `payg_link_subscription(team_id, customer_id, sub_id)` RPC — idempotent. - `payg_unlink_subscription(team_id, reason)` RPC — called on `subscription.deleted`. - AFTER-INSERT trigger on `teams` so every new signup gets a `payg_team_extensions` sidecar row from creation. - Backfill for existing teams without a sidecar row. - RLS: SELECT permissive (any team member), UPDATE restricted to LEADER. Service-role bypasses (backend reads + day-1 migration writes). ### V15 — payg_audit_logs.sql - `payg_meter_event_log` — backend audit of every Stripe meter event POST attempt (idempotency-key UNIQUE; index on unposted rows for nightly reconcile). - `payg_subscription_change_log` — written by V14 RPCs on every link/unlink. ### Entity updates - `PaygTeamExtensions.paygSubscriptionId` — read-only field; RPC functions are the only writers. - `PricingPolicy.freeTierUnitsPerCycle` — read by upcoming `PaygTeamUsageService` (PR-SB-4). ### Behaviour change **None yet.** The columns + functions sit unused until PR-SB-4 wires `PaygMeterReportingService` and the free-tier gate into `JobChargeService`. This PR is pure schema + JPA wiring. ## Test plan - [x] `./gradlew :saas:test` — BUILD SUCCESSFUL - [x] Manual schema review: column types, FK directions, RLS scope - [ ] Apply against v3-Supabase via `supabase db push` (after companion SaaS PR merges) - [ ] Smoke-test the trigger: `INSERT INTO teams(...)` → assert `payg_team_extensions` row appears - [ ] Smoke-test RPCs: SQL-only test of `payg_link_subscription` + `payg_unlink_subscription` produces expected row + audit entries ## References - `notes/PAYG_DESIGN.md` (revision note 2026-06-03) - `payg-stripe-supabase-plan.html` §3.1 (RPC functions), §3.5 (RLS), §3.10 (audit-log tables) --- .../saas/payg/policy/PaygTeamExtensions.java | 13 ++ .../saas/payg/policy/PricingPolicy.java | 14 ++ .../saas/V14__payg_subscription_state.sql | 189 ++++++++++++++++++ .../migration/saas/V15__payg_audit_logs.sql | 76 +++++++ 4 files changed, 292 insertions(+) create mode 100644 app/saas/src/main/resources/db/migration/saas/V14__payg_subscription_state.sql create mode 100644 app/saas/src/main/resources/db/migration/saas/V15__payg_audit_logs.sql diff --git a/app/saas/src/main/java/stirling/software/saas/payg/policy/PaygTeamExtensions.java b/app/saas/src/main/java/stirling/software/saas/payg/policy/PaygTeamExtensions.java index 2c0603fc80..7d6f1b4023 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/policy/PaygTeamExtensions.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/policy/PaygTeamExtensions.java @@ -58,6 +58,19 @@ public class PaygTeamExtensions implements Serializable { @Column(name = "stripe_customer_id", unique = true, length = 128) private String stripeCustomerId; + /** + * Stripe subscription id (sub_xxx) for this team's PAYG metered subscription. {@code null} + * means the team has not added a card yet — engine writes shadow rows only and free-tier gating + * applies. {@code non-null} means engine posts meter events to Stripe on every billable tool + * call. + * + *

This is the single switch that determines whether a team is billed. Mutated only by the + * {@code payg_link_subscription} / {@code payg_unlink_subscription} Postgres functions (V14) — + * never directly via JPA writes. Treat as read-only from Java. + */ + @Column(name = "payg_subscription_id", unique = true, length = 128) + private String paygSubscriptionId; + @CreationTimestamp @Column(name = "created_at", updatable = false) private LocalDateTime createdAt; diff --git a/app/saas/src/main/java/stirling/software/saas/payg/policy/PricingPolicy.java b/app/saas/src/main/java/stirling/software/saas/payg/policy/PricingPolicy.java index f1443664cc..743dcdc8ae 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/policy/PricingPolicy.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/policy/PricingPolicy.java @@ -73,6 +73,20 @@ public class PricingPolicy implements Serializable { @Column(name = "file_unit_cap", nullable = false) private Integer fileUnitCap = 1000; + /** + * Free-tier allowance — doc units a team on this policy can consume per cycle before they must + * add a card. {@code 0} (default) means no free tier; the team is blocked at 402 from their + * very first tool call until they pay. New-signup defaults will set this to a sensible positive + * value; the special launch policy used by the day-1 legacy migration script can override with + * a different size if product wants the original 10 customers to feel special. + * + *

Enforced by {@code PaygTeamUsageService} (PR-SB-4) on every tool call, gated on {@code + * payg_team_extensions.payg_subscription_id IS NULL} — teams with an active subscription bypass + * this check entirely. + */ + @Column(name = "free_tier_units_per_cycle", nullable = false) + private Long freeTierUnitsPerCycle = 0L; + /** * Max tool steps allowed in one process before it splits, keyed by the caller's {@link * JobSource}. Self-hosted teams typically get a higher limit via a per-team policy override. diff --git a/app/saas/src/main/resources/db/migration/saas/V14__payg_subscription_state.sql b/app/saas/src/main/resources/db/migration/saas/V14__payg_subscription_state.sql new file mode 100644 index 0000000000..eca2665de7 --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V14__payg_subscription_state.sql @@ -0,0 +1,189 @@ +-- PAYG subscription state — the column + functions that let new customers reach Stripe billing. +-- +-- This migration is half of the Stripe/Supabase wire-up (PR-SB-1 in `notes/PAYG_DESIGN.md` +-- revision note + `payg-stripe-supabase-plan.html`). It's strictly additive: +-- * one new column on payg_team_extensions (payg_subscription_id) +-- * one new column on pricing_policy (free_tier_units_per_cycle) +-- * two RPC functions (payg_link_subscription, payg_unlink_subscription) — the only writers +-- of subscription state, called by stripe-webhook + create-payg-team-subscription edge fns +-- * an AFTER-INSERT trigger on teams that auto-creates the payg_team_extensions sidecar row +-- so every new signup is PAYG-by-default +-- * an RLS policy that lets team LEADERs (and the service role) link subscriptions +-- +-- No behaviour change for the running app until PR-SB-4 wires PaygMeterReportingService and +-- the free-tier gate into JobChargeService. Until then this just exposes new state for the +-- edge functions in PR-SB-2 to write through to. +-- +-- Design references: +-- * notes/PAYG_DESIGN.md (revision note 2026-06-03 — "subscription presence is the gate") +-- * payg-stripe-supabase-plan.html §3.1 — RPC functions; §3.5 — RLS policy + +-- --------------------------------------------------------------------------------------------- +-- 1. New columns +-- --------------------------------------------------------------------------------------------- + +ALTER TABLE stirling_pdf.payg_team_extensions + ADD COLUMN IF NOT EXISTS payg_subscription_id VARCHAR(128) UNIQUE; + +COMMENT ON COLUMN stirling_pdf.payg_team_extensions.payg_subscription_id IS + 'Stripe subscription id (sub_xxx) for this team''s PAYG metered subscription. ' + 'NULL = team has not added a card yet; engine writes shadow rows only. ' + 'NOT NULL = engine posts meter events to Stripe on every billable tool call. ' + 'Mutated exclusively by payg_link_subscription / payg_unlink_subscription RPC functions.'; + +ALTER TABLE stirling_pdf.pricing_policy + ADD COLUMN IF NOT EXISTS free_tier_units_per_cycle BIGINT NOT NULL DEFAULT 0; + +COMMENT ON COLUMN stirling_pdf.pricing_policy.free_tier_units_per_cycle IS + 'Doc units a team on this policy can consume per cycle before they must add a card. ' + 'Default 0 = no free tier (block immediately). The seeded default policy will set this ' + 'to the launch free-tier size; the special "launch" policy used by the day-1 legacy ' + 'migration script (see PAYG_DESIGN.md §3.10 revised) can override.'; + +-- --------------------------------------------------------------------------------------------- +-- 2. RPC: payg_link_subscription +-- +-- Called by: +-- * supabase/functions/create-payg-team-subscription/index.ts (post-Stripe-Checkout, with +-- either user JWT [normal path, RLS-enforced] or service-role [day-1 migration script]) +-- * supabase/functions/stripe-webhook/handlers/payg-subscription.ts on +-- customer.subscription.created (idempotent — second invocation with same args is a no-op) +-- --------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION stirling_pdf.payg_link_subscription( + p_team_id BIGINT, + p_customer_id TEXT, + p_subscription_id TEXT +) RETURNS VOID +LANGUAGE plpgsql +SECURITY INVOKER +AS $$ +BEGIN + UPDATE stirling_pdf.payg_team_extensions + SET stripe_customer_id = p_customer_id, + payg_subscription_id = p_subscription_id, + updated_at = now() + WHERE team_id = p_team_id; + + IF NOT FOUND THEN + RAISE EXCEPTION 'payg_team_extensions row missing for team %', p_team_id + USING ERRCODE = 'foreign_key_violation'; + END IF; + + INSERT INTO stirling_pdf.payg_subscription_change_log(team_id, action, subscription_id) + VALUES (p_team_id, 'LINKED', p_subscription_id); +END $$; + +COMMENT ON FUNCTION stirling_pdf.payg_link_subscription(BIGINT, TEXT, TEXT) IS + 'Idempotent link of a Stripe subscription to a team. SECURITY INVOKER means RLS applies — ' + 'the caller must be a LEADER of the team (or hold the service-role bypass). ' + 'Writes an audit row to payg_subscription_change_log.'; + +-- --------------------------------------------------------------------------------------------- +-- 3. RPC: payg_unlink_subscription +-- +-- Called by stripe-webhook handlers/payg-subscription.ts on customer.subscription.deleted +-- (after Stripe's own retries have given up). Drops the team back to free-tier-then-block. +-- --------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION stirling_pdf.payg_unlink_subscription( + p_team_id BIGINT, + p_reason TEXT +) RETURNS VOID +LANGUAGE plpgsql +SECURITY INVOKER +AS $$ +BEGIN + UPDATE stirling_pdf.payg_team_extensions + SET payg_subscription_id = NULL, + updated_at = now() + WHERE team_id = p_team_id; + -- We deliberately keep stripe_customer_id — the team may add a new card later and we'd + -- like to reuse the existing Stripe customer record rather than create a duplicate. + + INSERT INTO stirling_pdf.payg_subscription_change_log(team_id, action, reason) + VALUES (p_team_id, 'UNLINKED', p_reason); +END $$; + +COMMENT ON FUNCTION stirling_pdf.payg_unlink_subscription(BIGINT, TEXT) IS + 'Drops the team back to free-tier-then-block derived state. Reason is logged for audit ' + '(typically subscription_deleted | admin | card_removed).'; + +-- --------------------------------------------------------------------------------------------- +-- 4. Auto-create payg_team_extensions row when a team is created +-- +-- Every new signup gets a payg_team_extensions row with NULL pricing_policy_id (which the +-- backend's PricingPolicyService resolves to the default policy). The free-tier gate kicks in +-- from the very first tool call. +-- --------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION stirling_pdf.payg_create_team_extensions_trigger() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + INSERT INTO stirling_pdf.payg_team_extensions(team_id) + VALUES (NEW.team_id) + ON CONFLICT (team_id) DO NOTHING; + RETURN NEW; +END $$; + +DROP TRIGGER IF EXISTS trg_payg_create_team_extensions ON stirling_pdf.teams; +CREATE TRIGGER trg_payg_create_team_extensions + AFTER INSERT ON stirling_pdf.teams + FOR EACH ROW + EXECUTE FUNCTION stirling_pdf.payg_create_team_extensions_trigger(); + +COMMENT ON TRIGGER trg_payg_create_team_extensions ON stirling_pdf.teams IS + 'Ensures every team has a payg_team_extensions sidecar row from creation. New customers ' + 'are PAYG-default from minute one — they consume free-tier units until they add a card.'; + +-- --------------------------------------------------------------------------------------------- +-- 5. Backfill: any existing team without a sidecar row gets one now +-- --------------------------------------------------------------------------------------------- + +INSERT INTO stirling_pdf.payg_team_extensions(team_id) +SELECT t.team_id + FROM stirling_pdf.teams t + WHERE NOT EXISTS ( + SELECT 1 FROM stirling_pdf.payg_team_extensions x WHERE x.team_id = t.team_id + ); + +-- --------------------------------------------------------------------------------------------- +-- 6. RLS policy +-- +-- Service-role bypasses RLS (backend reads + day-1 migration script writes via the service-role +-- key). For user-initiated writes via the frontend Add-Card flow, only team LEADERs can link a +-- subscription. SELECT remains permissive — anyone in the team can see the row. +-- --------------------------------------------------------------------------------------------- + +ALTER TABLE stirling_pdf.payg_team_extensions ENABLE ROW LEVEL SECURITY; + +-- Read: any team member can see their team's payg row. +DROP POLICY IF EXISTS payg_team_ext_select ON stirling_pdf.payg_team_extensions; +CREATE POLICY payg_team_ext_select + ON stirling_pdf.payg_team_extensions + FOR SELECT + USING ( + team_id IN ( + SELECT tm.team_id + FROM stirling_pdf.team_memberships tm + JOIN stirling_pdf.users u ON u.user_id = tm.user_id + WHERE u.supabase_auth_id = auth.uid() + ) + ); + +-- Update: only LEADERs of the team can update (i.e. link / unlink a subscription). +DROP POLICY IF EXISTS payg_team_ext_leader_update ON stirling_pdf.payg_team_extensions; +CREATE POLICY payg_team_ext_leader_update + ON stirling_pdf.payg_team_extensions + FOR UPDATE + USING ( + team_id IN ( + SELECT tm.team_id + FROM stirling_pdf.team_memberships tm + JOIN stirling_pdf.users u ON u.user_id = tm.user_id + WHERE u.supabase_auth_id = auth.uid() + AND tm.role = 'LEADER' + ) + ); diff --git a/app/saas/src/main/resources/db/migration/saas/V15__payg_audit_logs.sql b/app/saas/src/main/resources/db/migration/saas/V15__payg_audit_logs.sql new file mode 100644 index 0000000000..1b7e61e9c1 --- /dev/null +++ b/app/saas/src/main/resources/db/migration/saas/V15__payg_audit_logs.sql @@ -0,0 +1,76 @@ +-- PAYG audit-log tables. Two append-only logs: +-- +-- * payg_meter_event_log — written by the backend's PaygMeterReportingService +-- on every Stripe meter event POST attempt. Gives us a +-- record independent of Stripe's own logs so we can +-- replay-after-24h-window (Stripe's idempotency window) +-- and run nightly reconciliation against Stripe's +-- meter-event list. +-- +-- * payg_subscription_change_log — written by V14's two RPC functions on every +-- subscription link / unlink. Independent of Stripe's +-- webhook log; lets us diagnose "why is this team in +-- free-tier-block when their Stripe sub is active?" +-- without leaving our DB. +-- +-- Both are pure additive; nothing reads them yet. PR-SB-5 (nightly reconcile) wires the +-- meter-event log; the subscription change log is queried only from admin tooling. +-- +-- Design references: +-- * payg-stripe-supabase-plan.html §3.10 — twin migrations +-- * payg-stripe-supabase-plan.html §8 H5 — 24h idempotency window mitigation + +-- --------------------------------------------------------------------------------------------- +-- 1. payg_meter_event_log — backend-side audit of every Stripe meter event we tried to post. +-- --------------------------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS stirling_pdf.payg_meter_event_log ( + event_id BIGSERIAL PRIMARY KEY, + team_id BIGINT NOT NULL REFERENCES stirling_pdf.teams(team_id) ON DELETE CASCADE, + job_id UUID, + idempotency_key VARCHAR(128) NOT NULL UNIQUE, + units INTEGER NOT NULL, + occurred_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + posted_to_stripe_at TIMESTAMP, + -- NULL while pending; set when the meter-payg-units edge fn returns success. NULL after + -- 24h means the event never made it to Stripe — nightly reconcile retries with a fresh + -- idempotency-key suffix (see §8 H5 mitigation). + stripe_error_code VARCHAR(64), + stripe_error_body TEXT, + metadata JSONB +); + +CREATE INDEX IF NOT EXISTS idx_payg_meter_event_team_time + ON stirling_pdf.payg_meter_event_log (team_id, occurred_at); + +CREATE INDEX IF NOT EXISTS idx_payg_meter_event_unposted + ON stirling_pdf.payg_meter_event_log (occurred_at) + WHERE posted_to_stripe_at IS NULL; + +COMMENT ON TABLE stirling_pdf.payg_meter_event_log IS + 'Backend audit of every Stripe meter event POST attempt. Independent of Stripe meter ' + 'history. idempotency_key is the same one passed to Stripe; the UNIQUE constraint here ' + 'gives us safe at-least-once semantics even on backend retry. Rows older than 24h with ' + 'posted_to_stripe_at IS NULL are stuck and retried by the nightly reconcile job.'; + +-- --------------------------------------------------------------------------------------------- +-- 2. payg_subscription_change_log — written by V14's RPC functions on every link / unlink. +-- --------------------------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS stirling_pdf.payg_subscription_change_log ( + change_id BIGSERIAL PRIMARY KEY, + team_id BIGINT NOT NULL REFERENCES stirling_pdf.teams(team_id) ON DELETE CASCADE, + action VARCHAR(32) NOT NULL, + -- LINKED — payg_link_subscription written subscription_id + -- UNLINKED — payg_unlink_subscription cleared the subscription + subscription_id VARCHAR(128), + reason TEXT, + changed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_payg_sub_change_team_time + ON stirling_pdf.payg_subscription_change_log (team_id, changed_at); + +COMMENT ON TABLE stirling_pdf.payg_subscription_change_log IS + 'Append-only log of every subscription link / unlink. Written by V14 RPC functions; ' + 'never updated. Diagnostic value when reconciling against Stripe webhook history.'; From 1e739b6f6f0273ef877b2ddf84f08815441b8bbc Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:49:20 +0100 Subject: [PATCH 15/19] SaaS-aware API landing page (#6585) # Description of Changes OLD (and still current in selfhosted) image New (in SaaS mode) image --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .../web/ReactRoutingController.java | 28 ++ .../src/main/resources/static/modern-logo.svg | 4 + .../main/resources/static/saas-landing.html | 250 ++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 app/saas/src/main/resources/static/modern-logo.svg create mode 100644 app/saas/src/main/resources/static/saas-landing.html diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java b/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java index ce0f5edaf0..d6a95e66d4 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/web/ReactRoutingController.java @@ -41,6 +41,8 @@ public class ReactRoutingController { private boolean indexHtmlExists = false; private boolean useExternalIndexHtml = false; private boolean loggedMissingIndex = false; + private String cachedSaasLandingHtml; + private boolean saasLandingExists = false; @PostConstruct public void init() { @@ -49,6 +51,20 @@ public class ReactRoutingController { // Always initialize callback HTML (used for OAuth desktop flow) this.cachedCallbackHtml = buildCallbackHtml(); + // SaaS landing page: only present on the classpath when the :saas module is bundled + // (app/saas/src/main/resources/static/saas-landing.html). When present it replaces the + // root page so the SaaS API host shows its own landing instead of the OSS API-only page. + ClassPathResource saasLanding = new ClassPathResource("static/saas-landing.html"); + if (saasLanding.exists()) { + try (InputStream in = saasLanding.getInputStream()) { + this.cachedSaasLandingHtml = new String(in.readAllBytes(), StandardCharsets.UTF_8); + this.saasLandingExists = true; + log.info("SaaS landing page detected; serving it at '/' and '/index.html'"); + } catch (Exception ex) { + log.warn("Failed to read saas-landing.html; falling back to index.html", ex); + } + } + // Check for external index.html first (customFiles/static/) Path externalIndexPath = Paths.get(InstallationPathConfig.getStaticPath(), "index.html"); log.debug("Checking for custom index.html at: {}", externalIndexPath); @@ -132,6 +148,18 @@ public class ReactRoutingController { @GetMapping( value = {"/", "/index.html"}, produces = MediaType.TEXT_HTML_VALUE) + public ResponseEntity serveRootPage(HttpServletRequest request) { + // Swap ONLY the root page for SaaS. SPA entry points that delegate to serveIndexHtml + // (/auth/callback, /share/{token}, forwarded routes) keep serving the normal shell. + if (saasLandingExists && cachedSaasLandingHtml != null) { + return ResponseEntity.ok() + .cacheControl(CacheControl.noCache().mustRevalidate()) + .contentType(MediaType.TEXT_HTML) + .body(cachedSaasLandingHtml); + } + return serveIndexHtml(request); + } + public ResponseEntity serveIndexHtml(HttpServletRequest request) { try { if (indexHtmlExists && cachedIndexHtml != null) { diff --git a/app/saas/src/main/resources/static/modern-logo.svg b/app/saas/src/main/resources/static/modern-logo.svg new file mode 100644 index 0000000000..a4a1a1f87e --- /dev/null +++ b/app/saas/src/main/resources/static/modern-logo.svg @@ -0,0 +1,4 @@ + + + + diff --git a/app/saas/src/main/resources/static/saas-landing.html b/app/saas/src/main/resources/static/saas-landing.html new file mode 100644 index 0000000000..3ffef1d7bf --- /dev/null +++ b/app/saas/src/main/resources/static/saas-landing.html @@ -0,0 +1,250 @@ + + + + + + + Stirling PDF - Cloud API + + + + + + + + + + + +

+
+ + +
+

+ You've reached the Stirling PDF Cloud API. This endpoint serves the REST API that powers our apps and your integrations. +

+
+ + Open API Documentation + Read the Developer Docs + +
+ +
+

Just looking to edit a PDF?

+

+ This page is the API endpoint - it doesn't have an editor. If you want the full interface: +

+ +
+ +
+ +
+

Want to run your own?

+

+ Stirling PDF is open source. You can self-host the same toolkit on your own hardware or run it inside your network: +

+ +
+ +
+ +
+

Support

+ +
+
+
+ +
+ Powered by + Stirling PDF +
+ + From 1a0beaffc21d6db99b4f6dfa84e9efeaefc9c9a2 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:59:53 +0100 Subject: [PATCH 16/19] stop background flash on tab switches, unblock Audit/Usage demos (#6562) --- .../core/components/shared/AppConfigModal.tsx | 90 +++++++++++-------- .../src/core/tests/stubbed/settings.spec.ts | 75 ++++++++++++++++ .../shared/config/configNavSections.tsx | 29 +++--- 3 files changed, 141 insertions(+), 53 deletions(-) diff --git a/frontend/editor/src/core/components/shared/AppConfigModal.tsx b/frontend/editor/src/core/components/shared/AppConfigModal.tsx index a034898382..f4efe6d518 100644 --- a/frontend/editor/src/core/components/shared/AppConfigModal.tsx +++ b/frontend/editor/src/core/components/shared/AppConfigModal.tsx @@ -30,12 +30,26 @@ interface AppConfigModalProps { onClose: () => void; } +// Extract section from URL path (e.g., /settings/people -> people) +const getSectionFromPath = (pathname: string): NavKey | null => { + const match = pathname.match(/\/settings\/([^/]+)/); + if (match && match[1]) { + const section = match[1] as NavKey; + return VALID_NAV_KEYS.includes(section as NavKey) ? section : null; + } + return null; +}; + const AppConfigModalInner: React.FC = ({ opened, onClose, }) => { const { t } = useTranslation(); - const [active, setActive] = useState("general"); + // Initialize from the URL so a deep link (`/settings/people`) lands on the + // right tab without a one-frame "general" flicker. + const [active, setActive] = useState( + () => getSectionFromPath(window.location.pathname) ?? "general", + ); const isMobile = useIsMobile(); const navigate = useNavigate(); const location = useLocation(); @@ -44,17 +58,10 @@ const AppConfigModalInner: React.FC = ({ const { confirmIfDirty } = useUnsavedChanges(); const closeButtonRef = useRef(null); - // Extract section from URL path (e.g., /settings/people -> people) - const getSectionFromPath = (pathname: string): NavKey | null => { - const match = pathname.match(/\/settings\/([^/]+)/); - if (match && match[1]) { - const section = match[1] as NavKey; - return VALID_NAV_KEYS.includes(section as NavKey) ? section : null; - } - return null; - }; - - // Sync active state with URL path + // Sync active state with URL path. Runs on open, on external URL changes, + // and on the redirect path below - NOT on intra-modal tab clicks, because + // those update the URL via `history.replaceState` directly and never push + // a new React Router location. useEffect(() => { const section = getSectionFromPath(location.pathname); if (opened && section) { @@ -76,18 +83,41 @@ const AppConfigModalInner: React.FC = ({ } }, [opened]); - // Handle custom events for backwards compatibility. - // Use replace when already on /settings/* so external tab-switches - // don't pile up history entries that would break close-by-back. + // Switch tab without forcing every `useLocation()` subscriber (HomePage and + // its FileSidebar/Workbench/RightSidebar/FileManager tree) to re-render. + // + // First entry into /settings/* still goes through React Router so HomePage's + // location-watching effect opens the modal and pushes a real history entry - + // so the back button can close us. Subsequent tab clicks bypass React Router + // and mutate the URL bar via `history.replaceState`. The browser sees the + // URL update (deep-link / refresh still work) but React Router never fires a + // location change, so the layer behind the Mantine overlay never repaints + // and the backdrop-filter blur stops flashing. + const switchSection = useCallback( + (key: NavKey) => { + setActive(key); + const alreadyInSettings = + window.location.pathname.startsWith("/settings"); + if (alreadyInSettings) { + window.history.replaceState( + window.history.state, + "", + `/settings/${key}`, + ); + } else { + navigate(`/settings/${key}`); + } + }, + [navigate], + ); + + // Backwards-compat: external `appConfig:navigate` events route through the + // same switchSection path so they get the no-flash treatment too. useEffect(() => { const handler = (ev: Event) => { const detail = (ev as CustomEvent).detail as { key?: NavKey } | undefined; if (detail?.key) { - const alreadyInSettings = - window.location.pathname.startsWith("/settings"); - navigate(`/settings/${detail.key}`, { - replace: alreadyInSettings, - }); + switchSection(detail.key); } }; window.addEventListener("appConfig:navigate", handler as EventListener); @@ -96,7 +126,7 @@ const AppConfigModalInner: React.FC = ({ "appConfig:navigate", handler as EventListener, ); - }, [navigate]); + }, [switchSection]); const colors = useMemo( () => ({ @@ -165,23 +195,9 @@ const AppConfigModalInner: React.FC = ({ async (key: NavKey) => { const canProceed = await confirmIfDirty(); if (!canProceed) return; - - setActive(key); - // First in-modal nav (when current path isn't `/settings/*` yet) must - // PUSH so the originating page stays in history and close-by-back can - // return to it. Subsequent tab switches REPLACE so they don't pile up - // history entries that handleClose's navigate(-1) can't unwind. - // - // Read window.location.pathname directly (not the React hook's - // location.pathname) so rapid successive clicks pick up the URL - // change from the previous click immediately. The hook snapshot is - // stale between render cycles - relying on it lets a second click - // PUSH again before React re-renders, producing a history pile-up. - const alreadyInSettings = - window.location.pathname.startsWith("/settings"); - navigate(`/settings/${key}`, { replace: alreadyInSettings }); + switchSection(key); }, - [confirmIfDirty, navigate], + [confirmIfDirty, switchSection], ); return ( diff --git a/frontend/editor/src/core/tests/stubbed/settings.spec.ts b/frontend/editor/src/core/tests/stubbed/settings.spec.ts index 0bfd7b3ef9..cb95a1171e 100644 --- a/frontend/editor/src/core/tests/stubbed/settings.spec.ts +++ b/frontend/editor/src/core/tests/stubbed/settings.spec.ts @@ -121,6 +121,81 @@ test.describe("Settings dialog", () => { await restored.click(); }); + test("intra-modal tab switching updates URL via replaceState, never pushState", async ({ + page, + }) => { + // Mechanism test for the "background flash" fix. Before the fix, every + // tab click called `navigate(...)` which fired React Router's location + // subscribers - HomePage, QuickAccessBar, FileManagerView, ... - and the + // layer behind the Mantine overlay repainted, causing backdrop-filter + // blur to recompute and visibly flash. After the fix, only the very + // first nav into /settings/* is allowed to go through React Router (so + // HomePage's location-watching effect opens the modal and the back + // button has a real history entry to pop). Every subsequent tab click + // updates the URL bar via raw `window.history.replaceState`, which + // React Router does NOT subscribe to. We assert this directly by + // counting calls. + await page.addInitScript(() => { + const w = window as unknown as { + __historyOps: { push: number; replace: number }; + }; + w.__historyOps = { push: 0, replace: 0 }; + const origPush = window.history.pushState.bind(window.history); + const origReplace = window.history.replaceState.bind(window.history); + window.history.pushState = function (...args) { + w.__historyOps.push++; + return origPush( + ...(args as Parameters), + ); + }; + window.history.replaceState = function (...args) { + w.__historyOps.replace++; + return origReplace( + ...(args as Parameters), + ); + }; + }); + + await page.goto("/", { waitUntil: "domcontentloaded" }); + await openSettings(page); + + const generalNav = page.locator('[data-tour="admin-general-nav"]').first(); + const hotkeysNav = page.locator('[data-tour="admin-hotkeys-nav"]').first(); + await expect(generalNav).toBeVisible({ timeout: 5_000 }); + + // First nav into /settings/* takes the React Router path (push). We + // snapshot both counters AFTER this to isolate the intra-modal delta. + await generalNav.click(); + await page.waitForURL(/\/settings\/general/, { timeout: 5_000 }); + const baseline = await page.evaluate(() => { + const w = window as unknown as { + __historyOps: { push: number; replace: number }; + }; + return { ...w.__historyOps }; + }); + + // Now do 4 round-trips between two tabs - 8 intra-modal clicks total. + for (let i = 0; i < 4; i++) { + await hotkeysNav.click(); + await page.waitForURL(/\/settings\/hotkeys/, { timeout: 5_000 }); + await generalNav.click(); + await page.waitForURL(/\/settings\/general/, { timeout: 5_000 }); + } + + const after = await page.evaluate(() => { + const w = window as unknown as { + __historyOps: { push: number; replace: number }; + }; + return { ...w.__historyOps }; + }); + + // Zero pushes during 8 intra-modal clicks - the regression would show + // up here as `after.push - baseline.push >= 1`. + expect(after.push - baseline.push).toBe(0); + // Exactly 8 replaces - one per click. + expect(after.replace - baseline.replace).toBe(8); + }); + test("close returns to origin URL even after switching tabs (no history pile-up)", async ({ page, }) => { diff --git a/frontend/editor/src/proprietary/components/shared/config/configNavSections.tsx b/frontend/editor/src/proprietary/components/shared/config/configNavSections.tsx index 1ccfea9e7b..19fe77559a 100644 --- a/frontend/editor/src/proprietary/components/shared/config/configNavSections.tsx +++ b/frontend/editor/src/proprietary/components/shared/config/configNavSections.tsx @@ -71,10 +71,6 @@ export const useConfigNavSections = ( "settings.tooltips.enableLoginFirst", "Enable login mode first", ); - const requiresEnterpriseTooltip = t( - "settings.tooltips.requiresEnterprise", - "Requires Enterprise license", - ); // Workspace sections.push({ @@ -199,10 +195,10 @@ export const useConfigNavSections = ( label: t("settings.licensingAnalytics.audit", "Audit"), icon: "fact-check-rounded", component: , - disabled: !runningEE || requiresLogin, - disabledTooltip: requiresLogin - ? enableLoginTooltip - : requiresEnterpriseTooltip, + // Non-Enterprise users can still click in: AdminAuditSection + // renders a demo preview when `!hasEnterpriseLicense`. + disabled: requiresLogin, + disabledTooltip: requiresLogin ? enableLoginTooltip : undefined, }, { key: "adminUsage", @@ -212,10 +208,9 @@ export const useConfigNavSections = ( ), icon: "analytics-rounded", component: , - disabled: !runningEE || requiresLogin, - disabledTooltip: requiresLogin - ? enableLoginTooltip - : requiresEnterpriseTooltip, + // Same demo-preview story as adminAudit above. + disabled: requiresLogin, + disabledTooltip: requiresLogin ? enableLoginTooltip : undefined, }, ], }); @@ -441,20 +436,22 @@ export const createConfigNavSections = ( label: "Audit", icon: "fact-check-rounded", component: , - disabled: !runningEE || requiresLogin, + // Non-Enterprise users can click in to see the demo preview. + disabled: requiresLogin, disabledTooltip: requiresLogin ? "Enable login mode first" - : "Requires Enterprise license", + : undefined, }, { key: "adminUsage", label: "Usage Analytics", icon: "analytics-rounded", component: , - disabled: !runningEE || requiresLogin, + // Non-Enterprise users can click in to see the demo preview. + disabled: requiresLogin, disabledTooltip: requiresLogin ? "Enable login mode first" - : "Requires Enterprise license", + : undefined, }, ], }); From 502f6c1e4da8805c8380be761a09da1dfcedaa30 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:00:02 +0100 Subject: [PATCH 17/19] fix folder causing 500 toast when deleted on another machine (#6551) --- .../exception/GlobalExceptionHandler.java | 44 ++++ .../exception/GlobalExceptionHandlerTest.java | 15 ++ .../src/core/contexts/FolderContext.test.tsx | 195 +++++++++++++++++- .../src/core/contexts/FolderContext.tsx | 123 +++++++++-- 4 files changed, 350 insertions(+), 27 deletions(-) diff --git a/app/core/src/main/java/stirling/software/SPDF/exception/GlobalExceptionHandler.java b/app/core/src/main/java/stirling/software/SPDF/exception/GlobalExceptionHandler.java index 6b43b66ff6..504dd36f5b 100644 --- a/app/core/src/main/java/stirling/software/SPDF/exception/GlobalExceptionHandler.java +++ b/app/core/src/main/java/stirling/software/SPDF/exception/GlobalExceptionHandler.java @@ -24,6 +24,7 @@ import org.springframework.web.multipart.MaxUploadSizeExceededException; import org.springframework.web.multipart.support.MissingServletRequestPartException; import org.springframework.web.server.ResponseStatusException; import org.springframework.web.servlet.NoHandlerFoundException; +import org.springframework.web.servlet.resource.NoResourceFoundException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -993,6 +994,49 @@ public class GlobalExceptionHandler { .body(problemDetail); } + /** Unmapped path → clean 404 instead of falling through to the generic 500 catch-all. */ + @ExceptionHandler(NoResourceFoundException.class) + public ResponseEntity handleNoResourceFound( + NoResourceFoundException ex, HttpServletRequest request) { + // /api/* miss = likely missing controller (operator-relevant); other paths = favicons, + // robots.txt, scanner noise. Demote the latter so prod logs aren't flooded. + String uri = request.getRequestURI(); + if (uri != null && uri.startsWith("/api/")) { + log.warn("No resource at {}: {}", uri, ex.getMessage()); + } else { + log.debug("No resource at {}: {}", uri, ex.getMessage()); + } + + String title = getLocalizedMessage("error.notFound.title", ErrorTitles.NOT_FOUND_DEFAULT); + String detail = + getLocalizedMessage( + "error.notFound.detail", + String.format( + "No endpoint found for %s %s", + request.getMethod(), request.getRequestURI()), + request.getMethod(), + request.getRequestURI()); + + ProblemDetail problemDetail = + createBaseProblemDetail(HttpStatus.NOT_FOUND, detail, request); + problemDetail.setType(URI.create(ErrorTypes.NOT_FOUND)); + problemDetail.setTitle(title); + problemDetail.setProperty("title", title); + problemDetail.setProperty("method", request.getMethod()); + addStandardHints( + problemDetail, + "error.notFound.hints", + List.of( + "Verify the URL path and HTTP method are correct.", + "Check the API base path and version if applicable.", + "Ensure there are no typos in the endpoint path.")); + problemDetail.setProperty("actionRequired", "Use a valid endpoint URL and method."); + + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .contentType(PROBLEM_JSON) + .body(problemDetail); + } + /** * Handle IllegalArgumentException. * diff --git a/app/core/src/test/java/stirling/software/SPDF/exception/GlobalExceptionHandlerTest.java b/app/core/src/test/java/stirling/software/SPDF/exception/GlobalExceptionHandlerTest.java index 8e2275a728..4de6398e43 100644 --- a/app/core/src/test/java/stirling/software/SPDF/exception/GlobalExceptionHandlerTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/exception/GlobalExceptionHandlerTest.java @@ -18,6 +18,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.context.MessageSource; import org.springframework.core.env.Environment; +import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ProblemDetail; import org.springframework.http.ResponseEntity; @@ -27,6 +28,7 @@ import org.springframework.web.bind.MissingServletRequestParameterException; import org.springframework.web.multipart.MaxUploadSizeExceededException; import org.springframework.web.multipart.support.MissingServletRequestPartException; import org.springframework.web.servlet.NoHandlerFoundException; +import org.springframework.web.servlet.resource.NoResourceFoundException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -208,6 +210,19 @@ class GlobalExceptionHandlerTest { assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode()); } + // ---- NoResourceFoundException ---- + // Regression guard: was falling through to the 500 catch-all. + + @Test + void handleNoResourceFound_returns_404_not_500() { + when(request.getMethod()).thenReturn("GET"); + NoResourceFoundException ex = + new NoResourceFoundException(HttpMethod.GET, "/api/v1/storage/folders", ""); + ResponseEntity resp = handler.handleNoResourceFound(ex, request); + assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode()); + assertEquals("GET", resp.getBody().getProperties().get("method")); + } + // ---- IllegalArgumentException ---- @Test diff --git a/frontend/editor/src/core/contexts/FolderContext.test.tsx b/frontend/editor/src/core/contexts/FolderContext.test.tsx index 861ddd331b..46de8f6d71 100644 --- a/frontend/editor/src/core/contexts/FolderContext.test.tsx +++ b/frontend/editor/src/core/contexts/FolderContext.test.tsx @@ -3,6 +3,7 @@ import { describe, expect, test, vi, beforeEach } from "vitest"; import { render, screen, waitFor, act } from "@testing-library/react"; import { FolderProvider, useFolders } from "@app/contexts/FolderContext"; +import { createFolderId, FolderId, FolderRecord } from "@app/types/folder"; import { expectConsole } from "@app/tests/failOnConsole"; /** @@ -26,21 +27,43 @@ import { expectConsole } from "@app/tests/failOnConsole"; // require a running backend (folderSyncService) and a populated IDB // (folderStorage). Both are out of scope for testing the error gate. const mockList = vi.fn(); +const mockUpdate = vi.fn(); +const mockDelete = vi.fn(); vi.mock("@app/services/folderSyncService", () => ({ folderSyncService: { list: () => mockList(), create: vi.fn(), - update: vi.fn(), - delete: vi.fn(), + update: (...args: unknown[]) => mockUpdate(...args), + delete: (...args: unknown[]) => mockDelete(...args), }, })); +// Stateful IDB mock - the revision-driven refresh re-reads getAllFolders +// after every state change, so a stateless [] mock would clobber pull results. +const { mockIdb } = vi.hoisted(() => ({ + mockIdb: { folders: [] as { id: string }[] }, +})); vi.mock("@app/services/folderStorage", () => ({ folderStorage: { - getAllFolders: vi.fn().mockResolvedValue([]), - replaceAll: vi.fn().mockResolvedValue(undefined), - upsert: vi.fn().mockResolvedValue(undefined), + getAllFolders: vi.fn(() => Promise.resolve([...mockIdb.folders])), + replaceAll: vi.fn((next: { id: string }[]) => { + mockIdb.folders = [...next]; + return Promise.resolve(); + }), + upsert: vi.fn(), + upsertFolder: vi.fn((next: { id: string }) => { + mockIdb.folders = [ + ...mockIdb.folders.filter((f) => f.id !== next.id), + next, + ]; + return Promise.resolve(); + }), delete: vi.fn().mockResolvedValue(undefined), + removeFolders: vi.fn((ids: string[]) => { + const drop = new Set(ids); + mockIdb.folders = mockIdb.folders.filter((f) => !drop.has(f.id)); + return Promise.resolve(); + }), }, })); @@ -105,6 +128,8 @@ async function renderAndWaitForPull(): Promise { describe("FolderContext sync-banner gating", () => { beforeEach(() => { mockList.mockReset(); + mockUpdate.mockReset(); + mockDelete.mockReset(); }); test("401 (unauthorized) does NOT surface a banner", async () => { @@ -157,3 +182,163 @@ describe("FolderContext sync-banner gating", () => { expect(screen.getByTestId("reachable").textContent).toBe("false"); }); }); + +/** Per-folder mutation 404 = silent cleanup (drop subtree, no banner, pull). */ +describe("FolderContext stale-folder 404 cleanup", () => { + beforeEach(() => { + mockList.mockReset(); + mockUpdate.mockReset(); + mockDelete.mockReset(); + // Reset the stateful IDB mock so each test starts with an empty cache. + mockIdb.folders = []; + }); + + function makeFolder(name: string, parentFolderId: FolderId | null = null) { + return { + id: createFolderId(), + name, + parentFolderId, + createdAt: 0, + updatedAt: 0, + } as FolderRecord; + } + + type ProbeApi = { + error: string | null; + folderCount: number; + currentFolderId: FolderId | null; + setCurrentFolderId: (id: FolderId | null) => void; + rename: (id: FolderId, name: string) => Promise; + delete: (id: FolderId) => Promise; + }; + + function ApiProbe(props: { onReady: (api: ProbeApi) => void }) { + const f = useFolders(); + React.useEffect(() => { + props.onReady({ + error: f.error, + folderCount: f.folders.length, + currentFolderId: f.currentFolderId, + setCurrentFolderId: f.setCurrentFolderId, + rename: (id, name) => f.renameFolder(id, name), + delete: (id) => f.deleteFolder(id), + }); + }, [f, props]); + return ( + <> +
{f.error ?? ""}
+
{String(f.serverReachable)}
+
{f.folders.length}
+
{f.currentFolderId ?? ""}
+ + ); + } + + async function setupWithFolders( + initial: FolderRecord[], + ): Promise<{ current: ProbeApi }> { + // Ref (not plain object) so tests see the latest f-bound api after re-renders. + mockList.mockResolvedValueOnce(initial); + const apiRef: { current: ProbeApi | null } = { current: null }; + render( + + (apiRef.current = api)} /> + , + ); + await waitFor(() => + expect(screen.getByTestId("count").textContent).toBe( + String(initial.length), + ), + ); + if (!apiRef.current) throw new Error("ApiProbe never reported ready"); + return apiRef as { current: ProbeApi }; + } + + test("renameFolder 404 silently drops folder + descendants, no banner", async () => { + const parent = makeFolder("parent"); + const child = makeFolder("child", parent.id); + const sibling = makeFolder("sibling"); + const api = await setupWithFolders([parent, child, sibling]); + + // Convergence pull returns just sibling after the local drop. + mockList.mockResolvedValueOnce([sibling]); + mockUpdate.mockRejectedValueOnce(axiosError(404, "Folder not found")); + + await act(async () => { + const result = await api.current.rename(parent.id, "newname"); + expect(result).toBeNull(); + }); + + expect(screen.getByTestId("error").textContent).toBe(""); + await waitFor(() => + expect(screen.getByTestId("count").textContent).toBe("1"), + ); + expect(screen.getByTestId("reachable").textContent).toBe("true"); + }); + + test("deleteFolder 404 is treated as already-deleted, returns [id], no banner", async () => { + const target = makeFolder("target"); + const other = makeFolder("other"); + const api = await setupWithFolders([target, other]); + + mockList.mockResolvedValueOnce([other]); + mockDelete.mockRejectedValueOnce(axiosError(404, "Folder not found")); + + let result: unknown; + await act(async () => { + result = await api.current.delete(target.id); + }); + expect(result).toEqual([target.id]); + + expect(screen.getByTestId("error").textContent).toBe(""); + await waitFor(() => + expect(screen.getByTestId("count").textContent).toBe("1"), + ); + expect(screen.getByTestId("reachable").textContent).toBe("true"); + }); + + test("stale 404 strand-resets currentFolderId when user is inside the dropped subtree", async () => { + // User is parked inside the about-to-be-deleted subtree → must navigate out. + const parent = makeFolder("parent"); + const child = makeFolder("child", parent.id); + const sibling = makeFolder("sibling"); + const api = await setupWithFolders([parent, child, sibling]); + + act(() => { + api.current.setCurrentFolderId(child.id); + }); + await waitFor(() => + expect(screen.getByTestId("current").textContent).toBe(child.id), + ); + + mockList.mockResolvedValueOnce([sibling]); + mockUpdate.mockRejectedValueOnce(axiosError(404, "Folder not found")); + + await act(async () => { + await api.current.rename(parent.id, "doomed-rename"); + }); + + // ROOT_FOLDER_ID is null → probe renders "". + expect(screen.getByTestId("current").textContent).toBe(""); + expect(screen.getByTestId("error").textContent).toBe(""); + }); + + test("non-404 mutation errors still surface (regression guard)", async () => { + const target = makeFolder("target"); + const api = await setupWithFolders([target]); + + mockUpdate.mockRejectedValueOnce(axiosError(500, "boom")); + + let threw = false; + await act(async () => { + try { + await api.current.rename(target.id, "newname"); + } catch { + threw = true; + } + }); + expect(threw).toBe(true); + expect(screen.getByTestId("count").textContent).toBe("1"); + expect(screen.getByTestId("error").textContent).not.toBe(""); + }); +}); diff --git a/frontend/editor/src/core/contexts/FolderContext.tsx b/frontend/editor/src/core/contexts/FolderContext.tsx index 9c69fd0cf4..e218072ec1 100644 --- a/frontend/editor/src/core/contexts/FolderContext.tsx +++ b/frontend/editor/src/core/contexts/FolderContext.tsx @@ -178,6 +178,40 @@ function reachabilityFromError(err: unknown): boolean { return status !== undefined && status >= 400 && status < 500; } +/** Root + every local descendant via parentFolderId. Bounded for corrupted chains. */ +function collectLocalSubtreeIds( + rootId: FolderId, + folders: FolderRecord[], +): Set { + const childrenByParent = new Map(); + for (const f of folders) { + if (f.parentFolderId === null) continue; + const list = childrenByParent.get(f.parentFolderId) ?? []; + list.push(f.id); + childrenByParent.set(f.parentFolderId, list); + } + const result = new Set([rootId]); + const stack: FolderId[] = [rootId]; + const MAX_LOCAL_SUBTREE_NODES = 10_000; + while (stack.length > 0 && result.size < MAX_LOCAL_SUBTREE_NODES) { + const cur = stack.pop()!; + const children = childrenByParent.get(cur); + if (!children) continue; + for (const childId of children) { + if (!result.has(childId)) { + result.add(childId); + stack.push(childId); + } + } + } + return result; +} + +/** Extract HTTP status off an axios-style error, or undefined on network failure. */ +function errorStatus(err: unknown): number | undefined { + return (err as { response?: { status?: number } })?.response?.status; +} + /** * True if `currentId` or any of its ancestors is in `removedSet`. Walks up via * `parentFolderId` using the pre-removal `folders` snapshot so the chain is @@ -390,24 +424,58 @@ export function FolderProvider({ children }: FolderProviderProps) { // ─── mutations: server-first, cache update on success ────────────── + // Lifted up so handleStaleFolder below can reuse the file-detach primitive. + const { clearFolderForFiles } = useIndexedDB(); + + /** Treat a per-folder 404 as "deleted elsewhere": drop subtree, strand-reset, pull. */ + const handleStaleFolder = useCallback( + (staleId: FolderId, foldersSnapshot: FolderRecord[]) => { + const subtree = collectLocalSubtreeIds(staleId, foldersSnapshot); + if (mountedRef.current) { + setServerReachable(true); + setError(null); + setFolders((prev) => prev.filter((f) => !subtree.has(f.id))); + if ( + currentFolderId !== null && + shouldStrandedReset(currentFolderId, subtree, foldersSnapshot) + ) { + setCurrentFolderId(ROOT_FOLDER_ID); + } + } + const subtreeIds = [...subtree]; + // Best-effort - pullFromServer is authoritative if these miss. + void folderStorage + .removeFolders(subtreeIds) + .catch((e) => console.warn("[FolderContext] stale cache cleanup", e)); + void clearFolderForFiles(subtreeIds).catch((e) => + console.warn("[FolderContext] stale file-folder cleanup", e), + ); + bumpFolderRevision(); + void pullFromServer(); + }, + [bumpFolderRevision, clearFolderForFiles, currentFolderId, pullFromServer], + ); + /** - * Centralised mutation wrapper: - * 1. Calls the server op. - * 2. On success: flips `serverReachable=true`, updates in-memory state - * from the server response, then best-effort writes the cache (cache - * failure does NOT roll back; the in-memory truth came from the server). - * 3. On failure: updates `serverReachable` per the error class, surfaces - * via {@link setError}, re-throws so the caller's dialog can stay open. + * Server-first mutation wrapper. On 404 with `staleFolderId`, hands off to + * handleStaleFolder and resolves `null`; other errors surface + re-throw. */ const runFolderMutation = useCallback( async ( serverOp: () => Promise, onSuccess: (result: T) => void | Promise, - ): Promise => { + staleFolderId: FolderId | null = null, + ): Promise => { let result: T; try { result = await serverOp(); } catch (err) { + if (errorStatus(err) === 404 && staleFolderId !== null) { + // `folders` is a closure snapshot; functional setFolders + the pull + // keep this race-safe even if a concurrent mutation shifted state. + handleStaleFolder(staleFolderId, folders); + return null; + } if (mountedRef.current) { setServerReachable(reachabilityFromError(err)); setError(formatServerError(err)); @@ -421,18 +489,13 @@ export function FolderProvider({ children }: FolderProviderProps) { try { await onSuccess(result); } catch (cacheErr) { - // The server is authoritative - a cache write failure must not be - // surfaced as if the operation failed. Log + leave the in-memory - // state authoritative; next pullFromServer will re-seed the cache. - console.warn( - "[FolderContext] cache update failed after successful mutation", - cacheErr, - ); + // Cache write failure is non-fatal; next pull re-seeds. + console.warn("[FolderContext] cache update after mutation", cacheErr); } bumpFolderRevision(); return result; }, - [bumpFolderRevision], + [bumpFolderRevision, folders, handleStaleFolder], ); const createFolder = useCallback( @@ -441,10 +504,9 @@ export function FolderProvider({ children }: FolderProviderProps) { parentFolderId: FolderId | null = currentFolderId, ): Promise => { const color = pickFolderColor(name); - // Generate id client-side so the server's idempotency check makes - // retries safe (network blip → second POST returns the same row). + // Client-side id makes server idempotency check safe on retry. const id = createFolderId(); - return runFolderMutation( + const result = await runFolderMutation( () => folderSyncService.create({ id, @@ -460,6 +522,11 @@ export function FolderProvider({ children }: FolderProviderProps) { await folderStorage.upsertFolder(record); }, ); + // No staleFolderId passed → null branch can't fire; defensive throw. + if (result === null) { + throw new Error("createFolder unexpectedly returned null"); + } + return result; }, [currentFolderId, runFolderMutation], ); @@ -474,6 +541,7 @@ export function FolderProvider({ children }: FolderProviderProps) { ); await folderStorage.upsertFolder(record); }, + id, ); }, [runFolderMutation], @@ -493,6 +561,7 @@ export function FolderProvider({ children }: FolderProviderProps) { ); await folderStorage.upsertFolder(record); }, + id, ); }, [runFolderMutation], @@ -515,13 +584,12 @@ export function FolderProvider({ children }: FolderProviderProps) { ); await folderStorage.upsertFolder(record); }, + id, ); }, [runFolderMutation], ); - const { clearFolderForFiles } = useIndexedDB(); - const deleteFolder = useCallback( async (id: FolderId): Promise => { // Custom path (not runFolderMutation) because we have two best-effort @@ -532,6 +600,11 @@ export function FolderProvider({ children }: FolderProviderProps) { try { removed = await folderSyncService.delete(id); } catch (err) { + if (errorStatus(err) === 404) { + // Already gone; treat as success. Return [id]; pull is authoritative. + handleStaleFolder(id, folders); + return [id]; + } if (mountedRef.current) { setServerReachable(reachabilityFromError(err)); setError(formatServerError(err)); @@ -580,7 +653,13 @@ export function FolderProvider({ children }: FolderProviderProps) { } return removed; }, - [bumpFolderRevision, clearFolderForFiles, currentFolderId, folders], + [ + bumpFolderRevision, + clearFolderForFiles, + currentFolderId, + folders, + handleStaleFolder, + ], ); const value = useMemo( From 6478c400db5ae4480878e99ffd57954349819d92 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:00:20 +0100 Subject: [PATCH 18/19] Fix font loss in rearrange/overlay/autosplit/OCR from PDFBox (#6545) --- .../controller/api/PdfOverlayController.java | 6 +- .../api/RearrangePagesPDFController.java | 55 ++-- .../api/misc/AutoSplitPdfController.java | 6 +- .../controller/api/misc/OCRController.java | 11 +- .../api/RearrangePagesPDFControllerTest.java | 273 +++++++++--------- 5 files changed, 171 insertions(+), 180 deletions(-) diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/PdfOverlayController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/PdfOverlayController.java index 50b42d89dc..4aab03ff55 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/PdfOverlayController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/PdfOverlayController.java @@ -10,6 +10,7 @@ import java.util.Map; import org.apache.pdfbox.Loader; import org.apache.pdfbox.multipdf.Overlay; +import org.apache.pdfbox.pdfwriter.compress.CompressParameters; import org.apache.pdfbox.pdmodel.PDDocument; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; @@ -157,7 +158,10 @@ public class PdfOverlayController { PDDocument singlePageDocument = new PDDocument()) { singlePageDocument.addPage(overlayPdf.getPage(pageCountInCurrentOverlay)); File tempFile = Files.createTempFile("overlay-page-", ".pdf").toFile(); - singlePageDocument.save(tempFile); + // NO_COMPRESSION: this single-page doc holds a page copied from overlayPdf. + // PDFBox 3.0.7's compressed writer (PDFBOX-6203) drops shared resources imported + // across documents, corrupting overlay fonts. Revert once on 3.0.8. + singlePageDocument.save(tempFile, CompressParameters.NO_COMPRESSION); overlayGuide.put(basePageIndex, tempFile.getAbsolutePath()); tempFiles.add(tempFile); // Keep track of the temporary file for cleanup diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/RearrangePagesPDFController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/RearrangePagesPDFController.java index 1e87a15d57..6dd7aacd79 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/RearrangePagesPDFController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/RearrangePagesPDFController.java @@ -6,11 +6,9 @@ import java.util.Collections; import java.util.List; import java.util.Locale; -import org.apache.pdfbox.cos.COSName; import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDDocumentCatalog; import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm; +import org.apache.pdfbox.pdmodel.PDPageTree; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -262,38 +260,31 @@ public class RearrangePagesPDFController { } log.info("newPageOrder = {}", newPageOrder); log.info("totalPages = {}", totalPages); - // Create a new list to hold the pages in the new order - List newPages = new ArrayList<>(); - for (int i = 0; i < newPageOrder.size(); i++) { - newPages.add(document.getPage(newPageOrder.get(i))); + + // Snapshot the desired pages before mutating the source document's page tree. + List newPages = new ArrayList<>(newPageOrder.size()); + for (Integer idx : newPageOrder) { + newPages.add(document.getPage(idx)); } - // Create a new document based on the original one - try (PDDocument rearrangedDocument = - pdfDocumentFactory.createNewDocumentBasedOnOldDocument(document)) { - - // Add the pages in the new order - for (PDPage page : newPages) { - rearrangedDocument.addPage(page); - } - - PDDocumentCatalog sourceCatalog = document.getDocumentCatalog(); - if (sourceCatalog != null) { - PDAcroForm sourceForm = sourceCatalog.getAcroForm(null); - if (sourceForm != null) { - rearrangedDocument - .getDocumentCatalog() - .getCOSObject() - .setItem(COSName.ACRO_FORM, sourceForm.getCOSObject()); - } - } - - return WebResponseUtils.pdfDocToWebResponse( - rearrangedDocument, - GeneralUtils.generateFilename( - pdfFile.getOriginalFilename(), "_rearranged.pdf"), - tempFileManager); + // Rearrange in-place on the source document rather than copying pages into a + // freshly-created PDDocument. Copying pages across documents triggers a PDFBox + // 3.0.7 compressed-save regression (PDFBOX-6203, fixed for 3.0.8) where shared + // resource objects (fonts, etc.) imported from the source can be silently + // dropped from the output, producing pages with "font not found" errors. + PDPageTree pages = document.getPages(); + for (int i = totalPages - 1; i >= 0; i--) { + pages.remove(i); } + for (PDPage page : newPages) { + pages.add(page); + } + + return WebResponseUtils.pdfDocToWebResponse( + document, + GeneralUtils.generateFilename( + pdfFile.getOriginalFilename(), "_rearranged.pdf"), + tempFileManager); } } catch (IOException e) { ExceptionUtils.logException("document rearrangement", e); diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java index d9714f0618..ad5776b3cd 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java @@ -14,6 +14,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdfwriter.compress.CompressParameters; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject; @@ -357,7 +358,10 @@ public class AutoSplitPdfController { for (int i = 0; i < splitDocuments.size(); i++) { String fileName = filename + "_" + (i + 1) + ".pdf"; zipOut.putNextEntry(new ZipEntry(fileName)); - splitDocuments.get(i).save(zipOut); + // NO_COMPRESSION: split docs are built by addPage()-ing pages copied from the + // source document. PDFBox 3.0.7's compressed writer (PDFBOX-6203) drops shared + // resources imported across documents, corrupting fonts. Revert once on 3.0.8. + splitDocuments.get(i).save(zipOut, CompressParameters.NO_COMPRESSION); zipOut.closeEntry(); } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java index 64ed5b79ea..4660d5f128 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java @@ -17,6 +17,7 @@ import javax.imageio.ImageIO; import org.apache.pdfbox.io.IOUtils; import org.apache.pdfbox.multipdf.PDFMergerUtility; +import org.apache.pdfbox.pdfwriter.compress.CompressParameters; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.rendering.PDFRenderer; @@ -427,7 +428,10 @@ public class OCRController { // Save original page without OCR as fallback try (PDDocument pageDoc = new PDDocument()) { pageDoc.addPage(page); - pageDoc.save(pageOutputPath); + // NO_COMPRESSION: page is copied from another document; + // PDFBox 3.0.7 compressed writer (PDFBOX-6203) drops shared + // resources, corrupting fonts. Revert once on 3.0.8. + pageDoc.save(pageOutputPath, CompressParameters.NO_COMPRESSION); } } @@ -437,7 +441,10 @@ public class OCRController { // Save original page without OCR try (PDDocument pageDoc = new PDDocument()) { pageDoc.addPage(page); - pageDoc.save(pageOutputPath); + // NO_COMPRESSION: page is copied from another document; PDFBox 3.0.7 + // compressed writer (PDFBOX-6203) drops shared resources, corrupting + // fonts on retained text pages. Revert once on 3.0.8. + pageDoc.save(pageOutputPath, CompressParameters.NO_COMPRESSION); merger.addSource(pageOutputPath); } } diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/RearrangePagesPDFControllerTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/RearrangePagesPDFControllerTest.java index c31cae2958..a225c3fc52 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/RearrangePagesPDFControllerTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/RearrangePagesPDFControllerTest.java @@ -4,10 +4,14 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.*; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import org.apache.pdfbox.Loader; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.junit.jupiter.api.BeforeEach; @@ -56,6 +60,38 @@ class RearrangePagesPDFControllerTest { "fileInput", "test.pdf", MediaType.APPLICATION_PDF_VALUE, new byte[] {1, 2, 3}); } + /** Build a real, in-memory PDDocument with the requested number of blank pages. */ + private PDDocument buildRealPdf(int pageCount) throws IOException { + PDDocument doc = new PDDocument(); + for (int i = 0; i < pageCount; i++) { + doc.addPage(new PDPage()); + } + return doc; + } + + /** + * Returns the underlying {@link org.apache.pdfbox.cos.COSDictionary} for each page in document + * order. PDPageTree returns a fresh PDPage wrapper per get(), so comparing wrappers with + * assertSame is unreliable - the COSDictionary identity is the stable handle. + */ + private List snapshotCosPages(PDDocument doc) { + List snapshot = new ArrayList<>(); + for (PDPage p : doc.getPages()) { + snapshot.add(p.getCOSObject()); + } + return snapshot; + } + + private List reloadAndSnapshot(ResponseEntity response) throws IOException { + try (var in = response.getBody().getInputStream(); + var baos = new ByteArrayOutputStream()) { + in.transferTo(baos); + try (PDDocument out = Loader.loadPDF(baos.toByteArray())) { + return snapshotCosPages(out); + } + } + } + @Test void testDeletePages_Success() throws IOException { MockMultipartFile file = createMockPdf(); @@ -83,27 +119,23 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("REVERSE_ORDER"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); - PDPage page2 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(3)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(3); - when(mockDoc.getPage(0)).thenReturn(page0); - when(mockDoc.getPage(1)).thenReturn(page1); - when(mockDoc.getPage(2)).thenReturn(page2); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); - verify(mockNewDoc).addPage(page2); - verify(mockNewDoc).addPage(page1); - verify(mockNewDoc).addPage(page0); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + List finalOrder = reloadAndSnapshot(response); + assertEquals(3, finalOrder.size()); + // We can no longer compare references after a save/reload, so compare via + // the in-memory snapshot taken *after* the controller mutated the source. + List mutatedSource = snapshotCosPages(realDoc); + assertSame(originals.get(2), mutatedSource.get(0)); + assertSame(originals.get(1), mutatedSource.get(1)); + assertSame(originals.get(0), mutatedSource.get(2)); + } } @Test @@ -114,25 +146,18 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("REMOVE_FIRST"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); - PDPage page2 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(3)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(3); - when(mockDoc.getPage(1)).thenReturn(page1); - when(mockDoc.getPage(2)).thenReturn(page2); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - verify(mockNewDoc).addPage(page1); - verify(mockNewDoc).addPage(page2); - verify(mockNewDoc, never()).addPage(page0); + assertNotNull(response); + List mutated = snapshotCosPages(realDoc); + assertEquals(2, mutated.size()); + assertSame(originals.get(1), mutated.get(0)); + assertSame(originals.get(2), mutated.get(1)); + } } @Test @@ -143,23 +168,18 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("REMOVE_LAST"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(3)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(3); - when(mockDoc.getPage(0)).thenReturn(page0); - when(mockDoc.getPage(1)).thenReturn(page1); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - verify(mockNewDoc).addPage(page0); - verify(mockNewDoc).addPage(page1); + assertNotNull(response); + List mutated = snapshotCosPages(realDoc); + assertEquals(2, mutated.size()); + assertSame(originals.get(0), mutated.get(0)); + assertSame(originals.get(1), mutated.get(1)); + } } @Test @@ -170,21 +190,19 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("REMOVE_FIRST_AND_LAST"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page1 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(1)).thenReturn(page1); - when(mockDoc.getPage(2)).thenReturn(page1); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + List mutated = snapshotCosPages(realDoc); + assertEquals(2, mutated.size()); + assertSame(originals.get(1), mutated.get(0)); + assertSame(originals.get(2), mutated.get(1)); + } } @Test @@ -195,23 +213,15 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("DUPLEX_SORT"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); - PDPage page2 = mock(PDPage.class); - PDPage page3 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(anyInt())).thenReturn(page0); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + assertEquals(4, realDoc.getNumberOfPages()); + } } @Test @@ -222,20 +232,15 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("BOOKLET_SORT"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(anyInt())).thenReturn(page); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + assertEquals(4, realDoc.getNumberOfPages()); + } } @Test @@ -246,20 +251,15 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("ODD_EVEN_SPLIT"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(anyInt())).thenReturn(page); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + assertEquals(4, realDoc.getNumberOfPages()); + } } @Test @@ -270,24 +270,20 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers("3,1,2"); request.setCustomMode("custom"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page0 = mock(PDPage.class); - PDPage page1 = mock(PDPage.class); - PDPage page2 = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(3)) { + List originals = snapshotCosPages(realDoc); + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(3); - when(mockDoc.getPage(0)).thenReturn(page0); - when(mockDoc.getPage(1)).thenReturn(page1); - when(mockDoc.getPage(2)).thenReturn(page2); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + List mutated = snapshotCosPages(realDoc); + assertEquals(3, mutated.size()); + assertSame(originals.get(2), mutated.get(0)); + assertSame(originals.get(0), mutated.get(1)); + assertSame(originals.get(1), mutated.get(2)); + } } @Test @@ -298,21 +294,15 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers("3"); request.setCustomMode("DUPLICATE"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(2)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(2); - when(mockDoc.getPage(anyInt())).thenReturn(page); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - // 2 pages * 3 duplicates = 6 addPage calls - verify(mockNewDoc, times(6)).addPage(page); + assertNotNull(response); + // 2 pages * 3 duplicates = 6 final pages + assertEquals(6, realDoc.getNumberOfPages()); + } } @Test @@ -323,19 +313,14 @@ class RearrangePagesPDFControllerTest { request.setPageNumbers(""); request.setCustomMode("SIDE_STITCH_BOOKLET_SORT"); - PDDocument mockDoc = mock(PDDocument.class); - PDDocument mockNewDoc = mock(PDDocument.class); - PDPage page = mock(PDPage.class); + try (PDDocument realDoc = buildRealPdf(4)) { + when(pdfDocumentFactory.load(file)).thenReturn(realDoc); - when(pdfDocumentFactory.load(file)).thenReturn(mockDoc); - when(mockDoc.getNumberOfPages()).thenReturn(4); - when(mockDoc.getPage(anyInt())).thenReturn(page); - when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc)) - .thenReturn(mockNewDoc); + ResponseEntity response = controller.rearrangePages(request); - ResponseEntity response = controller.rearrangePages(request); - - assertNotNull(response); - assertEquals(200, response.getStatusCode().value()); + assertNotNull(response); + assertEquals(200, response.getStatusCode().value()); + assertEquals(4, realDoc.getNumberOfPages()); + } } } From 71361f0d334b95d212be21944a04b13f8129d746 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:00:34 +0100 Subject: [PATCH 19/19] Minor: Office doc changes (#6571) --- .../common/util/OfficeDocumentSanitizer.java | 310 +++++++++++++++ .../util/OfficeDocumentSanitizerTest.java | 370 ++++++++++++++++++ .../converters/ConvertOfficeController.java | 10 +- 3 files changed, 687 insertions(+), 3 deletions(-) create mode 100644 app/common/src/main/java/stirling/software/common/util/OfficeDocumentSanitizer.java create mode 100644 app/common/src/test/java/stirling/software/common/util/OfficeDocumentSanitizerTest.java diff --git a/app/common/src/main/java/stirling/software/common/util/OfficeDocumentSanitizer.java b/app/common/src/main/java/stirling/software/common/util/OfficeDocumentSanitizer.java new file mode 100644 index 0000000000..9cdf8cf53b --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/util/OfficeDocumentSanitizer.java @@ -0,0 +1,310 @@ +package stirling.software.common.util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.springframework.stereotype.Component; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import io.github.pixee.security.ZipSecurity; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.SsrfProtectionService; + +// Strips external refs from OOXML/ODF uploads so LibreOffice can't be made to fetch them. +@Component +@Slf4j +public class OfficeDocumentSanitizer { + + private static final Set OOXML_EXTENSIONS = + Set.of( + "docx", "docm", "dotx", "dotm", "xlsx", "xlsm", "xltx", "xltm", "pptx", "pptm", + "potx", "potm", "ppsx", "ppsm"); + + private static final Set ODF_EXTENSIONS = + Set.of( + "odt", "ott", "ods", "ots", "odp", "otp", "odg", "otg", "odf", "odc", "odi", + "odm"); + + private static final Set ODF_XML_PARTS = + Set.of("content.xml", "styles.xml", "meta.xml", "settings.xml"); + + private final SsrfProtectionService ssrfProtectionService; + private final ApplicationProperties applicationProperties; + + public OfficeDocumentSanitizer( + SsrfProtectionService ssrfProtectionService, + ApplicationProperties applicationProperties) { + this.ssrfProtectionService = ssrfProtectionService; + this.applicationProperties = applicationProperties; + } + + public boolean isSanitizableExtension(String extension) { + if (extension == null) { + return false; + } + String lower = extension.toLowerCase(Locale.ROOT); + return OOXML_EXTENSIONS.contains(lower) || ODF_EXTENSIONS.contains(lower); + } + + public byte[] sanitize(byte[] documentBytes, String extension) throws IOException { + if (documentBytes == null || documentBytes.length == 0) { + throw new IOException("Office document input is empty or null"); + } + if (applicationProperties.getSystem().isDisableSanitize()) { + log.debug("Office document sanitization disabled by configuration"); + return documentBytes; + } + if (!isSanitizableExtension(extension)) { + return documentBytes; + } + + ByteArrayOutputStream out = new ByteArrayOutputStream(documentBytes.length); + try (ZipInputStream zipIn = + ZipSecurity.createHardenedInputStream( + new ByteArrayInputStream(documentBytes)); + ZipOutputStream zipOut = new ZipOutputStream(out)) { + + ZipEntry entry; + while ((entry = zipIn.getNextEntry()) != null) { + String name = entry.getName(); + byte[] bytes = entry.isDirectory() ? new byte[0] : zipIn.readAllBytes(); + + if (!entry.isDirectory()) { + bytes = sanitizeEntry(name, bytes); + } + + ZipEntry outEntry = new ZipEntry(name); + if (entry.getComment() != null) { + outEntry.setComment(entry.getComment()); + } + if (entry.getExtra() != null) { + outEntry.setExtra(entry.getExtra()); + } + zipOut.putNextEntry(outEntry); + if (!entry.isDirectory()) { + zipOut.write(bytes); + } + zipOut.closeEntry(); + } + } + return out.toByteArray(); + } + + private byte[] sanitizeEntry(String entryName, byte[] entryBytes) { + String lower = entryName.toLowerCase(Locale.ROOT); + try { + if (lower.endsWith(".rels")) { + return sanitizeOoxmlRels(entryBytes); + } + if (isOdfXmlPart(lower)) { + return sanitizeOdfXml(entryBytes); + } + } catch (ParserConfigurationException + | SAXException + | IOException + | TransformerException e) { + log.warn( + "Failed to parse XML part '{}' for sanitization, leaving as-is: {}", + entryName, + e.getMessage()); + } + return entryBytes; + } + + private boolean isOdfXmlPart(String lowerName) { + int slash = lowerName.lastIndexOf('/'); + String base = slash >= 0 ? lowerName.substring(slash + 1) : lowerName; + return ODF_XML_PARTS.contains(base); + } + + private byte[] sanitizeOoxmlRels(byte[] xmlBytes) + throws IOException, ParserConfigurationException, SAXException, TransformerException { + Document doc = parseSecurely(xmlBytes); + Element root = doc.getDocumentElement(); + if (root == null) { + return xmlBytes; + } + NodeList relationships = root.getElementsByTagNameNS("*", "Relationship"); + List toRemove = new ArrayList<>(); + for (int i = 0; i < relationships.getLength(); i++) { + Node node = relationships.item(i); + NamedNodeMap attrs = node.getAttributes(); + if (attrs == null) { + continue; + } + Node targetMode = attrs.getNamedItem("TargetMode"); + if (targetMode == null || !"external".equalsIgnoreCase(targetMode.getNodeValue())) { + continue; + } + Node target = attrs.getNamedItem("Target"); + String targetValue = target == null ? "" : target.getNodeValue(); + if (isAdminAllowed(targetValue)) { + continue; + } + log.warn( + "Stripping OOXML external relationship target: {}", + truncateForLog(targetValue)); + toRemove.add(node); + } + if (toRemove.isEmpty()) { + return xmlBytes; + } + for (Node n : toRemove) { + n.getParentNode().removeChild(n); + } + return serializeDocument(doc); + } + + private byte[] sanitizeOdfXml(byte[] xmlBytes) + throws IOException, ParserConfigurationException, SAXException, TransformerException { + Document doc = parseSecurely(xmlBytes); + Element root = doc.getDocumentElement(); + if (root == null) { + return xmlBytes; + } + boolean modified = stripExternalHrefs(root); + if (!modified) { + return xmlBytes; + } + return serializeDocument(doc); + } + + private boolean stripExternalHrefs(Node node) { + boolean modified = false; + if (node.getNodeType() == Node.ELEMENT_NODE) { + NamedNodeMap attrs = node.getAttributes(); + List hrefAttrsToRemove = new ArrayList<>(); + for (int i = 0; i < attrs.getLength(); i++) { + Node attr = attrs.item(i); + String name = attr.getNodeName(); + if (name == null) { + continue; + } + String lower = name.toLowerCase(Locale.ROOT); + if (!(lower.equals("xlink:href") + || lower.endsWith(":href") + || lower.equals("href"))) { + continue; + } + String value = attr.getNodeValue(); + if (!isExternalUrl(value)) { + continue; + } + if (isAdminAllowed(value)) { + continue; + } + log.warn( + "Stripping ODF external href attribute ({}): {}", + name, + truncateForLog(value)); + hrefAttrsToRemove.add(name); + } + Element element = (Element) node; + for (String attrName : hrefAttrsToRemove) { + element.removeAttribute(attrName); + modified = true; + } + } + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + if (stripExternalHrefs(children.item(i))) { + modified = true; + } + } + return modified; + } + + private boolean isExternalUrl(String url) { + if (url == null) { + return false; + } + String trimmed = url.trim().toLowerCase(Locale.ROOT); + if (trimmed.isEmpty() || trimmed.startsWith("#") || trimmed.startsWith("../")) { + return false; + } + return trimmed.startsWith("http://") + || trimmed.startsWith("https://") + || trimmed.startsWith("ftp://") + || trimmed.startsWith("ftps://") + || trimmed.startsWith("file:") + || trimmed.startsWith("smb:") + || trimmed.startsWith("\\\\") + || trimmed.startsWith("//"); + } + + // Preserved only with an explicit allowedDomains entry; MEDIUM default would admit public URLs. + private boolean isAdminAllowed(String url) { + if (ssrfProtectionService == null || url == null || url.isBlank()) { + return false; + } + ApplicationProperties.Html.UrlSecurity config = + applicationProperties.getSystem().getHtml().getUrlSecurity(); + if (config == null + || config.getAllowedDomains() == null + || config.getAllowedDomains().isEmpty()) { + return false; + } + return ssrfProtectionService.isUrlAllowed(url); + } + + private Document parseSecurely(byte[] xmlBytes) + throws ParserConfigurationException, SAXException, IOException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + factory.setNamespaceAware(true); + DocumentBuilder builder = factory.newDocumentBuilder(); + return builder.parse(new ByteArrayInputStream(xmlBytes)); + } + + private byte[] serializeDocument(Document doc) throws TransformerException { + TransformerFactory tf = TransformerFactory.newInstance(); + tf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + transformer.setOutputProperty(OutputKeys.INDENT, "no"); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + transformer.transform(new DOMSource(doc), new StreamResult(baos)); + return baos.toByteArray(); + } + + private String truncateForLog(String value) { + if (value == null) { + return "null"; + } + return value.length() > 80 ? value.substring(0, 80) + "..." : value; + } +} diff --git a/app/common/src/test/java/stirling/software/common/util/OfficeDocumentSanitizerTest.java b/app/common/src/test/java/stirling/software/common/util/OfficeDocumentSanitizerTest.java new file mode 100644 index 0000000000..eb22919068 --- /dev/null +++ b/app/common/src/test/java/stirling/software/common/util/OfficeDocumentSanitizerTest.java @@ -0,0 +1,370 @@ +package stirling.software.common.util; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.SsrfProtectionService; + +class OfficeDocumentSanitizerTest { + + private static final String EXTERNAL_URL = "https://webhook.site/ssrf-callback"; + private static final String INTERNAL_TARGET = "media/image1.png"; + + private static final String DOCX_RELS = + "" + + "" + + "" + + "" + + ""; + + private static final String DOCX_DOCUMENT = + "" + + "" + + ""; + + private static final String ODF_CONTENT_EXTERNAL = + "" + + "" + + "" + + "" + + "" + + ""; + + private SsrfProtectionService ssrfProtectionService; + private ApplicationProperties applicationProperties; + private OfficeDocumentSanitizer sanitizer; + + @BeforeEach + void setUp() { + applicationProperties = new ApplicationProperties(); + ssrfProtectionService = mock(SsrfProtectionService.class); + sanitizer = new OfficeDocumentSanitizer(ssrfProtectionService, applicationProperties); + } + + @Test + void isSanitizableExtension_recognizesOoxmlAndOdf() { + assertTrue(sanitizer.isSanitizableExtension("docx")); + assertTrue(sanitizer.isSanitizableExtension("DOCX")); + assertTrue(sanitizer.isSanitizableExtension("xlsx")); + assertTrue(sanitizer.isSanitizableExtension("pptx")); + assertTrue(sanitizer.isSanitizableExtension("odt")); + assertTrue(sanitizer.isSanitizableExtension("ods")); + assertTrue(sanitizer.isSanitizableExtension("odp")); + assertFalse(sanitizer.isSanitizableExtension("pdf")); + assertFalse(sanitizer.isSanitizableExtension("html")); + assertFalse(sanitizer.isSanitizableExtension("")); + assertFalse(sanitizer.isSanitizableExtension(null)); + } + + @Test + void sanitize_stripsOoxmlExternalRelationship() throws IOException { + Map entries = new LinkedHashMap<>(); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + entries.put("word/document.xml", DOCX_DOCUMENT.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8); + assertFalse(rels.contains(EXTERNAL_URL), "External URL should be stripped from .rels"); + assertFalse( + rels.toLowerCase().contains("targetmode=\"external\""), + "TargetMode=External relationship should be removed"); + assertTrue(rels.contains(INTERNAL_TARGET), "Internal image target should be preserved"); + assertArrayEquals( + DOCX_DOCUMENT.getBytes(StandardCharsets.UTF_8), + result.get("word/document.xml"), + "Non-rels entries must be untouched"); + } + + @Test + void sanitize_pptxExternalImageRelStripped() throws IOException { + String pptxRels = + "" + + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put("ppt/slides/_rels/slide1.xml.rels", pptxRels.getBytes(StandardCharsets.UTF_8)); + byte[] pptx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(pptx, "pptx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("ppt/slides/_rels/slide1.xml.rels"), StandardCharsets.UTF_8); + assertFalse(rels.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_xlsxExternalImageRelStripped() throws IOException { + String xlsxRels = + "" + + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put( + "xl/drawings/_rels/drawing1.xml.rels", xlsxRels.getBytes(StandardCharsets.UTF_8)); + byte[] xlsx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(xlsx, "xlsx"); + + Map result = unzip(cleaned); + String rels = + new String( + result.get("xl/drawings/_rels/drawing1.xml.rels"), StandardCharsets.UTF_8); + assertFalse(rels.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_odtStripsExternalXlinkHrefButKeepsInternal() throws IOException { + Map entries = new LinkedHashMap<>(); + entries.put("content.xml", ODF_CONTENT_EXTERNAL.getBytes(StandardCharsets.UTF_8)); + String manifestXml = + ""; + entries.put("META-INF/manifest.xml", manifestXml.getBytes(StandardCharsets.UTF_8)); + byte[] odt = zip(entries); + + byte[] cleaned = sanitizer.sanitize(odt, "odt"); + + Map result = unzip(cleaned); + String content = new String(result.get("content.xml"), StandardCharsets.UTF_8); + assertFalse(content.contains(EXTERNAL_URL), "External xlink:href should be stripped"); + assertTrue(content.contains("Pictures/image1.png"), "Internal href should be preserved"); + } + + @Test + void sanitize_odsStripsExternalXlinkHref() throws IOException { + Map entries = new LinkedHashMap<>(); + entries.put("content.xml", ODF_CONTENT_EXTERNAL.getBytes(StandardCharsets.UTF_8)); + byte[] ods = zip(entries); + + byte[] cleaned = sanitizer.sanitize(ods, "ods"); + + Map result = unzip(cleaned); + String content = new String(result.get("content.xml"), StandardCharsets.UTF_8); + assertFalse(content.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_odpStripsExternalXlinkHrefInStylesXml() throws IOException { + String stylesXml = + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put("styles.xml", stylesXml.getBytes(StandardCharsets.UTF_8)); + byte[] odp = zip(entries); + + byte[] cleaned = sanitizer.sanitize(odp, "odp"); + + Map result = unzip(cleaned); + String content = new String(result.get("styles.xml"), StandardCharsets.UTF_8); + assertFalse(content.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_disabledByConfigReturnsOriginal() throws IOException { + applicationProperties.getSystem().setDisableSanitize(true); + Map entries = new LinkedHashMap<>(); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] result = sanitizer.sanitize(docx, "docx"); + assertArrayEquals(docx, result); + } + + @Test + void sanitize_unrecognizedExtensionReturnsOriginal() throws IOException { + byte[] original = "irrelevant".getBytes(StandardCharsets.UTF_8); + byte[] result = sanitizer.sanitize(original, "pdf"); + assertArrayEquals(original, result); + } + + @Test + void sanitize_emptyInputThrows() { + assertThrows(IOException.class, () -> sanitizer.sanitize(new byte[0], "docx")); + } + + @Test + void sanitize_nullInputThrows() { + assertThrows(IOException.class, () -> sanitizer.sanitize(null, "docx")); + } + + @Test + void sanitize_preservesEntryWithExternalRefWhenAdminAllowsDomain() throws IOException { + applicationProperties + .getSystem() + .getHtml() + .getUrlSecurity() + .getAllowedDomains() + .add("webhook.site"); + lenient().when(ssrfProtectionService.isUrlAllowed(eq(EXTERNAL_URL))).thenReturn(true); + + Map entries = new LinkedHashMap<>(); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8); + assertTrue(rels.contains(EXTERNAL_URL), "Allow-listed external URL should be preserved"); + } + + @Test + void sanitize_doesNotConsultSsrfServiceWhenAllowedDomainsEmpty() throws IOException { + // Even if mock would say allowed, we should not invoke it when there is no allow-list, + // because MEDIUM default would let public URLs through and re-introduce the vulnerability. + lenient().when(ssrfProtectionService.isUrlAllowed(eq(EXTERNAL_URL))).thenReturn(true); + + Map entries = new LinkedHashMap<>(); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8); + assertFalse(rels.contains(EXTERNAL_URL)); + } + + @Test + void sanitize_handlesNonXmlEntriesSafely() throws IOException { + Map entries = new LinkedHashMap<>(); + byte[] imageBytes = new byte[] {(byte) 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a}; + entries.put("word/media/image1.png", imageBytes); + entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + assertArrayEquals(imageBytes, result.get("word/media/image1.png")); + } + + @Test + void sanitize_internalLinksKeptWhenNoExternalPresent() throws IOException { + String internalOnlyRels = + "" + + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put( + "word/_rels/document.xml.rels", internalOnlyRels.getBytes(StandardCharsets.UTF_8)); + byte[] docx = zip(entries); + + byte[] cleaned = sanitizer.sanitize(docx, "docx"); + + Map result = unzip(cleaned); + String rels = + new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8); + assertTrue(rels.contains("media/image1.png")); + } + + @Test + void sanitize_corruptZipProducesSafeOutput() throws IOException { + byte[] garbage = "this is not a zip file".getBytes(StandardCharsets.UTF_8); + byte[] result = sanitizer.sanitize(garbage, "docx"); + Map entries = unzip(result); + assertTrue(entries.isEmpty(), "Garbage input must not yield exploitable entries"); + } + + @Test + void sanitize_relativeOdfPathsArePreserved() throws IOException { + String content = + "" + + "" + + "" + + "" + + ""; + Map entries = new LinkedHashMap<>(); + entries.put("content.xml", content.getBytes(StandardCharsets.UTF_8)); + byte[] odt = zip(entries); + + byte[] cleaned = sanitizer.sanitize(odt, "odt"); + + Map result = unzip(cleaned); + String out = new String(result.get("content.xml"), StandardCharsets.UTF_8); + assertTrue(out.contains("../Pictures/image1.png")); + assertTrue(out.contains("#anchor")); + } + + private static byte[] zip(Map entries) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + for (Map.Entry e : entries.entrySet()) { + ZipEntry entry = new ZipEntry(e.getKey()); + zos.putNextEntry(entry); + zos.write(e.getValue()); + zos.closeEntry(); + } + } + return baos.toByteArray(); + } + + private static Map unzip(byte[] data) throws IOException { + Map entries = new HashMap<>(); + try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(data))) { + ZipEntry e; + while ((e = zis.getNextEntry()) != null) { + entries.put(e.getName(), zis.readAllBytes()); + zis.closeEntry(); + } + } + return entries; + } +} diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java index 8bf53c79e1..4fc6669bc7 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java @@ -35,6 +35,7 @@ import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.CustomHtmlSanitizer; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.GeneralUtils; +import stirling.software.common.util.OfficeDocumentSanitizer; import stirling.software.common.util.ProcessExecutor; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; import stirling.software.common.util.RegexPatternUtils; @@ -50,6 +51,7 @@ public class ConvertOfficeController { private final CustomPDFDocumentFactory pdfDocumentFactory; private final RuntimePathConfig runtimePathConfig; private final CustomHtmlSanitizer customHtmlSanitizer; + private final OfficeDocumentSanitizer officeDocumentSanitizer; private final EndpointConfiguration endpointConfiguration; private final TempFileManager tempFileManager; @@ -83,14 +85,16 @@ public class ConvertOfficeController { Path inputPath = workDir.resolve(baseName + "." + extensionLower); Path outputPath = workDir.resolve(baseName + ".pdf"); - // Check if the file is HTML and apply sanitization if needed + // Sanitize input before LibreOffice sees it so embedded URLs can't trigger SSRF. if ("html".equals(extensionLower) || "htm".equals(extensionLower)) { - // Read and sanitize HTML content String htmlContent = new String(inputFile.getBytes(), StandardCharsets.UTF_8); String sanitizedHtml = customHtmlSanitizer.sanitize(htmlContent); Files.writeString(inputPath, sanitizedHtml, StandardCharsets.UTF_8); + } else if (officeDocumentSanitizer.isSanitizableExtension(extensionLower)) { + byte[] sanitized = + officeDocumentSanitizer.sanitize(inputFile.getBytes(), extensionLower); + Files.write(inputPath, sanitized); } else { - // copy file content Files.copy(inputFile.getInputStream(), inputPath, StandardCopyOption.REPLACE_EXISTING); }