Compare commits
31
Commits
fixCertSign
...
wt2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9b877d4f8d | ||
|
|
da4b84962c | ||
|
|
611468b972 | ||
|
|
5fca2f199a | ||
|
|
2aa6768921 | ||
|
|
f15e405759 | ||
|
|
3675db5907 | ||
|
|
e2536daeb8 | ||
|
|
e0fc5061de | ||
|
|
3ecd95b779 | ||
|
|
84aca12055 | ||
|
|
be914c7135 | ||
|
|
71361f0d33 | ||
|
|
6478c400db | ||
|
|
502f6c1e4d | ||
|
|
1a0beaffc2 | ||
|
|
1e739b6f6f | ||
|
|
98967bfa86 | ||
|
|
ff96a80947 | ||
|
|
347ae9ebbf | ||
|
|
800a411167 | ||
|
|
66f431a2b7 | ||
|
|
0e3cbb3cf2 | ||
|
|
002de06411 | ||
|
|
51478e5051 | ||
|
|
69e62d8949 | ||
|
|
2f6b113a13 | ||
|
|
af52134811 | ||
|
|
8a2474ff60 | ||
|
|
d202c9c32f | ||
|
|
1ef03c43b4 |
@@ -0,0 +1,136 @@
|
||||
name: Docker Compose Cucumber tests (saas / PAYG)
|
||||
|
||||
# Self-contained CI job for the PAYG shadow-mode cucumber scenarios.
|
||||
# Triggers only on PAYG-relevant paths so we don't add CI minutes to every PR
|
||||
# that doesn't touch the saas flavour.
|
||||
#
|
||||
# Companion to `docker-compose-tests.yml` (which runs against the
|
||||
# proprietary-flavour stack and skips features/payg via behave.ini's
|
||||
# exclude_re). Kept as a separate workflow so the saas matrix can fail and
|
||||
# succeed independently without touching the main cucumber harness.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "app/saas/**"
|
||||
- "testing/cucumber/features/payg/**"
|
||||
- "testing/cucumber/features/steps/payg_step_definitions.py"
|
||||
- "testing/cucumber/requirements.txt"
|
||||
- "testing/compose/docker-compose-saas.yml"
|
||||
- "testing/compose/payg/**"
|
||||
- "testing/test-payg.sh"
|
||||
- ".github/workflows/docker-compose-tests-saas.yml"
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "app/saas/**"
|
||||
- "testing/cucumber/features/payg/**"
|
||||
- "testing/cucumber/features/steps/payg_step_definitions.py"
|
||||
- "testing/cucumber/requirements.txt"
|
||||
- "testing/compose/docker-compose-saas.yml"
|
||||
- "testing/compose/payg/**"
|
||||
- "testing/test-payg.sh"
|
||||
- ".github/workflows/docker-compose-tests-saas.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
docker-compose-tests-saas:
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
checks: write
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up JDK 25
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
with:
|
||||
java-version: "25"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
~/.gradle/caches/modules-2/files-2.1
|
||||
~/.gradle/caches/modules-2/metadata-2.*
|
||||
key: gradle-deps-saas-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }}
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Expose GitHub runtime for Buildx cache
|
||||
uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0
|
||||
|
||||
# No "Install Docker Compose" step: Ubuntu runners ship with `docker compose`
|
||||
# v2 (built into the Docker CLI). test-payg.sh uses the v2 form throughout
|
||||
# (`docker compose …`, no hyphen), so the legacy v1 `docker-compose` binary
|
||||
# isn't needed. Avoids a `curl | sudo install` without checksum verification
|
||||
# (Aikido flagged this when copy-pasted from docker-compose-tests.yml).
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
cache-dependency-path: ./testing/cucumber/requirements.txt
|
||||
|
||||
- name: Pip requirements
|
||||
run: |
|
||||
pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt
|
||||
|
||||
- name: Run PAYG Cucumber Tests
|
||||
env:
|
||||
MAVEN_USER: ${{ secrets.MAVEN_USER }}
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
run: |
|
||||
chmod +x ./testing/test-payg.sh
|
||||
./testing/test-payg.sh
|
||||
|
||||
- name: Dump saas container logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
docker compose -f testing/compose/docker-compose-saas.yml logs --tail 500 stirling-pdf-saas || true
|
||||
docker compose -f testing/compose/docker-compose-saas.yml logs --tail 200 postgres-saas || true
|
||||
|
||||
- name: Upload PAYG Cucumber Report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: payg-cucumber-report
|
||||
path: testing/cucumber/report-payg.html
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: PAYG Cucumber Test Report
|
||||
if: always()
|
||||
uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
|
||||
with:
|
||||
name: PAYG Cucumber Tests
|
||||
path: testing/cucumber/junit-payg/*.xml
|
||||
reporter: java-junit
|
||||
fail-on-error: false
|
||||
@@ -3,3 +3,12 @@
|
||||
# intentionally in #6150 so engine/.env has a working default, with real
|
||||
# credentials overridden via engine/.env.local.
|
||||
engine/.env:generic-api-key:41
|
||||
|
||||
# MCP test fixtures / harness - no real secrets:
|
||||
# - test-only API key constant in an integration test
|
||||
# - JDBC URL + throwaway Keycloak creds in the local test compose
|
||||
# - placeholder / shell-variable Bearer headers in curl-based validation scripts
|
||||
app/proprietary/src/test/java/stirling/software/proprietary/mcp/security/McpApiKeyIntegrationTest.java:generic-api-key:40
|
||||
testing/compose/docker-compose-keycloak-mcp.yml:generic-api-key:25
|
||||
testing/compose/validate-mcp-apikey.sh:curl-auth-header:73
|
||||
testing/compose/validate-mcp-test.sh:curl-auth-header:92
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
version: '3'
|
||||
|
||||
vars:
|
||||
JLINK_MODULES: "java.base,java.compiler,java.desktop,java.instrument,java.logging,java.management,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.security.jgss,java.security.sasl,java.sql,java.transaction.xa,java.xml,java.xml.crypto,jdk.crypto.ec,jdk.crypto.cryptoki,jdk.unsupported"
|
||||
# jdk.dynalink is required by VeraPDF (PDF/A validation); without it the bundled JRE throws
|
||||
# NoClassDefFoundError: jdk/dynalink/Namespace at runtime in get-info-on-pdf and verify-pdf
|
||||
JLINK_MODULES: "java.base,java.compiler,java.desktop,java.instrument,java.logging,java.management,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.security.jgss,java.security.sasl,java.sql,java.transaction.xa,java.xml,java.xml.crypto,jdk.crypto.ec,jdk.crypto.cryptoki,jdk.unsupported,jdk.dynalink"
|
||||
|
||||
# Override via JPDFIUM_PLATFORMS env (csv of platform keys, or 'all').
|
||||
JPDFIUM_PLATFORMS:
|
||||
@@ -62,21 +64,21 @@ tasks:
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles app
|
||||
- npx tauri build --bundles app --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
|
||||
build:dev:windows:
|
||||
desc: "Build Tauri desktop NSIS installer (Windows)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles nsis
|
||||
- npx tauri build --bundles nsis --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
|
||||
build:dev:linux:
|
||||
desc: "Build Tauri desktop AppImage (Linux)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles appimage
|
||||
- npx tauri build --bundles appimage --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
|
||||
test:
|
||||
desc: "Run Tauri/Cargo tests"
|
||||
|
||||
@@ -212,3 +212,55 @@ tasks:
|
||||
desc: "Stop the SAML keycloak test environment"
|
||||
cmds:
|
||||
- docker compose -f testing/compose/docker-compose-keycloak-saml.yml down -v
|
||||
|
||||
mcp:up:
|
||||
desc: "Start the MCP keycloak test environment (Stirling as OAuth resource server)"
|
||||
summary: |
|
||||
Brings up Keycloak (OAuth authorization server) + Stirling configured as an
|
||||
MCP resource server, then you can exercise /mcp with real Keycloak tokens.
|
||||
Set LICENSE_KEY=<KEY> to skip the interactive license prompt:
|
||||
task e2e:mcp:up LICENSE_KEY=abc123
|
||||
Pass extra flags via -- :
|
||||
task e2e:mcp:up -- --validate --nobuild
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-mcp-test.sh {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
mcp:manual:
|
||||
desc: "Start the MCP keycloak test env in manual mode (prints URLs + a live token for your client)"
|
||||
summary: |
|
||||
Brings the stack up and prints copy-paste URLs/commands plus a freshly minted
|
||||
access token so you can drive your own MCP client (Inspector, curl, ...).
|
||||
task e2e:mcp:manual LICENSE_KEY=<your-license-key>
|
||||
Add --nobuild if the images are already built:
|
||||
task e2e:mcp:manual LICENSE_KEY=<your-license-key> -- --nobuild
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-mcp-test.sh --manual {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
mcp:apikey:
|
||||
desc: "Start the MCP test env in API-KEY manual mode (no OAuth/IdP): mints a key + prints client settings"
|
||||
summary: |
|
||||
Brings Stirling up in apikey auth mode and prints copy-paste client settings with a freshly
|
||||
minted X-API-KEY - ideal for clients whose OAuth layer can't reach localhost.
|
||||
task e2e:mcp:apikey LICENSE_KEY=<your-license-key>
|
||||
Add --nobuild if images are already built:
|
||||
task e2e:mcp:apikey LICENSE_KEY=<your-license-key> -- --nobuild
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-mcp-test.sh --apikey {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
mcp:validate:
|
||||
desc: "Validate the running MCP keycloak test environment end-to-end (oauth mode + real MCP SDK client)"
|
||||
cmds:
|
||||
- bash testing/compose/validate-mcp-test.sh
|
||||
|
||||
mcp:validate-apikey:
|
||||
desc: "Validate the MCP server in API-KEY auth mode (mints a key + real MCP SDK client), then restore oauth"
|
||||
cmds:
|
||||
- bash testing/compose/validate-mcp-apikey.sh
|
||||
|
||||
mcp:down:
|
||||
desc: "Stop the MCP keycloak test environment"
|
||||
cmds:
|
||||
- docker compose -f testing/compose/docker-compose-keycloak-mcp.yml down -v
|
||||
|
||||
+15
-2
@@ -1,5 +1,12 @@
|
||||
version: '3'
|
||||
|
||||
vars:
|
||||
# Engine-specific names to avoid overriding the root Taskfile's FIND_FREE_PORT_*
|
||||
# vars (Task merges included-file vars into the global scope).
|
||||
# Paths are relative to the engine/ include dir.
|
||||
ENGINE_FIND_FREE_PORT_SH: "bash ../scripts/find-free-port.sh"
|
||||
ENGINE_FIND_FREE_PORT_PS: "powershell -NoProfile -File ../scripts/find-free-port.ps1"
|
||||
|
||||
tasks:
|
||||
install:
|
||||
desc: "Install engine dependencies"
|
||||
@@ -29,7 +36,10 @@ tasks:
|
||||
ignore_error: true
|
||||
dir: src
|
||||
vars:
|
||||
PORT: '{{.PORT | default "5001"}}'
|
||||
# When PORT is provided (e.g. from dev:all), use it directly.
|
||||
# When running standalone, probe for a free port starting at 5001.
|
||||
PORT:
|
||||
sh: '{{if .PORT}}echo {{.PORT}}{{else if eq OS "windows"}}{{.ENGINE_FIND_FREE_PORT_PS}} 5001{{else}}{{.ENGINE_FIND_FREE_PORT_SH}} 5001{{end}}'
|
||||
env:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
cmds:
|
||||
@@ -41,7 +51,10 @@ tasks:
|
||||
ignore_error: true
|
||||
dir: src
|
||||
vars:
|
||||
PORT: '{{.PORT | default "5001"}}'
|
||||
# When PORT is provided (e.g. from dev:all), use it directly.
|
||||
# When running standalone, probe for a free port starting at 5001.
|
||||
PORT:
|
||||
sh: '{{if .PORT}}echo {{.PORT}}{{else if eq OS "windows"}}{{.ENGINE_FIND_FREE_PORT_PS}} 5001{{else}}{{.ENGINE_FIND_FREE_PORT_SH}} 5001{{end}}'
|
||||
env:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
cmds:
|
||||
|
||||
@@ -327,19 +327,19 @@ tasks:
|
||||
|
||||
test:
|
||||
desc: "Run tests"
|
||||
deps: [install]
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx vitest run --root editor
|
||||
|
||||
test:watch:
|
||||
desc: "Run tests in watch mode"
|
||||
deps: [install]
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx vitest --watch --root editor
|
||||
|
||||
test:coverage:
|
||||
desc: "Run tests with coverage (one-shot; CI-friendly)."
|
||||
deps: [install]
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
# `vitest run` makes this CI-safe (the bare `vitest` form enters watch
|
||||
# mode). Explicit reporter list because v8 + json-summary is what the
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link
|
||||
* LineAlignmentTableParser}. The first parser returning a result above {@link
|
||||
* #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page.
|
||||
*/
|
||||
@Service
|
||||
@Primary
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CompositeTableParser implements TableParser {
|
||||
|
||||
/** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */
|
||||
static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f;
|
||||
|
||||
private final TabulaTableParser tabulaParser;
|
||||
private final LineAlignmentTableParser lineAlignmentParser;
|
||||
|
||||
@Override
|
||||
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
|
||||
// Step 1: Tabula lattice mode (ruled/bordered tables).
|
||||
List<TableFragment> latticeResults = filterConfident(tabulaParser.parse(document, rawPage));
|
||||
if (!latticeResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using Tabula lattice ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
latticeResults.size());
|
||||
return latticeResults;
|
||||
}
|
||||
|
||||
// Step 2: Tabula stream mode (borderless/whitespace-delimited tables).
|
||||
// parseStream is not on the TableParser interface — this intentionally couples to the
|
||||
// concrete TabulaTableParser since stream mode is a Tabula-specific concept.
|
||||
List<TableFragment> streamResults =
|
||||
filterConfident(tabulaParser.parseStream(document, rawPage));
|
||||
if (!streamResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using Tabula stream ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
streamResults.size());
|
||||
return streamResults;
|
||||
}
|
||||
|
||||
// Step 3: Geometry-based line-alignment fallback.
|
||||
List<TableFragment> lineResults = lineAlignmentParser.parse(document, rawPage);
|
||||
if (!lineResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using LineAlignment ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
lineResults.size());
|
||||
return lineResults;
|
||||
}
|
||||
|
||||
return List.of();
|
||||
}
|
||||
|
||||
private List<TableFragment> filterConfident(List<TableFragment> tables) {
|
||||
return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList();
|
||||
}
|
||||
}
|
||||
-528
@@ -1,528 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Fallback {@link TableParser} for borderless financial tables using text geometry.
|
||||
*
|
||||
* <p>Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge
|
||||
* positions, groups vertically proximate anchor lines into table candidates, then scores each group
|
||||
* on column consistency and anchor density (confidence ceiling 0.85).
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LineAlignmentTableParser implements TableParser {
|
||||
|
||||
/** Width in points of each column position bucket. */
|
||||
static final float COLUMN_BUCKET_PT = 5f;
|
||||
|
||||
/** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */
|
||||
private static final int COLUMN_MATCH_BUCKETS = 2;
|
||||
|
||||
/** Maximum gap (as a multiple of modal line spacing) before splitting a group. */
|
||||
private static final float MAX_GAP_FACTOR = 2.5f;
|
||||
|
||||
/** Minimum anchor rows (numeric-heavy) to form a valid table. */
|
||||
static final int MIN_TABLE_ROWS = 3;
|
||||
|
||||
/** Minimum confirmed column positions to form a valid table. */
|
||||
static final int MIN_COLUMNS = 2;
|
||||
|
||||
/**
|
||||
* Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A
|
||||
* rows).
|
||||
*/
|
||||
private static final double COLUMN_MIN_FREQUENCY = 0.40;
|
||||
|
||||
/**
|
||||
* Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency,
|
||||
* percent, nil dashes.
|
||||
*/
|
||||
private static final Pattern NUMERIC =
|
||||
Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$");
|
||||
|
||||
/**
|
||||
* Lines within this y-distance are merged into one row (restores rows split by LineBuilder's
|
||||
* column-gap logic).
|
||||
*/
|
||||
static final float ROW_MERGE_TOLERANCE_PT = 2f;
|
||||
|
||||
// ── public API ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Override
|
||||
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
|
||||
List<RawLine> lines = rawPage.lines();
|
||||
if (lines.size() < MIN_TABLE_ROWS) return List.of();
|
||||
|
||||
float modalSpacing = computeModalSpacing(lines);
|
||||
List<TokenizedLine> tokenized =
|
||||
mergeCoincidentLines(lines.stream().map(this::tokenize).toList());
|
||||
|
||||
List<TokenizedLine> anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList();
|
||||
|
||||
if (anchors.size() < MIN_TABLE_ROWS) return List.of();
|
||||
|
||||
List<Float> columnGrid = buildColumnGrid(anchors);
|
||||
if (columnGrid.size() < MIN_COLUMNS) {
|
||||
log.debug(
|
||||
"Page {}: LineAlignment — fewer than {} confirmed columns, skipping",
|
||||
rawPage.pageNumber(),
|
||||
MIN_COLUMNS);
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<List<TokenizedLine>> groups = groupRows(tokenized, columnGrid, modalSpacing);
|
||||
|
||||
List<TableFragment> results = new ArrayList<>();
|
||||
for (int i = 0; i < groups.size(); i++) {
|
||||
buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i)
|
||||
.ifPresent(results::add);
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)",
|
||||
rawPage.pageNumber(),
|
||||
results.size(),
|
||||
anchors.size(),
|
||||
columnGrid.size());
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── coincident-line merging ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves
|
||||
* split by LineBuilder.
|
||||
*/
|
||||
List<TokenizedLine> mergeCoincidentLines(List<TokenizedLine> tokenized) {
|
||||
if (tokenized.size() < 2) return tokenized;
|
||||
|
||||
List<TokenizedLine> result = new ArrayList<>();
|
||||
int i = 0;
|
||||
|
||||
while (i < tokenized.size()) {
|
||||
float baseY = tokenized.get(i).line().bounds().y();
|
||||
int j = i + 1;
|
||||
while (j < tokenized.size()
|
||||
&& Math.abs(tokenized.get(j).line().bounds().y() - baseY)
|
||||
<= ROW_MERGE_TOLERANCE_PT) {
|
||||
j++;
|
||||
}
|
||||
|
||||
if (j == i + 1) {
|
||||
result.add(tokenized.get(i));
|
||||
} else {
|
||||
result.add(mergeGroup(tokenized.subList(i, j)));
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private TokenizedLine mergeGroup(List<TokenizedLine> group) {
|
||||
List<TextFragment> mergedFragments =
|
||||
group.stream()
|
||||
.flatMap(tl -> tl.line().fragments().stream())
|
||||
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
Bounds mergedBounds =
|
||||
group.stream()
|
||||
.map(tl -> tl.line().bounds())
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(group.get(0).line().bounds());
|
||||
|
||||
RawLine mergedLine =
|
||||
new RawLine(
|
||||
group.get(0).line().lineId(),
|
||||
mergedFragments,
|
||||
mergedBounds,
|
||||
group.get(0).line().pageNumber());
|
||||
|
||||
return tokenize(mergedLine);
|
||||
}
|
||||
|
||||
// ── tokenisation ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Splits fragments into word-level tokens; x-positions are estimated linearly within each
|
||||
* fragment.
|
||||
*/
|
||||
TokenizedLine tokenize(RawLine line) {
|
||||
List<LineToken> tokens = new ArrayList<>();
|
||||
for (TextFragment frag : line.fragments()) {
|
||||
tokens.addAll(tokensFromFragment(frag));
|
||||
}
|
||||
List<LineToken> numeric = tokens.stream().filter(LineToken::numeric).toList();
|
||||
return new TokenizedLine(line, tokens, numeric);
|
||||
}
|
||||
|
||||
private List<LineToken> tokensFromFragment(TextFragment frag) {
|
||||
String raw = frag.text();
|
||||
if (raw == null || raw.isBlank()) return List.of();
|
||||
|
||||
float fragX = frag.bounds().x();
|
||||
float fragWidth = frag.bounds().width();
|
||||
int rawLen = raw.length();
|
||||
|
||||
List<LineToken> result = new ArrayList<>();
|
||||
int offset = 0;
|
||||
for (String part : raw.split("\\s+")) {
|
||||
if (part.isEmpty()) {
|
||||
offset++;
|
||||
continue;
|
||||
}
|
||||
int idx = raw.indexOf(part, offset);
|
||||
if (idx < 0) idx = offset;
|
||||
|
||||
float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX;
|
||||
float tokenRight =
|
||||
rawLen > 0
|
||||
? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth
|
||||
: fragX + fragWidth;
|
||||
|
||||
result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches()));
|
||||
offset = idx + part.length();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── column grid ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Returns confirmed column right-edge positions — those appearing on ≥ {@value
|
||||
* #COLUMN_MIN_FREQUENCY} × N anchor lines.
|
||||
*/
|
||||
private List<Float> buildColumnGrid(List<TokenizedLine> anchors) {
|
||||
// bucket → set of line indices that contributed a numeric token to that bucket
|
||||
Map<Integer, List<Integer>> bucketLines = new HashMap<>();
|
||||
for (int i = 0; i < anchors.size(); i++) {
|
||||
for (LineToken t : anchors.get(i).numeric()) {
|
||||
int bucket = bucket(t.right());
|
||||
bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i);
|
||||
}
|
||||
}
|
||||
|
||||
int minHits =
|
||||
Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY));
|
||||
|
||||
// Confirmed buckets → average right-edge for that bucket
|
||||
TreeMap<Integer, Float> confirmed = new TreeMap<>();
|
||||
for (Map.Entry<Integer, List<Integer>> entry : bucketLines.entrySet()) {
|
||||
// Count distinct lines
|
||||
long distinctLines = entry.getValue().stream().distinct().count();
|
||||
if (distinctLines >= minHits) {
|
||||
double avg =
|
||||
entry.getValue().stream()
|
||||
.distinct() // weight each line equally regardless of token count
|
||||
.mapToDouble(
|
||||
lineIdx ->
|
||||
avgRightEdgeForBucket(
|
||||
anchors, lineIdx, entry.getKey()))
|
||||
.average()
|
||||
.orElse(entry.getKey() * (double) COLUMN_BUCKET_PT);
|
||||
confirmed.put(entry.getKey(), (float) avg);
|
||||
}
|
||||
}
|
||||
|
||||
return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code
|
||||
* targetBucket}, falling back to the bucket's nominal centre when no tokens match.
|
||||
*/
|
||||
private double avgRightEdgeForBucket(
|
||||
List<TokenizedLine> anchors, int lineIdx, int targetBucket) {
|
||||
return anchors.get(lineIdx).numeric().stream()
|
||||
.filter(t -> bucket(t.right()) == targetBucket)
|
||||
.mapToDouble(LineToken::right)
|
||||
.average()
|
||||
.orElse(targetBucket * (double) COLUMN_BUCKET_PT);
|
||||
}
|
||||
|
||||
// ── grouping ─────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Groups anchor lines into table candidates, including adjacent label rows; a gap >
|
||||
* MAX_GAP_FACTOR × modal spacing splits groups.
|
||||
*/
|
||||
private List<List<TokenizedLine>> groupRows(
|
||||
List<TokenizedLine> all, List<Float> columnGrid, float modalSpacing) {
|
||||
float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f;
|
||||
|
||||
List<List<TokenizedLine>> groups = new ArrayList<>();
|
||||
List<TokenizedLine> current = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < all.size(); i++) {
|
||||
TokenizedLine tl = all.get(i);
|
||||
boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid);
|
||||
|
||||
if (current.isEmpty()) {
|
||||
if (fits) current.add(tl);
|
||||
continue;
|
||||
}
|
||||
|
||||
float gap =
|
||||
tl.line().bounds().y()
|
||||
- current.get(current.size() - 1).line().bounds().bottom();
|
||||
|
||||
if (gap > maxGap) {
|
||||
groups.add(current);
|
||||
current = new ArrayList<>();
|
||||
if (fits) current.add(tl);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fits) {
|
||||
current.add(tl);
|
||||
} else if (!tl.line().text().isBlank()) {
|
||||
// Include non-anchor lines (labels) only if they have text and are within
|
||||
// proximity.
|
||||
current.add(tl);
|
||||
}
|
||||
}
|
||||
|
||||
if (!current.isEmpty()) groups.add(current);
|
||||
|
||||
return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList();
|
||||
}
|
||||
|
||||
private boolean hasEnoughAnchorRows(List<TokenizedLine> group, List<Float> columnGrid) {
|
||||
return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count()
|
||||
>= MIN_TABLE_ROWS;
|
||||
}
|
||||
|
||||
/** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */
|
||||
private boolean matchesGrid(TokenizedLine tl, List<Float> columnGrid) {
|
||||
if (tl.numeric().isEmpty()) return false;
|
||||
long matches =
|
||||
tl.numeric().stream()
|
||||
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
|
||||
.count();
|
||||
return (double) matches / tl.numeric().size() >= 0.60;
|
||||
}
|
||||
|
||||
private boolean hasInconsistentColumnMatch(TokenizedLine tl, List<Float> columnGrid) {
|
||||
if (tl.numeric().isEmpty()) return false;
|
||||
long hits =
|
||||
tl.numeric().stream()
|
||||
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
|
||||
.count();
|
||||
return (double) hits / tl.numeric().size() < 0.60;
|
||||
}
|
||||
|
||||
// ── fragment assembly ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private Optional<TableFragment> buildFragment(
|
||||
List<TokenizedLine> group, List<Float> columnGrid, int pageNumber, int tableIndex) {
|
||||
|
||||
long anchorCount =
|
||||
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
|
||||
if (anchorCount < MIN_TABLE_ROWS) return Optional.empty();
|
||||
|
||||
List<String> warnings = new ArrayList<>();
|
||||
List<List<String>> rawRows = new ArrayList<>();
|
||||
List<TableRow> rows = new ArrayList<>();
|
||||
|
||||
for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) {
|
||||
TokenizedLine tl = group.get(rowIdx);
|
||||
List<String> rawRow = buildRawRow(tl, columnGrid);
|
||||
rawRows.add(Collections.unmodifiableList(rawRow));
|
||||
rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid));
|
||||
}
|
||||
|
||||
// Column count = 1 label column + confirmed numeric columns
|
||||
int colCount = columnGrid.size() + 1;
|
||||
Bounds bounds = computeGroupBounds(group);
|
||||
float confidence = computeConfidence(group, columnGrid, warnings);
|
||||
|
||||
return Optional.of(
|
||||
new TableFragment(
|
||||
"tbl-la-p" + pageNumber + "-" + tableIndex,
|
||||
pageNumber,
|
||||
bounds,
|
||||
List.of(),
|
||||
Collections.unmodifiableList(rows),
|
||||
Collections.unmodifiableList(rawRows),
|
||||
colCount,
|
||||
confidence,
|
||||
Collections.unmodifiableList(warnings),
|
||||
null));
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values.
|
||||
*/
|
||||
private List<String> buildRawRow(TokenizedLine tl, List<Float> columnGrid) {
|
||||
String[] cells = new String[columnGrid.size() + 1];
|
||||
Arrays.fill(cells, "");
|
||||
|
||||
// Separate label tokens (those not landing in any confirmed column) from column tokens.
|
||||
List<String> labelParts = new ArrayList<>();
|
||||
for (LineToken token : tl.all()) {
|
||||
int col = nearestColumnIndex(token.right(), columnGrid);
|
||||
if (col >= 0 && token.numeric()) {
|
||||
int cellIdx = col + 1;
|
||||
cells[cellIdx] =
|
||||
cells[cellIdx].isEmpty()
|
||||
? token.text()
|
||||
: cells[cellIdx] + " " + token.text();
|
||||
} else {
|
||||
labelParts.add(token.text());
|
||||
}
|
||||
}
|
||||
cells[0] = String.join(" ", labelParts).trim();
|
||||
return Arrays.asList(cells);
|
||||
}
|
||||
|
||||
private TableRow buildTableRow(
|
||||
int rowIdx, TokenizedLine tl, List<String> rawRow, List<Float> columnGrid) {
|
||||
List<TableCell> cells = new ArrayList<>(rawRow.size());
|
||||
|
||||
// Label cell: use the line's full bounds as an approximation.
|
||||
cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds()));
|
||||
|
||||
for (int col = 0; col < columnGrid.size(); col++) {
|
||||
String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : "";
|
||||
float right = columnGrid.get(col);
|
||||
float left = col > 0 ? columnGrid.get(col - 1) : right - 50f;
|
||||
Bounds cellBounds =
|
||||
new Bounds(
|
||||
left,
|
||||
tl.line().bounds().y(),
|
||||
right - left,
|
||||
tl.line().bounds().height());
|
||||
cells.add(TableCell.of(col + 1, text, cellBounds));
|
||||
}
|
||||
return new TableRow(rowIdx, Collections.unmodifiableList(cells));
|
||||
}
|
||||
|
||||
// ── confidence scoring ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at
|
||||
* 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if
|
||||
* >30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors.
|
||||
*/
|
||||
private float computeConfidence(
|
||||
List<TokenizedLine> group, List<Float> columnGrid, List<String> warnings) {
|
||||
float score = 0.70f;
|
||||
|
||||
long anchorCount =
|
||||
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
|
||||
long totalRows = group.size();
|
||||
|
||||
// More columns
|
||||
int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2);
|
||||
score += extraCols * 0.05f;
|
||||
|
||||
// More anchor rows
|
||||
if (anchorCount >= 5) score += 0.05f;
|
||||
if (anchorCount >= 8) score += 0.05f;
|
||||
|
||||
// Inconsistent column matching
|
||||
long inconsistent =
|
||||
group.stream()
|
||||
.filter(TokenizedLine::isAnchor)
|
||||
.filter(tl -> hasInconsistentColumnMatch(tl, columnGrid))
|
||||
.count();
|
||||
if (inconsistent > anchorCount * 0.30) {
|
||||
score -= 0.15f;
|
||||
warnings.add(
|
||||
"Column match inconsistent on "
|
||||
+ inconsistent
|
||||
+ "/"
|
||||
+ anchorCount
|
||||
+ " anchor rows");
|
||||
}
|
||||
|
||||
// Label-heavy
|
||||
long nonAnchor = totalRows - anchorCount;
|
||||
if (nonAnchor > anchorCount) {
|
||||
score -= 0.10f;
|
||||
warnings.add(
|
||||
"Non-anchor rows ("
|
||||
+ nonAnchor
|
||||
+ ") outnumber anchor rows ("
|
||||
+ anchorCount
|
||||
+ ")");
|
||||
}
|
||||
|
||||
return Math.max(0f, Math.min(0.85f, score));
|
||||
}
|
||||
|
||||
// ── utility ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value
|
||||
* #COLUMN_MATCH_BUCKETS} buckets.
|
||||
*/
|
||||
private int nearestColumnIndex(float rightEdge, List<Float> grid) {
|
||||
int nearest = -1;
|
||||
float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f;
|
||||
for (int i = 0; i < grid.size(); i++) {
|
||||
float dist = Math.abs(rightEdge - grid.get(i));
|
||||
if (dist < minDist) {
|
||||
minDist = dist;
|
||||
nearest = i;
|
||||
}
|
||||
}
|
||||
return nearest;
|
||||
}
|
||||
|
||||
private Bounds computeGroupBounds(List<TokenizedLine> group) {
|
||||
return group.stream()
|
||||
.map(tl -> tl.line().bounds())
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(new Bounds(0, 0, 0, 0));
|
||||
}
|
||||
|
||||
/** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */
|
||||
private float computeModalSpacing(List<RawLine> lines) {
|
||||
if (lines.size() < 2) return 0f;
|
||||
Map<Float, Long> freq = new HashMap<>();
|
||||
for (int i = 1; i < lines.size(); i++) {
|
||||
float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom();
|
||||
if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum);
|
||||
}
|
||||
return freq.entrySet().stream()
|
||||
.max(Map.Entry.comparingByValue())
|
||||
.map(Map.Entry::getKey)
|
||||
.orElse(0f);
|
||||
}
|
||||
|
||||
private static int bucket(float x) {
|
||||
return Math.round(x / COLUMN_BUCKET_PT);
|
||||
}
|
||||
|
||||
// ── private data types ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** A word-level token with an approximate right-edge x-position. */
|
||||
record LineToken(String text, float x, float right, boolean numeric) {}
|
||||
|
||||
/** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */
|
||||
record TokenizedLine(RawLine line, List<LineToken> all, List<LineToken> numeric) {
|
||||
boolean isAnchor() {
|
||||
return numeric.size() >= 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity.
|
||||
*
|
||||
* <p>Fragments are on the same line when their baselines are within a font-size-derived tolerance.
|
||||
* A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code
|
||||
* max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text.
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LineBuilder {
|
||||
|
||||
/** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */
|
||||
private static final float BASELINE_TOLERANCE_FACTOR = 0.5f;
|
||||
|
||||
/** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */
|
||||
private static final float MIN_BASELINE_TOLERANCE = 2f;
|
||||
|
||||
/**
|
||||
* Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below
|
||||
* two-column gutters.
|
||||
*/
|
||||
static final float COLUMN_GAP_RATIO = 0.10f;
|
||||
|
||||
/** Floor for the column-gap threshold so narrow pages don't over-split lines. */
|
||||
static final float COLUMN_GAP_MIN_PT = 40f;
|
||||
|
||||
public List<RawLine> build(List<TextFragment> fragments, int pageNumber) {
|
||||
if (fragments.isEmpty()) return List.of();
|
||||
|
||||
float effectiveWidth = inferEffectiveWidth(fragments);
|
||||
float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT);
|
||||
log.debug(
|
||||
"LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt",
|
||||
pageNumber,
|
||||
effectiveWidth,
|
||||
columnGapThreshold);
|
||||
|
||||
// Sort top-to-bottom first, then left-to-right within the same baseline band.
|
||||
List<TextFragment> sorted =
|
||||
fragments.stream()
|
||||
.sorted(
|
||||
Comparator.comparingDouble(TextFragment::baseline)
|
||||
.thenComparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
List<List<TextFragment>> groups = groupByBaseline(sorted, columnGapThreshold);
|
||||
|
||||
List<RawLine> lines = new ArrayList<>(groups.size());
|
||||
for (int i = 0; i < groups.size(); i++) {
|
||||
List<TextFragment> group =
|
||||
groups.get(i).stream()
|
||||
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
Bounds lineBounds =
|
||||
group.stream()
|
||||
.map(TextFragment::bounds)
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(new Bounds(0, 0, 0, 0));
|
||||
|
||||
lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber));
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
private List<List<TextFragment>> groupByBaseline(
|
||||
List<TextFragment> sorted, float columnGapThreshold) {
|
||||
List<List<TextFragment>> groups = new ArrayList<>();
|
||||
List<TextFragment> current = new ArrayList<>();
|
||||
float currentBaseline = Float.NaN;
|
||||
|
||||
for (TextFragment fragment : sorted) {
|
||||
if (current.isEmpty()) {
|
||||
current.add(fragment);
|
||||
currentBaseline = fragment.baseline();
|
||||
continue;
|
||||
}
|
||||
|
||||
float maxFontSize =
|
||||
Math.max(
|
||||
fragment.fontSize(),
|
||||
(float)
|
||||
current.stream()
|
||||
.mapToDouble(TextFragment::fontSize)
|
||||
.max()
|
||||
.orElse(0));
|
||||
float tolerance =
|
||||
Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE);
|
||||
|
||||
boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance;
|
||||
boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold);
|
||||
|
||||
if (sameBaseline && !columnGap) {
|
||||
current.add(fragment);
|
||||
// Anchor to the weighted mean baseline so long lines stay stable.
|
||||
currentBaseline =
|
||||
(currentBaseline * (current.size() - 1) + fragment.baseline())
|
||||
/ current.size();
|
||||
} else {
|
||||
groups.add(current);
|
||||
current = new ArrayList<>();
|
||||
current.add(fragment);
|
||||
currentBaseline = fragment.baseline();
|
||||
}
|
||||
}
|
||||
|
||||
if (!current.isEmpty()) groups.add(current);
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code
|
||||
* threshold}.
|
||||
*/
|
||||
private static boolean hasColumnGap(
|
||||
TextFragment next, List<TextFragment> group, float threshold) {
|
||||
float lastRight = group.get(group.size() - 1).bounds().right();
|
||||
return next.bounds().x() - lastRight > threshold;
|
||||
}
|
||||
|
||||
/** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */
|
||||
private static float inferEffectiveWidth(List<TextFragment> fragments) {
|
||||
double maxRight =
|
||||
fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0);
|
||||
return (float) maxRight * 1.10f;
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} →
|
||||
* {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the
|
||||
* {@link PDDocument} lifecycle.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PdfIngester {
|
||||
|
||||
private final LineBuilder lineBuilder;
|
||||
private final TableParser tableParser;
|
||||
|
||||
public List<ParsedPage> parse(PDDocument document) throws IOException {
|
||||
return parse(document, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
public List<ParsedPage> parse(PDDocument document, int maxPages) throws IOException {
|
||||
int pageCount = Math.min(document.getNumberOfPages(), maxPages);
|
||||
List<ParsedPage> pages = new ArrayList<>(pageCount);
|
||||
long fragmentsMs = 0;
|
||||
long tablesMs = 0;
|
||||
long t0 = System.currentTimeMillis();
|
||||
|
||||
for (int p = 1; p <= pageCount; p++) {
|
||||
long ft = System.currentTimeMillis();
|
||||
List<TextFragment> fragments = extractFragments(document, p);
|
||||
fragmentsMs += System.currentTimeMillis() - ft;
|
||||
|
||||
PDPage page = document.getPage(p - 1);
|
||||
PDRectangle mediaBox = page.getMediaBox();
|
||||
List<RawLine> lines = lineBuilder.build(fragments, p);
|
||||
RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines);
|
||||
|
||||
long tt = System.currentTimeMillis();
|
||||
List<TableFragment> tables = tableParser.parse(document, rawPage);
|
||||
tablesMs += System.currentTimeMillis() - tt;
|
||||
|
||||
log.debug(
|
||||
"Page {}: {} fragments → {} lines, {} table(s)",
|
||||
p,
|
||||
fragments.size(),
|
||||
lines.size(),
|
||||
tables.size());
|
||||
pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines));
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[timing] parse pages={} total={}ms fragments={}ms tables={}ms",
|
||||
pageCount,
|
||||
System.currentTimeMillis() - t0,
|
||||
fragmentsMs,
|
||||
tablesMs);
|
||||
return pages;
|
||||
}
|
||||
|
||||
private List<TextFragment> extractFragments(PDDocument document, int pageNumber)
|
||||
throws IOException {
|
||||
WordExtractingStripper stripper = new WordExtractingStripper(pageNumber);
|
||||
stripper.getText(document);
|
||||
return stripper.getFragments();
|
||||
}
|
||||
}
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata.
|
||||
*
|
||||
* <p>Overrides {@link #writeString} to split each content-stream string into word-level {@link
|
||||
* TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in
|
||||
* PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline.
|
||||
*/
|
||||
class WordExtractingStripper extends PDFTextStripper {
|
||||
|
||||
private final int targetPage;
|
||||
private final List<TextFragment> fragments = new ArrayList<>();
|
||||
private int fragmentIndex = 0;
|
||||
|
||||
WordExtractingStripper(int pageNumber) throws IOException {
|
||||
this.targetPage = pageNumber;
|
||||
setStartPage(pageNumber);
|
||||
setEndPage(pageNumber);
|
||||
setSortByPosition(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startPage(PDPage page) throws IOException {
|
||||
super.startPage(page);
|
||||
fragments.clear();
|
||||
fragmentIndex = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
if (text == null || text.isBlank()) return;
|
||||
|
||||
// Fast path: no whitespace → emit one fragment (most financial PDFs have each
|
||||
// number as its own string operation, so this is the common case).
|
||||
if (text.indexOf(' ') < 0) {
|
||||
emitFragment(text, textPositions);
|
||||
return;
|
||||
}
|
||||
|
||||
// Per-word splitting requires 1:1 text-char to TextPosition correspondence.
|
||||
// Fall back to one fragment when sizes differ (ligatures, encoding edge cases).
|
||||
if (textPositions.size() != text.length()) {
|
||||
emitFragment(text, textPositions);
|
||||
return;
|
||||
}
|
||||
|
||||
// Emit one TextFragment per whitespace-delimited word with accurate per-word bounds.
|
||||
int start = 0;
|
||||
for (int i = 0; i <= text.length(); i++) {
|
||||
if (i == text.length() || text.charAt(i) == ' ') {
|
||||
if (start < i) {
|
||||
emitFragment(text.substring(start, i), textPositions.subList(start, i));
|
||||
}
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void emitFragment(String text, List<TextPosition> positions) {
|
||||
if (positions.isEmpty()) return;
|
||||
|
||||
float minX = Float.MAX_VALUE;
|
||||
float minY = Float.MAX_VALUE;
|
||||
float maxRight = -Float.MAX_VALUE;
|
||||
float maxBaseline = -Float.MAX_VALUE;
|
||||
TextPosition first = null;
|
||||
|
||||
for (TextPosition tp : positions) {
|
||||
if (tp == null) continue;
|
||||
if (first == null) first = tp;
|
||||
|
||||
float x = tp.getX();
|
||||
// getY() is the baseline; top of character = getY() - getHeight().
|
||||
float top = tp.getY() - tp.getHeight();
|
||||
float right = x + tp.getWidth();
|
||||
float baseline = tp.getY();
|
||||
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, top);
|
||||
maxRight = Math.max(maxRight, right);
|
||||
maxBaseline = Math.max(maxBaseline, baseline);
|
||||
}
|
||||
|
||||
if (first == null) return;
|
||||
|
||||
PDFont font = first.getFont();
|
||||
String fontName = font != null ? font.getName() : "";
|
||||
boolean bold = fontName != null && fontName.toLowerCase().contains("bold");
|
||||
// getHeight() gives the rendered glyph height, which is the most reliable visual size.
|
||||
float fontSize = first.getHeight();
|
||||
|
||||
Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY);
|
||||
String id = "tf-p" + targetPage + "-" + fragmentIndex++;
|
||||
fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold));
|
||||
}
|
||||
|
||||
List<TextFragment> getFragments() {
|
||||
return Collections.unmodifiableList(fragments);
|
||||
}
|
||||
}
|
||||
@@ -11,23 +11,39 @@ public interface FileStore {
|
||||
/** Stored file record. */
|
||||
record Stored(String fileId, long size) {}
|
||||
|
||||
/** Store the given stream and return a generated file id and total bytes written. */
|
||||
Stored store(InputStream in, String originalName) throws IOException;
|
||||
/**
|
||||
* Store the given stream and return a generated file id and total bytes written. {@code owner}
|
||||
* may be null to indicate the file has no associated user (anonymous / desktop / async job with
|
||||
* no propagated security context); a non-null value is persisted alongside the data so {@link
|
||||
* #getOwner(String)} can return it later for authorization checks.
|
||||
*/
|
||||
Stored store(InputStream in, String originalName, String owner) throws IOException;
|
||||
|
||||
/** Store with no owner. Equivalent to {@link #store(InputStream, String, String)} with null. */
|
||||
default Stored store(InputStream in, String originalName) throws IOException {
|
||||
return store(in, originalName, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the file at {@code source} and return a generated file id and total bytes written.
|
||||
*
|
||||
* <p>Default implementation opens {@code source} as a stream and delegates to {@link
|
||||
* #store(InputStream, String)}. Local-disk implementations should override to use a direct
|
||||
* file-to-file copy ({@code Files.copy(source, dest)} can use {@code sendfile(2)} on Linux),
|
||||
* which avoids the two-memory-copy hit of streaming a disk-backed upload through the JVM heap.
|
||||
* #store(InputStream, String, String)}. Local-disk implementations should override to use a
|
||||
* direct file-to-file copy ({@code Files.copy(source, dest)} can use {@code sendfile(2)} on
|
||||
* Linux), which avoids the two-memory-copy hit of streaming a disk-backed upload through the
|
||||
* JVM heap.
|
||||
*/
|
||||
default Stored store(Path source, String originalName) throws IOException {
|
||||
default Stored store(Path source, String originalName, String owner) throws IOException {
|
||||
try (InputStream in = Files.newInputStream(source)) {
|
||||
return store(in, originalName);
|
||||
return store(in, originalName, owner);
|
||||
}
|
||||
}
|
||||
|
||||
/** Store with no owner. Equivalent to {@link #store(Path, String, String)} with null. */
|
||||
default Stored store(Path source, String originalName) throws IOException {
|
||||
return store(source, originalName, null);
|
||||
}
|
||||
|
||||
/** Open the stored file for streaming reads. Caller closes. */
|
||||
InputStream retrieve(String fileId) throws IOException;
|
||||
|
||||
@@ -42,4 +58,12 @@ public interface FileStore {
|
||||
|
||||
/** Whether the file id exists in the store. */
|
||||
boolean exists(String fileId);
|
||||
|
||||
/**
|
||||
* Returns the owner identifier recorded at store time, or {@code null} if the file does not
|
||||
* exist or was stored without an owner. Implementations must not throw when the file is missing
|
||||
* or when the owner record is absent; they should return null so callers can treat "no owner"
|
||||
* as a non-authoritative case.
|
||||
*/
|
||||
String getOwner(String fileId) throws IOException;
|
||||
}
|
||||
|
||||
+100
-23
@@ -3,9 +3,12 @@ package stirling.software.common.cluster.inprocess;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@@ -15,33 +18,47 @@ import stirling.software.common.cluster.FileStore;
|
||||
@Slf4j
|
||||
public class LocalDiskFileStore implements FileStore {
|
||||
|
||||
private static final String OWNER_SUFFIX = ".owner";
|
||||
|
||||
// File ids are generated as random UUIDs; reject anything else so a tainted id can never reach
|
||||
// Files.* APIs (defence in depth on top of the resolve() prefix check, and silences CodeQL's
|
||||
// path-injection finding on the resolveOwner sidecar lookup).
|
||||
private static final Pattern UUID_PATTERN =
|
||||
Pattern.compile(
|
||||
"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
|
||||
|
||||
private final String baseDirPath;
|
||||
// Fixed-size lock stripes so concurrent store/delete on the same (or colliding) fileId
|
||||
// serialise the data-file + owner-sidecar pair as one critical section. Striped (not
|
||||
// per-id) so the map never has to be cleaned up; collisions across unrelated ids are
|
||||
// harmless contention.
|
||||
private static final int LOCK_STRIPES = 64;
|
||||
private final ReentrantLock[] stripes = new ReentrantLock[LOCK_STRIPES];
|
||||
|
||||
public LocalDiskFileStore(String baseDirPath) {
|
||||
this.baseDirPath = baseDirPath;
|
||||
for (int i = 0; i < LOCK_STRIPES; i++) {
|
||||
stripes[i] = new ReentrantLock();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stored store(InputStream in, String originalName) throws IOException {
|
||||
public Stored store(InputStream in, String originalName, String owner) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = resolve(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
ReentrantLock lock = acquire(fileId);
|
||||
boolean success = false;
|
||||
try {
|
||||
long size = Files.copy(in, filePath);
|
||||
writeOwner(fileId, owner);
|
||||
success = true;
|
||||
return new Stored(fileId, size);
|
||||
} finally {
|
||||
if (!success) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
cleanupAfterFailedStore(fileId, filePath);
|
||||
}
|
||||
release(fileId, lock);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,27 +69,44 @@ public class LocalDiskFileStore implements FileStore {
|
||||
* the source size before copying so the post-copy stat is unnecessary.
|
||||
*/
|
||||
@Override
|
||||
public Stored store(Path source, String originalName) throws IOException {
|
||||
public Stored store(Path source, String originalName, String owner) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = resolve(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
long size = Files.size(source);
|
||||
ReentrantLock lock = acquire(fileId);
|
||||
boolean success = false;
|
||||
try {
|
||||
Files.copy(source, filePath);
|
||||
writeOwner(fileId, owner);
|
||||
success = true;
|
||||
return new Stored(fileId, size);
|
||||
} finally {
|
||||
if (!success) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
cleanupAfterFailedStore(fileId, filePath);
|
||||
}
|
||||
release(fileId, lock);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeOwner(String fileId, String owner) throws IOException {
|
||||
if (owner == null || owner.isBlank()) {
|
||||
return;
|
||||
}
|
||||
Path ownerPath = resolveOwner(fileId);
|
||||
Files.write(ownerPath, owner.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
private void cleanupAfterFailedStore(String fileId, Path filePath) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn("Failed to clean up partial file {} after store failure", filePath, cleanupEx);
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(resolveOwner(fileId));
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn("Failed to clean up owner sidecar for {} after store failure", fileId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,11 +135,26 @@ public class LocalDiskFileStore implements FileStore {
|
||||
|
||||
@Override
|
||||
public boolean delete(String fileId) {
|
||||
ReentrantLock lock = acquire(fileId);
|
||||
try {
|
||||
return Files.deleteIfExists(resolve(fileId));
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
// Data first, owner second: a concurrent retrieve that observes the transient
|
||||
// (data-gone, owner-still-present) window simply fails with IOException; the inverse
|
||||
// order would briefly look like an unowned file and could grant cross-user access.
|
||||
boolean removed;
|
||||
try {
|
||||
removed = Files.deleteIfExists(resolve(fileId));
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(resolveOwner(fileId));
|
||||
} catch (IOException e) {
|
||||
log.warn("Error deleting owner sidecar for file ID: {}", fileId, e);
|
||||
}
|
||||
return removed;
|
||||
} finally {
|
||||
release(fileId, lock);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,8 +163,21 @@ public class LocalDiskFileStore implements FileStore {
|
||||
return Files.exists(resolve(fileId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOwner(String fileId) throws IOException {
|
||||
Path ownerPath = resolveOwner(fileId);
|
||||
if (!Files.exists(ownerPath)) {
|
||||
return null;
|
||||
}
|
||||
byte[] bytes = Files.readAllBytes(ownerPath);
|
||||
if (bytes.length == 0) {
|
||||
return null;
|
||||
}
|
||||
return new String(bytes, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public Path resolve(String fileId) {
|
||||
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
if (fileId == null || !UUID_PATTERN.matcher(fileId).matches()) {
|
||||
throw new IllegalArgumentException("Invalid file ID");
|
||||
}
|
||||
Path basePath = Path.of(baseDirPath).normalize().toAbsolutePath();
|
||||
@@ -125,4 +187,19 @@ public class LocalDiskFileStore implements FileStore {
|
||||
}
|
||||
return resolvedPath;
|
||||
}
|
||||
|
||||
private Path resolveOwner(String fileId) {
|
||||
Path data = resolve(fileId);
|
||||
return data.resolveSibling(data.getFileName().toString() + OWNER_SUFFIX);
|
||||
}
|
||||
|
||||
private ReentrantLock acquire(String fileId) {
|
||||
ReentrantLock lock = stripes[(fileId.hashCode() & Integer.MAX_VALUE) % LOCK_STRIPES];
|
||||
lock.lock();
|
||||
return lock;
|
||||
}
|
||||
|
||||
private void release(String fileId, ReentrantLock lock) {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,8 +77,10 @@ public class ApplicationProperties {
|
||||
private ProcessExecutor processExecutor = new ProcessExecutor();
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private Mcp mcp = new Mcp();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
private Cluster cluster = new Cluster();
|
||||
private Policies policies = new Policies();
|
||||
|
||||
@Bean
|
||||
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
|
||||
@@ -202,6 +204,45 @@ public class ApplicationProperties {
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Policies {
|
||||
/**
|
||||
* Absolute directories that policy folder input sources and output sinks may read from or
|
||||
* write to. Empty (the default) disables folder access entirely, so a policy can never be
|
||||
* pointed at an arbitrary server path. Stirling's own config directory is always
|
||||
* off-limits, and folder access is always disabled in SaaS mode regardless of this list.
|
||||
*/
|
||||
private List<String> allowedFolderRoots = new java.util.ArrayList<>();
|
||||
|
||||
/** How often (seconds) the schedule trigger checks for policies whose schedule is due. */
|
||||
private long scheduleSweepSeconds = 60;
|
||||
|
||||
/**
|
||||
* How often (seconds) the folder-watch trigger reconciles its watch registrations and
|
||||
* re-runs every folder-watch policy as a safety net for filesystem events that were missed
|
||||
* (NFS, bind mounts, inotify-queue overflow).
|
||||
*/
|
||||
private long watchReconcileSeconds = 300;
|
||||
|
||||
/**
|
||||
* How long (milliseconds) the folder-watch trigger keeps draining filesystem events after
|
||||
* the first, so a burst from a single file copy coalesces into one run instead of many.
|
||||
*/
|
||||
private long watchQuietPeriodMs = 500;
|
||||
|
||||
/**
|
||||
* SSE emitter timeout (milliseconds) for streamed runs; generous for long multi-step runs.
|
||||
*/
|
||||
private long streamTimeoutMs = 1800000;
|
||||
|
||||
/**
|
||||
* How long (minutes) a finished run's in-memory state is retained before eviction,
|
||||
* mirroring the job-result expiry so rich run state does not outlive the process. Active
|
||||
* and paused runs are kept regardless of age.
|
||||
*/
|
||||
private int runExpiryMinutes = 30;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class PdfEditor {
|
||||
private Cache cache = new Cache();
|
||||
@@ -256,6 +297,94 @@ public class ApplicationProperties {
|
||||
private int longRunningTimeoutSeconds = 600;
|
||||
}
|
||||
|
||||
/**
|
||||
* Model Context Protocol (MCP) server configuration. All keys live under the top-level {@code
|
||||
* mcp.*} prefix. {@link #enabled} defaults to {@code false}: when off, no MCP beans are wired,
|
||||
* no /mcp endpoint exists, and no protected-resource metadata is published.
|
||||
*/
|
||||
@Data
|
||||
public static class Mcp {
|
||||
|
||||
/** Master switch. When {@code false} (default), no MCP beans are wired. */
|
||||
private boolean enabled = false;
|
||||
|
||||
/**
|
||||
* When {@code true} (default), invocations require an OAuth scope: {@code mcp.tools.read}
|
||||
* for read-style operations and {@code mcp.tools.write} for write/destructive ones. When
|
||||
* {@code false}, scope checks are skipped (use only if your IdP issues a single coarse
|
||||
* scope).
|
||||
*/
|
||||
private boolean scopesEnabled = true;
|
||||
|
||||
/** How often to refresh the AI capabilities manifest from the engine. */
|
||||
private int engineCapabilityRefreshMinutes = 5;
|
||||
|
||||
/**
|
||||
* Tool allow-list (operation ids, e.g. {@code compress-pdf}). When non-empty, ONLY these
|
||||
* operations are exposed over MCP; everything else is hidden, undescribable, and
|
||||
* uninvocable - on top of the global endpoint enable/disable config. Empty = allow all.
|
||||
*/
|
||||
private List<String> allowedOperations = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* Tool deny-list (operation ids). Any operation listed here is removed from MCP even if it
|
||||
* would otherwise be allowed. Applied after {@link #allowedOperations}.
|
||||
*/
|
||||
private List<String> blockedOperations = new ArrayList<>();
|
||||
|
||||
/** Max MCP request body size in bytes; inline file uploads ride in the JSON-RPC body. */
|
||||
private long maxRequestBytes = 10L * 1024 * 1024;
|
||||
|
||||
/** Results up to this size return inline as base64; larger ones return a fileId only. */
|
||||
private long maxInlineResponseBytes = 10L * 1024 * 1024;
|
||||
|
||||
private Auth auth = new Auth();
|
||||
|
||||
@Data
|
||||
public static class Auth {
|
||||
/**
|
||||
* Authentication mode for the MCP endpoint. {@code oauth} (default) runs a full OAuth2
|
||||
* resource server (JWT, RFC 8707 audience, RFC 9728 metadata). {@code apikey} accepts a
|
||||
* Stirling per-user API key via the {@code X-API-KEY} header (or {@code Authorization:
|
||||
* Bearer <key>}) and binds the request to that user - the low-friction self-host path,
|
||||
* no external IdP required.
|
||||
*/
|
||||
private String mode = "oauth";
|
||||
|
||||
/** OAuth2 issuer URI, e.g. {@code http://localhost:9000}. Required when MCP is on. */
|
||||
private String issuerUri = "";
|
||||
|
||||
/**
|
||||
* JWKS URI. When blank, derived from the issuer's {@code
|
||||
* /.well-known/openid-configuration} document.
|
||||
*/
|
||||
private String jwksUri = "";
|
||||
|
||||
/**
|
||||
* RFC 8707 resource identifier of THIS MCP server, e.g. {@code
|
||||
* http://localhost:8080/mcp}. Tokens that do not list this id in their {@code aud}
|
||||
* claim are rejected with HTTP 401.
|
||||
*/
|
||||
private String resourceId = "";
|
||||
|
||||
/**
|
||||
* JWT claim whose value is matched against a provisioned Stirling username. Defaults to
|
||||
* {@code sub}; set to {@code email} or {@code preferred_username} to match how your IdP
|
||||
* maps users to Stirling accounts.
|
||||
*/
|
||||
private String usernameClaim = "sub";
|
||||
|
||||
/**
|
||||
* When {@code true} (default), a validated token is accepted only if its {@link
|
||||
* #usernameClaim} value resolves to an existing, enabled Stirling user account. Tokens
|
||||
* whose subject has no Stirling account (or a disabled one) are rejected with HTTP 403.
|
||||
* Set to {@code false} only if you intentionally want any IdP-valid token to use MCP
|
||||
* without a local account.
|
||||
*/
|
||||
private boolean requireExistingAccount = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cluster backplane configuration. All keys live under the top-level {@code cluster.*} prefix
|
||||
* (e.g. env var {@code CLUSTER_ENABLED}). The master switch is {@link #enabled} and defaults to
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import stirling.software.jpdfium.text.PageText;
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
final class HeadingDetector {
|
||||
|
||||
private HeadingDetector() {}
|
||||
|
||||
/** A heading is at most this many words; longer lines are treated as body text. */
|
||||
private static final int MAX_HEADING_WORDS = 12;
|
||||
|
||||
/**
|
||||
* Returns the Markdown heading prefix for a line. The decision combines several signals, never
|
||||
* text matching, so a plain line that merely shares text with a heading is never promoted:
|
||||
*
|
||||
* <ul>
|
||||
* <li><b>Size</b> — dominant glyph font size vs. the document body median (primary signal).
|
||||
* Some PDFs encode visual size in the text matrix, so every glyph reports ~1.0; for those
|
||||
* the line height is used as the proxy instead.
|
||||
* <li><b>Brevity</b> — headings are short labels; a line over {@value #MAX_HEADING_WORDS}
|
||||
* words is body text regardless of size.
|
||||
* <li><b>Not a sentence</b> — a line ending in {@code . ! ?} reads as prose, not a heading.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Boldness is deliberately <em>not</em> a heading signal — a bold-but-not-larger line is
|
||||
* emphasis, not a heading (see {@link #isBoldLabel}); promoting it to {@code #}/{@code ##} is
|
||||
* the main source of false-positive headings.
|
||||
*
|
||||
* <ul>
|
||||
* <li>size > baseline * 1.4 → {@code "# "}
|
||||
* <li>size > baseline * 1.2 → {@code "## "}
|
||||
* <li>otherwise → {@code ""}
|
||||
* </ul>
|
||||
*/
|
||||
static String headingPrefix(TextLine line, float medianBodySize, float medianBodyHeight) {
|
||||
String text = line.text().strip();
|
||||
if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
float dominant = dominantFontSize(line);
|
||||
float value;
|
||||
float baseline;
|
||||
if (dominant > 2f && medianBodySize > 2f) {
|
||||
value = dominant;
|
||||
baseline = medianBodySize;
|
||||
} else {
|
||||
value = line.height();
|
||||
baseline = medianBodyHeight;
|
||||
}
|
||||
if (baseline <= 0f) {
|
||||
return "";
|
||||
}
|
||||
|
||||
float ratio = value / baseline;
|
||||
if (ratio > 1.4f) {
|
||||
return "# ";
|
||||
}
|
||||
if (ratio > 1.2f) {
|
||||
return "## ";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a line should be emphasised as bold (rendered {@code **like this**}) rather than
|
||||
* promoted to a heading: it is bold, short, and not a full sentence. Used for bold labels that
|
||||
* are not large enough to be headings.
|
||||
*/
|
||||
static boolean isBoldLabel(TextLine line) {
|
||||
String text = line.text().strip();
|
||||
if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) {
|
||||
return false;
|
||||
}
|
||||
return isBold(line);
|
||||
}
|
||||
|
||||
private static int wordCount(String text) {
|
||||
return text.split("\\s+").length;
|
||||
}
|
||||
|
||||
private static boolean endsLikeSentence(String text) {
|
||||
char last = text.charAt(text.length() - 1);
|
||||
return last == '.' || last == '!' || last == '?';
|
||||
}
|
||||
|
||||
/** True when the line's dominant font is bold, inferred from PostScript font names. */
|
||||
private static boolean isBold(TextLine line) {
|
||||
Map<String, Integer> counts = new HashMap<>();
|
||||
for (TextWord word : line.words()) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (ch.isWhitespace() || ch.isNewline()) {
|
||||
continue;
|
||||
}
|
||||
String name = ch.fontName();
|
||||
if (name != null && !name.isBlank()) {
|
||||
counts.merge(name, 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
String dominantFont = "";
|
||||
int max = -1;
|
||||
for (Map.Entry<String, Integer> e : counts.entrySet()) {
|
||||
if (e.getValue() > max) {
|
||||
max = e.getValue();
|
||||
dominantFont = e.getKey();
|
||||
}
|
||||
}
|
||||
String lower = dominantFont.toLowerCase(java.util.Locale.ROOT);
|
||||
return lower.contains("bold")
|
||||
|| lower.contains("black")
|
||||
|| lower.contains("heavy")
|
||||
|| lower.contains("semibold");
|
||||
}
|
||||
|
||||
/** Computes the median glyph font size across all pages. */
|
||||
static float medianFontSize(List<PageText> allPages) {
|
||||
List<Float> sizes = new ArrayList<>();
|
||||
for (PageText page : allPages) {
|
||||
for (TextChar ch : page.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
|
||||
sizes.add(ch.fontSize());
|
||||
}
|
||||
}
|
||||
}
|
||||
return median(sizes, 12f);
|
||||
}
|
||||
|
||||
/** Computes the median TextLine height across all pages. Used when font size is degenerate. */
|
||||
static float medianLineHeight(List<PageText> allPages) {
|
||||
List<Float> heights = new ArrayList<>();
|
||||
for (PageText page : allPages) {
|
||||
for (TextLine line : page.lines()) {
|
||||
if (line.height() > 0f && !line.text().isBlank()) {
|
||||
heights.add(line.height());
|
||||
}
|
||||
}
|
||||
}
|
||||
return median(heights, 12f);
|
||||
}
|
||||
|
||||
private static float median(List<Float> values, float fallback) {
|
||||
if (values.isEmpty()) {
|
||||
return fallback;
|
||||
}
|
||||
Collections.sort(values);
|
||||
int mid = values.size() / 2;
|
||||
if (values.size() % 2 == 0) {
|
||||
return (values.get(mid - 1) + values.get(mid)) / 2f;
|
||||
}
|
||||
return values.get(mid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the font size that appears most often (by character count) in the given line. Ties
|
||||
* are broken in favour of the larger size.
|
||||
*/
|
||||
private static float dominantFontSize(TextLine line) {
|
||||
Map<Float, Integer> counts = new HashMap<>();
|
||||
for (TextWord word : line.words()) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
|
||||
counts.merge(ch.fontSize(), 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (counts.isEmpty()) {
|
||||
return 0f;
|
||||
}
|
||||
float dominant = 0f;
|
||||
int maxCount = -1;
|
||||
for (Map.Entry<Float, Integer> entry : counts.entrySet()) {
|
||||
int count = entry.getValue();
|
||||
float size = entry.getKey();
|
||||
if (count > maxCount || (count == maxCount && size > dominant)) {
|
||||
maxCount = count;
|
||||
dominant = size;
|
||||
}
|
||||
}
|
||||
return dominant;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,82 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import stirling.software.jpdfium.text.Table;
|
||||
|
||||
final class TableRenderer {
|
||||
private TableRenderer() {}
|
||||
|
||||
/** Renders a Table as a GitHub-Flavoured Markdown table string. */
|
||||
static String render(Table table) {
|
||||
if (table.rowCount() == 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
String[][] grid = table.asGrid();
|
||||
|
||||
if (table.rowCount() < 2) {
|
||||
// No separator row possible — return plain lines
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int c = 0; c < grid[0].length; c++) {
|
||||
if (c > 0) sb.append('\n');
|
||||
sb.append(escape(grid[0][c].trim()));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
int cols = grid[0].length;
|
||||
|
||||
// Compute column widths: max(3, max content length across all rows)
|
||||
int[] widths = new int[cols];
|
||||
for (int c = 0; c < cols; c++) {
|
||||
widths[c] = 3;
|
||||
}
|
||||
for (String[] row : grid) {
|
||||
for (int c = 0; c < cols; c++) {
|
||||
String cell = c < row.length ? row[c].trim() : "";
|
||||
widths[c] = Math.max(widths[c], escape(cell).length());
|
||||
}
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// Header row
|
||||
sb.append(buildRow(grid[0], widths, cols));
|
||||
sb.append('\n');
|
||||
|
||||
// Separator row
|
||||
sb.append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
|
||||
}
|
||||
sb.append('\n');
|
||||
|
||||
// Data rows
|
||||
for (int r = 1; r < grid.length; r++) {
|
||||
sb.append(buildRow(grid[r], widths, cols));
|
||||
if (r < grid.length - 1) {
|
||||
sb.append('\n');
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String buildRow(String[] row, int[] widths, int cols) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
String cell = c < row.length ? escape(row[c].trim()) : "";
|
||||
sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String escape(String cell) {
|
||||
return cell.replace("|", "\\|");
|
||||
}
|
||||
|
||||
private static String padRight(String s, int width) {
|
||||
if (s.length() >= width) return s;
|
||||
return s + " ".repeat(width - s.length());
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PipedInputStream;
|
||||
import java.io.PipedOutputStream;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
@@ -17,6 +18,7 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
import stirling.software.common.util.JobContext;
|
||||
|
||||
/**
|
||||
* Service for storing and retrieving files with unique file IDs. Used by the AutoJobPostMapping
|
||||
@@ -32,8 +34,10 @@ public class FileStorage {
|
||||
|
||||
private final FileOrUploadService fileOrUploadService;
|
||||
private final FileStore fileStore;
|
||||
private final Optional<JobOwnershipService> jobOwnershipService;
|
||||
|
||||
public String storeFile(MultipartFile file) throws IOException {
|
||||
String owner = resolveOwner();
|
||||
// Fast path: when Spring buffered the multipart to disk (typical for large uploads), the
|
||||
// backing Resource exposes a real File. Hand the Path to the FileStore so it can do a
|
||||
// file-to-file copy (Linux sendfile, no copy through Java heap) rather than streaming
|
||||
@@ -48,7 +52,7 @@ public class FileStorage {
|
||||
if (res != null && res.isFile()) {
|
||||
try {
|
||||
FileStore.Stored stored =
|
||||
fileStore.store(res.getFile().toPath(), file.getOriginalFilename());
|
||||
fileStore.store(res.getFile().toPath(), file.getOriginalFilename(), owner);
|
||||
log.debug("Stored file with ID: {} (fast path)", stored.fileId());
|
||||
return stored.fileId();
|
||||
} catch (IOException ex) {
|
||||
@@ -57,40 +61,45 @@ public class FileStorage {
|
||||
}
|
||||
}
|
||||
try (InputStream in = file.getInputStream()) {
|
||||
FileStore.Stored stored = fileStore.store(in, file.getOriginalFilename());
|
||||
FileStore.Stored stored = fileStore.store(in, file.getOriginalFilename(), owner);
|
||||
log.debug("Stored file with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
}
|
||||
|
||||
public String storeBytes(byte[] bytes, String originalName) throws IOException {
|
||||
FileStore.Stored stored = fileStore.store(new ByteArrayInputStream(bytes), originalName);
|
||||
FileStore.Stored stored =
|
||||
fileStore.store(new ByteArrayInputStream(bytes), originalName, resolveOwner());
|
||||
log.debug("Stored byte array with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
|
||||
public MultipartFile retrieveFile(String fileId) throws IOException {
|
||||
enforceOwnership(fileId);
|
||||
byte[] fileData = fileStore.retrieveBytes(fileId);
|
||||
return fileOrUploadService.toMockMultipartFile(fileId, fileData);
|
||||
}
|
||||
|
||||
public byte[] retrieveBytes(String fileId) throws IOException {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.retrieveBytes(fileId);
|
||||
}
|
||||
|
||||
public InputStream retrieveInputStream(String fileId) throws IOException {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.retrieve(fileId);
|
||||
}
|
||||
|
||||
public StoredFile storeInputStream(InputStream inputStream, String originalName)
|
||||
throws IOException {
|
||||
FileStore.Stored stored = fileStore.store(inputStream, originalName);
|
||||
FileStore.Stored stored = fileStore.store(inputStream, originalName, resolveOwner());
|
||||
log.debug("Stored input stream with ID: {}", stored.fileId());
|
||||
return new StoredFile(stored.fileId(), stored.size());
|
||||
}
|
||||
|
||||
public String storeFromStreamingBody(StreamingResponseBody body, String originalName)
|
||||
throws IOException {
|
||||
String owner = resolveOwner();
|
||||
// Hold Throwable not IOException: an unchecked failure (NPE, IllegalState, OOM, etc.)
|
||||
// from the body writer would otherwise close the pipe with EOF and the consumer would
|
||||
// return a truncated file with no error surfaced to the caller.
|
||||
@@ -115,7 +124,7 @@ public class FileStorage {
|
||||
}
|
||||
}
|
||||
});
|
||||
FileStore.Stored stored = fileStore.store(in, originalName);
|
||||
FileStore.Stored stored = fileStore.store(in, originalName, owner);
|
||||
Throwable writerErr = bodyError.get();
|
||||
if (writerErr != null) {
|
||||
// Body failed mid-write: the FileStore persisted a truncated entry.
|
||||
@@ -159,21 +168,62 @@ public class FileStorage {
|
||||
|
||||
public String storeFromResource(Resource resource, String originalName) throws IOException {
|
||||
try (InputStream in = resource.getInputStream()) {
|
||||
FileStore.Stored stored = fileStore.store(in, originalName);
|
||||
FileStore.Stored stored = fileStore.store(in, originalName, resolveOwner());
|
||||
log.debug("Stored Resource with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
}
|
||||
|
||||
public boolean deleteFile(String fileId) {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.delete(fileId);
|
||||
}
|
||||
|
||||
public boolean fileExists(String fileId) {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.exists(fileId);
|
||||
}
|
||||
|
||||
public long getFileSize(String fileId) throws IOException {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.size(fileId);
|
||||
}
|
||||
|
||||
private String resolveOwner() {
|
||||
String propagated = JobContext.getOwner();
|
||||
if (propagated != null) {
|
||||
return propagated;
|
||||
}
|
||||
return jobOwnershipService.flatMap(JobOwnershipService::getCurrentUserId).orElse(null);
|
||||
}
|
||||
|
||||
private void enforceOwnership(String fileId) {
|
||||
if (jobOwnershipService.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Optional<String> currentUser = jobOwnershipService.get().getCurrentUserId();
|
||||
if (currentUser.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String owner;
|
||||
try {
|
||||
owner = fileStore.getOwner(fileId);
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to read owner for file {}: {}", fileId, e.getMessage());
|
||||
throw new SecurityException(
|
||||
"Access denied: could not verify ownership of the requested file");
|
||||
}
|
||||
if (owner == null) {
|
||||
return;
|
||||
}
|
||||
if (!owner.equals(currentUser.get())) {
|
||||
log.warn(
|
||||
"Access denied: user {} attempted to access file {} owned by {}",
|
||||
currentUser.get(),
|
||||
fileId,
|
||||
owner);
|
||||
throw new SecurityException(
|
||||
"Access denied: you do not have permission to access this file");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +89,11 @@ public class JobExecutorService {
|
||||
|
||||
String jobId = scopedJobKey;
|
||||
|
||||
final String jobOwner =
|
||||
jobOwnershipService != null
|
||||
? jobOwnershipService.getCurrentUserId().orElse(null)
|
||||
: null;
|
||||
|
||||
long timeoutToUse = customTimeoutMs > 0 ? customTimeoutMs : effectiveTimeoutMs;
|
||||
|
||||
log.debug(
|
||||
@@ -119,6 +124,7 @@ public class JobExecutorService {
|
||||
try {
|
||||
stirling.software.common.util.JobContext.setJobId(
|
||||
capturedJobIdForQueue);
|
||||
stirling.software.common.util.JobContext.setOwner(jobOwner);
|
||||
Object result = work.get();
|
||||
processJobResult(capturedJobIdForQueue, result);
|
||||
return result;
|
||||
@@ -153,6 +159,7 @@ public class JobExecutorService {
|
||||
timeoutToUse);
|
||||
|
||||
stirling.software.common.util.JobContext.setJobId(capturedJobId);
|
||||
stirling.software.common.util.JobContext.setOwner(jobOwner);
|
||||
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
|
||||
processJobResult(capturedJobId, result);
|
||||
} catch (TimeoutException te) {
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Provides metadata about tool endpoints for internal dispatch. */
|
||||
public interface ToolMetadataService {
|
||||
|
||||
/** Returns true if the given operation path accepts multiple input files. */
|
||||
boolean isMultiInput(String operationPath);
|
||||
|
||||
/**
|
||||
* Returns the file extensions (lowercase, no leading dot, e.g. {@code "pdf"}) that the
|
||||
* operation accepts as input ({@code output=false}) or produces as output ({@code
|
||||
* output=true}), derived from the endpoint's declared type. Returns {@code null} when the
|
||||
* endpoint declares no specific type, which callers should treat as "any type accepted".
|
||||
*/
|
||||
List<String> getExtensionTypes(boolean output, String operationPath);
|
||||
|
||||
/**
|
||||
* Returns true when the endpoint's ZIP response is a transport for multiple typed results and
|
||||
* should be unpacked: multi-output endpoints (Type:SIMO / Type:MIMO) and wrapper declarations
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
/** Thread-local context for passing job ID across async boundaries */
|
||||
/** Thread-local context for passing job ID and owner across async boundaries */
|
||||
public class JobContext {
|
||||
private static final ThreadLocal<String> CURRENT_JOB_ID = new ThreadLocal<>();
|
||||
private static final ThreadLocal<String> CURRENT_OWNER = new ThreadLocal<>();
|
||||
|
||||
public static void setJobId(String jobId) {
|
||||
CURRENT_JOB_ID.set(jobId);
|
||||
@@ -12,7 +13,16 @@ public class JobContext {
|
||||
return CURRENT_JOB_ID.get();
|
||||
}
|
||||
|
||||
public static void setOwner(String owner) {
|
||||
CURRENT_OWNER.set(owner);
|
||||
}
|
||||
|
||||
public static String getOwner() {
|
||||
return CURRENT_OWNER.get();
|
||||
}
|
||||
|
||||
public static void clear() {
|
||||
CURRENT_JOB_ID.remove();
|
||||
CURRENT_OWNER.remove();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import javax.xml.XMLConstants;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.transform.OutputKeys;
|
||||
import javax.xml.transform.Transformer;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.transform.TransformerFactory;
|
||||
import javax.xml.transform.dom.DOMSource;
|
||||
import javax.xml.transform.stream.StreamResult;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NamedNodeMap;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import io.github.pixee.security.ZipSecurity;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.SsrfProtectionService;
|
||||
|
||||
// Strips external refs from OOXML/ODF uploads so LibreOffice can't be made to fetch them.
|
||||
@Component
|
||||
@Slf4j
|
||||
public class OfficeDocumentSanitizer {
|
||||
|
||||
private static final Set<String> OOXML_EXTENSIONS =
|
||||
Set.of(
|
||||
"docx", "docm", "dotx", "dotm", "xlsx", "xlsm", "xltx", "xltm", "pptx", "pptm",
|
||||
"potx", "potm", "ppsx", "ppsm");
|
||||
|
||||
private static final Set<String> ODF_EXTENSIONS =
|
||||
Set.of(
|
||||
"odt", "ott", "ods", "ots", "odp", "otp", "odg", "otg", "odf", "odc", "odi",
|
||||
"odm");
|
||||
|
||||
private static final Set<String> ODF_XML_PARTS =
|
||||
Set.of("content.xml", "styles.xml", "meta.xml", "settings.xml");
|
||||
|
||||
private final SsrfProtectionService ssrfProtectionService;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
public OfficeDocumentSanitizer(
|
||||
SsrfProtectionService ssrfProtectionService,
|
||||
ApplicationProperties applicationProperties) {
|
||||
this.ssrfProtectionService = ssrfProtectionService;
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
|
||||
public boolean isSanitizableExtension(String extension) {
|
||||
if (extension == null) {
|
||||
return false;
|
||||
}
|
||||
String lower = extension.toLowerCase(Locale.ROOT);
|
||||
return OOXML_EXTENSIONS.contains(lower) || ODF_EXTENSIONS.contains(lower);
|
||||
}
|
||||
|
||||
public byte[] sanitize(byte[] documentBytes, String extension) throws IOException {
|
||||
if (documentBytes == null || documentBytes.length == 0) {
|
||||
throw new IOException("Office document input is empty or null");
|
||||
}
|
||||
if (applicationProperties.getSystem().isDisableSanitize()) {
|
||||
log.debug("Office document sanitization disabled by configuration");
|
||||
return documentBytes;
|
||||
}
|
||||
if (!isSanitizableExtension(extension)) {
|
||||
return documentBytes;
|
||||
}
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream(documentBytes.length);
|
||||
try (ZipInputStream zipIn =
|
||||
ZipSecurity.createHardenedInputStream(
|
||||
new ByteArrayInputStream(documentBytes));
|
||||
ZipOutputStream zipOut = new ZipOutputStream(out)) {
|
||||
|
||||
ZipEntry entry;
|
||||
while ((entry = zipIn.getNextEntry()) != null) {
|
||||
String name = entry.getName();
|
||||
byte[] bytes = entry.isDirectory() ? new byte[0] : zipIn.readAllBytes();
|
||||
|
||||
if (!entry.isDirectory()) {
|
||||
bytes = sanitizeEntry(name, bytes);
|
||||
}
|
||||
|
||||
ZipEntry outEntry = new ZipEntry(name);
|
||||
if (entry.getComment() != null) {
|
||||
outEntry.setComment(entry.getComment());
|
||||
}
|
||||
if (entry.getExtra() != null) {
|
||||
outEntry.setExtra(entry.getExtra());
|
||||
}
|
||||
zipOut.putNextEntry(outEntry);
|
||||
if (!entry.isDirectory()) {
|
||||
zipOut.write(bytes);
|
||||
}
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private byte[] sanitizeEntry(String entryName, byte[] entryBytes) {
|
||||
String lower = entryName.toLowerCase(Locale.ROOT);
|
||||
try {
|
||||
if (lower.endsWith(".rels")) {
|
||||
return sanitizeOoxmlRels(entryBytes);
|
||||
}
|
||||
if (isOdfXmlPart(lower)) {
|
||||
return sanitizeOdfXml(entryBytes);
|
||||
}
|
||||
} catch (ParserConfigurationException
|
||||
| SAXException
|
||||
| IOException
|
||||
| TransformerException e) {
|
||||
log.warn(
|
||||
"Failed to parse XML part '{}' for sanitization, leaving as-is: {}",
|
||||
entryName,
|
||||
e.getMessage());
|
||||
}
|
||||
return entryBytes;
|
||||
}
|
||||
|
||||
private boolean isOdfXmlPart(String lowerName) {
|
||||
int slash = lowerName.lastIndexOf('/');
|
||||
String base = slash >= 0 ? lowerName.substring(slash + 1) : lowerName;
|
||||
return ODF_XML_PARTS.contains(base);
|
||||
}
|
||||
|
||||
private byte[] sanitizeOoxmlRels(byte[] xmlBytes)
|
||||
throws IOException, ParserConfigurationException, SAXException, TransformerException {
|
||||
Document doc = parseSecurely(xmlBytes);
|
||||
Element root = doc.getDocumentElement();
|
||||
if (root == null) {
|
||||
return xmlBytes;
|
||||
}
|
||||
NodeList relationships = root.getElementsByTagNameNS("*", "Relationship");
|
||||
List<Node> toRemove = new ArrayList<>();
|
||||
for (int i = 0; i < relationships.getLength(); i++) {
|
||||
Node node = relationships.item(i);
|
||||
NamedNodeMap attrs = node.getAttributes();
|
||||
if (attrs == null) {
|
||||
continue;
|
||||
}
|
||||
Node targetMode = attrs.getNamedItem("TargetMode");
|
||||
if (targetMode == null || !"external".equalsIgnoreCase(targetMode.getNodeValue())) {
|
||||
continue;
|
||||
}
|
||||
Node target = attrs.getNamedItem("Target");
|
||||
String targetValue = target == null ? "" : target.getNodeValue();
|
||||
if (isAdminAllowed(targetValue)) {
|
||||
continue;
|
||||
}
|
||||
log.warn(
|
||||
"Stripping OOXML external relationship target: {}",
|
||||
truncateForLog(targetValue));
|
||||
toRemove.add(node);
|
||||
}
|
||||
if (toRemove.isEmpty()) {
|
||||
return xmlBytes;
|
||||
}
|
||||
for (Node n : toRemove) {
|
||||
n.getParentNode().removeChild(n);
|
||||
}
|
||||
return serializeDocument(doc);
|
||||
}
|
||||
|
||||
private byte[] sanitizeOdfXml(byte[] xmlBytes)
|
||||
throws IOException, ParserConfigurationException, SAXException, TransformerException {
|
||||
Document doc = parseSecurely(xmlBytes);
|
||||
Element root = doc.getDocumentElement();
|
||||
if (root == null) {
|
||||
return xmlBytes;
|
||||
}
|
||||
boolean modified = stripExternalHrefs(root);
|
||||
if (!modified) {
|
||||
return xmlBytes;
|
||||
}
|
||||
return serializeDocument(doc);
|
||||
}
|
||||
|
||||
private boolean stripExternalHrefs(Node node) {
|
||||
boolean modified = false;
|
||||
if (node.getNodeType() == Node.ELEMENT_NODE) {
|
||||
NamedNodeMap attrs = node.getAttributes();
|
||||
List<String> hrefAttrsToRemove = new ArrayList<>();
|
||||
for (int i = 0; i < attrs.getLength(); i++) {
|
||||
Node attr = attrs.item(i);
|
||||
String name = attr.getNodeName();
|
||||
if (name == null) {
|
||||
continue;
|
||||
}
|
||||
String lower = name.toLowerCase(Locale.ROOT);
|
||||
if (!(lower.equals("xlink:href")
|
||||
|| lower.endsWith(":href")
|
||||
|| lower.equals("href"))) {
|
||||
continue;
|
||||
}
|
||||
String value = attr.getNodeValue();
|
||||
if (!isExternalUrl(value)) {
|
||||
continue;
|
||||
}
|
||||
if (isAdminAllowed(value)) {
|
||||
continue;
|
||||
}
|
||||
log.warn(
|
||||
"Stripping ODF external href attribute ({}): {}",
|
||||
name,
|
||||
truncateForLog(value));
|
||||
hrefAttrsToRemove.add(name);
|
||||
}
|
||||
Element element = (Element) node;
|
||||
for (String attrName : hrefAttrsToRemove) {
|
||||
element.removeAttribute(attrName);
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
NodeList children = node.getChildNodes();
|
||||
for (int i = 0; i < children.getLength(); i++) {
|
||||
if (stripExternalHrefs(children.item(i))) {
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
return modified;
|
||||
}
|
||||
|
||||
private boolean isExternalUrl(String url) {
|
||||
if (url == null) {
|
||||
return false;
|
||||
}
|
||||
String trimmed = url.trim().toLowerCase(Locale.ROOT);
|
||||
if (trimmed.isEmpty() || trimmed.startsWith("#") || trimmed.startsWith("../")) {
|
||||
return false;
|
||||
}
|
||||
return trimmed.startsWith("http://")
|
||||
|| trimmed.startsWith("https://")
|
||||
|| trimmed.startsWith("ftp://")
|
||||
|| trimmed.startsWith("ftps://")
|
||||
|| trimmed.startsWith("file:")
|
||||
|| trimmed.startsWith("smb:")
|
||||
|| trimmed.startsWith("\\\\")
|
||||
|| trimmed.startsWith("//");
|
||||
}
|
||||
|
||||
// Preserved only with an explicit allowedDomains entry; MEDIUM default would admit public URLs.
|
||||
private boolean isAdminAllowed(String url) {
|
||||
if (ssrfProtectionService == null || url == null || url.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
ApplicationProperties.Html.UrlSecurity config =
|
||||
applicationProperties.getSystem().getHtml().getUrlSecurity();
|
||||
if (config == null
|
||||
|| config.getAllowedDomains() == null
|
||||
|| config.getAllowedDomains().isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
return ssrfProtectionService.isUrlAllowed(url);
|
||||
}
|
||||
|
||||
private Document parseSecurely(byte[] xmlBytes)
|
||||
throws ParserConfigurationException, SAXException, IOException {
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
|
||||
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
|
||||
factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
|
||||
factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
|
||||
factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
|
||||
factory.setXIncludeAware(false);
|
||||
factory.setExpandEntityReferences(false);
|
||||
factory.setNamespaceAware(true);
|
||||
DocumentBuilder builder = factory.newDocumentBuilder();
|
||||
return builder.parse(new ByteArrayInputStream(xmlBytes));
|
||||
}
|
||||
|
||||
private byte[] serializeDocument(Document doc) throws TransformerException {
|
||||
TransformerFactory tf = TransformerFactory.newInstance();
|
||||
tf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
|
||||
Transformer transformer = tf.newTransformer();
|
||||
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
|
||||
transformer.setOutputProperty(OutputKeys.INDENT, "no");
|
||||
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no");
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
transformer.transform(new DOMSource(doc), new StreamResult(baos));
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
private String truncateForLog(String value) {
|
||||
if (value == null) {
|
||||
return "null";
|
||||
}
|
||||
return value.length() > 80 ? value.substring(0, 80) + "..." : value;
|
||||
}
|
||||
}
|
||||
-153
@@ -1,153 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and
|
||||
* column-grid construction.
|
||||
*/
|
||||
class LineAlignmentTableParserTest {
|
||||
|
||||
private final LineAlignmentTableParser parser = new LineAlignmentTableParser();
|
||||
|
||||
// ── mergeCoincidentLines ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_singleLine_unchanged() {
|
||||
var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue")));
|
||||
assertThat(parser.mergeCoincidentLines(lines)).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_distinctYLines_unchanged() {
|
||||
// Two lines at different y positions — must NOT be merged.
|
||||
var lines =
|
||||
List.of(
|
||||
tokenized(rawLine(10f, 100f, "Revenue")),
|
||||
tokenized(rawLine(10f, 115f, "Cost")));
|
||||
assertThat(parser.mergeCoincidentLines(lines)).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_sameY_merged() {
|
||||
// Simulates a financial-table row split by LineBuilder at the column gap:
|
||||
// label fragment at x=72 → "Revenue"
|
||||
// value fragment at x=350 → "1,234"
|
||||
// Both have y=100. After merge they should form one TokenizedLine.
|
||||
var label = rawLine(72f, 100f, "Revenue");
|
||||
var value = rawLine(350f, 100f, "1,234");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
|
||||
|
||||
assertThat(merged).hasSize(1);
|
||||
// The merged line should contain tokens from both halves.
|
||||
var tokens = merged.get(0).all();
|
||||
assertThat(tokens.stream().map(t -> t.text()).toList())
|
||||
.containsExactlyInAnyOrder("Revenue", "1,234");
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() {
|
||||
var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114
|
||||
var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
|
||||
|
||||
var bounds = merged.get(0).line().bounds();
|
||||
assertThat(bounds.x()).isEqualTo(72f);
|
||||
assertThat(bounds.right()).isEqualTo(380f);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_withinTolerance_merged() {
|
||||
// Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge.
|
||||
var a = rawLine(10f, 100.0f, "Alpha");
|
||||
var b = rawLine(200f, 101.5f, "99");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
|
||||
assertThat(merged).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_beyondTolerance_notMerged() {
|
||||
// Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge.
|
||||
var a = rawLine(10f, 100.0f, "Alpha");
|
||||
var b = rawLine(200f, 103.0f, "99");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
|
||||
assertThat(merged).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_threeCoincident_allMerged() {
|
||||
// Three fragments at the same y (e.g. wide financial table with two value columns).
|
||||
var a = rawLine(72f, 100f, "Revenue");
|
||||
var b = rawLine(300f, 100f, "1,234");
|
||||
var c = rawLine(400f, 100f, "5,678");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
|
||||
assertThat(merged).hasSize(1);
|
||||
assertThat(merged.get(0).all()).hasSize(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() {
|
||||
var a = rawLine(72f, 100f, "Revenue");
|
||||
var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a
|
||||
var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
|
||||
assertThat(merged).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() {
|
||||
// After merging, the combined line should be an anchor (≥2 numeric tokens).
|
||||
// "Revenue" alone → not an anchor. "1,234 567" alone → anchor.
|
||||
// Merged → anchor with at least 2 numerics.
|
||||
var label = rawLine(72f, 100f, "Revenue");
|
||||
var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f);
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values)));
|
||||
|
||||
assertThat(merged).hasSize(1);
|
||||
assertThat(merged.get(0).isAnchor()).isTrue();
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Creates a RawLine with a single TextFragment of the given text at the given position. */
|
||||
private static RawLine rawLine(float x, float y, String text) {
|
||||
float width = text.length() * 6f; // ~6pt per char — rough but consistent
|
||||
float height = 12f;
|
||||
Bounds bounds = new Bounds(x, y, width, height);
|
||||
TextFragment fragment =
|
||||
new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false);
|
||||
return new RawLine("ln-test", List.of(fragment), bounds, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a RawLine with two TextFragments representing two words separated by a small gap.
|
||||
* Used to simulate a values-only line with multiple numeric tokens.
|
||||
*/
|
||||
private static RawLine rawLineMultiWord(
|
||||
float x, float y, String word1, float w1, String word2, float w2) {
|
||||
float height = 12f;
|
||||
Bounds b1 = new Bounds(x, y, w1, height);
|
||||
Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height);
|
||||
TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false);
|
||||
TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false);
|
||||
Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height);
|
||||
return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1);
|
||||
}
|
||||
|
||||
/** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */
|
||||
private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) {
|
||||
return parser.tokenize(line);
|
||||
}
|
||||
}
|
||||
+44
@@ -3,11 +3,13 @@ package stirling.software.common.cluster.inprocess;
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -39,4 +41,46 @@ class LocalDiskFileStoreTest {
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("a/b"));
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("a\\b"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ownerSidecarCannotBeReadAsFileId(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
FileStore.Stored stored =
|
||||
store.store(new ByteArrayInputStream("hi".getBytes()), "f.bin", "alice");
|
||||
String sidecarId = stored.fileId() + ".owner";
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve(sidecarId));
|
||||
assertThrows(IllegalArgumentException.class, () -> store.retrieveBytes(sidecarId));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ownerIsPersistedAndReturnedByGetOwner(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
FileStore.Stored stored =
|
||||
store.store(new ByteArrayInputStream("hi".getBytes()), "f.bin", "alice");
|
||||
assertEquals("alice", store.getOwner(stored.fileId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void getOwnerReturnsNullWhenNoOwnerWasRecorded(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
FileStore.Stored stored =
|
||||
store.store(new ByteArrayInputStream("hi".getBytes()), "f.bin", null);
|
||||
assertNull(store.getOwner(stored.fileId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void getOwnerReturnsNullForUnknownFileId(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
assertNull(store.getOwner("00000000-0000-0000-0000-000000000000"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteRemovesOwnerSidecar(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
FileStore.Stored stored =
|
||||
store.store(new ByteArrayInputStream("hi".getBytes()), "f.bin", "alice");
|
||||
assertTrue(store.delete(stored.fileId()));
|
||||
assertFalse(Files.exists(dir.resolve(stored.fileId() + ".owner")));
|
||||
assertNull(store.getOwner(stored.fileId()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Accuracy and robustness tests for {@link PdfMarkdownConverter}, comparing conversion output
|
||||
* against hand-authored golden Markdown for a set of owned/synthetic fixtures.
|
||||
*
|
||||
* <p>The {@link #gatedFixtures()} set is enforced in CI: those fixtures currently convert within
|
||||
* the accuracy threshold and guard against regressions. Fixtures still being iterated on live in
|
||||
* {@link #wipFixtures()} under a {@link Disabled} test so the goldens stay in the tree without
|
||||
* breaking the build. Enable the WIP test locally to see per-fixture scores while working on the
|
||||
* converter.
|
||||
*/
|
||||
class PdfMarkdownConverterTest {
|
||||
|
||||
/** Accuracy threshold: output must share at least this fraction of content with the golden. */
|
||||
private static final double THRESHOLD = 0.95;
|
||||
|
||||
@TempDir Path tmp;
|
||||
|
||||
/** Fixtures that meet the accuracy threshold today and therefore gate CI. */
|
||||
static Stream<Arguments> gatedFixtures() {
|
||||
return Stream.of(
|
||||
Arguments.of("multi-column-test_lorem.pdf", "multi-column-test_lorem.md"),
|
||||
Arguments.of("bordered-table-test_widget.pdf", "bordered-table-test_widget.md"),
|
||||
Arguments.of("many-tables-test_stress.pdf", "many-tables-test_stress.md"));
|
||||
}
|
||||
|
||||
/** Fixtures still below the threshold; tracked here, enable locally to iterate. */
|
||||
static Stream<Arguments> wipFixtures() {
|
||||
return Stream.of(
|
||||
Arguments.of(
|
||||
"wrapped-cell-test_expense-report.pdf",
|
||||
"wrapped-cell-test_expense-report.md"));
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@MethodSource("gatedFixtures")
|
||||
void convertMatchesGoldenMarkdown(String pdfName, String mdName) throws IOException {
|
||||
assertConversionMatchesGolden(pdfName, mdName);
|
||||
}
|
||||
|
||||
@Disabled("WIP fixtures below the accuracy threshold; enable locally to iterate")
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@MethodSource("wipFixtures")
|
||||
void convertMatchesGoldenMarkdownWip(String pdfName, String mdName) throws IOException {
|
||||
assertConversionMatchesGolden(pdfName, mdName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Degenerate/extreme geometry must not crash the converter. A crafted or malformed PDF can
|
||||
* position text anywhere via a text matrix, so a row's words can span from near the origin to a
|
||||
* coordinate beyond {@link Integer#MAX_VALUE}. The old column-detection code sized an {@code
|
||||
* int[]} straight from {@code (int) Math.ceil(maxX) - lo}, which either allocated a multi-GB
|
||||
* array (OutOfMemoryError) or overflowed to a negative length (NegativeArraySizeException) —
|
||||
* taking down the request thread. Detection must instead bail out and return no columns.
|
||||
*/
|
||||
@Test
|
||||
void columnDetectionSurvivesDegenerateGeometry() {
|
||||
// x ≈ 2.5e9 is past Integer.MAX_VALUE; combined with a near-origin word it yields an
|
||||
// implausible span that the pre-fix code turned into a fatal array allocation.
|
||||
List<TextLine> rows = new ArrayList<>();
|
||||
for (int r = 0; r < 4; r++) {
|
||||
float y = 400f - r * 12f;
|
||||
TextWord near = new TextWord(List.of(), 50f, y, 30f, 10f);
|
||||
TextWord far = new TextWord(List.of(), 2_500_000_000f, y, 30f, 10f);
|
||||
rows.add(new TextLine(List.of(near, far), 50f, y, 2_499_999_980f, 10f));
|
||||
}
|
||||
|
||||
List<float[]> columns =
|
||||
assertDoesNotThrow(() -> PdfMarkdownConverter.findColumnRangesFromLines(rows));
|
||||
assertTrue(
|
||||
columns.isEmpty(),
|
||||
"implausible page span should disable column detection, not allocate from it");
|
||||
}
|
||||
|
||||
private void assertConversionMatchesGolden(String pdfName, String mdName) throws IOException {
|
||||
Path pdfPath = tmp.resolve(pdfName);
|
||||
try (InputStream in =
|
||||
getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + pdfName)) {
|
||||
if (in == null) {
|
||||
fail("Fixture not found on classpath: /pdf-ingestion-fixtures/" + pdfName);
|
||||
}
|
||||
Files.copy(in, pdfPath);
|
||||
}
|
||||
|
||||
String actual;
|
||||
try (PdfDocument doc = PdfDocument.open(pdfPath)) {
|
||||
actual = new PdfMarkdownConverter().convert(doc);
|
||||
}
|
||||
|
||||
String expected;
|
||||
try (InputStream in = getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + mdName)) {
|
||||
if (in == null) {
|
||||
fail("Golden file not found on classpath: /pdf-ingestion-fixtures/" + mdName);
|
||||
}
|
||||
expected = new String(in.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
// Image placeholders are not scored: their body text is a TODO ("ideally, add the info
|
||||
// available about the image...") rather than real content, so comparing it would penalise
|
||||
// output for matching a placeholder we intend to replace. Drop those lines from both sides.
|
||||
expected = stripImagePlaceholders(expected);
|
||||
actual = stripImagePlaceholders(actual);
|
||||
|
||||
double similarity = similarity(expected, actual);
|
||||
if (similarity < THRESHOLD) {
|
||||
fail(
|
||||
String.format(
|
||||
"Markdown output differs from golden file '%s' by %.1f%% (threshold %.0f%%):%n%s",
|
||||
mdName,
|
||||
(1.0 - similarity) * 100,
|
||||
(1.0 - THRESHOLD) * 100,
|
||||
unifiedDiff(expected, actual)));
|
||||
}
|
||||
}
|
||||
|
||||
/** Substring identifying an image-placeholder line, which is excluded from scoring. */
|
||||
private static final String IMAGE_PLACEHOLDER_MARKER = "Image intentionally redacted";
|
||||
|
||||
/**
|
||||
* Removes non-content lines from the comparison: image placeholders (TODO text we intend to
|
||||
* replace) and GFM table separator rows (the {@code |---|---|} divider, whose exact dash count
|
||||
* is cosmetic — any run of three or more dashes is valid Markdown).
|
||||
*/
|
||||
private static String stripImagePlaceholders(String md) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String line : md.split("\n", -1)) {
|
||||
if (line.contains(IMAGE_PLACEHOLDER_MARKER)
|
||||
|| line.strip().startsWith("<image redacted")
|
||||
|| isTableSeparatorRow(line)) {
|
||||
continue;
|
||||
}
|
||||
if (sb.length() > 0) {
|
||||
sb.append('\n');
|
||||
}
|
||||
sb.append(line);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/** True for a GFM table separator row, e.g. {@code |---|:--:|---|} (only |, -, :, space). */
|
||||
private static boolean isTableSeparatorRow(String line) {
|
||||
String t = line.strip();
|
||||
if (!t.contains("-")) {
|
||||
return false;
|
||||
}
|
||||
return t.chars().allMatch(c -> c == '|' || c == '-' || c == ':' || c == ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Character-level similarity: proportion of expected characters that appear in the LCS. O(n*m)
|
||||
* but golden files are small enough that this is fine.
|
||||
*/
|
||||
private static double similarity(String expected, String actual) {
|
||||
if (expected.isEmpty() && actual.isEmpty()) return 1.0;
|
||||
if (expected.isEmpty() || actual.isEmpty()) return 0.0;
|
||||
// Strip all whitespace for a content-focused comparison
|
||||
String e = expected.replaceAll("\\s+", " ").strip();
|
||||
String a = actual.replaceAll("\\s+", " ").strip();
|
||||
int lcs = lcsLength(e, a);
|
||||
return (double) lcs / Math.max(e.length(), a.length());
|
||||
}
|
||||
|
||||
private static int lcsLength(String a, String b) {
|
||||
// Use two-row DP to keep memory reasonable
|
||||
int m = a.length(), n = b.length();
|
||||
int[] prev = new int[n + 1];
|
||||
int[] curr = new int[n + 1];
|
||||
for (int i = 1; i <= m; i++) {
|
||||
for (int j = 1; j <= n; j++) {
|
||||
if (a.charAt(i - 1) == b.charAt(j - 1)) {
|
||||
curr[j] = prev[j - 1] + 1;
|
||||
} else {
|
||||
curr[j] = Math.max(curr[j - 1], prev[j]);
|
||||
}
|
||||
}
|
||||
int[] tmp = prev;
|
||||
prev = curr;
|
||||
curr = tmp;
|
||||
java.util.Arrays.fill(curr, 0);
|
||||
}
|
||||
return prev[n];
|
||||
}
|
||||
|
||||
private static String unifiedDiff(String expected, String actual) {
|
||||
String[] expectedLines = expected.split("\n", -1);
|
||||
String[] actualLines = actual.split("\n", -1);
|
||||
|
||||
List<String> diff = new ArrayList<>();
|
||||
diff.add("--- expected");
|
||||
diff.add("+++ actual");
|
||||
|
||||
int maxLines = Math.max(expectedLines.length, actualLines.length);
|
||||
int context = 3;
|
||||
boolean inHunk = false;
|
||||
int hunkStart = -1;
|
||||
List<String> hunkLines = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < maxLines; i++) {
|
||||
String exp = i < expectedLines.length ? expectedLines[i] : null;
|
||||
String act = i < actualLines.length ? actualLines[i] : null;
|
||||
|
||||
boolean changed = exp == null || act == null || !exp.equals(act);
|
||||
if (changed) {
|
||||
if (!inHunk) {
|
||||
inHunk = true;
|
||||
hunkStart = Math.max(0, i - context);
|
||||
// add context lines before change
|
||||
for (int c = hunkStart; c < i; c++) {
|
||||
hunkLines.add(" " + (c < expectedLines.length ? expectedLines[c] : ""));
|
||||
}
|
||||
}
|
||||
if (exp != null) hunkLines.add("-" + exp);
|
||||
if (act != null) hunkLines.add("+" + act);
|
||||
} else {
|
||||
if (inHunk) {
|
||||
hunkLines.add(" " + exp);
|
||||
// check if we're far enough past the last change to close the hunk
|
||||
boolean moreChanges = false;
|
||||
for (int j = i + 1; j < Math.min(i + context, maxLines); j++) {
|
||||
String e2 = j < expectedLines.length ? expectedLines[j] : null;
|
||||
String a2 = j < actualLines.length ? actualLines[j] : null;
|
||||
if (e2 == null || a2 == null || !e2.equals(a2)) {
|
||||
moreChanges = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!moreChanges && (i - hunkStart) >= context) {
|
||||
diff.add("@@ -" + (hunkStart + 1) + " @@");
|
||||
diff.addAll(hunkLines);
|
||||
hunkLines.clear();
|
||||
inHunk = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (inHunk && !hunkLines.isEmpty()) {
|
||||
diff.add("@@ -" + (hunkStart + 1) + " @@");
|
||||
diff.addAll(hunkLines);
|
||||
}
|
||||
|
||||
return String.join("\n", diff);
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -5,6 +5,7 @@ import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
@@ -19,7 +20,8 @@ class FileStorageDelegationTest {
|
||||
FileStorage fs =
|
||||
new FileStorage(
|
||||
mock(FileOrUploadService.class),
|
||||
new LocalDiskFileStore(tempDir.toString()));
|
||||
new LocalDiskFileStore(tempDir.toString()),
|
||||
Optional.empty());
|
||||
byte[] payload = "round-trip".getBytes();
|
||||
String id = fs.storeBytes(payload, "x.bin");
|
||||
assertArrayEquals(payload, fs.retrieveBytes(id));
|
||||
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import stirling.software.common.cluster.inprocess.LocalDiskFileStore;
|
||||
import stirling.software.common.util.JobContext;
|
||||
|
||||
class FileStorageOwnershipTest {
|
||||
|
||||
private FileStorage newStorageWithoutSecurity(Path tempDir) {
|
||||
return new FileStorage(
|
||||
mock(FileOrUploadService.class),
|
||||
new LocalDiskFileStore(tempDir.toString()),
|
||||
Optional.empty());
|
||||
}
|
||||
|
||||
private FileStorage newStorageWithCurrentUser(Path tempDir, AtomicReference<String> userRef) {
|
||||
JobOwnershipService svc = mock(JobOwnershipService.class);
|
||||
when(svc.getCurrentUserId()).thenAnswer(invocation -> Optional.ofNullable(userRef.get()));
|
||||
return new FileStorage(
|
||||
mock(FileOrUploadService.class),
|
||||
new LocalDiskFileStore(tempDir.toString()),
|
||||
Optional.of(svc));
|
||||
}
|
||||
|
||||
@Test
|
||||
void desktopMode_noOwnershipService_storesAndRetrievesWithoutChecks(@TempDir Path tempDir)
|
||||
throws IOException {
|
||||
FileStorage fs = newStorageWithoutSecurity(tempDir);
|
||||
byte[] payload = "desktop".getBytes();
|
||||
String id = fs.storeBytes(payload, "x.bin");
|
||||
assertArrayEquals(payload, fs.retrieveBytes(id));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sameUserStoresAndRetrieves_allowed(@TempDir Path tempDir) throws IOException {
|
||||
AtomicReference<String> user = new AtomicReference<>("alice");
|
||||
FileStorage fs = newStorageWithCurrentUser(tempDir, user);
|
||||
byte[] payload = "alice's file".getBytes();
|
||||
String id = fs.storeBytes(payload, "x.bin");
|
||||
assertArrayEquals(payload, fs.retrieveBytes(id));
|
||||
}
|
||||
|
||||
@Test
|
||||
void differentUserRetrieves_throwsSecurityException(@TempDir Path tempDir) throws IOException {
|
||||
AtomicReference<String> user = new AtomicReference<>("alice");
|
||||
FileStorage fs = newStorageWithCurrentUser(tempDir, user);
|
||||
String id = fs.storeBytes("alice's file".getBytes(), "x.bin");
|
||||
user.set("bob");
|
||||
assertThrows(SecurityException.class, () -> fs.retrieveBytes(id));
|
||||
assertThrows(SecurityException.class, () -> fs.retrieveInputStream(id));
|
||||
assertThrows(SecurityException.class, () -> fs.getFileSize(id));
|
||||
assertThrows(SecurityException.class, () -> fs.fileExists(id));
|
||||
assertThrows(SecurityException.class, () -> fs.deleteFile(id));
|
||||
}
|
||||
|
||||
@Test
|
||||
void anonymousRetrieveOfOwnedFile_allowed_noCurrentUserMeansNoCompare(@TempDir Path tempDir)
|
||||
throws IOException {
|
||||
AtomicReference<String> user = new AtomicReference<>("alice");
|
||||
FileStorage fs = newStorageWithCurrentUser(tempDir, user);
|
||||
byte[] payload = "alice's file".getBytes();
|
||||
String id = fs.storeBytes(payload, "x.bin");
|
||||
user.set(null);
|
||||
assertArrayEquals(payload, fs.retrieveBytes(id));
|
||||
}
|
||||
|
||||
@Test
|
||||
void authedRetrieveOfAnonymousFile_allowed_noOwnerOnFile(@TempDir Path tempDir)
|
||||
throws IOException {
|
||||
AtomicReference<String> user = new AtomicReference<>(null);
|
||||
FileStorage fs = newStorageWithCurrentUser(tempDir, user);
|
||||
byte[] payload = "no-owner".getBytes();
|
||||
String id = fs.storeBytes(payload, "x.bin");
|
||||
user.set("alice");
|
||||
assertArrayEquals(payload, fs.retrieveBytes(id));
|
||||
}
|
||||
|
||||
@Test
|
||||
void propagatedOwner_scopesAsyncWriteWithNoLiveUser(@TempDir Path tempDir) throws IOException {
|
||||
AtomicReference<String> user = new AtomicReference<>(null);
|
||||
FileStorage fs = newStorageWithCurrentUser(tempDir, user);
|
||||
byte[] payload = "alice's async result".getBytes();
|
||||
String id;
|
||||
try {
|
||||
JobContext.setOwner("alice");
|
||||
id = fs.storeBytes(payload, "x.bin");
|
||||
} finally {
|
||||
JobContext.clear();
|
||||
}
|
||||
user.set("alice");
|
||||
assertArrayEquals(payload, fs.retrieveBytes(id));
|
||||
user.set("bob");
|
||||
assertThrows(SecurityException.class, () -> fs.retrieveBytes(id));
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -37,7 +39,10 @@ class FileStorageTest {
|
||||
void setUp() throws IOException {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
fileStorage =
|
||||
new FileStorage(fileOrUploadService, new LocalDiskFileStore(tempDir.toString()));
|
||||
new FileStorage(
|
||||
fileOrUploadService,
|
||||
new LocalDiskFileStore(tempDir.toString()),
|
||||
Optional.empty());
|
||||
|
||||
// Create a mock MultipartFile
|
||||
mockFile = mock(MultipartFile.class);
|
||||
@@ -79,7 +84,7 @@ class FileStorageTest {
|
||||
void testRetrieveFile() throws IOException {
|
||||
// Arrange
|
||||
byte[] fileContent = "Test PDF content".getBytes();
|
||||
String fileId = "test-file-1";
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = tempDir.resolve(fileId);
|
||||
Files.write(filePath, fileContent);
|
||||
|
||||
@@ -99,7 +104,7 @@ class FileStorageTest {
|
||||
void testRetrieveBytes() throws IOException {
|
||||
// Arrange
|
||||
byte[] fileContent = "Test PDF content".getBytes();
|
||||
String fileId = "test-file-2";
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = tempDir.resolve(fileId);
|
||||
Files.write(filePath, fileContent);
|
||||
|
||||
@@ -113,7 +118,7 @@ class FileStorageTest {
|
||||
@Test
|
||||
void testRetrieveFile_FileNotFound() {
|
||||
// Arrange
|
||||
String nonExistentFileId = "non-existent-file";
|
||||
String nonExistentFileId = UUID.randomUUID().toString();
|
||||
|
||||
// Act & Assert
|
||||
assertThrows(IOException.class, () -> fileStorage.retrieveFile(nonExistentFileId));
|
||||
@@ -122,7 +127,7 @@ class FileStorageTest {
|
||||
@Test
|
||||
void testRetrieveBytes_FileNotFound() {
|
||||
// Arrange
|
||||
String nonExistentFileId = "non-existent-file";
|
||||
String nonExistentFileId = UUID.randomUUID().toString();
|
||||
|
||||
// Act & Assert
|
||||
assertThrows(IOException.class, () -> fileStorage.retrieveBytes(nonExistentFileId));
|
||||
@@ -132,7 +137,7 @@ class FileStorageTest {
|
||||
void testDeleteFile() throws IOException {
|
||||
// Arrange
|
||||
byte[] fileContent = "Test PDF content".getBytes();
|
||||
String fileId = "test-file-3";
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = tempDir.resolve(fileId);
|
||||
Files.write(filePath, fileContent);
|
||||
|
||||
@@ -147,7 +152,7 @@ class FileStorageTest {
|
||||
@Test
|
||||
void testDeleteFile_FileNotFound() {
|
||||
// Arrange
|
||||
String nonExistentFileId = "non-existent-file";
|
||||
String nonExistentFileId = UUID.randomUUID().toString();
|
||||
|
||||
// Act
|
||||
boolean result = fileStorage.deleteFile(nonExistentFileId);
|
||||
@@ -160,7 +165,7 @@ class FileStorageTest {
|
||||
void testFileExists() throws IOException {
|
||||
// Arrange
|
||||
byte[] fileContent = "Test PDF content".getBytes();
|
||||
String fileId = "test-file-4";
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = tempDir.resolve(fileId);
|
||||
Files.write(filePath, fileContent);
|
||||
|
||||
@@ -174,7 +179,7 @@ class FileStorageTest {
|
||||
@Test
|
||||
void testFileExists_FileNotFound() {
|
||||
// Arrange
|
||||
String nonExistentFileId = "non-existent-file";
|
||||
String nonExistentFileId = UUID.randomUUID().toString();
|
||||
|
||||
// Act
|
||||
boolean result = fileStorage.fileExists(nonExistentFileId);
|
||||
|
||||
+370
@@ -0,0 +1,370 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.SsrfProtectionService;
|
||||
|
||||
class OfficeDocumentSanitizerTest {
|
||||
|
||||
private static final String EXTERNAL_URL = "https://webhook.site/ssrf-callback";
|
||||
private static final String INTERNAL_TARGET = "media/image1.png";
|
||||
|
||||
private static final String DOCX_RELS =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
+ "<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\">"
|
||||
+ "<Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image\""
|
||||
+ " Target=\""
|
||||
+ EXTERNAL_URL
|
||||
+ "\" TargetMode=\"External\"/>"
|
||||
+ "<Relationship Id=\"rId2\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image\""
|
||||
+ " Target=\""
|
||||
+ INTERNAL_TARGET
|
||||
+ "\"/>"
|
||||
+ "</Relationships>";
|
||||
|
||||
private static final String DOCX_DOCUMENT =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
+ "<w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\">"
|
||||
+ "<w:body><w:p/></w:body></w:document>";
|
||||
|
||||
private static final String ODF_CONTENT_EXTERNAL =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
+ "<office:document-content"
|
||||
+ " xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
|
||||
+ " xmlns:draw=\"urn:oasis:names:tc:opendocument:xmlns:drawing:1.0\""
|
||||
+ " xmlns:xlink=\"http://www.w3.org/1999/xlink\">"
|
||||
+ "<office:body><office:text>"
|
||||
+ "<draw:frame><draw:image xlink:href=\""
|
||||
+ EXTERNAL_URL
|
||||
+ "\" xlink:type=\"simple\"/></draw:frame>"
|
||||
+ "<draw:frame><draw:image xlink:href=\"Pictures/image1.png\" xlink:type=\"simple\"/></draw:frame>"
|
||||
+ "</office:text></office:body></office:document-content>";
|
||||
|
||||
private SsrfProtectionService ssrfProtectionService;
|
||||
private ApplicationProperties applicationProperties;
|
||||
private OfficeDocumentSanitizer sanitizer;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
applicationProperties = new ApplicationProperties();
|
||||
ssrfProtectionService = mock(SsrfProtectionService.class);
|
||||
sanitizer = new OfficeDocumentSanitizer(ssrfProtectionService, applicationProperties);
|
||||
}
|
||||
|
||||
@Test
|
||||
void isSanitizableExtension_recognizesOoxmlAndOdf() {
|
||||
assertTrue(sanitizer.isSanitizableExtension("docx"));
|
||||
assertTrue(sanitizer.isSanitizableExtension("DOCX"));
|
||||
assertTrue(sanitizer.isSanitizableExtension("xlsx"));
|
||||
assertTrue(sanitizer.isSanitizableExtension("pptx"));
|
||||
assertTrue(sanitizer.isSanitizableExtension("odt"));
|
||||
assertTrue(sanitizer.isSanitizableExtension("ods"));
|
||||
assertTrue(sanitizer.isSanitizableExtension("odp"));
|
||||
assertFalse(sanitizer.isSanitizableExtension("pdf"));
|
||||
assertFalse(sanitizer.isSanitizableExtension("html"));
|
||||
assertFalse(sanitizer.isSanitizableExtension(""));
|
||||
assertFalse(sanitizer.isSanitizableExtension(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_stripsOoxmlExternalRelationship() throws IOException {
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8));
|
||||
entries.put("word/document.xml", DOCX_DOCUMENT.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] docx = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(docx, "docx");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String rels =
|
||||
new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8);
|
||||
assertFalse(rels.contains(EXTERNAL_URL), "External URL should be stripped from .rels");
|
||||
assertFalse(
|
||||
rels.toLowerCase().contains("targetmode=\"external\""),
|
||||
"TargetMode=External relationship should be removed");
|
||||
assertTrue(rels.contains(INTERNAL_TARGET), "Internal image target should be preserved");
|
||||
assertArrayEquals(
|
||||
DOCX_DOCUMENT.getBytes(StandardCharsets.UTF_8),
|
||||
result.get("word/document.xml"),
|
||||
"Non-rels entries must be untouched");
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_pptxExternalImageRelStripped() throws IOException {
|
||||
String pptxRels =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
+ "<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\">"
|
||||
+ "<Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image\""
|
||||
+ " Target=\""
|
||||
+ EXTERNAL_URL
|
||||
+ "\" TargetMode=\"External\"/>"
|
||||
+ "</Relationships>";
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("ppt/slides/_rels/slide1.xml.rels", pptxRels.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] pptx = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(pptx, "pptx");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String rels =
|
||||
new String(result.get("ppt/slides/_rels/slide1.xml.rels"), StandardCharsets.UTF_8);
|
||||
assertFalse(rels.contains(EXTERNAL_URL));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_xlsxExternalImageRelStripped() throws IOException {
|
||||
String xlsxRels =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
+ "<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\">"
|
||||
+ "<Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image\""
|
||||
+ " Target=\""
|
||||
+ EXTERNAL_URL
|
||||
+ "\" TargetMode=\"External\"/>"
|
||||
+ "</Relationships>";
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put(
|
||||
"xl/drawings/_rels/drawing1.xml.rels", xlsxRels.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] xlsx = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(xlsx, "xlsx");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String rels =
|
||||
new String(
|
||||
result.get("xl/drawings/_rels/drawing1.xml.rels"), StandardCharsets.UTF_8);
|
||||
assertFalse(rels.contains(EXTERNAL_URL));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_odtStripsExternalXlinkHrefButKeepsInternal() throws IOException {
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("content.xml", ODF_CONTENT_EXTERNAL.getBytes(StandardCharsets.UTF_8));
|
||||
String manifestXml =
|
||||
"<?xml version=\"1.0\"?><manifest:manifest"
|
||||
+ " xmlns:manifest=\"urn:oasis:names:tc:opendocument:xmlns:manifest:1.0\"/>";
|
||||
entries.put("META-INF/manifest.xml", manifestXml.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] odt = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(odt, "odt");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String content = new String(result.get("content.xml"), StandardCharsets.UTF_8);
|
||||
assertFalse(content.contains(EXTERNAL_URL), "External xlink:href should be stripped");
|
||||
assertTrue(content.contains("Pictures/image1.png"), "Internal href should be preserved");
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_odsStripsExternalXlinkHref() throws IOException {
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("content.xml", ODF_CONTENT_EXTERNAL.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] ods = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(ods, "ods");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String content = new String(result.get("content.xml"), StandardCharsets.UTF_8);
|
||||
assertFalse(content.contains(EXTERNAL_URL));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_odpStripsExternalXlinkHrefInStylesXml() throws IOException {
|
||||
String stylesXml =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
+ "<office:document-styles"
|
||||
+ " xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
|
||||
+ " xmlns:draw=\"urn:oasis:names:tc:opendocument:xmlns:drawing:1.0\""
|
||||
+ " xmlns:xlink=\"http://www.w3.org/1999/xlink\">"
|
||||
+ "<draw:image xlink:href=\""
|
||||
+ EXTERNAL_URL
|
||||
+ "\"/></office:document-styles>";
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("styles.xml", stylesXml.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] odp = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(odp, "odp");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String content = new String(result.get("styles.xml"), StandardCharsets.UTF_8);
|
||||
assertFalse(content.contains(EXTERNAL_URL));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_disabledByConfigReturnsOriginal() throws IOException {
|
||||
applicationProperties.getSystem().setDisableSanitize(true);
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] docx = zip(entries);
|
||||
|
||||
byte[] result = sanitizer.sanitize(docx, "docx");
|
||||
assertArrayEquals(docx, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_unrecognizedExtensionReturnsOriginal() throws IOException {
|
||||
byte[] original = "irrelevant".getBytes(StandardCharsets.UTF_8);
|
||||
byte[] result = sanitizer.sanitize(original, "pdf");
|
||||
assertArrayEquals(original, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_emptyInputThrows() {
|
||||
assertThrows(IOException.class, () -> sanitizer.sanitize(new byte[0], "docx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_nullInputThrows() {
|
||||
assertThrows(IOException.class, () -> sanitizer.sanitize(null, "docx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_preservesEntryWithExternalRefWhenAdminAllowsDomain() throws IOException {
|
||||
applicationProperties
|
||||
.getSystem()
|
||||
.getHtml()
|
||||
.getUrlSecurity()
|
||||
.getAllowedDomains()
|
||||
.add("webhook.site");
|
||||
lenient().when(ssrfProtectionService.isUrlAllowed(eq(EXTERNAL_URL))).thenReturn(true);
|
||||
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] docx = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(docx, "docx");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String rels =
|
||||
new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8);
|
||||
assertTrue(rels.contains(EXTERNAL_URL), "Allow-listed external URL should be preserved");
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_doesNotConsultSsrfServiceWhenAllowedDomainsEmpty() throws IOException {
|
||||
// Even if mock would say allowed, we should not invoke it when there is no allow-list,
|
||||
// because MEDIUM default would let public URLs through and re-introduce the vulnerability.
|
||||
lenient().when(ssrfProtectionService.isUrlAllowed(eq(EXTERNAL_URL))).thenReturn(true);
|
||||
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] docx = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(docx, "docx");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String rels =
|
||||
new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8);
|
||||
assertFalse(rels.contains(EXTERNAL_URL));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_handlesNonXmlEntriesSafely() throws IOException {
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
byte[] imageBytes = new byte[] {(byte) 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a};
|
||||
entries.put("word/media/image1.png", imageBytes);
|
||||
entries.put("word/_rels/document.xml.rels", DOCX_RELS.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] docx = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(docx, "docx");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
assertArrayEquals(imageBytes, result.get("word/media/image1.png"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_internalLinksKeptWhenNoExternalPresent() throws IOException {
|
||||
String internalOnlyRels =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
+ "<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\">"
|
||||
+ "<Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image\""
|
||||
+ " Target=\"media/image1.png\"/>"
|
||||
+ "</Relationships>";
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put(
|
||||
"word/_rels/document.xml.rels", internalOnlyRels.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] docx = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(docx, "docx");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String rels =
|
||||
new String(result.get("word/_rels/document.xml.rels"), StandardCharsets.UTF_8);
|
||||
assertTrue(rels.contains("media/image1.png"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_corruptZipProducesSafeOutput() throws IOException {
|
||||
byte[] garbage = "this is not a zip file".getBytes(StandardCharsets.UTF_8);
|
||||
byte[] result = sanitizer.sanitize(garbage, "docx");
|
||||
Map<String, byte[]> entries = unzip(result);
|
||||
assertTrue(entries.isEmpty(), "Garbage input must not yield exploitable entries");
|
||||
}
|
||||
|
||||
@Test
|
||||
void sanitize_relativeOdfPathsArePreserved() throws IOException {
|
||||
String content =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
+ "<office:document-content"
|
||||
+ " xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
|
||||
+ " xmlns:draw=\"urn:oasis:names:tc:opendocument:xmlns:drawing:1.0\""
|
||||
+ " xmlns:xlink=\"http://www.w3.org/1999/xlink\">"
|
||||
+ "<draw:image xlink:href=\"../Pictures/image1.png\"/>"
|
||||
+ "<draw:image xlink:href=\"#anchor\"/>"
|
||||
+ "</office:document-content>";
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("content.xml", content.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] odt = zip(entries);
|
||||
|
||||
byte[] cleaned = sanitizer.sanitize(odt, "odt");
|
||||
|
||||
Map<String, byte[]> result = unzip(cleaned);
|
||||
String out = new String(result.get("content.xml"), StandardCharsets.UTF_8);
|
||||
assertTrue(out.contains("../Pictures/image1.png"));
|
||||
assertTrue(out.contains("#anchor"));
|
||||
}
|
||||
|
||||
private static byte[] zip(Map<String, byte[]> entries) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
|
||||
for (Map.Entry<String, byte[]> e : entries.entrySet()) {
|
||||
ZipEntry entry = new ZipEntry(e.getKey());
|
||||
zos.putNextEntry(entry);
|
||||
zos.write(e.getValue());
|
||||
zos.closeEntry();
|
||||
}
|
||||
}
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
private static Map<String, byte[]> unzip(byte[] data) throws IOException {
|
||||
Map<String, byte[]> entries = new HashMap<>();
|
||||
try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(data))) {
|
||||
ZipEntry e;
|
||||
while ((e = zis.getNextEntry()) != null) {
|
||||
entries.put(e.getName(), zis.readAllBytes());
|
||||
zis.closeEntry();
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
# Widget Inventory Report
|
||||
|
||||
This report lists current stock levels for each warehouse.
|
||||
|
||||
| Region | Units | Status |
|
||||
|---|---|---|
|
||||
| North | 1200 | OK |
|
||||
| South | 950 | Low |
|
||||
| East | 1430 | OK |
|
||||
| West | 875 | Low |
|
||||
@@ -0,0 +1,74 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20260603003133+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603003133+01'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 500
|
||||
>>
|
||||
stream
|
||||
Gas1[9i&Y\%))C:pc(6t3;pQ8%0T@tD+-Nf,MP8j(><jDruNO^VsZ'm=ue7^Ia=;l(`h-+jUU3'Knh#YjH-MEIFj6r!oqf+FjMh;(rCq>R4tMCHL4NI*\1+UNiI'V9NC%VeJKn/YI0J];XQt&X83?=ihrg<*Mcn1n!1nWcDaQPe\P"9gnJuHl(jf]JQgZ[,&^uobI4QF',k"*^S)3c;)GMWC(T'=",ErnS#U=YCUN0&q4+*KmK1Zd*NI\GQDiZUG7;PTja8lulb"\PWWO#WcfI[ZB:6s*3g$be%?JH<b[c(?4J1!2SLDmbmkh8.c&CU)B?Xu,cp$<hP@=fLc>(n`oaEJ[XE'%QW=HE04M<,;ERm[MS=uYF=nN3jG'f@#?O48Ia,6Y-3m&tTWVq1?DeiBkp.Ug*;lVZX`Z=P.eklHhNV;!R_?QOuoeJ<0%7idG7GM8boU$^>N.N,2^;25]0Z8M<<]XMCct>noC'Qfb?`*[Mo+,F9#>t~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 9
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000514 00000 n
|
||||
0000000582 00000 n
|
||||
0000000862 00000 n
|
||||
0000000921 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<ee376a59c53e3f2af87024f65fd4222c><ee376a59c53e3f2af87024f65fd4222c>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 6 0 R
|
||||
/Root 5 0 R
|
||||
/Size 9
|
||||
>>
|
||||
startxref
|
||||
1511
|
||||
%%EOF
|
||||
@@ -0,0 +1,222 @@
|
||||
Intro paragraph for section 1.
|
||||
|
||||
| Name | Qty |
|
||||
|---|---|
|
||||
| alpha | 101 |
|
||||
| delta | 201 |
|
||||
|
||||
# Section 2 Heading
|
||||
|
||||
| Name | Qty | Price |
|
||||
|---|---|---|
|
||||
| alpha | 101 | charlie |
|
||||
| delta | 201 | foxtrot |
|
||||
| golf | 301 | india |
|
||||
|
||||
## Section 3 Heading
|
||||
|
||||
| Name | Qty | Price | Region |
|
||||
|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 |
|
||||
| delta | 201 | foxtrot | 13 |
|
||||
| golf | 301 | india | 23 |
|
||||
| juliet | 401 | lima | 33 |
|
||||
|
||||
Intro paragraph for section 4.
|
||||
|
||||
| Name | Qty | Price | Region | Status |
|
||||
|---|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 | echo |
|
||||
| delta | 201 | foxtrot | 13 | hotel |
|
||||
| golf | 301 | india | 23 | kilo |
|
||||
| juliet | 401 | lima | 33 | november |
|
||||
| mike | 501 | oscar | 43 | alpha |
|
||||
|
||||
# Section 5 Heading
|
||||
|
||||
| Name | Qty |
|
||||
|---|---|
|
||||
| alpha | 101 |
|
||||
| delta | 201 |
|
||||
| golf | 301 |
|
||||
| juliet | 401 |
|
||||
| mike | 501 |
|
||||
| papa | 601 |
|
||||
|
||||
| Name | Qty | Price |
|
||||
|---|---|---|
|
||||
| alpha | 101 | charlie |
|
||||
| delta | 201 | foxtrot |
|
||||
|
||||
# Section 7 Heading
|
||||
|
||||
Intro paragraph for section 7.
|
||||
|
||||
| Name | Qty | Price | Region |
|
||||
|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 |
|
||||
| delta | 201 | foxtrot | 13 |
|
||||
| golf | 301 | india | 23 |
|
||||
|
||||
## Section 8 Heading
|
||||
|
||||
| Name | Qty | Price | Region | Status |
|
||||
|---|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 | echo |
|
||||
| delta | 201 | foxtrot | 13 | hotel |
|
||||
| golf | 301 | india | 23 | kilo |
|
||||
| juliet | 401 | lima | 33 | november |
|
||||
|
||||
| Name | Qty |
|
||||
|---|---|
|
||||
| alpha | 101 |
|
||||
| delta | 201 |
|
||||
| golf | 301 |
|
||||
| juliet | 401 |
|
||||
| mike | 501 |
|
||||
|
||||
# Section 10 Heading
|
||||
|
||||
Intro paragraph for section 10.
|
||||
|
||||
| Name | Qty | Price |
|
||||
|---|---|---|
|
||||
| alpha | 101 | charlie |
|
||||
| delta | 201 | foxtrot |
|
||||
| golf | 301 | india |
|
||||
| juliet | 401 | lima |
|
||||
| mike | 501 | oscar |
|
||||
| papa | 601 | bravo |
|
||||
|
||||
| Name | Qty | Price | Region |
|
||||
|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 |
|
||||
| delta | 201 | foxtrot | 13 |
|
||||
|
||||
# Section 12 Heading
|
||||
|
||||
| Name | Qty | Price | Region | Status |
|
||||
|---|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 | echo |
|
||||
| delta | 201 | foxtrot | 13 | hotel |
|
||||
| golf | 301 | india | 23 | kilo |
|
||||
|
||||
## Section 13 Heading
|
||||
|
||||
Intro paragraph for section 13.
|
||||
|
||||
| Name | Qty |
|
||||
|---|---|
|
||||
| alpha | 101 |
|
||||
| delta | 201 |
|
||||
| golf | 301 |
|
||||
| juliet | 401 |
|
||||
|
||||
| Name | Qty | Price |
|
||||
|---|---|---|
|
||||
| alpha | 101 | charlie |
|
||||
| delta | 201 | foxtrot |
|
||||
| golf | 301 | india |
|
||||
| juliet | 401 | lima |
|
||||
| mike | 501 | oscar |
|
||||
|
||||
# Section 15 Heading
|
||||
|
||||
| Name | Qty | Price | Region |
|
||||
|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 |
|
||||
| delta | 201 | foxtrot | 13 |
|
||||
| golf | 301 | india | 23 |
|
||||
| juliet | 401 | lima | 33 |
|
||||
| mike | 501 | oscar | 43 |
|
||||
| papa | 601 | bravo | 53 |
|
||||
|
||||
Intro paragraph for section 16.
|
||||
|
||||
| Name | Qty | Price | Region | Status |
|
||||
|---|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 | echo |
|
||||
| delta | 201 | foxtrot | 13 | hotel |
|
||||
|
||||
# Section 17 Heading
|
||||
|
||||
| Name | Qty |
|
||||
|---|---|
|
||||
| alpha | 101 |
|
||||
| delta | 201 |
|
||||
| golf | 301 |
|
||||
|
||||
## Section 18 Heading
|
||||
|
||||
| Name | Qty | Price |
|
||||
|---|---|---|
|
||||
| alpha | 101 | charlie |
|
||||
| delta | 201 | foxtrot |
|
||||
| golf | 301 | india |
|
||||
| juliet | 401 | lima |
|
||||
|
||||
Intro paragraph for section 19.
|
||||
|
||||
| Name | Qty | Price | Region |
|
||||
|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 |
|
||||
| delta | 201 | foxtrot | 13 |
|
||||
| golf | 301 | india | 23 |
|
||||
| juliet | 401 | lima | 33 |
|
||||
| mike | 501 | oscar | 43 |
|
||||
|
||||
# Section 20 Heading
|
||||
|
||||
| Name | Qty | Price | Region | Status |
|
||||
|---|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 | echo |
|
||||
| delta | 201 | foxtrot | 13 | hotel |
|
||||
| golf | 301 | india | 23 | kilo |
|
||||
| juliet | 401 | lima | 33 | november |
|
||||
| mike | 501 | oscar | 43 | alpha |
|
||||
| papa | 601 | bravo | 53 | delta |
|
||||
|
||||
| Name | Qty |
|
||||
|---|---|
|
||||
| alpha | 101 |
|
||||
| delta | 201 |
|
||||
|
||||
# Section 22 Heading
|
||||
|
||||
Intro paragraph for section 22.
|
||||
|
||||
| Name | Qty | Price |
|
||||
|---|---|---|
|
||||
| alpha | 101 | charlie |
|
||||
| delta | 201 | foxtrot |
|
||||
| golf | 301 | india |
|
||||
|
||||
## Section 23 Heading
|
||||
|
||||
| Name | Qty | Price | Region |
|
||||
|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 |
|
||||
| delta | 201 | foxtrot | 13 |
|
||||
| golf | 301 | india | 23 |
|
||||
| juliet | 401 | lima | 33 |
|
||||
|
||||
| Name | Qty | Price | Region | Status |
|
||||
|---|---|---|---|---|
|
||||
| alpha | 101 | charlie | 3 | echo |
|
||||
| delta | 201 | foxtrot | 13 | hotel |
|
||||
| golf | 301 | india | 23 | kilo |
|
||||
| juliet | 401 | lima | 33 | november |
|
||||
| mike | 501 | oscar | 43 | alpha |
|
||||
|
||||
# Section 25 Heading
|
||||
|
||||
Intro paragraph for section 25.
|
||||
|
||||
| Name | Qty |
|
||||
|---|---|
|
||||
| alpha | 101 |
|
||||
| delta | 201 |
|
||||
| golf | 301 |
|
||||
| juliet | 401 |
|
||||
| mike | 501 |
|
||||
| papa | 601 |
|
||||
@@ -0,0 +1,169 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 13 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/Contents 14 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Contents 15 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Contents 16 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Contents 17 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Contents 18 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 12 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
11 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20260603005358+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603005358+01'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
12 0 obj
|
||||
<<
|
||||
/Count 6 /Kids [ 4 0 R 5 0 R 6 0 R 7 0 R 8 0 R 9 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
13 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1007
|
||||
>>
|
||||
stream
|
||||
GauHKgN)$k&:O:Sn<io8&IUqd[51N1e*/]GIR%m`>DZmeMUK]*(BdhV"2[X,j0;*\%_E*0HXJRc`<Z.PlPcH-)*5ON#Qr4-#h+U(ll5egXa[FW7^<l-+Mko1pU57W'%??d'1Q-,]\qWS80;JOp0HFGa1T[+D>q%4S#/VM`O[Rh4i5T.Phi,$]aT$05!q:PBuCQU/.p2]@1koN*Tb*K9k5G\ht+Dr\K=+8\NZ"alMaOEo**@OK:8-.O1X3-?Gg`@m3%,ti3'">T-&c=M&Wuu?cDbGDp.gO<KniopM)A\M"ajn9N]6p@tefHlX67:iW3SE/[O\LV1TKe.i9es,%kWY+1-s,ft.u27]:^>F0!r6&&CLC$tM?fIR"M37["/*k9@YkpKKSS@Np"1/4#R]`I^(g*1Sc,9L3Qt6N(T@F`A>oBGgL-!r#Y4)R\G`D,iVtFeJUE9u4iuUQ?D%C&SB4kkp.>D5tii>nDKJ"Y07jANhOb$R(_$=U7Stjs)-/KZ(6IBm`6<Z:IO1/g<Q6n^@fdK*]SO#g&Wj4B)g1,#O?30",57V_ONl0p`%uQN]'EAZV$475Z"'DApH2)Sg6UQ75XC?4^'<&6&d?YC`OF9F3eE;WIMt@a?'q)c\3oWQ_BD]d`4=O\,'aSUFV4HsE7)O:lPOi6Ht1,[dTm6JqQM6es[?9G4VC(YieF*)uM6[SZT#[TN2,BIir^1;/Ku"E1laH[\"M+F'8=n0!;?_#hHd!hkpV0u\Ij\4VC-km.[F&JhGmbc`6@I1=mf>u<3;i.Bh4+MFJ"H:.XWUQX6%LU(sg4Tt$_":5`p.2,gZkpUfdg,Hd(qR1)9\ltF2^8b5,,XbY%VfhD52O7A.c.u]dhTc5t%0<0L5E38`Bq+i;"%J7kcc#@i)@okdN-qiaA"33DdPgPrUs7;j%+47_cnGN@&ug9/dqGOAHA4f.N*&guHspf?E;GIc-Wt>:<(m1AmcS_Zc2VlEI'_S_>@#!MF^m1$$<mB1`gt\X~>endstream
|
||||
endobj
|
||||
14 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 987
|
||||
>>
|
||||
stream
|
||||
GatU3gN&c;&:O:SkV;H,eI!J\f?X"LSPIZ'"4Y.F#oAAY?N.Y?8Iu:s)eV;,aNG1Lea>G$!MSc\rU6m7jF'AOI09V1,^TSTi$A+f4sZVQ%2^>uj$40\AI>WbZo9q!NL^CS9P3E9X.:XEMb$a/X"qH9hN6fY,]D!OEOVT,722%RJnVqK4f'[4d3(/hbJWJRs,>>28jk5A:nCl/%FlC#LntsAC5cE$q`QO_@Pk%Lp>7V23")NNrFkcXfuoI=SJJ-`g)_+K\@,SQ@8ORGg(_(B.[u[WXD%Q8@I";9d(]M62K_)Q+-<\kHY5,)o99%BB2bcQVkd:+MZVH=$Z`S@BhH]-XfPANk@qBN.:Jc'?.Kn\p7(aPI<Cb@g:XRZ9Km+RD-e6S\N*kQMA?P5>QOI(du7U]WDrNNPW%d"8P_j^Y8d%G<t1\CJ_5-m+>,'JJLXrV.UD!ah19&9b#*fgR2o730<9)QX)X0"EK6u;mBJs3/fl\d&K$B3bXI5R>F*E,^\%+b?0o8s$o.CuX4uDAT_?G0(p4N3La,8qfi'j?e`e88KrZ8*NDgc:62'icCb],4B]Z"SO&e/BRC*C$2PmbSKAjc\$FLD8?&qHVH3G4p834/77n[%()-p/JDRBapcO2b\,3dW%#U9u!ilSkd%2'rr&g[u".1O]W%0J<@#(ptUD;%0:^.^ls:.#.V6NgR[`2\RUW!k$-*GZIf3DO<K8np=hCA>BiB_Mu"=LVGlO\1Z]0pF(@UaiVQ=dh?h5kfQ_gj>$jHW$8TCt:C.jVU98ne.XoiMtoV;q8XhnlOE;3a]=dGJ8KL&+Khj"&P4Q=P-:nHsDprY"B4+#A3dF,<W=NC>;b&gf(sT\Ts*iH)f!KhfHV;\mMSHej.9.XrBG41_<:b)MG;-Y(:&nCKS0F]r&CGl>EfMc0td\0GHScl3Lr;?OUoo6619Qdr)S8S);]7L5rmWW&(5t4Dil"USDhhY/Y=!=!X:85**$:~>endstream
|
||||
endobj
|
||||
15 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 976
|
||||
>>
|
||||
stream
|
||||
GatU3gN&c;'R]XVkgAc"70atdYFXp#3h7VV#7/=-%N$"G?N.YO&iX#cVsS_l_5i8)eum:1-)(7`mXJ7LnhqY0hGZal,We=q^e"$M]Lu;_=4AoIN$QOjOF.@i*Sg!_rLL=-'?kk;m;M&R`$D>'HCL7<qMHOgP.\NgCm:Zf5tLAgarh6YCKLONS%?!`3T6F7YKNJ(dBFh(U1"e7D6YdC:71/`?cjR"QrKtsGg*:'=Ck>]$A,-S`H1I/1T]4\B3=gQ9:I4j9f]+af*Z7*GH5H+;j73Z-T#DAcgpO<SFuA['l7@MF*6c^<pB]o84rG$&I@Cs3`KIG,N/bG0f<k/M@gUKb"%\W-lZ#AU+i=gUKkm46K-b>W(/'b?XF9+X+'-Dqi"+1U7?6=ZRJNBSJ6ojDTa+]o4RF^26C+i0s(9G2o;$@RNbs\(f75],L(SVpI0M!'7JVhZ"$t/%kb1+)F\p54/@jpVJF2+)7^F=gVcp8(h*m%F#'p9c%9Ep9?b7g^F5>(>I3t^d_"f'aue%^E\X4kN:g\P[;sC=g3h,nD/n9*`94uoM?sBCL<kr$AOqh:,Y&c90,npSDXN*bEfY(k^0LZ_\I@Y+L2nn64-qsuX5D'=o'B2"8rCsRJ?h/-4Ur6khb;ph%X"8-mXTUU5-!Uh$iVuRi1.=>4;XilfMPT4]&FC.U%DI(c6uf3*XPJP2MRdq@Ag3uYF().nqLFAE2Nih'(^p\5FnSNa>n+cI?U65A61_N1<9ZXH4Xr)Fgq(6md3H9[MQ3Q^%aq>E?XM:O:RKGjc+QV2MB!.I.17QpSt]X[C*6ap.`#'o;8F(ebZ.VMM6k`BkZB\I,2MNoX]J"k]Qd";,@($aWX&29q"6k;>Let)@]QHm:i/lf[DB_&U@ROq-0MLQp;@j+]?$"E)brXVun52AVQo$"[a7@!?!LT*>$&:5X_.R;9)'$l-S.>WjLQIVTo.JU%(H\,-*pl_kP9~>endstream
|
||||
endobj
|
||||
16 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1024
|
||||
>>
|
||||
stream
|
||||
GauHKflEQ9'Rf^WgrHc4<#\/Qm7c-rFIIq+TFSD%Z#L'6o(Nl&^_!k0ds=.%-s&pt#i0QBKE91*<^/MJJCcfoHA_e;aL?\&_BAj]Dt;HW$6(8Ql\%O7-5%uT8<qrc9!'@hZ!'`C3a&iu\ZDZi+5lF*E@>0ZIL#\l)(+[ABOTNKmq!2F,S:.F93Z6QIu[nf\Q[qCE`Y-=.8k$67iCR!7=El+50a@f&b^6'Yg'mqlcRQ-5j:%[K@'5!A_m0L)&VlW1T50^X)"3Ma3,U1P8Di$>uPZ)Zj_igBc0lm]WE0#e>*M53(Yh6\9jHYh6B+3bM\b[Q9j'O_9:!:W0<K\bX;'ZV[%iHrL=:rH.Wh5O-II8-Rh3..Sd^\dNItUUd<nli^`$=A@?\WW4/7Wnp>$Ijm,#(bKn_D?UYMGJ7LLqC0\[$l<8U*O5^GnN;!N6YB'?d[d8DAPt^>+E:r0<2$'VL/TtuNg;C@5iM#%KdS-GU->0]`/20eLXVW#qnjsU.P:Xj&=enVE[mpqDCn2!qDRuQhI/#cZ-#m6`U9'SI4"9\<WSWoVPaf@]<NRl?FrTPOd:V_=>"r:4p.\W>IK_$#C,EkU#uO1g6*HoUK':XiJjtQfI'Sdc)B3J1eqgGJ+l?"$<J+^?VHgiq^90uD"@6ar+/_WhKG+8t!Slo>//&C$^f'TA!2:K!0pd;^Zif)c[:f!C/8hNKc7+'+1WbOFKr9T7c_G$s*O634O2Zs:"iN>j,pWU+:kEQ>5YbF=$27*59iJE<glrr=EAMssR&N"lVT1:*a,OcI=8\"6/>A//7(KsL\pUG.G[+JB-r8t;Sih+(W.A'8Q*LZK*I9WjCLjEt>lT%2RFJZg0Ps:5c,HdK#tSNI:#Rp!]O$Ic=T%K0[._E_Fcjn>OcM,iSAf9iF7,,+Qe:@o'Y4o_:54t*l+TAg<i_F?ke9:\G<n<b*t@"_`_;08ft@[9;qtKC#H0Z6cJ+d)MdidY-OFV1K"=!=K=^?hK-Q.s+?\[I98%IXRiAI.!:AVTf`~>endstream
|
||||
endobj
|
||||
17 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1075
|
||||
>>
|
||||
stream
|
||||
GauHKgN&c;&:O:SkV;H,6&Q<FCM0)BVUK5trr=4p-3<-<P.#jI+b[BoQ`]$,GaorC,Bfp>Z[5)3-pL]0bpl':D91fR,o"F49.1/bfmFt32ljt6eO[j7!K]\nZ:^QQASu[l@3no(6*HL'@3I!Qfi7%,.N\.>A91O)D\n5*o@JWSOI@ng5p/"_I`g*Y$j0+5PWaso'25ltcglFd,QIaO'dPO0Yr\h5/,=m\PiP:P+Gg0&N;_5iLN$]BD.#VL+h,tSe\fL&,;+2fg&%\pAJiV#eVhq@ro%%/`[(&8n29YiYd>bDF^Bg[AVf/n>[FE(au"ZAe&>%Q[7/n3-QhsPXuPbp#<$d"UR8u-nBnr!Q$Wjd1;9WN/,KG8])Ca5rrDrI'Ue'*I0C%pn+f`Hd"4sB_&2*)S$pX=F/6"DC"TU`bMtd/m7u]?]=J5L[SNBuE:6Ri,NmW7.*qUp)97Tt+a84b\os`Ti8/uRD<mg2;n&q)23L#\f:[QJk+sYYhIC_O\TNS/;3<qfN@GE^L#jl^i&)"7_/%Cb;-M;;ooT&`R;C&\WTZ4"\YIcTZDa;0CKTX+?@ra\94$H:Q<l)r*NHFJWjMr7[tCBb^PX,G]H!fs0g3,5^'AZ%<4Zqp(JCd+;^lkoDJ%q(Bb84V=:PCgOtM#Rr-=oHW&DmO?/!mYrV;g]MR?14cGuH_H*]73!34(()[A.)c<V3:@I>lhrl*Xrn(a0\,%4%*k^rL\k3%6ako`&A3MoN^CXV77D!Qg<S_OEMIh5(T*nFt'&4]_38'Q<s*V7!;A%5-@W.E/(oCu;?^7kb[#R/-P*GF@KHDm6E*dLMg/4Q$W(<Ta[fVpGB5td>%=j9Ge+0B@dZDHELb2fDU(,iiXXRl^*U,U#Fld!R72UIpK<KU`cn&R/_eYS//1PE/$d7p:t-BP62fL<3/QQ4%=NoD?Tlf:A,GpJc<]*uYaL6fSAqm"-E.#%h]idd9$6\DZKKp8Vd[>GWu#-DXi653a?U$fqCs18gt(5/#U%!u,gkMK;>OT_/='S:kn,iD@u]<8-rlTRu'+\7L$U7-#!5-5\WQn(n:q@-p24u!~>endstream
|
||||
endobj
|
||||
18 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 590
|
||||
>>
|
||||
stream
|
||||
GasbXgMWc?&;KY!MRgt!"n`^Bf\9$c*`Q-Sb6tfd8P(q-dT1eng=QV-!OG*\kiWo2AGBdKLqI^%h,XNB)-gDk+GFVBa?g*a%:!J\97Vc8-jH8>8P>0S@Wr)o*"Hu<6qPp`EF:M3'l4@S\c2]`&[J%dO@5p$<(([H6:SlB;FS8L`&pO%6Y\V"/O7F]E%L@d.%>L@"4bK_XXs&";n_.W^Nr847HH,X@$.pC"4bYC?9RH,dN?h8<BTH9l89:QMq:es\ePUpFRc4?hV'h3b4`sE_b4r->a+8/!?Q=D\F41Nc@'B`';4XgMeh!;U23C+>AbMQD-o--lJ/":m#(Yt,X5(`1KL,GTMBLlr6=-1mi#k.-Ou\\T4$Pun2dEU(\?$&;1@T4dm^t!KuOD-U@N_AMr"&uVpsGm,+8I7B*f!%9.o4cC1<Nr^$d5!dH-#UIr,n6Z;JUFYWCTVA\,or>a[CZ12(hd>1*0bU`k2-MXo1[Gor#kmXGIM'#R49X#NSAOpdf'0ilUH4:M(^Snc3;m/+><UlJ1jSN&ZuoQ68F8`c/Y-e0[!aljuioC5fR7-VUAX`%g<0P5J7"CFlQJT1~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 19
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000516 00000 n
|
||||
0000000711 00000 n
|
||||
0000000906 00000 n
|
||||
0000001101 00000 n
|
||||
0000001296 00000 n
|
||||
0000001491 00000 n
|
||||
0000001561 00000 n
|
||||
0000001842 00000 n
|
||||
0000001932 00000 n
|
||||
0000003031 00000 n
|
||||
0000004109 00000 n
|
||||
0000005176 00000 n
|
||||
0000006292 00000 n
|
||||
0000007459 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<ba877f458edd6b06b69a7e843c67586d><ba877f458edd6b06b69a7e843c67586d>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 11 0 R
|
||||
/Root 10 0 R
|
||||
/Size 19
|
||||
>>
|
||||
startxref
|
||||
8140
|
||||
%%EOF
|
||||
@@ -0,0 +1,25 @@
|
||||
# Lorem Ipsum in Two Columns
|
||||
|
||||
## 1. Origins
|
||||
|
||||
Lorem ipsum dolor sit amet consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
|
||||
|
||||
## 2. Structure
|
||||
|
||||
Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
|
||||
|
||||
## 3. Usage
|
||||
|
||||
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
|
||||
|
||||
## 4. Variations
|
||||
|
||||
Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum.
|
||||
|
||||
## 5. Typography
|
||||
|
||||
Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam various turpis et commodo pharetra est.
|
||||
|
||||
## 6. Conclusion
|
||||
|
||||
Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum.
|
||||
@@ -0,0 +1,74 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20260603021636+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603021636+01'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 861
|
||||
>>
|
||||
stream
|
||||
Gat=i>Ak00&;B$5/*8Q/Z)fmNp(^2^OK'MC[dT6#Oq.&b^4c4;1No6>+D%TC.n+fice+l9R0[L%'(o<DJBpB+r$jsRcu23$RjI].E7rS*BB"3t\qRKaS-SYt[_<-]),Cp@'$i;6dQuJ#Q*+lGY?X<phgEWPa?op]_1g@kf_>sQ?dQP@AUa.A:T=#%ZIIl18hB7Zf<piVi-V+;]ct^jS<=Z4l:IOd[,o"%q.hK[[k]r1!^oX'%44os7T9[m%YrbZ3KU/;aR0$cV$;D._".OT8)jeEBKU'l<A`'aD&R'L"Bo]Q)R+=cKOEld8s#M0ZYZN4[lRm1*F6^d9-Gfl5>I(hVU+?`]UK%;aabAH:q>9NUGQq^.=u])Wt-ETI))C'a[FE@elj!RSrf2Q'F>URAO.C,!DneTPqrj#4e2kb9%1"4qfZ)"#&^0j9nHQ?9nF!j7mVPP5\*Uq'_jMVS]9%`kQB\8*AF_bpr/hGj;HCUOSQU-%5:6S79Ud\b!*tPbr_'pCr$Ea#(FYP31NFhSX.-("1M:$cgH#hX8L(2]R3Q>'BYHCS%pI!;=WdJp,'ii[`QPZ_9mcd\baZ2U(_;c\-p,8EoIEpQ*lstL>]LE;C#\dLnT2R:)BM-fTc['3_He[U,k'!Bo".uERd>SkhRj^J+koSIrZ_dEf_5L'/1h.`+DTK(R:P,WH)h5\se=SZ"L/5b8b..,e/E\o+4YQ+*im^C>AERG/TieEK\)#>U@HXnJ,H0A9-MqhhkDp8%.6Lr,OrK*lih;B<-opZ8%EU?,$r^jmCDAQ`-0/-8/`[p]7Fm%0f:E&S*FV)DX2>#q\bRqA=^_`43#8EA$u%8r6F5`rc9>K@q>E4q^*~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 9
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000514 00000 n
|
||||
0000000582 00000 n
|
||||
0000000862 00000 n
|
||||
0000000921 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<21a9fbd0a0991a91b6e6e2db0856056e><21a9fbd0a0991a91b6e6e2db0856056e>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 6 0 R
|
||||
/Root 5 0 R
|
||||
/Size 9
|
||||
>>
|
||||
startxref
|
||||
1872
|
||||
%%EOF
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
# Employee Expense Report
|
||||
|
||||
Reimbursement Request
|
||||
|
||||
EMP-1047
|
||||
|
||||
**Report Header**
|
||||
|
||||
| Employee Name | Michael Tran |
|
||||
|---|---|
|
||||
| Employee ID | EMP-1047 |
|
||||
| Department | Client Services |
|
||||
| Report Date | January 20th, 2026 |
|
||||
| Reporting Period | January 5th–16th, 2026 |
|
||||
| Manager Approver | Laura Simmons |
|
||||
|
||||
**Company Information**
|
||||
|
||||
| Company | Summit Consulting Partners |
|
||||
|---|---|
|
||||
| Company Address | 88 Riverside Plaza, Suite 1400, New York, NY 10069 |
|
||||
| Accounting Department Email | expenses@example.com |
|
||||
|
||||
**Trip Purpose**
|
||||
|
||||
The trip was undertaken for client onsite meetings with Atlantic Energy Solutions in Boston, MA.
|
||||
|
||||
**Expense Details**
|
||||
|
||||
| Description | Amount | Date | Category |
|
||||
|---|---|---|---|
|
||||
| Flight (NYC to Boston roundtrip) | $325.40 | January 5th, 2026 | Airline ticket |
|
||||
| Hotel (3 nights at Harborview Hotel) | $822.75 | January 5th–8th, 2026 | Lodging |
|
||||
| Taxi from airport to hotel | $48.00 | January 5th, 2026 | Ground transportation |
|
||||
| Client dinner (3 attendees) | $186.20 | January 6th, 2026 | Meals |
|
||||
| Parking at JFK Airport | $72.00 | January 5th–8th, 2026 | Parking |
|
||||
| Breakfast (per diem not used) | $18.50 | January 7th, 2026 | Meals |
|
||||
|
||||
| Description | Amount | Date | Category |
|
||||
|---|---|---|---|
|
||||
| Uber to client office | $22.10 | January 7th, 2026 | Ground transportation |
|
||||
| Printing + presentation materials | $46.90 | January 8th, 2026 | Materials |
|
||||
| Lunch with client | $39.75 | January 8th, 2026 | Meals |
|
||||
| Office supplies (notebooks, pens) | $27.60 | January 10th, 2026 | Supplies |
|
||||
| Mileage reimbursement (client visit in NJ, 42 miles @ $0.67/mile) | $28.14 | January 14th, 2026 | Mileage |
|
||||
| Team lunch meeting (internal) | $64.30 | January 15th, 2026 | Meals |
|
||||
|
||||
Total Expenses $1,701.64
|
||||
|
||||
Reimbursement Method
|
||||
|
||||
Reimbursement method Direct deposit
|
||||
|
||||
Notes
|
||||
|
||||
All receipts are attached. Expenses are business-related and comply with company travel policy.
|
||||
|
||||
**Approval**
|
||||
|
||||
Michael Tran, Employee
|
||||
|
||||
Laura Simmons, Manager
|
||||
BIN
Binary file not shown.
@@ -18,6 +18,7 @@ import io.swagger.v3.oas.models.media.StringSchema;
|
||||
import io.swagger.v3.oas.models.security.SecurityRequirement;
|
||||
import io.swagger.v3.oas.models.security.SecurityScheme;
|
||||
import io.swagger.v3.oas.models.servers.Server;
|
||||
import io.swagger.v3.oas.models.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -60,6 +61,15 @@ public class OpenApiConfig {
|
||||
|
||||
OpenAPI openAPI = new OpenAPI().info(info).openapi("3.0.3");
|
||||
|
||||
// Register a single global "AI" tag so every AI endpoint groups under it in the docs.
|
||||
// The AI controllers are currently @Hidden, so they don't emit this tag themselves yet;
|
||||
// defining it here keeps the grouping ready for when those endpoints are unhidden.
|
||||
openAPI.addTagsItem(
|
||||
new Tag()
|
||||
.name("AI")
|
||||
.description(
|
||||
"AI-powered document creation, editing, and assistant endpoints."));
|
||||
|
||||
// Add server configuration from environment variable
|
||||
String swaggerServerUrl = System.getenv("SWAGGER_SERVER_URL");
|
||||
Server server;
|
||||
|
||||
+5
-1
@@ -10,6 +10,7 @@ import java.util.Map;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.multipdf.Overlay;
|
||||
import org.apache.pdfbox.pdfwriter.compress.CompressParameters;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
@@ -157,7 +158,10 @@ public class PdfOverlayController {
|
||||
PDDocument singlePageDocument = new PDDocument()) {
|
||||
singlePageDocument.addPage(overlayPdf.getPage(pageCountInCurrentOverlay));
|
||||
File tempFile = Files.createTempFile("overlay-page-", ".pdf").toFile();
|
||||
singlePageDocument.save(tempFile);
|
||||
// NO_COMPRESSION: this single-page doc holds a page copied from overlayPdf.
|
||||
// PDFBox 3.0.7's compressed writer (PDFBOX-6203) drops shared resources imported
|
||||
// across documents, corrupting overlay fonts. Revert once on 3.0.8.
|
||||
singlePageDocument.save(tempFile, CompressParameters.NO_COMPRESSION);
|
||||
|
||||
overlayGuide.put(basePageIndex, tempFile.getAbsolutePath());
|
||||
tempFiles.add(tempFile); // Keep track of the temporary file for cleanup
|
||||
|
||||
+23
-32
@@ -6,11 +6,9 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@@ -262,38 +260,31 @@ public class RearrangePagesPDFController {
|
||||
}
|
||||
log.info("newPageOrder = {}", newPageOrder);
|
||||
log.info("totalPages = {}", totalPages);
|
||||
// Create a new list to hold the pages in the new order
|
||||
List<PDPage> newPages = new ArrayList<>();
|
||||
for (int i = 0; i < newPageOrder.size(); i++) {
|
||||
newPages.add(document.getPage(newPageOrder.get(i)));
|
||||
|
||||
// Snapshot the desired pages before mutating the source document's page tree.
|
||||
List<PDPage> newPages = new ArrayList<>(newPageOrder.size());
|
||||
for (Integer idx : newPageOrder) {
|
||||
newPages.add(document.getPage(idx));
|
||||
}
|
||||
|
||||
// Create a new document based on the original one
|
||||
try (PDDocument rearrangedDocument =
|
||||
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(document)) {
|
||||
|
||||
// Add the pages in the new order
|
||||
for (PDPage page : newPages) {
|
||||
rearrangedDocument.addPage(page);
|
||||
}
|
||||
|
||||
PDDocumentCatalog sourceCatalog = document.getDocumentCatalog();
|
||||
if (sourceCatalog != null) {
|
||||
PDAcroForm sourceForm = sourceCatalog.getAcroForm(null);
|
||||
if (sourceForm != null) {
|
||||
rearrangedDocument
|
||||
.getDocumentCatalog()
|
||||
.getCOSObject()
|
||||
.setItem(COSName.ACRO_FORM, sourceForm.getCOSObject());
|
||||
}
|
||||
}
|
||||
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
rearrangedDocument,
|
||||
GeneralUtils.generateFilename(
|
||||
pdfFile.getOriginalFilename(), "_rearranged.pdf"),
|
||||
tempFileManager);
|
||||
// Rearrange in-place on the source document rather than copying pages into a
|
||||
// freshly-created PDDocument. Copying pages across documents triggers a PDFBox
|
||||
// 3.0.7 compressed-save regression (PDFBOX-6203, fixed for 3.0.8) where shared
|
||||
// resource objects (fonts, etc.) imported from the source can be silently
|
||||
// dropped from the output, producing pages with "font not found" errors.
|
||||
PDPageTree pages = document.getPages();
|
||||
for (int i = totalPages - 1; i >= 0; i--) {
|
||||
pages.remove(i);
|
||||
}
|
||||
for (PDPage page : newPages) {
|
||||
pages.add(page);
|
||||
}
|
||||
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
document,
|
||||
GeneralUtils.generateFilename(
|
||||
pdfFile.getOriginalFilename(), "_rearranged.pdf"),
|
||||
tempFileManager);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
ExceptionUtils.logException("document rearrangement", e);
|
||||
|
||||
+7
-3
@@ -35,6 +35,7 @@ import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.CustomHtmlSanitizer;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.OfficeDocumentSanitizer;
|
||||
import stirling.software.common.util.ProcessExecutor;
|
||||
import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult;
|
||||
import stirling.software.common.util.RegexPatternUtils;
|
||||
@@ -50,6 +51,7 @@ public class ConvertOfficeController {
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final RuntimePathConfig runtimePathConfig;
|
||||
private final CustomHtmlSanitizer customHtmlSanitizer;
|
||||
private final OfficeDocumentSanitizer officeDocumentSanitizer;
|
||||
private final EndpointConfiguration endpointConfiguration;
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
@@ -83,14 +85,16 @@ public class ConvertOfficeController {
|
||||
Path inputPath = workDir.resolve(baseName + "." + extensionLower);
|
||||
Path outputPath = workDir.resolve(baseName + ".pdf");
|
||||
|
||||
// Check if the file is HTML and apply sanitization if needed
|
||||
// Sanitize input before LibreOffice sees it so embedded URLs can't trigger SSRF.
|
||||
if ("html".equals(extensionLower) || "htm".equals(extensionLower)) {
|
||||
// Read and sanitize HTML content
|
||||
String htmlContent = new String(inputFile.getBytes(), StandardCharsets.UTF_8);
|
||||
String sanitizedHtml = customHtmlSanitizer.sanitize(htmlContent);
|
||||
Files.writeString(inputPath, sanitizedHtml, StandardCharsets.UTF_8);
|
||||
} else if (officeDocumentSanitizer.isSanitizableExtension(extensionLower)) {
|
||||
byte[] sanitized =
|
||||
officeDocumentSanitizer.sanitize(inputFile.getBytes(), extensionLower);
|
||||
Files.write(inputPath, sanitized);
|
||||
} else {
|
||||
// copy file content
|
||||
Files.copy(inputFile.getInputStream(), inputPath, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
|
||||
|
||||
+5
-1
@@ -14,6 +14,7 @@ import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdfwriter.compress.CompressParameters;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
@@ -357,7 +358,10 @@ public class AutoSplitPdfController {
|
||||
for (int i = 0; i < splitDocuments.size(); i++) {
|
||||
String fileName = filename + "_" + (i + 1) + ".pdf";
|
||||
zipOut.putNextEntry(new ZipEntry(fileName));
|
||||
splitDocuments.get(i).save(zipOut);
|
||||
// NO_COMPRESSION: split docs are built by addPage()-ing pages copied from the
|
||||
// source document. PDFBox 3.0.7's compressed writer (PDFBOX-6203) drops shared
|
||||
// resources imported across documents, corrupting fonts. Revert once on 3.0.8.
|
||||
splitDocuments.get(i).save(zipOut, CompressParameters.NO_COMPRESSION);
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
}
|
||||
|
||||
+9
-2
@@ -17,6 +17,7 @@ import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.pdfbox.io.IOUtils;
|
||||
import org.apache.pdfbox.multipdf.PDFMergerUtility;
|
||||
import org.apache.pdfbox.pdfwriter.compress.CompressParameters;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
@@ -427,7 +428,10 @@ public class OCRController {
|
||||
// Save original page without OCR as fallback
|
||||
try (PDDocument pageDoc = new PDDocument()) {
|
||||
pageDoc.addPage(page);
|
||||
pageDoc.save(pageOutputPath);
|
||||
// NO_COMPRESSION: page is copied from another document;
|
||||
// PDFBox 3.0.7 compressed writer (PDFBOX-6203) drops shared
|
||||
// resources, corrupting fonts. Revert once on 3.0.8.
|
||||
pageDoc.save(pageOutputPath, CompressParameters.NO_COMPRESSION);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -437,7 +441,10 @@ public class OCRController {
|
||||
// Save original page without OCR
|
||||
try (PDDocument pageDoc = new PDDocument()) {
|
||||
pageDoc.addPage(page);
|
||||
pageDoc.save(pageOutputPath);
|
||||
// NO_COMPRESSION: page is copied from another document; PDFBox 3.0.7
|
||||
// compressed writer (PDFBOX-6203) drops shared resources, corrupting
|
||||
// fonts on retained text pages. Revert once on 3.0.8.
|
||||
pageDoc.save(pageOutputPath, CompressParameters.NO_COMPRESSION);
|
||||
merger.addSource(pageOutputPath);
|
||||
}
|
||||
}
|
||||
|
||||
+5
@@ -25,6 +25,7 @@ import stirling.software.common.annotations.api.MiscApi;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.SvgSanitizer;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
@@ -36,6 +37,7 @@ public class OverlayImageController {
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final SvgSanitizer svgSanitizer;
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
@@ -61,6 +63,9 @@ public class OverlayImageController {
|
||||
byte[] imageBytes = imageFile.getBytes();
|
||||
|
||||
boolean isSvg = SvgOverlayUtil.isSvgImage(imageBytes);
|
||||
if (isSvg) {
|
||||
imageBytes = svgSanitizer.sanitize(imageBytes);
|
||||
}
|
||||
|
||||
try (PDDocument document = pdfDocumentFactory.load(pdfBytes)) {
|
||||
int pages = document.getNumberOfPages();
|
||||
|
||||
+28
@@ -41,6 +41,8 @@ public class ReactRoutingController {
|
||||
private boolean indexHtmlExists = false;
|
||||
private boolean useExternalIndexHtml = false;
|
||||
private boolean loggedMissingIndex = false;
|
||||
private String cachedSaasLandingHtml;
|
||||
private boolean saasLandingExists = false;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
@@ -49,6 +51,20 @@ public class ReactRoutingController {
|
||||
// Always initialize callback HTML (used for OAuth desktop flow)
|
||||
this.cachedCallbackHtml = buildCallbackHtml();
|
||||
|
||||
// SaaS landing page: only present on the classpath when the :saas module is bundled
|
||||
// (app/saas/src/main/resources/static/saas-landing.html). When present it replaces the
|
||||
// root page so the SaaS API host shows its own landing instead of the OSS API-only page.
|
||||
ClassPathResource saasLanding = new ClassPathResource("static/saas-landing.html");
|
||||
if (saasLanding.exists()) {
|
||||
try (InputStream in = saasLanding.getInputStream()) {
|
||||
this.cachedSaasLandingHtml = new String(in.readAllBytes(), StandardCharsets.UTF_8);
|
||||
this.saasLandingExists = true;
|
||||
log.info("SaaS landing page detected; serving it at '/' and '/index.html'");
|
||||
} catch (Exception ex) {
|
||||
log.warn("Failed to read saas-landing.html; falling back to index.html", ex);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for external index.html first (customFiles/static/)
|
||||
Path externalIndexPath = Paths.get(InstallationPathConfig.getStaticPath(), "index.html");
|
||||
log.debug("Checking for custom index.html at: {}", externalIndexPath);
|
||||
@@ -132,6 +148,18 @@ public class ReactRoutingController {
|
||||
@GetMapping(
|
||||
value = {"/", "/index.html"},
|
||||
produces = MediaType.TEXT_HTML_VALUE)
|
||||
public ResponseEntity<String> serveRootPage(HttpServletRequest request) {
|
||||
// Swap ONLY the root page for SaaS. SPA entry points that delegate to serveIndexHtml
|
||||
// (/auth/callback, /share/{token}, forwarded routes) keep serving the normal shell.
|
||||
if (saasLandingExists && cachedSaasLandingHtml != null) {
|
||||
return ResponseEntity.ok()
|
||||
.cacheControl(CacheControl.noCache().mustRevalidate())
|
||||
.contentType(MediaType.TEXT_HTML)
|
||||
.body(cachedSaasLandingHtml);
|
||||
}
|
||||
return serveIndexHtml(request);
|
||||
}
|
||||
|
||||
public ResponseEntity<String> serveIndexHtml(HttpServletRequest request) {
|
||||
try {
|
||||
if (indexHtmlExists && cachedIndexHtml != null) {
|
||||
|
||||
@@ -24,6 +24,7 @@ import org.springframework.web.multipart.MaxUploadSizeExceededException;
|
||||
import org.springframework.web.multipart.support.MissingServletRequestPartException;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import org.springframework.web.servlet.NoHandlerFoundException;
|
||||
import org.springframework.web.servlet.resource.NoResourceFoundException;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
@@ -993,6 +994,49 @@ public class GlobalExceptionHandler {
|
||||
.body(problemDetail);
|
||||
}
|
||||
|
||||
/** Unmapped path → clean 404 instead of falling through to the generic 500 catch-all. */
|
||||
@ExceptionHandler(NoResourceFoundException.class)
|
||||
public ResponseEntity<ProblemDetail> handleNoResourceFound(
|
||||
NoResourceFoundException ex, HttpServletRequest request) {
|
||||
// /api/* miss = likely missing controller (operator-relevant); other paths = favicons,
|
||||
// robots.txt, scanner noise. Demote the latter so prod logs aren't flooded.
|
||||
String uri = request.getRequestURI();
|
||||
if (uri != null && uri.startsWith("/api/")) {
|
||||
log.warn("No resource at {}: {}", uri, ex.getMessage());
|
||||
} else {
|
||||
log.debug("No resource at {}: {}", uri, ex.getMessage());
|
||||
}
|
||||
|
||||
String title = getLocalizedMessage("error.notFound.title", ErrorTitles.NOT_FOUND_DEFAULT);
|
||||
String detail =
|
||||
getLocalizedMessage(
|
||||
"error.notFound.detail",
|
||||
String.format(
|
||||
"No endpoint found for %s %s",
|
||||
request.getMethod(), request.getRequestURI()),
|
||||
request.getMethod(),
|
||||
request.getRequestURI());
|
||||
|
||||
ProblemDetail problemDetail =
|
||||
createBaseProblemDetail(HttpStatus.NOT_FOUND, detail, request);
|
||||
problemDetail.setType(URI.create(ErrorTypes.NOT_FOUND));
|
||||
problemDetail.setTitle(title);
|
||||
problemDetail.setProperty("title", title);
|
||||
problemDetail.setProperty("method", request.getMethod());
|
||||
addStandardHints(
|
||||
problemDetail,
|
||||
"error.notFound.hints",
|
||||
List.of(
|
||||
"Verify the URL path and HTTP method are correct.",
|
||||
"Check the API base path and version if applicable.",
|
||||
"Ensure there are no typos in the endpoint path."));
|
||||
problemDetail.setProperty("actionRequired", "Use a valid endpoint URL and method.");
|
||||
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND)
|
||||
.contentType(PROBLEM_JSON)
|
||||
.body(problemDetail);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle IllegalArgumentException.
|
||||
*
|
||||
|
||||
+27
-5
@@ -1,11 +1,13 @@
|
||||
package stirling.software.SPDF.model.api.converters;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.github.pixee.security.Filenames;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -15,8 +17,11 @@ import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.ConvertApi;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
import stirling.software.common.util.PDFToFile;
|
||||
import stirling.software.common.pdf.PdfMarkdownConverter;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
@ConvertApi
|
||||
@RequiredArgsConstructor
|
||||
@@ -33,10 +38,27 @@ public class ConvertPDFToMarkdown {
|
||||
summary = "Convert PDF to Markdown",
|
||||
description =
|
||||
"This endpoint converts a PDF file to Markdown format. Input:PDF Output:Markdown Type:SISO")
|
||||
public ResponseEntity<Resource> processPdfToMarkdown(@ModelAttribute PDFFile file)
|
||||
public ResponseEntity<byte[]> processPdfToMarkdown(@ModelAttribute PDFFile file)
|
||||
throws Exception {
|
||||
MultipartFile inputFile = file.getFileInput();
|
||||
PDFToFile pdfToFile = new PDFToFile(tempFileManager);
|
||||
return pdfToFile.processPdfToMarkdown(inputFile);
|
||||
|
||||
String originalName = Filenames.toSimpleFileName(inputFile.getOriginalFilename());
|
||||
String baseName =
|
||||
originalName.contains(".")
|
||||
? originalName.substring(0, originalName.lastIndexOf('.'))
|
||||
: originalName;
|
||||
|
||||
String markdown;
|
||||
try (TempFile tempInput = new TempFile(tempFileManager, ".pdf")) {
|
||||
inputFile.transferTo(tempInput.getFile());
|
||||
try (PdfDocument doc = PdfDocument.open(tempInput.getPath())) {
|
||||
markdown = new PdfMarkdownConverter().convert(doc);
|
||||
}
|
||||
}
|
||||
|
||||
return WebResponseUtils.bytesToWebResponse(
|
||||
markdown.getBytes(StandardCharsets.UTF_8),
|
||||
baseName + ".md",
|
||||
MediaType.valueOf("text/markdown"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,6 +63,7 @@ public class ApiDocService implements stirling.software.common.service.ToolMetad
|
||||
return "http://localhost:" + port + contextPath + "/v1/api-docs";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getExtensionTypes(boolean output, String operationName) {
|
||||
if (outputToFileTypes.isEmpty()) {
|
||||
outputToFileTypes.put("PDF", List.of("pdf"));
|
||||
|
||||
@@ -10,6 +10,7 @@ import org.apache.batik.bridge.GVTBuilder;
|
||||
import org.apache.batik.bridge.UserAgent;
|
||||
import org.apache.batik.bridge.UserAgentAdapter;
|
||||
import org.apache.batik.gvt.GraphicsNode;
|
||||
import org.apache.batik.util.ParsedURL;
|
||||
import org.apache.batik.util.XMLResourceDescriptor;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
@@ -39,7 +40,16 @@ public class SvgOverlayUtil {
|
||||
svgDoc = factory.createSVGDocument("file:///overlay.svg", inputStream);
|
||||
}
|
||||
|
||||
UserAgent userAgent = new UserAgentAdapter();
|
||||
UserAgent userAgent =
|
||||
new UserAgentAdapter() {
|
||||
@Override
|
||||
public void checkLoadExternalResource(
|
||||
ParsedURL resourceURL, ParsedURL docURL) {
|
||||
throw new SecurityException(
|
||||
"External resource loading is disabled for SVG overlays: "
|
||||
+ resourceURL);
|
||||
}
|
||||
};
|
||||
DocumentLoader loader = new DocumentLoader(userAgent);
|
||||
BridgeContext ctx = new BridgeContext(userAgent, loader);
|
||||
ctx.setDynamicState(BridgeContext.DYNAMIC);
|
||||
|
||||
@@ -93,6 +93,11 @@ posthog.host=https://eu.i.posthog.com
|
||||
|
||||
spring.main.allow-bean-definition-overriding=true
|
||||
|
||||
# spring-data-redis is on the classpath only for the optional Valkey backplane (which wires its own
|
||||
# factory); exclude Spring Boot's stock Redis auto-config so a default install doesn't create a dead
|
||||
# localhost:6379 factory that flips /actuator/health to DOWN.
|
||||
spring.autoconfigure.exclude=org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration
|
||||
|
||||
# Set up a consistent temporary directory location
|
||||
java.io.tmpdir=${stirling.tempfiles.directory:${java.io.tmpdir}/stirling-pdf}
|
||||
|
||||
|
||||
@@ -364,6 +364,36 @@ aiEngine:
|
||||
url: http://localhost:5001 # URL of the Python AI engine
|
||||
timeoutSeconds: 120 # Timeout in seconds for AI engine requests
|
||||
|
||||
policies:
|
||||
# Folder automations can read from and write to the directories you allow here, so treat this as a
|
||||
# security boundary. Leave allowedFolderRoots empty (default) to disable folder sources/outputs
|
||||
# entirely; list absolute directories to permit folder access only within them. Stirling's own
|
||||
# config directory is always off-limits, and folder access is always disabled in SaaS mode.
|
||||
allowedFolderRoots: [] # e.g. ["/data/inbox", "/data/outbox"]
|
||||
scheduleSweepSeconds: 60 # How often (seconds) scheduled policies are checked for being due
|
||||
watchReconcileSeconds: 300 # How often (seconds) folder-watch re-syncs watches and re-runs as a safety net for missed events
|
||||
watchQuietPeriodMs: 500 # How long (ms) folder-watch coalesces a burst of file events into a single run
|
||||
streamTimeoutMs: 1800000 # SSE timeout (ms) for live run-progress streams
|
||||
runExpiryMinutes: 30 # How long (minutes) a finished run's in-memory state is kept before eviction
|
||||
|
||||
# Model Context Protocol (MCP) server. Exposes Stirling's PDF tools (grouped by namespace)
|
||||
# plus the AI agents to MCP clients (Inspector, Claude Desktop, custom). OAuth-protected.
|
||||
# Disabled by default - enable explicitly per deployment after configuring mcp.auth.
|
||||
mcp:
|
||||
enabled: false # Master switch. 'false' (default) means no /mcp endpoint, no metadata, no beans wired.
|
||||
scopesEnabled: true # Enforce mcp.tools.read / mcp.tools.write scopes derived from operation category
|
||||
allowedOperations: [] # Tool allow-list (operation ids, e.g. ['compress-pdf']). Empty = all. When set, ONLY these are exposed over MCP.
|
||||
blockedOperations: [] # Tool deny-list (operation ids). Always removed from MCP even if otherwise allowed.
|
||||
auth:
|
||||
mode: oauth # 'oauth' (full OAuth2 resource server) or 'apikey' (Stirling per-user API key via X-API-KEY header; no external IdP needed - the low-friction self-host option)
|
||||
issuerUri: "" # OAuth2 issuer URI (e.g. http://localhost:9000). Required when mode=oauth.
|
||||
jwksUri: "" # JWKS URI. Blank -> derived from issuer's /.well-known/openid-configuration.
|
||||
resourceId: "" # RFC 8707 resource identifier of THIS MCP server (e.g. http://localhost:8080/mcp).
|
||||
# Required: tokens must list this id in `aud` or the request is rejected.
|
||||
usernameClaim: sub # JWT claim matched against a Stirling username (e.g. 'sub', 'email', 'preferred_username')
|
||||
requireExistingAccount: true # Reject tokens whose subject has no enabled Stirling account (recommended)
|
||||
engineCapabilityRefreshMinutes: 5 # How often to refresh the AI capabilities manifest from the engine
|
||||
|
||||
# Cluster configuration. NOT YET ENABLED - scaffolding for later work. Leave at defaults.
|
||||
cluster:
|
||||
enabled: false # Master switch. 'false' (default) wires the in-process backplane and skips all cluster checks. Single-instance installs do not need to change anything here.
|
||||
|
||||
+129
-144
@@ -4,10 +4,14 @@ import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -56,6 +60,38 @@ class RearrangePagesPDFControllerTest {
|
||||
"fileInput", "test.pdf", MediaType.APPLICATION_PDF_VALUE, new byte[] {1, 2, 3});
|
||||
}
|
||||
|
||||
/** Build a real, in-memory PDDocument with the requested number of blank pages. */
|
||||
private PDDocument buildRealPdf(int pageCount) throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
for (int i = 0; i < pageCount; i++) {
|
||||
doc.addPage(new PDPage());
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the underlying {@link org.apache.pdfbox.cos.COSDictionary} for each page in document
|
||||
* order. PDPageTree returns a fresh PDPage wrapper per get(), so comparing wrappers with
|
||||
* assertSame is unreliable - the COSDictionary identity is the stable handle.
|
||||
*/
|
||||
private List<Object> snapshotCosPages(PDDocument doc) {
|
||||
List<Object> snapshot = new ArrayList<>();
|
||||
for (PDPage p : doc.getPages()) {
|
||||
snapshot.add(p.getCOSObject());
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
private List<Object> reloadAndSnapshot(ResponseEntity<Resource> response) throws IOException {
|
||||
try (var in = response.getBody().getInputStream();
|
||||
var baos = new ByteArrayOutputStream()) {
|
||||
in.transferTo(baos);
|
||||
try (PDDocument out = Loader.loadPDF(baos.toByteArray())) {
|
||||
return snapshotCosPages(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDeletePages_Success() throws IOException {
|
||||
MockMultipartFile file = createMockPdf();
|
||||
@@ -83,27 +119,23 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("");
|
||||
request.setCustomMode("REVERSE_ORDER");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page0 = mock(PDPage.class);
|
||||
PDPage page1 = mock(PDPage.class);
|
||||
PDPage page2 = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(3)) {
|
||||
List<Object> originals = snapshotCosPages(realDoc);
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(3);
|
||||
when(mockDoc.getPage(0)).thenReturn(page0);
|
||||
when(mockDoc.getPage(1)).thenReturn(page1);
|
||||
when(mockDoc.getPage(2)).thenReturn(page2);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
verify(mockNewDoc).addPage(page2);
|
||||
verify(mockNewDoc).addPage(page1);
|
||||
verify(mockNewDoc).addPage(page0);
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
List<Object> finalOrder = reloadAndSnapshot(response);
|
||||
assertEquals(3, finalOrder.size());
|
||||
// We can no longer compare references after a save/reload, so compare via
|
||||
// the in-memory snapshot taken *after* the controller mutated the source.
|
||||
List<Object> mutatedSource = snapshotCosPages(realDoc);
|
||||
assertSame(originals.get(2), mutatedSource.get(0));
|
||||
assertSame(originals.get(1), mutatedSource.get(1));
|
||||
assertSame(originals.get(0), mutatedSource.get(2));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -114,25 +146,18 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("");
|
||||
request.setCustomMode("REMOVE_FIRST");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page0 = mock(PDPage.class);
|
||||
PDPage page1 = mock(PDPage.class);
|
||||
PDPage page2 = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(3)) {
|
||||
List<Object> originals = snapshotCosPages(realDoc);
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(3);
|
||||
when(mockDoc.getPage(1)).thenReturn(page1);
|
||||
when(mockDoc.getPage(2)).thenReturn(page2);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
verify(mockNewDoc).addPage(page1);
|
||||
verify(mockNewDoc).addPage(page2);
|
||||
verify(mockNewDoc, never()).addPage(page0);
|
||||
assertNotNull(response);
|
||||
List<Object> mutated = snapshotCosPages(realDoc);
|
||||
assertEquals(2, mutated.size());
|
||||
assertSame(originals.get(1), mutated.get(0));
|
||||
assertSame(originals.get(2), mutated.get(1));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -143,23 +168,18 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("");
|
||||
request.setCustomMode("REMOVE_LAST");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page0 = mock(PDPage.class);
|
||||
PDPage page1 = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(3)) {
|
||||
List<Object> originals = snapshotCosPages(realDoc);
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(3);
|
||||
when(mockDoc.getPage(0)).thenReturn(page0);
|
||||
when(mockDoc.getPage(1)).thenReturn(page1);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
verify(mockNewDoc).addPage(page0);
|
||||
verify(mockNewDoc).addPage(page1);
|
||||
assertNotNull(response);
|
||||
List<Object> mutated = snapshotCosPages(realDoc);
|
||||
assertEquals(2, mutated.size());
|
||||
assertSame(originals.get(0), mutated.get(0));
|
||||
assertSame(originals.get(1), mutated.get(1));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -170,21 +190,19 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("");
|
||||
request.setCustomMode("REMOVE_FIRST_AND_LAST");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page1 = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(4)) {
|
||||
List<Object> originals = snapshotCosPages(realDoc);
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(4);
|
||||
when(mockDoc.getPage(1)).thenReturn(page1);
|
||||
when(mockDoc.getPage(2)).thenReturn(page1);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
List<Object> mutated = snapshotCosPages(realDoc);
|
||||
assertEquals(2, mutated.size());
|
||||
assertSame(originals.get(1), mutated.get(0));
|
||||
assertSame(originals.get(2), mutated.get(1));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -195,23 +213,15 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("");
|
||||
request.setCustomMode("DUPLEX_SORT");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page0 = mock(PDPage.class);
|
||||
PDPage page1 = mock(PDPage.class);
|
||||
PDPage page2 = mock(PDPage.class);
|
||||
PDPage page3 = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(4)) {
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(4);
|
||||
when(mockDoc.getPage(anyInt())).thenReturn(page0);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertEquals(4, realDoc.getNumberOfPages());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -222,20 +232,15 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("");
|
||||
request.setCustomMode("BOOKLET_SORT");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(4)) {
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(4);
|
||||
when(mockDoc.getPage(anyInt())).thenReturn(page);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertEquals(4, realDoc.getNumberOfPages());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -246,20 +251,15 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("");
|
||||
request.setCustomMode("ODD_EVEN_SPLIT");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(4)) {
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(4);
|
||||
when(mockDoc.getPage(anyInt())).thenReturn(page);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertEquals(4, realDoc.getNumberOfPages());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -270,24 +270,20 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("3,1,2");
|
||||
request.setCustomMode("custom");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page0 = mock(PDPage.class);
|
||||
PDPage page1 = mock(PDPage.class);
|
||||
PDPage page2 = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(3)) {
|
||||
List<Object> originals = snapshotCosPages(realDoc);
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(3);
|
||||
when(mockDoc.getPage(0)).thenReturn(page0);
|
||||
when(mockDoc.getPage(1)).thenReturn(page1);
|
||||
when(mockDoc.getPage(2)).thenReturn(page2);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
List<Object> mutated = snapshotCosPages(realDoc);
|
||||
assertEquals(3, mutated.size());
|
||||
assertSame(originals.get(2), mutated.get(0));
|
||||
assertSame(originals.get(0), mutated.get(1));
|
||||
assertSame(originals.get(1), mutated.get(2));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -298,21 +294,15 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("3");
|
||||
request.setCustomMode("DUPLICATE");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(2)) {
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(2);
|
||||
when(mockDoc.getPage(anyInt())).thenReturn(page);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
// 2 pages * 3 duplicates = 6 addPage calls
|
||||
verify(mockNewDoc, times(6)).addPage(page);
|
||||
assertNotNull(response);
|
||||
// 2 pages * 3 duplicates = 6 final pages
|
||||
assertEquals(6, realDoc.getNumberOfPages());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -323,19 +313,14 @@ class RearrangePagesPDFControllerTest {
|
||||
request.setPageNumbers("");
|
||||
request.setCustomMode("SIDE_STITCH_BOOKLET_SORT");
|
||||
|
||||
PDDocument mockDoc = mock(PDDocument.class);
|
||||
PDDocument mockNewDoc = mock(PDDocument.class);
|
||||
PDPage page = mock(PDPage.class);
|
||||
try (PDDocument realDoc = buildRealPdf(4)) {
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(realDoc);
|
||||
|
||||
when(pdfDocumentFactory.load(file)).thenReturn(mockDoc);
|
||||
when(mockDoc.getNumberOfPages()).thenReturn(4);
|
||||
when(mockDoc.getPage(anyInt())).thenReturn(page);
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(mockDoc))
|
||||
.thenReturn(mockNewDoc);
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
ResponseEntity<Resource> response = controller.rearrangePages(request);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertNotNull(response);
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
assertEquals(4, realDoc.getNumberOfPages());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+48
@@ -32,6 +32,7 @@ import org.springframework.mock.web.MockMultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.misc.OverlayImageRequest;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.SvgSanitizer;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
@@ -52,6 +53,7 @@ class OverlayImageControllerTest {
|
||||
|
||||
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
@Mock private SvgSanitizer svgSanitizer;
|
||||
|
||||
@InjectMocks private OverlayImageController controller;
|
||||
|
||||
@@ -205,6 +207,52 @@ class OverlayImageControllerTest {
|
||||
mockDoc.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void overlayImage_svgInput_sanitizedBeforeOverlay() throws Exception {
|
||||
byte[] maliciousSvg =
|
||||
("<svg xmlns=\"http://www.w3.org/2000/svg\""
|
||||
+ " xmlns:xlink=\"http://www.w3.org/1999/xlink\""
|
||||
+ " width=\"10\" height=\"10\">"
|
||||
+ "<image x=\"0\" y=\"0\" width=\"10\" height=\"10\""
|
||||
+ " xlink:href=\"file:///etc/passwd\"/>"
|
||||
+ "</svg>")
|
||||
.getBytes();
|
||||
byte[] sanitized =
|
||||
("<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"10\" height=\"10\">"
|
||||
+ "<image x=\"0\" y=\"0\" width=\"10\" height=\"10\"/>"
|
||||
+ "</svg>")
|
||||
.getBytes();
|
||||
when(svgSanitizer.sanitize(maliciousSvg)).thenReturn(sanitized);
|
||||
|
||||
MockMultipartFile svgFile =
|
||||
new MockMultipartFile("imageFile", "overlay.svg", "image/svg+xml", maliciousSvg);
|
||||
OverlayImageRequest request = new OverlayImageRequest();
|
||||
request.setFileInput(pdfFile);
|
||||
request.setImageFile(svgFile);
|
||||
request.setX(0);
|
||||
request.setY(0);
|
||||
request.setEveryPage(false);
|
||||
|
||||
PDDocument mockDoc = new PDDocument();
|
||||
mockDoc.addPage(new PDPage(PDRectangle.A4));
|
||||
when(pdfDocumentFactory.load(any(byte[].class))).thenReturn(mockDoc);
|
||||
|
||||
try (MockedStatic<WebResponseUtils> mockedWebResponse =
|
||||
mockStatic(WebResponseUtils.class)) {
|
||||
mockedWebResponse
|
||||
.when(
|
||||
() ->
|
||||
WebResponseUtils.pdfFileToWebResponse(
|
||||
any(TempFile.class), anyString()))
|
||||
.thenReturn(streamingOk("result".getBytes()));
|
||||
|
||||
controller.overlayImage(request);
|
||||
}
|
||||
mockDoc.close();
|
||||
|
||||
verify(svgSanitizer).sanitize(maliciousSvg);
|
||||
}
|
||||
|
||||
@Test
|
||||
void overlayImage_withCoordinates_usesXY() throws Exception {
|
||||
OverlayImageRequest request = new OverlayImageRequest();
|
||||
|
||||
+15
@@ -18,6 +18,7 @@ import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.context.MessageSource;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ProblemDetail;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@@ -27,6 +28,7 @@ import org.springframework.web.bind.MissingServletRequestParameterException;
|
||||
import org.springframework.web.multipart.MaxUploadSizeExceededException;
|
||||
import org.springframework.web.multipart.support.MissingServletRequestPartException;
|
||||
import org.springframework.web.servlet.NoHandlerFoundException;
|
||||
import org.springframework.web.servlet.resource.NoResourceFoundException;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
@@ -208,6 +210,19 @@ class GlobalExceptionHandlerTest {
|
||||
assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode());
|
||||
}
|
||||
|
||||
// ---- NoResourceFoundException ----
|
||||
// Regression guard: was falling through to the 500 catch-all.
|
||||
|
||||
@Test
|
||||
void handleNoResourceFound_returns_404_not_500() {
|
||||
when(request.getMethod()).thenReturn("GET");
|
||||
NoResourceFoundException ex =
|
||||
new NoResourceFoundException(HttpMethod.GET, "/api/v1/storage/folders", "");
|
||||
ResponseEntity<ProblemDetail> resp = handler.handleNoResourceFound(ex, request);
|
||||
assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode());
|
||||
assertEquals("GET", resp.getBody().getProperties().get("method"));
|
||||
}
|
||||
|
||||
// ---- IllegalArgumentException ----
|
||||
|
||||
@Test
|
||||
|
||||
+48
-46
@@ -1,16 +1,17 @@
|
||||
package stirling.software.SPDF.model.api.converters;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.MockedConstruction;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
@@ -21,9 +22,10 @@ import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.common.util.PDFToFile;
|
||||
import stirling.software.common.pdf.PdfMarkdownConverter;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
class ConvertPDFToMarkdownTest {
|
||||
|
||||
@@ -47,68 +49,68 @@ class ConvertPDFToMarkdownTest {
|
||||
@Test
|
||||
void pdfToMarkdownReturnsMarkdownBytes() throws Exception {
|
||||
byte[] md = "# heading\n\ncontent\n".getBytes(StandardCharsets.UTF_8);
|
||||
String expectedMd = "# heading\n\ncontent\n";
|
||||
|
||||
try (MockedConstruction<PDFToFile> construction =
|
||||
Mockito.mockConstruction(
|
||||
PDFToFile.class,
|
||||
(mock, ctx) -> {
|
||||
when(mock.processPdfToMarkdown(any(MultipartFile.class)))
|
||||
.thenAnswer(
|
||||
inv ->
|
||||
ResponseEntity.ok()
|
||||
.header("Content-Type", "text/markdown")
|
||||
.body(new ByteArrayResource(md)));
|
||||
})) {
|
||||
File tmpFile = File.createTempFile("test", ".pdf");
|
||||
tmpFile.deleteOnExit();
|
||||
|
||||
MockMvc mvc = mockMvc();
|
||||
try (MockedConstruction<TempFile> tempMock =
|
||||
Mockito.mockConstruction(
|
||||
TempFile.class,
|
||||
(mock, ctx) -> {
|
||||
when(mock.getFile()).thenReturn(tmpFile);
|
||||
when(mock.getPath()).thenReturn(tmpFile.toPath());
|
||||
});
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
|
||||
MockedConstruction<PdfMarkdownConverter> converterMock =
|
||||
Mockito.mockConstruction(
|
||||
PdfMarkdownConverter.class,
|
||||
(mock, ctx) -> when(mock.convert(any())).thenReturn(expectedMd))) {
|
||||
|
||||
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
|
||||
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
|
||||
|
||||
MockMultipartFile file =
|
||||
new MockMultipartFile(
|
||||
"fileInput", // must match the field name in PDFFile
|
||||
"input.pdf",
|
||||
"application/pdf",
|
||||
new byte[] {1, 2, 3});
|
||||
"fileInput", "input.pdf", "application/pdf", new byte[] {1, 2, 3});
|
||||
|
||||
// ResponseEntity<Resource> is written synchronously on the request thread,
|
||||
// so there is no async dispatch to wait for (unlike the old StreamingResponseBody
|
||||
// path).
|
||||
mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
|
||||
mockMvc()
|
||||
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(header().string("Content-Type", "text/markdown"))
|
||||
.andExpect(content().bytes(md));
|
||||
|
||||
// Verify that exactly one instance was created
|
||||
assert construction.constructed().size() == 1;
|
||||
|
||||
// And that the uploaded file was passed to processPdfToMarkdown()
|
||||
PDFToFile created = construction.constructed().get(0);
|
||||
ArgumentCaptor<MultipartFile> captor = ArgumentCaptor.forClass(MultipartFile.class);
|
||||
verify(created, times(1)).processPdfToMarkdown(captor.capture());
|
||||
MultipartFile passed = captor.getValue();
|
||||
|
||||
// Minimal plausibility checks
|
||||
assertEquals("input.pdf", passed.getOriginalFilename());
|
||||
assertEquals("application/pdf", passed.getContentType());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void pdfToMarkdownWhenServiceThrowsReturns500() throws Exception {
|
||||
try (MockedConstruction<PDFToFile> ignored =
|
||||
Mockito.mockConstruction(
|
||||
PDFToFile.class,
|
||||
(mock, ctx) -> {
|
||||
when(mock.processPdfToMarkdown(any(MultipartFile.class)))
|
||||
.thenThrow(new RuntimeException("boom"));
|
||||
})) {
|
||||
File tmpFile = File.createTempFile("test", ".pdf");
|
||||
tmpFile.deleteOnExit();
|
||||
|
||||
MockMvc mvc = mockMvc();
|
||||
try (MockedConstruction<TempFile> tempMock =
|
||||
Mockito.mockConstruction(
|
||||
TempFile.class,
|
||||
(mock, ctx) -> {
|
||||
when(mock.getFile()).thenReturn(tmpFile);
|
||||
when(mock.getPath()).thenReturn(tmpFile.toPath());
|
||||
});
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
|
||||
MockedConstruction<PdfMarkdownConverter> converterMock =
|
||||
Mockito.mockConstruction(
|
||||
PdfMarkdownConverter.class,
|
||||
(mock, ctx) ->
|
||||
when(mock.convert(any()))
|
||||
.thenThrow(new RuntimeException("boom")))) {
|
||||
|
||||
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
|
||||
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
|
||||
|
||||
MockMultipartFile file =
|
||||
new MockMultipartFile(
|
||||
"fileInput", "x.pdf", "application/pdf", new byte[] {0x01});
|
||||
|
||||
mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
|
||||
mockMvc()
|
||||
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
|
||||
.andExpect(status().isInternalServerError());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,10 @@ dependencies {
|
||||
api 'org.springframework.boot:spring-boot-starter-security'
|
||||
api 'org.springframework.boot:spring-boot-starter-data-jpa'
|
||||
api 'org.springframework.boot:spring-boot-starter-security-oauth2-client'
|
||||
// MCP server (RFC 8707 audience binding + RFC 9728 metadata) - resource-server side only.
|
||||
// Brings nimbus-jose-jwt onto the proprietary classpath if not already transitive via
|
||||
// oauth2-client; on Boot 4.0.6 the delta is around 130KB because nimbus is already pulled.
|
||||
api 'org.springframework.boot:spring-boot-starter-oauth2-resource-server'
|
||||
api 'org.springframework.boot:spring-boot-starter-mail'
|
||||
api 'org.springframework.boot:spring-boot-starter-cache'
|
||||
api 'com.github.ben-manes.caffeine:caffeine'
|
||||
|
||||
+41
-5
@@ -6,6 +6,8 @@ import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
@@ -35,6 +37,7 @@ import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||
public class S3FileStore implements FileStore, AutoCloseable {
|
||||
|
||||
public static final String DEFAULT_KEY_PREFIX = "transient/";
|
||||
static final String OWNER_METADATA_KEY = "owner";
|
||||
|
||||
private final S3Client s3Client;
|
||||
private final String bucket;
|
||||
@@ -64,7 +67,7 @@ public class S3FileStore implements FileStore, AutoCloseable {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stored store(InputStream in, String originalName) throws IOException {
|
||||
public Stored store(InputStream in, String originalName, String owner) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
// S3 PUT requires a known content-length; spool to a temp file first so memory stays
|
||||
// bounded for large payloads, then stream the file to S3 via RequestBody.fromFile.
|
||||
@@ -75,10 +78,13 @@ public class S3FileStore implements FileStore, AutoCloseable {
|
||||
Files.copy(src, tempFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
size = Files.size(tempFile);
|
||||
PutObjectRequest request =
|
||||
PutObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
|
||||
PutObjectRequest.Builder builder =
|
||||
PutObjectRequest.builder().bucket(bucket).key(resolveKey(fileId));
|
||||
if (owner != null && !owner.isBlank()) {
|
||||
builder.metadata(Map.of(OWNER_METADATA_KEY, owner));
|
||||
}
|
||||
try {
|
||||
s3Client.putObject(request, RequestBody.fromFile(tempFile));
|
||||
s3Client.putObject(builder.build(), RequestBody.fromFile(tempFile));
|
||||
} catch (SdkException e) {
|
||||
throw new IOException("Failed to upload object to S3", e);
|
||||
}
|
||||
@@ -185,6 +191,36 @@ public class S3FileStore implements FileStore, AutoCloseable {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOwner(String fileId) throws IOException {
|
||||
try {
|
||||
validateFileId(fileId);
|
||||
} catch (IllegalArgumentException e) {
|
||||
return null;
|
||||
}
|
||||
HeadObjectRequest request =
|
||||
HeadObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
|
||||
try {
|
||||
HeadObjectResponse response = s3Client.headObject(request);
|
||||
Map<String, String> metadata =
|
||||
Optional.ofNullable(response.metadata()).orElse(Collections.emptyMap());
|
||||
String owner = metadata.get(OWNER_METADATA_KEY);
|
||||
if (owner != null && !owner.isBlank()) {
|
||||
return owner;
|
||||
}
|
||||
return null;
|
||||
} catch (NoSuchKeyException e) {
|
||||
return null;
|
||||
} catch (S3Exception e) {
|
||||
if (e.statusCode() == 404) {
|
||||
return null;
|
||||
}
|
||||
throw new IOException("Failed to read owner metadata from S3", e);
|
||||
} catch (SdkException e) {
|
||||
throw new IOException("Failed to read owner metadata from S3", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!ownsClient) {
|
||||
@@ -205,7 +241,7 @@ public class S3FileStore implements FileStore, AutoCloseable {
|
||||
if (fileId == null || fileId.isBlank()) {
|
||||
throw new IllegalArgumentException("File ID must not be blank");
|
||||
}
|
||||
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
if (fileId.contains(".") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
throw new IllegalArgumentException("Invalid file ID");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package stirling.software.proprietary.mcp;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
/** Per-call context: resolved Stirling identity and granted scopes for an {@link McpTool#call}. */
|
||||
public record McpCallContext(
|
||||
String stirlingUserId, Set<String> grantedScopes, boolean scopesEnabled) {
|
||||
|
||||
public boolean hasScope(String required) {
|
||||
if (!scopesEnabled) {
|
||||
return true;
|
||||
}
|
||||
return required == null || required.isBlank() || grantedScopes.contains(required);
|
||||
}
|
||||
}
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
package stirling.software.proprietary.mcp;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.http.converter.HttpMessageNotReadableException;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.mcp.jsonrpc.JsonRpcError;
|
||||
import stirling.software.proprietary.mcp.jsonrpc.JsonRpcRequest;
|
||||
import stirling.software.proprietary.mcp.jsonrpc.JsonRpcResponse;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/** Streamable-HTTP MCP server endpoint serving JSON-RPC 2.0 frames on {@code POST /mcp}. */
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class McpServerController {
|
||||
|
||||
private static final String PREFERRED_PROTOCOL_VERSION = "2025-06-18";
|
||||
private static final Set<String> SUPPORTED_PROTOCOL_VERSIONS =
|
||||
Set.of("2025-06-18", "2025-03-26", "2024-11-05");
|
||||
private static final String SERVER_NAME = "stirling-pdf-mcp";
|
||||
|
||||
private final ObjectMapper mapper;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final Map<String, McpTool> toolsByName;
|
||||
|
||||
public McpServerController(
|
||||
ObjectMapper mapper, ApplicationProperties applicationProperties, List<McpTool> tools) {
|
||||
this.mapper = mapper;
|
||||
this.applicationProperties = applicationProperties;
|
||||
this.toolsByName = new HashMap<>();
|
||||
for (McpTool tool : tools) {
|
||||
this.toolsByName.put(tool.name(), tool);
|
||||
}
|
||||
log.info(
|
||||
"MCP server controller wired with {} tool(s): {}",
|
||||
toolsByName.size(),
|
||||
toolsByName.keySet());
|
||||
}
|
||||
|
||||
@PostMapping(
|
||||
path = "/mcp",
|
||||
consumes = MediaType.APPLICATION_JSON_VALUE,
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public ResponseEntity<?> handle(@RequestBody JsonNode body) {
|
||||
JsonRpcRequest request = decode(body);
|
||||
if (request == null) {
|
||||
// Valid JSON but not a JSON-RPC request -> Invalid Request, not Parse error.
|
||||
return ResponseEntity.badRequest()
|
||||
.body(
|
||||
JsonRpcResponse.failure(
|
||||
null,
|
||||
JsonRpcError.invalidRequest(
|
||||
"Body is not a valid JSON-RPC 2.0 request")));
|
||||
}
|
||||
if (request.isNotification()) {
|
||||
log.debug("Notification received: {}", sanitizeForLog(request.method()));
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).build();
|
||||
}
|
||||
JsonRpcResponse response;
|
||||
try {
|
||||
response = dispatch(request);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn(
|
||||
"MCP dispatch failed for method {}: {}",
|
||||
sanitizeForLog(request.method()),
|
||||
e.getMessage(),
|
||||
e);
|
||||
response =
|
||||
JsonRpcResponse.failure(
|
||||
request.id(),
|
||||
JsonRpcError.internalError(
|
||||
"Internal error handling " + request.method()));
|
||||
}
|
||||
return ResponseEntity.ok(response);
|
||||
}
|
||||
|
||||
/** Wrap malformed-JSON failures (caught before {@link #handle}) as a JSON-RPC Parse error. */
|
||||
@ExceptionHandler(HttpMessageNotReadableException.class)
|
||||
public ResponseEntity<JsonRpcResponse> handleUnreadable(HttpMessageNotReadableException ex) {
|
||||
return ResponseEntity.badRequest()
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.body(
|
||||
JsonRpcResponse.failure(
|
||||
null, JsonRpcError.parseError("Request body is not valid JSON")));
|
||||
}
|
||||
|
||||
private static String sanitizeForLog(String value) {
|
||||
return value == null ? null : value.replace('\r', ' ').replace('\n', ' ');
|
||||
}
|
||||
|
||||
private JsonRpcRequest decode(JsonNode body) {
|
||||
if (body == null || !body.isObject()) {
|
||||
return null;
|
||||
}
|
||||
JsonNode jsonrpc = body.get("jsonrpc");
|
||||
JsonNode method = body.get("method");
|
||||
if (jsonrpc == null || !"2.0".equals(jsonrpc.asText())) {
|
||||
return null;
|
||||
}
|
||||
if (method == null || !method.isTextual()) {
|
||||
return null;
|
||||
}
|
||||
return new JsonRpcRequest(
|
||||
jsonrpc.asText(), body.get("id"), method.asText(), body.get("params"));
|
||||
}
|
||||
|
||||
private JsonRpcResponse dispatch(JsonRpcRequest request) {
|
||||
return switch (request.method()) {
|
||||
case "initialize" ->
|
||||
JsonRpcResponse.success(request.id(), initializeResult(request.params()));
|
||||
case "tools/list" -> JsonRpcResponse.success(request.id(), toolsListResult());
|
||||
case "tools/call" -> handleToolsCall(request);
|
||||
case "ping" -> JsonRpcResponse.success(request.id(), mapper.createObjectNode());
|
||||
case "notifications/initialized" ->
|
||||
JsonRpcResponse.success(request.id(), mapper.createObjectNode());
|
||||
default ->
|
||||
JsonRpcResponse.failure(
|
||||
request.id(), JsonRpcError.methodNotFound(request.method()));
|
||||
};
|
||||
}
|
||||
|
||||
private ObjectNode initializeResult(JsonNode params) {
|
||||
ObjectNode result = mapper.createObjectNode();
|
||||
// Echo the client's requested protocolVersion when supported, else advertise our preferred.
|
||||
String requested =
|
||||
params != null && params.hasNonNull("protocolVersion")
|
||||
? params.get("protocolVersion").asText()
|
||||
: null;
|
||||
String negotiated =
|
||||
requested != null && SUPPORTED_PROTOCOL_VERSIONS.contains(requested)
|
||||
? requested
|
||||
: PREFERRED_PROTOCOL_VERSION;
|
||||
result.put("protocolVersion", negotiated);
|
||||
ObjectNode caps = result.putObject("capabilities");
|
||||
caps.putObject("tools");
|
||||
ObjectNode info = result.putObject("serverInfo");
|
||||
info.put("name", SERVER_NAME);
|
||||
info.put("version", applicationProperties.getAutomaticallyGenerated().getAppVersion());
|
||||
return result;
|
||||
}
|
||||
|
||||
private ObjectNode toolsListResult() {
|
||||
ObjectNode result = mapper.createObjectNode();
|
||||
ArrayNode tools = result.putArray("tools");
|
||||
for (McpTool t : toolsByName.values()) {
|
||||
ObjectNode entry = mapper.createObjectNode();
|
||||
entry.put("name", t.name());
|
||||
entry.put("description", t.description());
|
||||
entry.set("inputSchema", t.inputSchema());
|
||||
tools.add(entry);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private JsonRpcResponse handleToolsCall(JsonRpcRequest request) {
|
||||
JsonNode params = request.params();
|
||||
if (params == null || !params.isObject()) {
|
||||
return JsonRpcResponse.failure(
|
||||
request.id(), JsonRpcError.invalidParams("Missing params for tools/call"));
|
||||
}
|
||||
JsonNode nameNode = params.get("name");
|
||||
if (nameNode == null || !nameNode.isTextual()) {
|
||||
return JsonRpcResponse.failure(
|
||||
request.id(), JsonRpcError.invalidParams("Missing tool name"));
|
||||
}
|
||||
McpTool tool = toolsByName.get(nameNode.asText());
|
||||
if (tool == null) {
|
||||
return JsonRpcResponse.failure(
|
||||
request.id(), JsonRpcError.invalidParams("Unknown tool: " + nameNode.asText()));
|
||||
}
|
||||
JsonNode args = params.get("arguments");
|
||||
McpCallContext context = resolveContext();
|
||||
ObjectNode toolResult = tool.call(args == null ? mapper.createObjectNode() : args, context);
|
||||
return JsonRpcResponse.success(request.id(), toolResult);
|
||||
}
|
||||
|
||||
private McpCallContext resolveContext() {
|
||||
boolean scopesEnabled = applicationProperties.getMcp().isScopesEnabled();
|
||||
org.springframework.security.core.Authentication auth =
|
||||
org.springframework.security.core.context.SecurityContextHolder.getContext()
|
||||
.getAuthentication();
|
||||
// Fail closed: no/unauthenticated principal yields an empty context so scoped ops are
|
||||
// refused.
|
||||
if (auth == null || !auth.isAuthenticated() || auth.getName() == null) {
|
||||
return new McpCallContext(null, Set.of(), scopesEnabled);
|
||||
}
|
||||
java.util.Set<String> scopes = new java.util.HashSet<>();
|
||||
for (org.springframework.security.core.GrantedAuthority ga : auth.getAuthorities()) {
|
||||
String authority = ga.getAuthority();
|
||||
if (authority != null && authority.startsWith("SCOPE_")) {
|
||||
scopes.add(authority.substring("SCOPE_".length()));
|
||||
}
|
||||
}
|
||||
return new McpCallContext(auth.getName(), scopes, scopesEnabled);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package stirling.software.proprietary.mcp;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/** Contract every MCP tool registered with the server must satisfy. */
|
||||
public interface McpTool {
|
||||
|
||||
String name();
|
||||
|
||||
String description();
|
||||
|
||||
/** The tool's {@code inputSchema} (an object JSON Schema) published in {@code tools/list}. */
|
||||
ObjectNode inputSchema();
|
||||
|
||||
/** Execute the tool; the controller wraps any thrown exception as an MCP internal error. */
|
||||
ObjectNode call(JsonNode arguments, McpCallContext context);
|
||||
}
|
||||
+266
@@ -0,0 +1,266 @@
|
||||
package stirling.software.proprietary.mcp.catalog;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.event.ContextRefreshedEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.core.MethodParameter;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.bind.annotation.RequestMethod;
|
||||
import org.springframework.web.method.HandlerMethod;
|
||||
import org.springframework.web.servlet.mvc.method.RequestMappingInfo;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Discovers MCP-exposable operations and caches a per-op {@link OperationMeta}. Refreshed on {@link
|
||||
* ContextRefreshedEvent} and filtered on read by {@link
|
||||
* EndpointConfiguration#isEndpointEnabledForUri}. AI capabilities are fed in via {@link
|
||||
* #replaceAiCapabilities}.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class McpToolCatalog {
|
||||
|
||||
private static final String WRITE_SCOPE = "mcp.tools.write";
|
||||
|
||||
private final ApplicationContext applicationContext;
|
||||
private final EndpointConfiguration endpointConfiguration;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final SimpleSchemaGenerator schemaGenerator;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
// Concurrent: written on the boot thread, read on request threads, AI map replaced at runtime.
|
||||
private final Map<String, OperationMeta> pdfOps = new ConcurrentHashMap<>();
|
||||
|
||||
// Engine-driven AI capabilities. Replaced wholesale by the scheduled refresh task on a
|
||||
// background thread while request threads read via findByOperationId/enabledOps. The volatile
|
||||
// reference makes the swap publication-safe; readers either see the old or the new snapshot,
|
||||
// never a partially-merged one.
|
||||
private volatile Map<String, OperationMeta> aiOps = new ConcurrentHashMap<>();
|
||||
|
||||
public McpToolCatalog(
|
||||
ApplicationContext applicationContext,
|
||||
EndpointConfiguration endpointConfiguration,
|
||||
ApplicationProperties applicationProperties,
|
||||
ObjectMapper objectMapper) {
|
||||
this.applicationContext = applicationContext;
|
||||
this.endpointConfiguration = endpointConfiguration;
|
||||
this.applicationProperties = applicationProperties;
|
||||
this.schemaGenerator = new SimpleSchemaGenerator(objectMapper);
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
/** Admin tool filter: non-empty allow list is a whitelist; block list always removes. */
|
||||
private boolean isOperationAllowed(String id) {
|
||||
ApplicationProperties.Mcp mcp = applicationProperties.getMcp();
|
||||
List<String> allowed = mcp.getAllowedOperations();
|
||||
List<String> blocked = mcp.getBlockedOperations();
|
||||
if (blocked != null && blocked.contains(id)) {
|
||||
return false;
|
||||
}
|
||||
if (allowed != null && !allowed.isEmpty()) {
|
||||
return allowed.contains(id);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@EventListener(ContextRefreshedEvent.class)
|
||||
public void discover() {
|
||||
pdfOps.clear();
|
||||
for (RequestMappingHandlerMapping mapping :
|
||||
applicationContext.getBeansOfType(RequestMappingHandlerMapping.class).values()) {
|
||||
for (Map.Entry<RequestMappingInfo, HandlerMethod> e :
|
||||
mapping.getHandlerMethods().entrySet()) {
|
||||
indexOne(e.getKey(), e.getValue());
|
||||
}
|
||||
}
|
||||
log.info("MCP tool catalog discovered {} PDF operation(s)", pdfOps.size());
|
||||
}
|
||||
|
||||
private void indexOne(RequestMappingInfo info, HandlerMethod handler) {
|
||||
Set<String> patterns = extractPatterns(info);
|
||||
if (patterns.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Set<RequestMethod> methods = info.getMethodsCondition().getMethods();
|
||||
if (!isInvocableMethod(methods)) {
|
||||
return;
|
||||
}
|
||||
for (String pattern : patterns) {
|
||||
OperationCategory category = OperationCategory.fromUrl(pattern);
|
||||
if (category == null) {
|
||||
continue;
|
||||
}
|
||||
String opId = extractOpId(pattern, category);
|
||||
if (opId == null) {
|
||||
continue;
|
||||
}
|
||||
OperationMeta meta = buildMeta(opId, category, pattern, handler);
|
||||
// First handler wins on duplicate URLs.
|
||||
pdfOps.putIfAbsent(opId, meta);
|
||||
}
|
||||
}
|
||||
|
||||
private OperationMeta buildMeta(
|
||||
String opId, OperationCategory category, String url, HandlerMethod handler) {
|
||||
Method method = handler.getMethod();
|
||||
Operation opAnno = method.getAnnotation(Operation.class);
|
||||
String summary =
|
||||
opAnno != null && !opAnno.summary().isBlank()
|
||||
? opAnno.summary()
|
||||
: prettifyOpId(opId);
|
||||
ObjectNode schema = paramSchemaFor(handler);
|
||||
// Every mutating endpoint requires the write scope.
|
||||
return new OperationMeta(
|
||||
opId,
|
||||
category,
|
||||
summary,
|
||||
schema,
|
||||
WRITE_SCOPE,
|
||||
OperationMeta.Target.JAVA_ENDPOINT,
|
||||
url,
|
||||
handler);
|
||||
}
|
||||
|
||||
private ObjectNode paramSchemaFor(HandlerMethod handler) {
|
||||
Optional<Class<?>> bodyType = firstComplexParamType(handler);
|
||||
return bodyType.map(schemaGenerator::toSchema).orElseGet(() -> emptyObjectSchema());
|
||||
}
|
||||
|
||||
private ObjectNode emptyObjectSchema() {
|
||||
ObjectNode out = objectMapper.createObjectNode();
|
||||
out.put("type", "object");
|
||||
out.put("additionalProperties", true);
|
||||
return out;
|
||||
}
|
||||
|
||||
private Optional<Class<?>> firstComplexParamType(HandlerMethod handler) {
|
||||
for (MethodParameter p : handler.getMethodParameters()) {
|
||||
Class<?> type = p.getParameterType();
|
||||
if (type.isPrimitive() || type == String.class || type.getName().startsWith("java.")) {
|
||||
continue;
|
||||
}
|
||||
// Skip Spring-managed parameter types (HttpServletRequest, Principal, etc.).
|
||||
String pkg = type.getPackageName();
|
||||
if (pkg.startsWith("jakarta.") || pkg.startsWith("org.springframework.")) {
|
||||
continue;
|
||||
}
|
||||
return Optional.of(type);
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
public List<OperationMeta> enabledOps(OperationCategory category) {
|
||||
if (category == OperationCategory.AI) {
|
||||
List<OperationMeta> ai = new ArrayList<>();
|
||||
for (OperationMeta m : aiOps.values()) {
|
||||
if (isOperationAllowed(m.id())) {
|
||||
ai.add(m);
|
||||
}
|
||||
}
|
||||
return ai;
|
||||
}
|
||||
List<OperationMeta> out = new ArrayList<>();
|
||||
for (OperationMeta m : pdfOps.values()) {
|
||||
if (m.category() == category
|
||||
&& isOperationAllowed(m.id())
|
||||
&& endpointConfiguration.isEndpointEnabledForUri(m.endpointPath())) {
|
||||
out.add(m);
|
||||
}
|
||||
}
|
||||
out.sort((a, b) -> a.id().compareTo(b.id()));
|
||||
return out;
|
||||
}
|
||||
|
||||
public Optional<OperationMeta> findByOperationId(String id) {
|
||||
if (!isOperationAllowed(id)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
// A disabled PDF op returns empty rather than falling through to a same-id AI capability.
|
||||
OperationMeta meta = pdfOps.get(id);
|
||||
if (meta != null) {
|
||||
boolean enabled =
|
||||
meta.target() != OperationMeta.Target.JAVA_ENDPOINT
|
||||
|| endpointConfiguration.isEndpointEnabledForUri(meta.endpointPath());
|
||||
return enabled ? Optional.of(meta) : Optional.empty();
|
||||
}
|
||||
return Optional.ofNullable(aiOps.get(id));
|
||||
}
|
||||
|
||||
/** Replace the AI capabilities snapshot. Called by the engine refresh task. */
|
||||
public void replaceAiCapabilities(Map<String, OperationMeta> updated) {
|
||||
// Build a fresh map then swap atomically via the volatile reference. The previous
|
||||
// implementation did putAll-then-retainAll on a shared ConcurrentHashMap, which left a
|
||||
// transient window where readers could observe stale entries that should have been
|
||||
// removed (race between the two structural updates).
|
||||
Map<String, OperationMeta> next = new ConcurrentHashMap<>(updated);
|
||||
this.aiOps = next;
|
||||
log.info("MCP tool catalog AI capabilities replaced: {} entries", next.size());
|
||||
}
|
||||
|
||||
/** Only POST/PUT endpoints are exposed as tools; DELETE and GET are excluded. */
|
||||
static boolean isInvocableMethod(Set<RequestMethod> methods) {
|
||||
return methods.contains(RequestMethod.POST) || methods.contains(RequestMethod.PUT);
|
||||
}
|
||||
|
||||
private static String extractOpId(String pattern, OperationCategory category) {
|
||||
if (category.urlPrefix() == null || !pattern.startsWith(category.urlPrefix())) {
|
||||
return null;
|
||||
}
|
||||
String tail = pattern.substring(category.urlPrefix().length());
|
||||
if (tail.isBlank() || tail.contains("/") || tail.contains("{")) {
|
||||
// Skip nested paths and path-variable templates.
|
||||
return null;
|
||||
}
|
||||
return tail;
|
||||
}
|
||||
|
||||
private static String prettifyOpId(String id) {
|
||||
return id.replace('-', ' ');
|
||||
}
|
||||
|
||||
private static Set<String> extractPatterns(RequestMappingInfo info) {
|
||||
try {
|
||||
Method getDirectPaths = info.getClass().getMethod("getDirectPaths");
|
||||
Object result = getDirectPaths.invoke(info);
|
||||
if (result instanceof Set<?> set) {
|
||||
Set<String> patterns = new TreeSet<>();
|
||||
for (Object v : set) {
|
||||
if (v instanceof String s) {
|
||||
patterns.add(s);
|
||||
}
|
||||
}
|
||||
return patterns;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.trace("getDirectPaths unavailable on RequestMappingInfo", e);
|
||||
}
|
||||
return Collections.emptySet();
|
||||
}
|
||||
|
||||
public Map<String, OperationMeta> snapshotPdfOps() {
|
||||
return new LinkedHashMap<>(pdfOps);
|
||||
}
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
package stirling.software.proprietary.mcp.catalog;
|
||||
|
||||
/** MCP tool categories; {@link #urlPrefix} maps a {@code /api/v1/} namespace to a category. */
|
||||
public enum OperationCategory {
|
||||
CONVERT("/api/v1/convert/", "stirling_convert"),
|
||||
PAGES("/api/v1/general/", "stirling_pages"),
|
||||
MISC("/api/v1/misc/", "stirling_misc"),
|
||||
SECURITY("/api/v1/security/", "stirling_security"),
|
||||
AI(null, "stirling_ai");
|
||||
|
||||
private final String urlPrefix;
|
||||
private final String toolName;
|
||||
|
||||
OperationCategory(String urlPrefix, String toolName) {
|
||||
this.urlPrefix = urlPrefix;
|
||||
this.toolName = toolName;
|
||||
}
|
||||
|
||||
public String urlPrefix() {
|
||||
return urlPrefix;
|
||||
}
|
||||
|
||||
public String toolName() {
|
||||
return toolName;
|
||||
}
|
||||
|
||||
public static OperationCategory fromUrl(String url) {
|
||||
if (url == null) {
|
||||
return null;
|
||||
}
|
||||
for (OperationCategory c : values()) {
|
||||
if (c.urlPrefix != null && url.startsWith(c.urlPrefix)) {
|
||||
return c;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package stirling.software.proprietary.mcp.catalog;
|
||||
|
||||
import org.springframework.web.method.HandlerMethod;
|
||||
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/** Metadata for one MCP-exposed operation (PDF endpoint or AI capability). */
|
||||
public record OperationMeta(
|
||||
String id,
|
||||
OperationCategory category,
|
||||
String summary,
|
||||
ObjectNode paramSchema,
|
||||
String requiredScope,
|
||||
Target target,
|
||||
String endpointPath,
|
||||
HandlerMethod handlerMethod) {
|
||||
|
||||
public enum Target {
|
||||
JAVA_ENDPOINT,
|
||||
ENGINE_CAPABILITY
|
||||
}
|
||||
}
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
package stirling.software.proprietary.mcp.catalog;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.ParameterizedType;
|
||||
import java.lang.reflect.Type;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Reflection-based JSON Schema generator for controller request-body classes. {@link MultipartFile}
|
||||
* fields are emitted as {@code "type":"string"} with a {@code "format":"file-id"} hint.
|
||||
*/
|
||||
public final class SimpleSchemaGenerator {
|
||||
|
||||
private final ObjectMapper mapper;
|
||||
|
||||
public SimpleSchemaGenerator(ObjectMapper mapper) {
|
||||
this.mapper = mapper;
|
||||
}
|
||||
|
||||
public ObjectNode toSchema(Class<?> type) {
|
||||
return toSchema(type, new HashSet<>());
|
||||
}
|
||||
|
||||
private ObjectNode toSchema(Class<?> type, Set<Class<?>> visited) {
|
||||
ObjectNode schema = mapper.createObjectNode();
|
||||
schema.put("type", "object");
|
||||
schema.put("additionalProperties", false);
|
||||
ObjectNode properties = schema.putObject("properties");
|
||||
ArrayNode required = mapper.createArrayNode();
|
||||
if (!visited.add(type)) {
|
||||
// Cycle: emit a loose object and bail.
|
||||
schema.put("additionalProperties", true);
|
||||
return schema;
|
||||
}
|
||||
|
||||
Set<String> seen = new HashSet<>();
|
||||
for (Field field : collectFields(type)) {
|
||||
if (java.lang.reflect.Modifier.isStatic(field.getModifiers())
|
||||
|| java.lang.reflect.Modifier.isTransient(field.getModifiers())) {
|
||||
continue;
|
||||
}
|
||||
// Skip fields Jackson won't (de)serialize.
|
||||
if (field.isAnnotationPresent(JsonIgnore.class)) {
|
||||
continue;
|
||||
}
|
||||
String name = jsonPropertyName(field);
|
||||
if (!seen.add(name)) {
|
||||
continue;
|
||||
}
|
||||
properties.set(name, typeSchema(field.getGenericType(), visited));
|
||||
if (isRequired(field)) {
|
||||
required.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
if (!required.isEmpty()) {
|
||||
schema.set("required", required);
|
||||
}
|
||||
return schema;
|
||||
}
|
||||
|
||||
private List<Field> collectFields(Class<?> type) {
|
||||
List<Field> all = new ArrayList<>();
|
||||
for (Class<?> c = type; c != null && c != Object.class; c = c.getSuperclass()) {
|
||||
for (Field f : c.getDeclaredFields()) {
|
||||
all.add(f);
|
||||
}
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
private static String jsonPropertyName(Field field) {
|
||||
JsonProperty ann = field.getAnnotation(JsonProperty.class);
|
||||
if (ann != null && !ann.value().isEmpty()) {
|
||||
return ann.value();
|
||||
}
|
||||
return field.getName();
|
||||
}
|
||||
|
||||
private boolean isRequired(Field field) {
|
||||
JsonProperty json = field.getAnnotation(JsonProperty.class);
|
||||
if (json != null && json.required()) {
|
||||
return true;
|
||||
}
|
||||
return field.isAnnotationPresent(jakarta.validation.constraints.NotNull.class)
|
||||
|| field.isAnnotationPresent(jakarta.validation.constraints.NotBlank.class)
|
||||
|| field.isAnnotationPresent(jakarta.validation.constraints.NotEmpty.class);
|
||||
}
|
||||
|
||||
private ObjectNode typeSchema(Type t, Set<Class<?>> visited) {
|
||||
ObjectNode out = mapper.createObjectNode();
|
||||
if (t instanceof Class<?> c) {
|
||||
populatePrimitive(out, c, visited);
|
||||
} else if (t instanceof ParameterizedType pt) {
|
||||
Type raw = pt.getRawType();
|
||||
if (raw instanceof Class<?> rawClass) {
|
||||
if (java.util.Collection.class.isAssignableFrom(rawClass)) {
|
||||
out.put("type", "array");
|
||||
Type[] args = pt.getActualTypeArguments();
|
||||
if (args.length == 1) {
|
||||
out.set("items", typeSchema(args[0], visited));
|
||||
}
|
||||
} else if (java.util.Map.class.isAssignableFrom(rawClass)) {
|
||||
out.put("type", "object");
|
||||
out.put("additionalProperties", true);
|
||||
} else {
|
||||
populatePrimitive(out, rawClass, visited);
|
||||
}
|
||||
} else {
|
||||
out.put("type", "object");
|
||||
}
|
||||
} else {
|
||||
out.put("type", "object");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private void populatePrimitive(ObjectNode out, Class<?> c, Set<Class<?>> visited) {
|
||||
if (MultipartFile.class.isAssignableFrom(c)) {
|
||||
out.put("type", "string");
|
||||
out.put("format", "file-id");
|
||||
out.put(
|
||||
"description",
|
||||
"Reference to a previously-uploaded file in Stirling's job store.");
|
||||
return;
|
||||
}
|
||||
if (c.isArray()) {
|
||||
out.put("type", "array");
|
||||
out.set("items", typeSchema(c.getComponentType(), visited));
|
||||
return;
|
||||
}
|
||||
if (c == String.class) {
|
||||
out.put("type", "string");
|
||||
} else if (c == boolean.class || c == Boolean.class) {
|
||||
out.put("type", "boolean");
|
||||
} else if (c == int.class
|
||||
|| c == Integer.class
|
||||
|| c == long.class
|
||||
|| c == Long.class
|
||||
|| c == short.class
|
||||
|| c == Short.class
|
||||
|| c == byte.class
|
||||
|| c == Byte.class) {
|
||||
out.put("type", "integer");
|
||||
} else if (c == float.class || c == Float.class || c == double.class || c == Double.class) {
|
||||
out.put("type", "number");
|
||||
} else if (c.isEnum()) {
|
||||
out.put("type", "string");
|
||||
ArrayNode values = out.putArray("enum");
|
||||
for (Object constant : c.getEnumConstants()) {
|
||||
values.add(constant.toString());
|
||||
}
|
||||
} else if (c == java.util.UUID.class) {
|
||||
out.put("type", "string");
|
||||
out.put("format", "uuid");
|
||||
} else if (java.time.temporal.Temporal.class.isAssignableFrom(c)
|
||||
|| c == java.util.Date.class) {
|
||||
out.put("type", "string");
|
||||
out.put("format", "date-time");
|
||||
} else {
|
||||
// Complex bean: recurse with the shared visited set.
|
||||
ObjectNode nested = toSchema(c, visited);
|
||||
out.setAll(nested);
|
||||
}
|
||||
}
|
||||
}
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
package stirling.software.proprietary.mcp.engine;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.mcp.catalog.McpToolCatalog;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationCategory;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationMeta;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Pulls the engine's capabilities manifest at boot and on a schedule, feeding it into the shared
|
||||
* {@link McpToolCatalog}.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class EngineCapabilityClient {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final McpToolCatalog catalog;
|
||||
private final ObjectMapper mapper;
|
||||
private final HttpClient httpClient;
|
||||
private final String sharedSecret;
|
||||
|
||||
private ScheduledExecutorService scheduler;
|
||||
|
||||
public EngineCapabilityClient(
|
||||
ApplicationProperties applicationProperties,
|
||||
McpToolCatalog catalog,
|
||||
ObjectMapper mapper) {
|
||||
this.applicationProperties = applicationProperties;
|
||||
this.catalog = catalog;
|
||||
this.mapper = mapper;
|
||||
this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build();
|
||||
this.sharedSecret = System.getenv("STIRLING_ENGINE_SHARED_SECRET");
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
void start() {
|
||||
scheduler =
|
||||
Executors.newSingleThreadScheduledExecutor(
|
||||
r -> {
|
||||
Thread t = new Thread(r, "mcp-engine-capability-refresh");
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
});
|
||||
}
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void onReady() {
|
||||
long minutes =
|
||||
Math.max(1, applicationProperties.getMcp().getEngineCapabilityRefreshMinutes());
|
||||
// First refresh immediately, then on the configured cadence.
|
||||
scheduler.schedule(this::refreshSafely, 0, TimeUnit.SECONDS);
|
||||
scheduler.scheduleAtFixedRate(this::refreshSafely, minutes, minutes, TimeUnit.MINUTES);
|
||||
log.info("MCP engine capability refresh scheduled every {} minute(s)", minutes);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
void stop() {
|
||||
if (scheduler != null) {
|
||||
scheduler.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private void refreshSafely() {
|
||||
try {
|
||||
refresh();
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"MCP engine capability refresh failed ({}). AI tool enum stays at the last"
|
||||
+ " known state until the next successful pull.",
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/** Visible for testing. */
|
||||
public void refresh() throws IOException, InterruptedException {
|
||||
if (!applicationProperties.getAiEngine().isEnabled()) {
|
||||
log.debug("AI engine disabled; skipping MCP capability refresh");
|
||||
catalog.replaceAiCapabilities(Map.of());
|
||||
return;
|
||||
}
|
||||
// Trim whitespace and any trailing slash to avoid a malformed URI.
|
||||
String base = applicationProperties.getAiEngine().getUrl().strip().replaceAll("/+$", "");
|
||||
URI uri = URI.create(base + "/api/v1/agents/capabilities");
|
||||
HttpRequest.Builder reqBuilder =
|
||||
HttpRequest.newBuilder()
|
||||
.uri(uri)
|
||||
.timeout(Duration.ofSeconds(10))
|
||||
.header("Accept", "application/json")
|
||||
.GET();
|
||||
if (sharedSecret != null && !sharedSecret.isBlank()) {
|
||||
reqBuilder.header("X-Engine-Auth", sharedSecret);
|
||||
}
|
||||
HttpResponse<String> response =
|
||||
httpClient.send(reqBuilder.build(), HttpResponse.BodyHandlers.ofString());
|
||||
if (response.statusCode() != 200) {
|
||||
throw new IOException(
|
||||
"Engine capabilities endpoint returned HTTP " + response.statusCode());
|
||||
}
|
||||
Map<String, OperationMeta> parsed = parseManifest(response.body());
|
||||
catalog.replaceAiCapabilities(parsed);
|
||||
}
|
||||
|
||||
private Map<String, OperationMeta> parseManifest(String body) throws IOException {
|
||||
JsonNode root = mapper.readTree(body);
|
||||
JsonNode capabilities = root.get("capabilities");
|
||||
if (capabilities == null || !capabilities.isArray()) {
|
||||
throw new IOException("Manifest missing 'capabilities' array");
|
||||
}
|
||||
Map<String, OperationMeta> out = new LinkedHashMap<>();
|
||||
for (JsonNode entry : capabilities) {
|
||||
JsonNode id = entry.get("id");
|
||||
JsonNode desc = entry.get("description");
|
||||
JsonNode schema = entry.get("input_schema");
|
||||
JsonNode scope = entry.get("required_scope");
|
||||
JsonNode route = entry.get("route");
|
||||
if (id == null || !id.isTextual() || schema == null || !schema.isObject()) {
|
||||
log.warn("Skipping malformed capability entry: {}", entry);
|
||||
continue;
|
||||
}
|
||||
String routeValue = route == null || !route.isTextual() ? null : route.asText();
|
||||
if (routeValue != null && !isSafeRelativeRoute(routeValue)) {
|
||||
// Defence in depth: a tampered manifest must not steer Java at an arbitrary
|
||||
// host/path.
|
||||
log.warn(
|
||||
"Skipping capability '{}' with unsafe route '{}' (must be a server-relative"
|
||||
+ " /api path with no scheme, authority, or '..')",
|
||||
id.asText(),
|
||||
routeValue);
|
||||
continue;
|
||||
}
|
||||
// Fail safe: default to the stricter write scope when the manifest omits one.
|
||||
String requiredScope =
|
||||
scope != null && scope.isTextual() && !scope.asText().isBlank()
|
||||
? scope.asText()
|
||||
: WRITE_SCOPE;
|
||||
ObjectNode schemaCopy = (ObjectNode) schema.deepCopy();
|
||||
out.put(
|
||||
id.asText(),
|
||||
new OperationMeta(
|
||||
id.asText(),
|
||||
OperationCategory.AI,
|
||||
desc == null ? id.asText() : desc.asText(),
|
||||
schemaCopy,
|
||||
requiredScope,
|
||||
OperationMeta.Target.ENGINE_CAPABILITY,
|
||||
routeValue,
|
||||
null));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static final String WRITE_SCOPE = "mcp.tools.write";
|
||||
|
||||
/**
|
||||
* True only for a server-relative {@code /api/} path with no scheme, authority, {@code ..}, or
|
||||
* control chars (blocks SSRF / path escape).
|
||||
*/
|
||||
static boolean isSafeRelativeRoute(String route) {
|
||||
if (route == null || route.isBlank() || !route.startsWith("/api/")) {
|
||||
return false;
|
||||
}
|
||||
if (route.startsWith("//")
|
||||
|| route.contains("..")
|
||||
|| route.contains("@")
|
||||
|| route.contains("\\")
|
||||
|| route.contains(":")) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < route.length(); i++) {
|
||||
char c = route.charAt(i);
|
||||
if (c <= ' ') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package stirling.software.proprietary.mcp.jsonrpc;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
/** JSON-RPC 2.0 error object. */
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public record JsonRpcError(int code, String message, JsonNode data) {
|
||||
|
||||
public static final int PARSE_ERROR = -32700;
|
||||
public static final int INVALID_REQUEST = -32600;
|
||||
public static final int METHOD_NOT_FOUND = -32601;
|
||||
public static final int INVALID_PARAMS = -32602;
|
||||
public static final int INTERNAL_ERROR = -32603;
|
||||
|
||||
public static JsonRpcError parseError(String message) {
|
||||
return new JsonRpcError(PARSE_ERROR, message, null);
|
||||
}
|
||||
|
||||
public static JsonRpcError invalidRequest(String message) {
|
||||
return new JsonRpcError(INVALID_REQUEST, message, null);
|
||||
}
|
||||
|
||||
public static JsonRpcError methodNotFound(String method) {
|
||||
return new JsonRpcError(METHOD_NOT_FOUND, "Method not found: " + method, null);
|
||||
}
|
||||
|
||||
public static JsonRpcError invalidParams(String message) {
|
||||
return new JsonRpcError(INVALID_PARAMS, message, null);
|
||||
}
|
||||
|
||||
public static JsonRpcError internalError(String message) {
|
||||
return new JsonRpcError(INTERNAL_ERROR, message, null);
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package stirling.software.proprietary.mcp.jsonrpc;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
/** JSON-RPC 2.0 request frame; a null {@code id} marks a notification. */
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public record JsonRpcRequest(String jsonrpc, JsonNode id, String method, JsonNode params) {
|
||||
|
||||
public boolean isNotification() {
|
||||
return id == null || id.isNull();
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package stirling.software.proprietary.mcp.jsonrpc;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
/** JSON-RPC 2.0 response; exactly one of {@code result} or {@code error} is non-null. */
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public record JsonRpcResponse(String jsonrpc, JsonNode id, Object result, JsonRpcError error) {
|
||||
|
||||
public static JsonRpcResponse success(JsonNode id, Object result) {
|
||||
return new JsonRpcResponse("2.0", id, result, null);
|
||||
}
|
||||
|
||||
public static JsonRpcResponse failure(JsonNode id, JsonRpcError error) {
|
||||
return new JsonRpcResponse("2.0", id, null, error);
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package stirling.software.proprietary.mcp.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.security.authentication.AnonymousAuthenticationToken;
|
||||
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.GrantedAuthority;
|
||||
import org.springframework.security.core.authority.SimpleGrantedAuthority;
|
||||
import org.springframework.security.core.context.SecurityContext;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.web.filter.OncePerRequestFilter;
|
||||
|
||||
import jakarta.servlet.FilterChain;
|
||||
import jakarta.servlet.ServletException;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
/**
|
||||
* API-key auth for the MCP endpoint: validates a Stirling per-user API key and binds the request to
|
||||
* that user with the MCP scopes.
|
||||
*/
|
||||
@Slf4j
|
||||
public class McpApiKeyAuthFilter extends OncePerRequestFilter {
|
||||
|
||||
private static final List<GrantedAuthority> MCP_SCOPES =
|
||||
List.of(
|
||||
new SimpleGrantedAuthority("SCOPE_mcp.tools.read"),
|
||||
new SimpleGrantedAuthority("SCOPE_mcp.tools.write"));
|
||||
|
||||
private final UserService userService;
|
||||
|
||||
public McpApiKeyAuthFilter(UserService userService) {
|
||||
this.userService = userService;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doFilterInternal(
|
||||
HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
|
||||
throws ServletException, IOException {
|
||||
Authentication existing = SecurityContextHolder.getContext().getAuthentication();
|
||||
// Treat an anonymous token as not authenticated so the key is still processed.
|
||||
boolean unauthenticated =
|
||||
existing == null
|
||||
|| existing instanceof AnonymousAuthenticationToken
|
||||
|| !existing.isAuthenticated();
|
||||
if (unauthenticated) {
|
||||
String apiKey = extractKey(request);
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
Optional<User> user = userService.getUserByApiKey(apiKey);
|
||||
if (user.isPresent() && user.get().isEnabled()) {
|
||||
UsernamePasswordAuthenticationToken auth =
|
||||
new UsernamePasswordAuthenticationToken(
|
||||
user.get().getUsername(), null, MCP_SCOPES);
|
||||
SecurityContext context = SecurityContextHolder.createEmptyContext();
|
||||
context.setAuthentication(auth);
|
||||
SecurityContextHolder.setContext(context);
|
||||
} else {
|
||||
log.warn(
|
||||
"MCP access denied: presented API key did not match an active account");
|
||||
}
|
||||
}
|
||||
}
|
||||
filterChain.doFilter(request, response);
|
||||
}
|
||||
|
||||
private String extractKey(HttpServletRequest request) {
|
||||
String headerKey = request.getHeader("X-API-KEY");
|
||||
if (headerKey != null && !headerKey.isBlank()) {
|
||||
return headerKey.trim();
|
||||
}
|
||||
String authz = request.getHeader("Authorization");
|
||||
if (authz != null && authz.regionMatches(true, 0, "Bearer ", 0, 7)) {
|
||||
return authz.substring(7).trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package stirling.software.proprietary.mcp.security;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.security.oauth2.core.OAuth2Error;
|
||||
import org.springframework.security.oauth2.core.OAuth2TokenValidator;
|
||||
import org.springframework.security.oauth2.core.OAuth2TokenValidatorResult;
|
||||
import org.springframework.security.oauth2.jwt.Jwt;
|
||||
|
||||
/**
|
||||
* RFC 8707 audience binding: a JWT at the MCP endpoint must list this server's resource id in its
|
||||
* {@code aud} claim. Fails closed when the resource id is unset.
|
||||
*/
|
||||
public class McpAudienceValidator implements OAuth2TokenValidator<Jwt> {
|
||||
|
||||
private final String expectedResourceId;
|
||||
|
||||
public McpAudienceValidator(String expectedResourceId) {
|
||||
this.expectedResourceId = expectedResourceId == null ? "" : expectedResourceId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public OAuth2TokenValidatorResult validate(Jwt token) {
|
||||
if (expectedResourceId.isBlank()) {
|
||||
return OAuth2TokenValidatorResult.failure(
|
||||
new OAuth2Error(
|
||||
"invalid_token",
|
||||
"MCP server has no resource id configured; rejecting all tokens"
|
||||
+ " until mcp.auth.resource-id is set.",
|
||||
null));
|
||||
}
|
||||
List<String> aud = token.getAudience();
|
||||
if (aud == null || !aud.contains(expectedResourceId)) {
|
||||
return OAuth2TokenValidatorResult.failure(
|
||||
new OAuth2Error(
|
||||
"invalid_token",
|
||||
"Token audience does not include this server's resource id ("
|
||||
+ expectedResourceId
|
||||
+ ").",
|
||||
null));
|
||||
}
|
||||
return OAuth2TokenValidatorResult.success();
|
||||
}
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package stirling.software.proprietary.mcp.security;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.security.core.AuthenticationException;
|
||||
import org.springframework.security.web.AuthenticationEntryPoint;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
|
||||
/**
|
||||
* Emits 401 + {@code WWW-Authenticate: Bearer resource_metadata="..."} (RFC 9728), preferring
|
||||
* X-Forwarded-* headers to build the public-facing metadata URL.
|
||||
*/
|
||||
public class McpAuthenticationEntryPoint implements AuthenticationEntryPoint {
|
||||
|
||||
private final String metadataPath;
|
||||
|
||||
public McpAuthenticationEntryPoint(String metadataPath) {
|
||||
this.metadataPath =
|
||||
metadataPath == null ? "/.well-known/oauth-protected-resource" : metadataPath;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void commence(
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response,
|
||||
AuthenticationException authException)
|
||||
throws IOException {
|
||||
String scheme = firstForwarded(request, "X-Forwarded-Proto", request.getScheme());
|
||||
String authority = forwardedHost(request, scheme);
|
||||
String metadataUrl = scheme + "://" + authority + metadataPath;
|
||||
response.setHeader(
|
||||
"WWW-Authenticate",
|
||||
"Bearer error=\"invalid_token\", resource_metadata=\"" + metadataUrl + "\"");
|
||||
response.sendError(HttpStatus.UNAUTHORIZED.value(), "Unauthorized");
|
||||
}
|
||||
|
||||
/** host[:port] from forwarded headers when present, else the servlet host/port. */
|
||||
private static String forwardedHost(HttpServletRequest request, String scheme) {
|
||||
String host = firstForwarded(request, "X-Forwarded-Host", null);
|
||||
if (host != null && !host.isBlank()) {
|
||||
// X-Forwarded-Host may already carry a port.
|
||||
if (host.contains(":")) {
|
||||
return host;
|
||||
}
|
||||
String fwdPort = firstForwarded(request, "X-Forwarded-Port", null);
|
||||
if (fwdPort != null && !isDefaultPort(scheme, fwdPort)) {
|
||||
return host + ":" + fwdPort;
|
||||
}
|
||||
return host;
|
||||
}
|
||||
String authority = request.getServerName();
|
||||
int port = request.getServerPort();
|
||||
if (port > 0 && !isDefaultPort(scheme, Integer.toString(port))) {
|
||||
authority = authority + ":" + port;
|
||||
}
|
||||
return authority;
|
||||
}
|
||||
|
||||
/** First (client-most) value of a possibly comma-listed forwarded header, trimmed. */
|
||||
private static String firstForwarded(HttpServletRequest request, String name, String fallback) {
|
||||
String value = request.getHeader(name);
|
||||
if (value == null || value.isBlank()) {
|
||||
return fallback;
|
||||
}
|
||||
int comma = value.indexOf(',');
|
||||
return (comma >= 0 ? value.substring(0, comma) : value).trim();
|
||||
}
|
||||
|
||||
private static boolean isDefaultPort(String scheme, String port) {
|
||||
return ("http".equals(scheme) && "80".equals(port))
|
||||
|| ("https".equals(scheme) && "443".equals(port));
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
package stirling.software.proprietary.mcp.security;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.springframework.web.filter.OncePerRequestFilter;
|
||||
|
||||
import jakarta.servlet.FilterChain;
|
||||
import jakarta.servlet.ReadListener;
|
||||
import jakarta.servlet.ServletException;
|
||||
import jakarta.servlet.ServletInputStream;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletRequestWrapper;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
|
||||
/**
|
||||
* Caps MCP request body size (via Content-Length and by buffering up to the cap) and rejects
|
||||
* oversized bodies with a clean 413 before JSON parsing.
|
||||
*/
|
||||
public class McpRequestSizeFilter extends OncePerRequestFilter {
|
||||
|
||||
private final long maxBodyBytes;
|
||||
|
||||
public McpRequestSizeFilter(long maxBodyBytes) {
|
||||
this.maxBodyBytes = maxBodyBytes > 0 ? maxBodyBytes : 256L * 1024L;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doFilterInternal(
|
||||
HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
|
||||
throws ServletException, IOException {
|
||||
long declared = request.getContentLengthLong();
|
||||
if (declared > maxBodyBytes) {
|
||||
tooLarge(response);
|
||||
return;
|
||||
}
|
||||
byte[] body;
|
||||
try {
|
||||
body = readUpTo(request.getInputStream(), maxBodyBytes);
|
||||
} catch (BodyTooLargeException e) {
|
||||
tooLarge(response);
|
||||
return;
|
||||
}
|
||||
filterChain.doFilter(new CachedBodyRequest(request, body), response);
|
||||
}
|
||||
|
||||
private static byte[] readUpTo(InputStream in, long max) throws IOException {
|
||||
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
|
||||
byte[] chunk = new byte[8192];
|
||||
long total = 0;
|
||||
int n;
|
||||
while ((n = in.read(chunk)) != -1) {
|
||||
total += n;
|
||||
if (total > max) {
|
||||
throw new BodyTooLargeException();
|
||||
}
|
||||
buffer.write(chunk, 0, n);
|
||||
}
|
||||
return buffer.toByteArray();
|
||||
}
|
||||
|
||||
private void tooLarge(HttpServletResponse response) throws IOException {
|
||||
response.setStatus(HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE);
|
||||
response.setContentType("application/json");
|
||||
response.getWriter()
|
||||
.write(
|
||||
"{\"error\":\"payload_too_large\",\"message\":\"MCP request body exceeds the"
|
||||
+ " configured limit of "
|
||||
+ maxBodyBytes
|
||||
+ " bytes.\"}");
|
||||
}
|
||||
|
||||
private static final class BodyTooLargeException extends IOException {}
|
||||
|
||||
/** Re-serves the buffered body to the controller. */
|
||||
private static final class CachedBodyRequest extends HttpServletRequestWrapper {
|
||||
private final byte[] body;
|
||||
|
||||
CachedBodyRequest(HttpServletRequest request, byte[] body) {
|
||||
super(request);
|
||||
this.body = body;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServletInputStream getInputStream() {
|
||||
ByteArrayInputStream source = new ByteArrayInputStream(body);
|
||||
return new ServletInputStream() {
|
||||
@Override
|
||||
public int read() {
|
||||
return source.read();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) {
|
||||
return source.read(b, off, len);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isFinished() {
|
||||
return source.available() == 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isReady() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setReadListener(ReadListener readListener) {
|
||||
// Synchronous buffered body; no async reads.
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public BufferedReader getReader() {
|
||||
String enc = getCharacterEncoding();
|
||||
Charset cs = enc == null ? StandardCharsets.UTF_8 : Charset.forName(enc);
|
||||
return new BufferedReader(new InputStreamReader(new ByteArrayInputStream(body), cs));
|
||||
}
|
||||
}
|
||||
}
|
||||
+262
@@ -0,0 +1,262 @@
|
||||
package stirling.software.proprietary.mcp.security;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.core.convert.converter.Converter;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.security.authentication.AbstractAuthenticationToken;
|
||||
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
|
||||
import org.springframework.security.config.http.SessionCreationPolicy;
|
||||
import org.springframework.security.core.GrantedAuthority;
|
||||
import org.springframework.security.core.authority.SimpleGrantedAuthority;
|
||||
import org.springframework.security.oauth2.core.DelegatingOAuth2TokenValidator;
|
||||
import org.springframework.security.oauth2.core.OAuth2TokenValidator;
|
||||
import org.springframework.security.oauth2.jwt.Jwt;
|
||||
import org.springframework.security.oauth2.jwt.JwtDecoder;
|
||||
import org.springframework.security.oauth2.jwt.JwtValidators;
|
||||
import org.springframework.security.oauth2.jwt.NimbusJwtDecoder;
|
||||
import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter;
|
||||
import org.springframework.security.oauth2.server.resource.authentication.JwtGrantedAuthoritiesConverter;
|
||||
import org.springframework.security.oauth2.server.resource.web.authentication.BearerTokenAuthenticationFilter;
|
||||
import org.springframework.security.web.SecurityFilterChain;
|
||||
import org.springframework.security.web.access.intercept.AuthorizationFilter;
|
||||
import org.springframework.security.web.authentication.AnonymousAuthenticationFilter;
|
||||
import org.springframework.web.cors.CorsConfigurationSource;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
/**
|
||||
* MCP security chain: validates JWTs (JWKS + RFC 8707 audience), maps scope claims to authorities,
|
||||
* and fails closed when the issuer is unset.
|
||||
*/
|
||||
@Slf4j
|
||||
@Configuration
|
||||
@Order(Ordered.HIGHEST_PRECEDENCE)
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class McpSecurityConfig {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final UserService userService;
|
||||
|
||||
// Reuse the app's CORS config; ObjectProvider so the chain still wires when no CORS bean
|
||||
// exists.
|
||||
private final ObjectProvider<CorsConfigurationSource> corsConfigurationSource;
|
||||
|
||||
private static final String BASE_PATH = "/mcp";
|
||||
|
||||
public McpSecurityConfig(
|
||||
ApplicationProperties applicationProperties,
|
||||
@Lazy UserService userService,
|
||||
ObjectProvider<CorsConfigurationSource> corsConfigurationSource) {
|
||||
this.applicationProperties = applicationProperties;
|
||||
this.userService = userService;
|
||||
this.corsConfigurationSource = corsConfigurationSource;
|
||||
}
|
||||
|
||||
/** Enable CORS on the MCP chain using the app-wide source when available. */
|
||||
private void applyCors(HttpSecurity http) throws Exception {
|
||||
CorsConfigurationSource source = corsConfigurationSource.getIfAvailable();
|
||||
if (source != null) {
|
||||
http.cors(cors -> cors.configurationSource(source));
|
||||
}
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
void warnIfMisconfigured() {
|
||||
ApplicationProperties.Mcp mcp = applicationProperties.getMcp();
|
||||
if (isApiKeyMode()) {
|
||||
log.info(
|
||||
"MCP auth mode = apikey: clients authenticate with a Stirling per-user API key"
|
||||
+ " (X-API-KEY header). No OAuth issuer required.");
|
||||
} else {
|
||||
if (mcp.getAuth().getIssuerUri().isBlank()) {
|
||||
log.warn(
|
||||
"MCP enabled but mcp.auth.issuer-uri is blank - JWT decoder will reject"
|
||||
+ " every token (fail-closed). Set mcp.auth.issuer-uri and"
|
||||
+ " mcp.auth.resource-id before exposing /mcp to clients.");
|
||||
}
|
||||
if (mcp.getAuth().getResourceId().isBlank()) {
|
||||
log.warn(
|
||||
"MCP enabled but mcp.auth.resource-id is blank - audience validator will"
|
||||
+ " reject every token. Set this to the public URL of the MCP"
|
||||
+ " endpoint (RFC 8707).");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Order(0)
|
||||
SecurityFilterChain mcpSecurityFilterChain(HttpSecurity http, JwtDecoder mcpJwtDecoder)
|
||||
throws Exception {
|
||||
ApplicationProperties.Mcp.Auth auth = applicationProperties.getMcp().getAuth();
|
||||
if (isApiKeyMode()) {
|
||||
return apiKeyFilterChain(http);
|
||||
}
|
||||
return oauthFilterChain(http, mcpJwtDecoder, auth);
|
||||
}
|
||||
|
||||
private boolean isApiKeyMode() {
|
||||
return "apikey".equalsIgnoreCase(applicationProperties.getMcp().getAuth().getMode());
|
||||
}
|
||||
|
||||
/**
|
||||
* API-key chain: a Stirling per-user API key is validated by {@link McpApiKeyAuthFilter};
|
||||
* otherwise 401.
|
||||
*/
|
||||
private SecurityFilterChain apiKeyFilterChain(HttpSecurity http) throws Exception {
|
||||
applyCors(http);
|
||||
http.securityMatcher(BASE_PATH, BASE_PATH + "/**")
|
||||
// CSRF intentionally disabled: /mcp is a stateless JSON-RPC API authenticated by an
|
||||
// out-of-band X-API-KEY header (or Authorization: Bearer <key>). No cookies, no
|
||||
// session, no form submissions; a browser cannot trick a victim into sending the
|
||||
// header cross-origin, so the CSRF attack model does not apply. CodeQL flags this
|
||||
// generically; the SessionCreationPolicy.STATELESS below is the relevant guarantee.
|
||||
.csrf(csrf -> csrf.disable())
|
||||
.sessionManagement(s -> s.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
|
||||
.authorizeHttpRequests(a -> a.anyRequest().authenticated())
|
||||
.exceptionHandling(
|
||||
e ->
|
||||
e.authenticationEntryPoint(
|
||||
(request, response, ex) -> {
|
||||
response.setStatus(401);
|
||||
response.setHeader(
|
||||
"WWW-Authenticate",
|
||||
"Bearer realm=\"Stirling MCP (API key)\"");
|
||||
response.setContentType("application/json");
|
||||
response.getWriter()
|
||||
.write(
|
||||
"{\"error\":\"unauthorized\",\"message\":\"Provide a valid Stirling API key via the X-API-KEY header (or Authorization: Bearer <key>).\"}");
|
||||
}))
|
||||
.addFilterBefore(
|
||||
new McpRequestSizeFilter(
|
||||
applicationProperties.getMcp().getMaxRequestBytes()),
|
||||
AuthorizationFilter.class)
|
||||
// Authenticate before the anonymous filter sets an anonymous token.
|
||||
.addFilterBefore(
|
||||
new McpApiKeyAuthFilter(userService), AnonymousAuthenticationFilter.class);
|
||||
return http.build();
|
||||
}
|
||||
|
||||
/** OAuth2 resource-server chain (JWT, RFC 8707 audience, RFC 9728 metadata). */
|
||||
private SecurityFilterChain oauthFilterChain(
|
||||
HttpSecurity http, JwtDecoder mcpJwtDecoder, ApplicationProperties.Mcp.Auth auth)
|
||||
throws Exception {
|
||||
String metadataPath = "/.well-known/oauth-protected-resource";
|
||||
applyCors(http);
|
||||
http.securityMatcher(BASE_PATH, BASE_PATH + "/**", metadataPath)
|
||||
// CSRF intentionally disabled: /mcp is a stateless JSON-RPC resource server
|
||||
// authenticated by OAuth2 Bearer JWTs (Authorization header). No cookies, no
|
||||
// session, no form submissions; CSRF requires browser-attached ambient credentials
|
||||
// and the bearer token is supplied per-request by the MCP client. CodeQL flags
|
||||
// this generically; the SessionCreationPolicy.STATELESS below is the actual
|
||||
// guarantee, and the .well-known metadata endpoint only serves GET.
|
||||
.csrf(csrf -> csrf.disable())
|
||||
.sessionManagement(s -> s.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
|
||||
.authorizeHttpRequests(
|
||||
a ->
|
||||
a.requestMatchers(HttpMethod.GET, metadataPath)
|
||||
.permitAll()
|
||||
.anyRequest()
|
||||
.authenticated())
|
||||
// Cap body size pre-auth, then bind the validated token to a Stirling user after
|
||||
// the bearer filter.
|
||||
.addFilterBefore(
|
||||
new McpRequestSizeFilter(
|
||||
applicationProperties.getMcp().getMaxRequestBytes()),
|
||||
BearerTokenAuthenticationFilter.class)
|
||||
.addFilterAfter(
|
||||
new McpUserBindingFilter(
|
||||
userService,
|
||||
auth.getUsernameClaim(),
|
||||
auth.isRequireExistingAccount()),
|
||||
BearerTokenAuthenticationFilter.class)
|
||||
.oauth2ResourceServer(
|
||||
oauth2 ->
|
||||
oauth2.authenticationEntryPoint(
|
||||
new McpAuthenticationEntryPoint(metadataPath))
|
||||
// RFC 9728 protected-resource metadata for OAuth discovery.
|
||||
.protectedResourceMetadata(
|
||||
prm ->
|
||||
prm.protectedResourceMetadataCustomizer(
|
||||
builder -> {
|
||||
if (!auth.getResourceId()
|
||||
.isBlank()) {
|
||||
builder.resource(
|
||||
auth
|
||||
.getResourceId());
|
||||
}
|
||||
if (!auth.getIssuerUri()
|
||||
.isBlank()) {
|
||||
builder.authorizationServer(
|
||||
auth
|
||||
.getIssuerUri());
|
||||
}
|
||||
builder.scope("mcp.tools.read");
|
||||
builder.scope(
|
||||
"mcp.tools.write");
|
||||
}))
|
||||
.jwt(
|
||||
jwt ->
|
||||
jwt.decoder(mcpJwtDecoder)
|
||||
.jwtAuthenticationConverter(
|
||||
mcpJwtAuthenticationConverter())));
|
||||
return http.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
JwtDecoder mcpJwtDecoder() {
|
||||
ApplicationProperties.Mcp.Auth auth = applicationProperties.getMcp().getAuth();
|
||||
if (auth.getIssuerUri().isBlank()) {
|
||||
// Fail-closed decoder: rejects every token until the issuer is set.
|
||||
return token -> {
|
||||
throw new org.springframework.security.oauth2.jwt.BadJwtException(
|
||||
"mcp.auth.issuer-uri is not configured");
|
||||
};
|
||||
}
|
||||
String jwksUri = auth.getJwksUri();
|
||||
NimbusJwtDecoder decoder =
|
||||
jwksUri.isBlank()
|
||||
? NimbusJwtDecoder.withIssuerLocation(auth.getIssuerUri()).build()
|
||||
: NimbusJwtDecoder.withJwkSetUri(jwksUri).build();
|
||||
OAuth2TokenValidator<Jwt> defaultValidators =
|
||||
JwtValidators.createDefaultWithIssuer(auth.getIssuerUri());
|
||||
OAuth2TokenValidator<Jwt> combined =
|
||||
new DelegatingOAuth2TokenValidator<>(
|
||||
defaultValidators, new McpAudienceValidator(auth.getResourceId()));
|
||||
decoder.setJwtValidator(combined);
|
||||
return decoder;
|
||||
}
|
||||
|
||||
private Converter<Jwt, AbstractAuthenticationToken> mcpJwtAuthenticationConverter() {
|
||||
JwtGrantedAuthoritiesConverter scopes = new JwtGrantedAuthoritiesConverter();
|
||||
scopes.setAuthorityPrefix("SCOPE_");
|
||||
scopes.setAuthoritiesClaimName("scope");
|
||||
JwtAuthenticationConverter converter = new JwtAuthenticationConverter();
|
||||
converter.setJwtGrantedAuthoritiesConverter(
|
||||
jwt -> {
|
||||
Collection<GrantedAuthority> out = new ArrayList<>(scopes.convert(jwt));
|
||||
List<String> aud = jwt.getAudience();
|
||||
if (aud != null) {
|
||||
for (String a : aud) {
|
||||
out.add(new SimpleGrantedAuthority("AUDIENCE_" + a));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
});
|
||||
return converter;
|
||||
}
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
package stirling.software.proprietary.mcp.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.context.SecurityContext;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.security.oauth2.jwt.Jwt;
|
||||
import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken;
|
||||
import org.springframework.web.filter.OncePerRequestFilter;
|
||||
|
||||
import jakarta.servlet.FilterChain;
|
||||
import jakarta.servlet.ServletException;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.security.service.UserService;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Binds an MCP-validated JWT to a provisioned Stirling user: optionally rejects subjects with no
|
||||
* enabled account, then rebinds the principal to the canonical Stirling username (scope authorities
|
||||
* only) so audit/metering attribute correctly.
|
||||
*/
|
||||
@Slf4j
|
||||
public class McpUserBindingFilter extends OncePerRequestFilter {
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private final UserService userService;
|
||||
private final String usernameClaim;
|
||||
private final boolean requireExistingAccount;
|
||||
|
||||
public McpUserBindingFilter(
|
||||
UserService userService, String usernameClaim, boolean requireExistingAccount) {
|
||||
this.userService = userService;
|
||||
this.usernameClaim =
|
||||
(usernameClaim == null || usernameClaim.isBlank()) ? "sub" : usernameClaim;
|
||||
this.requireExistingAccount = requireExistingAccount;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doFilterInternal(
|
||||
HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
|
||||
throws ServletException, IOException {
|
||||
Authentication current = SecurityContextHolder.getContext().getAuthentication();
|
||||
|
||||
// Only act on a JWT-authenticated request; everything else passes through.
|
||||
if (current instanceof JwtAuthenticationToken jwtAuth && jwtAuth.isAuthenticated()) {
|
||||
Jwt jwt = jwtAuth.getToken();
|
||||
String username = jwt.getClaimAsString(usernameClaim);
|
||||
|
||||
if (username == null || username.isBlank()) {
|
||||
reject(
|
||||
response,
|
||||
"Token is missing the '"
|
||||
+ usernameClaim
|
||||
+ "' claim used to map to a"
|
||||
+ " Stirling user.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Prefer the canonical username from the account record; fall back to the claim when
|
||||
// binding is off.
|
||||
String boundUsername = username;
|
||||
if (requireExistingAccount) {
|
||||
Optional<User> account = userService.findByUsernameIgnoreCase(username);
|
||||
if (account.isEmpty() || !account.get().isEnabled()) {
|
||||
log.warn(
|
||||
"MCP access denied: token subject '{}' has no active Stirling account",
|
||||
sanitizeForLog(username));
|
||||
reject(
|
||||
response,
|
||||
"MCP access requires a provisioned, enabled Stirling account for this"
|
||||
+ " subject.");
|
||||
return;
|
||||
}
|
||||
boundUsername = account.get().getUsername();
|
||||
}
|
||||
|
||||
// Rebind to the Stirling username, carrying only the OAuth scope authorities.
|
||||
UsernamePasswordAuthenticationToken bound =
|
||||
new UsernamePasswordAuthenticationToken(
|
||||
boundUsername, null, jwtAuth.getAuthorities());
|
||||
bound.setDetails(jwtAuth.getDetails());
|
||||
SecurityContext context = SecurityContextHolder.createEmptyContext();
|
||||
context.setAuthentication(bound);
|
||||
SecurityContextHolder.setContext(context);
|
||||
}
|
||||
|
||||
filterChain.doFilter(request, response);
|
||||
}
|
||||
|
||||
/** Strip CR/LF so a crafted claim value can't forge log lines. */
|
||||
private static String sanitizeForLog(String value) {
|
||||
return value == null ? null : value.replace('\r', ' ').replace('\n', ' ');
|
||||
}
|
||||
|
||||
private void reject(HttpServletResponse response, String message) throws IOException {
|
||||
SecurityContextHolder.clearContext();
|
||||
response.setStatus(HttpServletResponse.SC_FORBIDDEN);
|
||||
response.setContentType("application/json");
|
||||
ObjectNode body = MAPPER.createObjectNode();
|
||||
body.put("error", "insufficient_account");
|
||||
body.put("message", message);
|
||||
response.getWriter().write(MAPPER.writeValueAsString(body));
|
||||
}
|
||||
}
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
|
||||
import stirling.software.proprietary.mcp.McpCallContext;
|
||||
import stirling.software.proprietary.mcp.McpTool;
|
||||
import stirling.software.proprietary.mcp.catalog.McpToolCatalog;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationCategory;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationMeta;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Common scaffolding for the PDF category tools. Operation ids and summaries come from the live
|
||||
* {@link McpToolCatalog}.
|
||||
*/
|
||||
abstract class AbstractCategoryTool implements McpTool {
|
||||
|
||||
protected final ObjectMapper mapper;
|
||||
protected final ObjectProvider<McpToolCatalog> catalogProvider;
|
||||
protected final ObjectProvider<McpOperationExecutor> executorProvider;
|
||||
|
||||
protected AbstractCategoryTool(
|
||||
ObjectMapper mapper,
|
||||
ObjectProvider<McpToolCatalog> catalog,
|
||||
ObjectProvider<McpOperationExecutor> executor) {
|
||||
this.mapper = mapper;
|
||||
this.catalogProvider = catalog;
|
||||
this.executorProvider = executor;
|
||||
}
|
||||
|
||||
protected abstract OperationCategory category();
|
||||
|
||||
protected List<OperationMeta> enabledOperations() {
|
||||
McpToolCatalog catalog = catalogProvider.getIfAvailable();
|
||||
if (catalog == null) {
|
||||
return List.of();
|
||||
}
|
||||
return catalog.enabledOps(category());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode inputSchema() {
|
||||
ObjectNode schema = mapper.createObjectNode();
|
||||
schema.put("type", "object");
|
||||
schema.put("additionalProperties", false);
|
||||
|
||||
ObjectNode props = schema.putObject("properties");
|
||||
|
||||
ObjectNode op = props.putObject("operation");
|
||||
op.put("type", "string");
|
||||
List<OperationMeta> enabled = enabledOperations();
|
||||
StringBuilder opDesc = new StringBuilder();
|
||||
opDesc.append(
|
||||
"Operation id from this category. Call stirling_describe_operation first to learn"
|
||||
+ " the exact parameters schema. Available operations:\n");
|
||||
ArrayNode opEnum = op.putArray("enum");
|
||||
for (OperationMeta m : enabled) {
|
||||
opEnum.add(m.id());
|
||||
opDesc.append("- ").append(m.id()).append(" - ").append(m.summary()).append('\n');
|
||||
}
|
||||
op.put("description", opDesc.toString().trim());
|
||||
|
||||
ObjectNode params = props.putObject("parameters");
|
||||
params.put("type", "object");
|
||||
params.put(
|
||||
"description",
|
||||
"Per-operation parameters. Schema available via stirling_describe_operation.");
|
||||
params.put("additionalProperties", true);
|
||||
|
||||
McpToolSupport.stringProperty(
|
||||
props,
|
||||
"file",
|
||||
"Base64-encoded file content to process. The recommended way to provide a file for"
|
||||
+ " most uses. Bounded by the MCP request size limit; for very large files"
|
||||
+ " use 'fileId' instead.");
|
||||
McpToolSupport.stringProperty(
|
||||
props,
|
||||
"fileName",
|
||||
"Optional original filename (with extension) for the input; helps operations that"
|
||||
+ " key off file type.");
|
||||
McpToolSupport.stringProperty(
|
||||
props,
|
||||
"fileId",
|
||||
"Reference to a file already stored via stirling_upload. Recommended only for large"
|
||||
+ " files or multi-step workflows; most users should pass the file inline"
|
||||
+ " via 'file' instead.");
|
||||
|
||||
ArrayNode required = schema.putArray("required");
|
||||
required.add("operation");
|
||||
return schema;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode call(JsonNode arguments, McpCallContext context) {
|
||||
JsonNode opNode = arguments == null ? null : arguments.get("operation");
|
||||
// No operation chosen: return this category's operation list.
|
||||
if (opNode == null || !opNode.isTextual() || opNode.asText().isBlank()) {
|
||||
return operationListError(null);
|
||||
}
|
||||
String opId = opNode.asText();
|
||||
McpToolCatalog catalog = catalogProvider.getIfAvailable();
|
||||
if (catalog == null) {
|
||||
return McpResponses.error(mapper, "MCP catalog is not available");
|
||||
}
|
||||
OperationMeta meta = catalog.findByOperationId(opId).orElse(null);
|
||||
// Invalid/disabled/wrong-category op: return this category's operations.
|
||||
if (meta == null || meta.category() != category()) {
|
||||
return operationListError(opId);
|
||||
}
|
||||
if (!context.hasScope(meta.requiredScope())) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
"Insufficient scope: this operation requires '" + meta.requiredScope() + "'.");
|
||||
}
|
||||
McpOperationExecutor executor = executorProvider.getIfAvailable();
|
||||
if (executor == null) {
|
||||
return McpResponses.error(mapper, "MCP execution is not available.");
|
||||
}
|
||||
return executor.execute(meta, arguments);
|
||||
}
|
||||
|
||||
/**
|
||||
* Error for a missing/unknown operation, listing this category's available operation ids and
|
||||
* summaries.
|
||||
*/
|
||||
private ObjectNode operationListError(String badOpId) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
if (badOpId == null) {
|
||||
sb.append("Missing required argument 'operation' for ").append(category().toolName());
|
||||
} else {
|
||||
sb.append("Unknown or disabled operation '")
|
||||
.append(badOpId)
|
||||
.append("' for ")
|
||||
.append(category().toolName());
|
||||
}
|
||||
List<OperationMeta> ops = enabledOperations();
|
||||
if (ops.isEmpty()) {
|
||||
sb.append(". No operations are currently available in this category.");
|
||||
} else {
|
||||
sb.append(". Available operations:");
|
||||
for (OperationMeta m : ops) {
|
||||
sb.append("\n- ").append(m.id()).append(" - ").append(m.summary());
|
||||
}
|
||||
sb.append("\nRe-call this tool with a valid 'operation'.");
|
||||
}
|
||||
return McpResponses.error(mapper, sb.toString());
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.proprietary.mcp.McpCallContext;
|
||||
import stirling.software.proprietary.mcp.McpTool;
|
||||
import stirling.software.proprietary.mcp.catalog.McpToolCatalog;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationMeta;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/** Returns the JSON Schema for one operation's parameters, from the live {@link McpToolCatalog}. */
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class DescribeOperationTool implements McpTool {
|
||||
|
||||
private final ObjectMapper mapper;
|
||||
private final ObjectProvider<McpToolCatalog> catalogProvider;
|
||||
|
||||
public DescribeOperationTool(ObjectMapper mapper, ObjectProvider<McpToolCatalog> catalog) {
|
||||
this.mapper = mapper;
|
||||
this.catalogProvider = catalog;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "stirling_describe_operation";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return "Return the full JSON Schema for one Stirling operation's parameters. Call this "
|
||||
+ "before invoking a category tool to learn the exact shape of `parameters`. "
|
||||
+ "Argument: { operation: <op-id> } where <op-id> appears in the enum of any "
|
||||
+ "category tool (stirling_convert, _pages, _misc, _security, _ai).";
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode inputSchema() {
|
||||
ObjectNode schema = mapper.createObjectNode();
|
||||
schema.put("type", "object");
|
||||
schema.put("additionalProperties", false);
|
||||
ObjectNode props = schema.putObject("properties");
|
||||
ObjectNode op = props.putObject("operation");
|
||||
op.put("type", "string");
|
||||
op.put(
|
||||
"description",
|
||||
"Operation id (e.g. compress-pdf, pdf-to-word, q-and-a). See category tool enums.");
|
||||
ArrayNode required = schema.putArray("required");
|
||||
required.add("operation");
|
||||
return schema;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode call(JsonNode arguments, McpCallContext context) {
|
||||
JsonNode opNode = arguments == null ? null : arguments.get("operation");
|
||||
if (opNode == null || !opNode.isTextual() || opNode.asText().isBlank()) {
|
||||
return McpResponses.error(mapper, "Missing required argument: operation");
|
||||
}
|
||||
String opId = opNode.asText();
|
||||
McpToolCatalog catalog = catalogProvider.getIfAvailable();
|
||||
if (catalog == null) {
|
||||
return McpResponses.error(mapper, "MCP catalog is not available");
|
||||
}
|
||||
OperationMeta meta = catalog.findByOperationId(opId).orElse(null);
|
||||
if (meta == null) {
|
||||
return McpResponses.error(mapper, "Unknown or disabled operation: " + opId);
|
||||
}
|
||||
|
||||
ObjectNode payload = mapper.createObjectNode();
|
||||
payload.put("operation", meta.id());
|
||||
payload.put("category", meta.category().toolName());
|
||||
payload.put("summary", meta.summary());
|
||||
payload.put("endpoint", meta.endpointPath());
|
||||
payload.put("requiredScope", meta.requiredScope());
|
||||
payload.set("parametersSchema", meta.paramSchema());
|
||||
return McpResponses.json(mapper, payload);
|
||||
}
|
||||
}
|
||||
+255
@@ -0,0 +1,255 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RestClientResponseException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.FileStorage;
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.service.InternalApiTimeoutException;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationMeta;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Runs a JAVA_ENDPOINT operation: resolves the input file (inline base64 or a fileId), dispatches
|
||||
* to the Stirling endpoint over the loopback via {@link InternalApiClient}, and stores the result.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class McpOperationExecutor {
|
||||
|
||||
private final ObjectMapper mapper;
|
||||
private final InternalApiClient internalApiClient;
|
||||
private final FileStorage fileStorage;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
public McpOperationExecutor(
|
||||
ObjectMapper mapper,
|
||||
InternalApiClient internalApiClient,
|
||||
FileStorage fileStorage,
|
||||
ApplicationProperties applicationProperties) {
|
||||
this.mapper = mapper;
|
||||
this.internalApiClient = internalApiClient;
|
||||
this.fileStorage = fileStorage;
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
|
||||
public ObjectNode execute(OperationMeta meta, JsonNode arguments) {
|
||||
String fileName = McpToolSupport.textArg(arguments, "fileName");
|
||||
String fileId = McpToolSupport.textArg(arguments, "fileId");
|
||||
byte[] inputBytes;
|
||||
String inputName;
|
||||
if (fileId != null) {
|
||||
try {
|
||||
if (!fileStorage.fileExists(fileId)) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
"Unknown or inaccessible fileId '"
|
||||
+ fileId
|
||||
+ "'. Re-upload with stirling_upload.");
|
||||
}
|
||||
inputBytes = fileStorage.retrieveBytes(fileId);
|
||||
} catch (SecurityException e) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
"Unknown or inaccessible fileId '"
|
||||
+ fileId
|
||||
+ "'. Re-upload with stirling_upload.");
|
||||
} catch (IOException e) {
|
||||
return McpResponses.error(mapper, "Could not read fileId '" + fileId + "'.");
|
||||
}
|
||||
inputName = fileName != null ? fileName : fileId;
|
||||
} else {
|
||||
String base64 = McpToolSupport.textArg(arguments, "file");
|
||||
if (base64 == null) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
"This operation needs an input file. Pass 'file' as base64 (recommended for"
|
||||
+ " most files), or 'fileId' from stirling_upload for large files.");
|
||||
}
|
||||
inputBytes = McpToolSupport.decodeBase64OrNull(base64);
|
||||
if (inputBytes == null) {
|
||||
return McpResponses.error(mapper, "The 'file' argument is not valid base64.");
|
||||
}
|
||||
inputName = fileName != null ? fileName : "input.pdf";
|
||||
}
|
||||
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("fileInput", bytesResource(inputBytes, inputName));
|
||||
addParameters(body, arguments == null ? null : arguments.get("parameters"));
|
||||
|
||||
ResponseEntity<Resource> response;
|
||||
try {
|
||||
response = internalApiClient.post(meta.endpointPath(), body);
|
||||
} catch (InternalApiTimeoutException e) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
meta.id()
|
||||
+ " timed out after "
|
||||
+ e.getReadTimeout().toSeconds()
|
||||
+ "s. Try a smaller file or a different approach.");
|
||||
} catch (RestClientResponseException e) {
|
||||
log.warn(
|
||||
"MCP {} upstream error: HTTP {} - {}",
|
||||
meta.id(),
|
||||
e.getStatusCode().value(),
|
||||
snippet(e.getResponseBodyAsString()));
|
||||
return McpResponses.error(
|
||||
mapper, meta.id() + " failed: HTTP " + e.getStatusCode().value() + ".");
|
||||
} catch (SecurityException e) {
|
||||
return McpResponses.error(
|
||||
mapper, meta.id() + " endpoint is not permitted for MCP dispatch.");
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("MCP execution of {} failed", meta.id(), e);
|
||||
return McpResponses.error(
|
||||
mapper, meta.id() + " failed unexpectedly. See server logs for details.");
|
||||
}
|
||||
return buildResult(meta, response);
|
||||
}
|
||||
|
||||
private ObjectNode buildResult(OperationMeta meta, ResponseEntity<Resource> response) {
|
||||
Resource body = response.getBody();
|
||||
if (body == null) {
|
||||
return McpResponses.error(mapper, meta.id() + " returned an empty response.");
|
||||
}
|
||||
MediaType contentType = response.getHeaders().getContentType();
|
||||
|
||||
// A JSON body is a structured report (e.g. get-info), not a file.
|
||||
if (contentType != null && MediaType.APPLICATION_JSON.isCompatibleWith(contentType)) {
|
||||
try (InputStream is = body.getInputStream()) {
|
||||
return McpResponses.text(
|
||||
mapper, new String(is.readAllBytes(), StandardCharsets.UTF_8));
|
||||
} catch (IOException e) {
|
||||
return McpResponses.error(mapper, "Failed to read " + meta.id() + " result.");
|
||||
}
|
||||
}
|
||||
|
||||
String filename =
|
||||
body.getFilename() == null || body.getFilename().isBlank()
|
||||
? meta.id()
|
||||
: body.getFilename();
|
||||
String mimeType =
|
||||
contentType != null
|
||||
? contentType.toString()
|
||||
: MediaType.APPLICATION_OCTET_STREAM_VALUE;
|
||||
long maxInline = applicationProperties.getMcp().getMaxInlineResponseBytes();
|
||||
try {
|
||||
long size = body.contentLength();
|
||||
byte[] inline = null;
|
||||
if (size >= 0 && size <= maxInline) {
|
||||
try (InputStream is = body.getInputStream()) {
|
||||
inline = is.readAllBytes();
|
||||
}
|
||||
}
|
||||
String fileId =
|
||||
inline != null
|
||||
? fileStorage.storeBytes(inline, filename)
|
||||
: storeStreamed(body, filename);
|
||||
String summary =
|
||||
meta.id()
|
||||
+ " succeeded. Result: "
|
||||
+ filename
|
||||
+ " ("
|
||||
+ size
|
||||
+ " bytes), fileId="
|
||||
+ fileId
|
||||
+ ". ";
|
||||
if (inline != null) {
|
||||
return McpResponses.result(
|
||||
mapper,
|
||||
false,
|
||||
McpResponses.textBlock(
|
||||
mapper, summary + "The file is included inline below."),
|
||||
McpResponses.resourceBlock(
|
||||
mapper,
|
||||
"stirling://file/" + fileId,
|
||||
mimeType,
|
||||
Base64.getEncoder().encodeToString(inline)));
|
||||
}
|
||||
return McpResponses.result(
|
||||
mapper,
|
||||
false,
|
||||
McpResponses.textBlock(
|
||||
mapper,
|
||||
summary
|
||||
+ "Large result - fetch it with stirling_download {\"fileId\":\""
|
||||
+ fileId
|
||||
+ "\"}, or pass this fileId to another operation."));
|
||||
} catch (IOException e) {
|
||||
return McpResponses.error(mapper, "Failed to store " + meta.id() + " result.");
|
||||
}
|
||||
}
|
||||
|
||||
private String storeStreamed(Resource body, String filename) throws IOException {
|
||||
try (InputStream is = body.getInputStream()) {
|
||||
return fileStorage.storeInputStream(is, filename).fileId();
|
||||
}
|
||||
}
|
||||
|
||||
private void addParameters(MultiValueMap<String, Object> body, JsonNode params) {
|
||||
if (params == null || !params.isObject()) {
|
||||
return;
|
||||
}
|
||||
Map<String, Object> map =
|
||||
mapper.convertValue(params, new TypeReference<Map<String, Object>>() {});
|
||||
for (Map.Entry<String, Object> entry : map.entrySet()) {
|
||||
Object value = entry.getValue();
|
||||
if (value == null) {
|
||||
continue;
|
||||
}
|
||||
if (value instanceof List<?> list) {
|
||||
if (containsStructured(list)) {
|
||||
body.add(entry.getKey(), mapper.writeValueAsString(list));
|
||||
} else {
|
||||
list.forEach(item -> body.add(entry.getKey(), item));
|
||||
}
|
||||
} else if (value instanceof Map<?, ?>) {
|
||||
body.add(entry.getKey(), mapper.writeValueAsString(value));
|
||||
} else {
|
||||
body.add(entry.getKey(), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean containsStructured(List<?> list) {
|
||||
return list.stream().anyMatch(item -> item instanceof Map<?, ?> || item instanceof List<?>);
|
||||
}
|
||||
|
||||
private static Resource bytesResource(byte[] bytes, String filename) {
|
||||
return new ByteArrayResource(bytes) {
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static String snippet(String body) {
|
||||
if (body == null || body.isBlank()) {
|
||||
return "(no body)";
|
||||
}
|
||||
String trimmed = body.strip();
|
||||
return trimmed.length() > 300 ? trimmed.substring(0, 300) + "..." : trimmed;
|
||||
}
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/** Helpers for the MCP {@code CallToolResult} response shape. */
|
||||
public final class McpResponses {
|
||||
|
||||
private McpResponses() {}
|
||||
|
||||
/** Plain-text content block. */
|
||||
public static ObjectNode text(ObjectMapper mapper, String text) {
|
||||
ObjectNode block = mapper.createObjectNode();
|
||||
block.put("type", "text");
|
||||
block.put("text", text);
|
||||
return wrap(mapper, block, false);
|
||||
}
|
||||
|
||||
/** Plain-text error ({@code isError:true}). */
|
||||
public static ObjectNode error(ObjectMapper mapper, String message) {
|
||||
ObjectNode block = mapper.createObjectNode();
|
||||
block.put("type", "text");
|
||||
block.put("text", message);
|
||||
return wrap(mapper, block, true);
|
||||
}
|
||||
|
||||
/** JSON payload as embedded text. */
|
||||
public static ObjectNode json(ObjectMapper mapper, ObjectNode payload) {
|
||||
ObjectNode block = mapper.createObjectNode();
|
||||
block.put("type", "text");
|
||||
block.put("text", payload.toString());
|
||||
return wrap(mapper, block, false);
|
||||
}
|
||||
|
||||
/** A text content block (unwrapped). */
|
||||
public static ObjectNode textBlock(ObjectMapper mapper, String text) {
|
||||
ObjectNode block = mapper.createObjectNode();
|
||||
block.put("type", "text");
|
||||
block.put("text", text);
|
||||
return block;
|
||||
}
|
||||
|
||||
/** An embedded-resource content block carrying base64 file content. */
|
||||
public static ObjectNode resourceBlock(
|
||||
ObjectMapper mapper, String uri, String mimeType, String base64) {
|
||||
ObjectNode block = mapper.createObjectNode();
|
||||
block.put("type", "resource");
|
||||
ObjectNode res = block.putObject("resource");
|
||||
res.put("uri", uri);
|
||||
if (mimeType != null) {
|
||||
res.put("mimeType", mimeType);
|
||||
}
|
||||
res.put("blob", base64);
|
||||
return block;
|
||||
}
|
||||
|
||||
/** Build a result from explicit content blocks. */
|
||||
public static ObjectNode result(ObjectMapper mapper, boolean isError, ObjectNode... blocks) {
|
||||
ObjectNode result = mapper.createObjectNode();
|
||||
ArrayNode content = result.putArray("content");
|
||||
for (ObjectNode b : blocks) {
|
||||
content.add(b);
|
||||
}
|
||||
if (isError) {
|
||||
result.put("isError", true);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static ObjectNode wrap(ObjectMapper mapper, ObjectNode block, boolean isError) {
|
||||
ObjectNode result = mapper.createObjectNode();
|
||||
ArrayNode content = result.putArray("content");
|
||||
content.add(block);
|
||||
if (isError) {
|
||||
result.put("isError", true);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import java.util.Base64;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/** Shared helpers for MCP tools: argument parsing and JSON-Schema building. */
|
||||
final class McpToolSupport {
|
||||
|
||||
private McpToolSupport() {}
|
||||
|
||||
/** Trimmed text value of an argument, or null if absent, blank, or not a string. */
|
||||
static String textArg(JsonNode args, String field) {
|
||||
if (args == null) {
|
||||
return null;
|
||||
}
|
||||
JsonNode node = args.get(field);
|
||||
if (node == null || !node.isTextual()) {
|
||||
return null;
|
||||
}
|
||||
String value = node.asText().trim();
|
||||
return value.isEmpty() ? null : value;
|
||||
}
|
||||
|
||||
/** Decode base64 content, or null if the input is not valid base64. */
|
||||
static byte[] decodeBase64OrNull(String base64) {
|
||||
try {
|
||||
return Base64.getDecoder().decode(base64);
|
||||
} catch (IllegalArgumentException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a {@code string} property with a description to a JSON-Schema {@code properties} node.
|
||||
*/
|
||||
static void stringProperty(ObjectNode properties, String name, String description) {
|
||||
ObjectNode prop = properties.putObject(name);
|
||||
prop.put("type", "string");
|
||||
prop.put("description", description);
|
||||
}
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.mcp.McpCallContext;
|
||||
import stirling.software.proprietary.mcp.McpTool;
|
||||
import stirling.software.proprietary.mcp.catalog.McpToolCatalog;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationCategory;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationMeta;
|
||||
import stirling.software.proprietary.service.AiEngineClient;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Exposes curated Python agent capabilities as a single MCP tool, sourced from the engine
|
||||
* capabilities manifest.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class StirlingAiTool implements McpTool {
|
||||
|
||||
private final ObjectMapper mapper;
|
||||
private final ObjectProvider<McpToolCatalog> catalogProvider;
|
||||
private final ObjectProvider<AiEngineClient> engineClientProvider;
|
||||
|
||||
public StirlingAiTool(
|
||||
ObjectMapper mapper,
|
||||
ObjectProvider<McpToolCatalog> catalog,
|
||||
ObjectProvider<AiEngineClient> engineClient) {
|
||||
this.mapper = mapper;
|
||||
this.catalogProvider = catalog;
|
||||
this.engineClientProvider = engineClient;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "stirling_ai";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return "Invoke a Stirling AI agent capability (Q&A about a PDF, edit-plan generation,"
|
||||
+ " inline comments, math audit, draft-spec helper). Call"
|
||||
+ " stirling_describe_operation with the chosen capability id to get its"
|
||||
+ " parameters schema before invoking this tool. Some capabilities return content"
|
||||
+ " inline; others return a job reference that resolves to a file when ready.";
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode inputSchema() {
|
||||
ObjectNode schema = mapper.createObjectNode();
|
||||
schema.put("type", "object");
|
||||
schema.put("additionalProperties", false);
|
||||
ObjectNode props = schema.putObject("properties");
|
||||
|
||||
ObjectNode op = props.putObject("operation");
|
||||
op.put("type", "string");
|
||||
StringBuilder desc = new StringBuilder();
|
||||
desc.append("Capability id from the engine manifest. Available capabilities:\n");
|
||||
ArrayNode opEnum = op.putArray("enum");
|
||||
for (OperationMeta m : aiOps()) {
|
||||
opEnum.add(m.id());
|
||||
desc.append("- ").append(m.id()).append(" - ").append(m.summary()).append('\n');
|
||||
}
|
||||
op.put("description", desc.toString().trim());
|
||||
|
||||
ObjectNode params = props.putObject("parameters");
|
||||
params.put("type", "object");
|
||||
params.put("description", "Per-capability parameters.");
|
||||
params.put("additionalProperties", true);
|
||||
|
||||
ObjectNode fileId = props.putObject("fileId");
|
||||
fileId.put("type", "string");
|
||||
fileId.put(
|
||||
"description",
|
||||
"Reference to a previously-uploaded PDF in Stirling's job store. Required for"
|
||||
+ " capabilities that consume a document.");
|
||||
|
||||
ArrayNode required = schema.putArray("required");
|
||||
required.add("operation");
|
||||
return schema;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode call(JsonNode arguments, McpCallContext context) {
|
||||
JsonNode opNode = arguments == null ? null : arguments.get("operation");
|
||||
if (opNode == null || !opNode.isTextual() || opNode.asText().isBlank()) {
|
||||
return McpResponses.error(mapper, "Missing required argument: operation");
|
||||
}
|
||||
String opId = opNode.asText();
|
||||
McpToolCatalog catalog = catalogProvider.getIfAvailable();
|
||||
if (catalog == null) {
|
||||
return McpResponses.error(mapper, "MCP catalog is not available");
|
||||
}
|
||||
OperationMeta meta = catalog.findByOperationId(opId).orElse(null);
|
||||
if (meta == null || meta.category() != OperationCategory.AI) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
"Unknown AI capability '"
|
||||
+ opId
|
||||
+ "'. The engine manifest may not be loaded yet - retry shortly or"
|
||||
+ " confirm the engine is reachable.");
|
||||
}
|
||||
if (!context.hasScope(meta.requiredScope())) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
"Insufficient scope: this capability requires '" + meta.requiredScope() + "'.");
|
||||
}
|
||||
AiEngineClient client = engineClientProvider.getIfAvailable();
|
||||
if (client == null) {
|
||||
return McpResponses.error(
|
||||
mapper, "AI engine client is not configured - enable aiEngine in settings.");
|
||||
}
|
||||
if (meta.endpointPath() == null) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
"Capability '" + opId + "' has no route configured in the engine manifest.");
|
||||
}
|
||||
JsonNode params = arguments.get("parameters");
|
||||
String body = (params == null ? mapper.createObjectNode() : params).toString();
|
||||
try {
|
||||
String response = client.post(meta.endpointPath(), body, context.stirlingUserId());
|
||||
return McpResponses.text(mapper, response);
|
||||
} catch (IOException e) {
|
||||
log.warn("MCP AI capability '{}' engine request failed", opId, e);
|
||||
return McpResponses.error(
|
||||
mapper, "Engine request failed for capability '" + opId + "'.");
|
||||
}
|
||||
}
|
||||
|
||||
private List<OperationMeta> aiOps() {
|
||||
McpToolCatalog catalog = catalogProvider.getIfAvailable();
|
||||
if (catalog == null) {
|
||||
return List.of();
|
||||
}
|
||||
return catalog.enabledOps(OperationCategory.AI);
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.proprietary.mcp.catalog.McpToolCatalog;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationCategory;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/** Exposes the {@code /api/v1/convert/*} namespace as a single MCP tool. */
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class StirlingConvertTool extends AbstractCategoryTool {
|
||||
|
||||
public StirlingConvertTool(
|
||||
ObjectMapper mapper,
|
||||
ObjectProvider<McpToolCatalog> catalog,
|
||||
ObjectProvider<McpOperationExecutor> executor) {
|
||||
super(mapper, catalog, executor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "stirling_convert";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return "Convert files between PDF and other formats (PDF<->Word, PDF<->image, HTML->PDF,"
|
||||
+ " etc.). Inspect the `operation` enum, then call stirling_describe_operation"
|
||||
+ " with the chosen op to get its parameters JSON Schema before calling this"
|
||||
+ " tool.";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected OperationCategory category() {
|
||||
return OperationCategory.CONVERT;
|
||||
}
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Base64;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.FileStorage;
|
||||
import stirling.software.proprietary.mcp.McpCallContext;
|
||||
import stirling.software.proprietary.mcp.McpTool;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Fetches a stored file's content by fileId, returned inline as base64. For large results that were
|
||||
* not returned inline by an operation.
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class StirlingDownloadTool implements McpTool {
|
||||
|
||||
private final ObjectMapper mapper;
|
||||
private final FileStorage fileStorage;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
public StirlingDownloadTool(
|
||||
ObjectMapper mapper,
|
||||
FileStorage fileStorage,
|
||||
ApplicationProperties applicationProperties) {
|
||||
this.mapper = mapper;
|
||||
this.fileStorage = fileStorage;
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "stirling_download";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return "Fetch a stored file's content by fileId (e.g. an operation result), returned inline"
|
||||
+ " as base64. Recommended only when a result was too large to be returned inline."
|
||||
+ " Argument: { fileId: <id> }.";
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode inputSchema() {
|
||||
ObjectNode schema = mapper.createObjectNode();
|
||||
schema.put("type", "object");
|
||||
schema.put("additionalProperties", false);
|
||||
ObjectNode props = schema.putObject("properties");
|
||||
McpToolSupport.stringProperty(
|
||||
props, "fileId", "Id of a stored file (e.g. an operation result's fileId).");
|
||||
schema.putArray("required").add("fileId");
|
||||
return schema;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode call(JsonNode arguments, McpCallContext context) {
|
||||
if (!context.hasScope("mcp.tools.read")) {
|
||||
return McpResponses.error(
|
||||
mapper, "Insufficient scope: stirling_download requires 'mcp.tools.read'.");
|
||||
}
|
||||
String fileId = McpToolSupport.textArg(arguments, "fileId");
|
||||
if (fileId == null) {
|
||||
return McpResponses.error(mapper, "Missing required argument: fileId.");
|
||||
}
|
||||
long maxInline = applicationProperties.getMcp().getMaxInlineResponseBytes();
|
||||
try {
|
||||
if (!fileStorage.fileExists(fileId)) {
|
||||
return McpResponses.error(
|
||||
mapper, "Unknown or inaccessible fileId '" + fileId + "'.");
|
||||
}
|
||||
long size = fileStorage.getFileSize(fileId);
|
||||
if (size > maxInline) {
|
||||
return McpResponses.error(
|
||||
mapper,
|
||||
"File is "
|
||||
+ size
|
||||
+ " bytes, over the inline limit of "
|
||||
+ maxInline
|
||||
+ " bytes. Raise mcp.maxInlineResponseBytes or retrieve it via the"
|
||||
+ " Stirling UI/API.");
|
||||
}
|
||||
byte[] bytes = fileStorage.retrieveBytes(fileId);
|
||||
return McpResponses.result(
|
||||
mapper,
|
||||
false,
|
||||
McpResponses.textBlock(
|
||||
mapper,
|
||||
"File "
|
||||
+ fileId
|
||||
+ " ("
|
||||
+ bytes.length
|
||||
+ " bytes) included inline below."),
|
||||
McpResponses.resourceBlock(
|
||||
mapper,
|
||||
"stirling://file/" + fileId,
|
||||
MediaType.APPLICATION_OCTET_STREAM_VALUE,
|
||||
Base64.getEncoder().encodeToString(bytes)));
|
||||
} catch (SecurityException e) {
|
||||
return McpResponses.error(mapper, "Unknown or inaccessible fileId '" + fileId + "'.");
|
||||
} catch (IOException e) {
|
||||
return McpResponses.error(mapper, "Failed to read fileId '" + fileId + "'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.proprietary.mcp.catalog.McpToolCatalog;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationCategory;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/** Exposes the {@code /api/v1/misc/*} namespace as a single MCP tool. */
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class StirlingMiscTool extends AbstractCategoryTool {
|
||||
|
||||
public StirlingMiscTool(
|
||||
ObjectMapper mapper,
|
||||
ObjectProvider<McpToolCatalog> catalog,
|
||||
ObjectProvider<McpOperationExecutor> executor) {
|
||||
super(mapper, catalog, executor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "stirling_misc";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return "Miscellaneous PDF operations: compress, OCR, stamp / watermark, edit metadata,"
|
||||
+ " flatten, repair, and similar utilities. Call stirling_describe_operation with"
|
||||
+ " the chosen op to get its parameters schema before invoking this tool.";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected OperationCategory category() {
|
||||
return OperationCategory.MISC;
|
||||
}
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.proprietary.mcp.catalog.McpToolCatalog;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationCategory;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/** Exposes the {@code /api/v1/general/*} (page operations) namespace as a single MCP tool. */
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class StirlingPagesTool extends AbstractCategoryTool {
|
||||
|
||||
public StirlingPagesTool(
|
||||
ObjectMapper mapper,
|
||||
ObjectProvider<McpToolCatalog> catalog,
|
||||
ObjectProvider<McpOperationExecutor> executor) {
|
||||
super(mapper, catalog, executor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "stirling_pages";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return "Manipulate PDF pages: merge, split, rotate, rearrange, crop, delete, overlay,"
|
||||
+ " add blank pages. Call stirling_describe_operation with the chosen op to get"
|
||||
+ " its parameters schema before invoking this tool.";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected OperationCategory category() {
|
||||
return OperationCategory.PAGES;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.proprietary.mcp.catalog.McpToolCatalog;
|
||||
import stirling.software.proprietary.mcp.catalog.OperationCategory;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/** Exposes the {@code /api/v1/security/*} namespace as a single MCP tool. */
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class StirlingSecurityTool extends AbstractCategoryTool {
|
||||
|
||||
public StirlingSecurityTool(
|
||||
ObjectMapper mapper,
|
||||
ObjectProvider<McpToolCatalog> catalog,
|
||||
ObjectProvider<McpOperationExecutor> executor) {
|
||||
super(mapper, catalog, executor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "stirling_security";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return "Security-related PDF operations: password add/remove, redact, sanitize, certify"
|
||||
+ " / sign with cert, validate signature, add watermark. Call"
|
||||
+ " stirling_describe_operation with the chosen op to get its parameters schema"
|
||||
+ " before invoking this tool.";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected OperationCategory category() {
|
||||
return OperationCategory.SECURITY;
|
||||
}
|
||||
}
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
package stirling.software.proprietary.mcp.tools;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.service.FileStorage;
|
||||
import stirling.software.proprietary.mcp.McpCallContext;
|
||||
import stirling.software.proprietary.mcp.McpTool;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Stores a file server-side and returns a fileId. For large files or multi-step workflows only -
|
||||
* most operations accept the file inline via their {@code file} argument.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "mcp.enabled", havingValue = "true")
|
||||
public class StirlingUploadTool implements McpTool {
|
||||
|
||||
private final ObjectMapper mapper;
|
||||
private final FileStorage fileStorage;
|
||||
|
||||
public StirlingUploadTool(ObjectMapper mapper, FileStorage fileStorage) {
|
||||
this.mapper = mapper;
|
||||
this.fileStorage = fileStorage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "stirling_upload";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return "Store a file server-side and get back a fileId to reuse across operations."
|
||||
+ " Recommended only for large files or multi-step workflows; for a single"
|
||||
+ " operation on a typical file, pass the file inline via the operation's `file`"
|
||||
+ " argument instead. Argument: { file: <base64>, fileName?: <name> }.";
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode inputSchema() {
|
||||
ObjectNode schema = mapper.createObjectNode();
|
||||
schema.put("type", "object");
|
||||
schema.put("additionalProperties", false);
|
||||
ObjectNode props = schema.putObject("properties");
|
||||
McpToolSupport.stringProperty(props, "file", "Base64-encoded file content.");
|
||||
McpToolSupport.stringProperty(
|
||||
props, "fileName", "Optional original filename (with extension).");
|
||||
schema.putArray("required").add("file");
|
||||
return schema;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectNode call(JsonNode arguments, McpCallContext context) {
|
||||
if (!context.hasScope("mcp.tools.write")) {
|
||||
return McpResponses.error(
|
||||
mapper, "Insufficient scope: stirling_upload requires 'mcp.tools.write'.");
|
||||
}
|
||||
String base64 = McpToolSupport.textArg(arguments, "file");
|
||||
if (base64 == null) {
|
||||
return McpResponses.error(
|
||||
mapper, "Missing required argument: file (base64-encoded content).");
|
||||
}
|
||||
byte[] bytes = McpToolSupport.decodeBase64OrNull(base64);
|
||||
if (bytes == null) {
|
||||
return McpResponses.error(mapper, "The 'file' argument is not valid base64.");
|
||||
}
|
||||
String name = McpToolSupport.textArg(arguments, "fileName");
|
||||
if (name == null) {
|
||||
name = "upload.bin";
|
||||
}
|
||||
try {
|
||||
String fileId = fileStorage.storeBytes(bytes, name);
|
||||
return McpResponses.text(
|
||||
mapper,
|
||||
"Stored '"
|
||||
+ name
|
||||
+ "' ("
|
||||
+ bytes.length
|
||||
+ " bytes) as fileId="
|
||||
+ fileId
|
||||
+ ". Pass this fileId to a Stirling operation's 'fileId' argument.");
|
||||
} catch (IOException e) {
|
||||
log.warn("MCP upload failed to store file", e);
|
||||
return McpResponses.error(mapper, "Failed to store the uploaded file.");
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -21,7 +21,8 @@ public enum AiWorkflowOutcome {
|
||||
COMPLETED("completed"),
|
||||
UNSUPPORTED_CAPABILITY("unsupported_capability"),
|
||||
CANNOT_CONTINUE("cannot_continue"),
|
||||
GENERATE_FILE("generate_file");
|
||||
GENERATE_FILE("generate_file"),
|
||||
CONVERT_MARKDOWN("convert_markdown");
|
||||
|
||||
private final String value;
|
||||
|
||||
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
package stirling.software.proprietary.policy.config;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.common.configuration.InstallationPathConfig;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
|
||||
/**
|
||||
* The single authority on which filesystem locations a policy may read from or write to. Folder
|
||||
* sources and sinks take a configured directory, so without this a user who can save a policy could
|
||||
* point one at Stirling's own config/secrets directory and exfiltrate (or overwrite) it. Every
|
||||
* folder source and sink runs its directory through {@link #requirePermitted(Path)} at save time
|
||||
* and again at run time.
|
||||
*
|
||||
* <p>Enforced fail-closed, in order:
|
||||
*
|
||||
* <ul>
|
||||
* <li><b>Disabled in SaaS</b> - folder access is never allowed when the {@code saas} profile is
|
||||
* active; a tenant must not reach the host filesystem at all.
|
||||
* <li><b>Protected paths</b> - Stirling's own config directory (settings, database, keys,
|
||||
* backups) is always rejected, even if an allowed root were misconfigured to contain it.
|
||||
* <li><b>Allowlist</b> - the directory must resolve within one of {@code
|
||||
* policies.allowedFolderRoots}; with none configured, all folder access is refused.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Paths are compared after normalisation, so {@code ..} segments cannot walk out of an allowed
|
||||
* root. (Symlink escape is not defended here; an operator who configures an allowed root containing
|
||||
* a symlink to a sensitive location is trusted.)
|
||||
*/
|
||||
@Component
|
||||
public class FolderAccessGuard {
|
||||
|
||||
public static final String FOLDER_TYPE = "folder";
|
||||
|
||||
private final boolean saasActive;
|
||||
private final List<Path> allowedRoots;
|
||||
private final List<Path> protectedRoots;
|
||||
|
||||
public FolderAccessGuard(ApplicationProperties applicationProperties, Environment environment) {
|
||||
this.saasActive = Arrays.asList(environment.getActiveProfiles()).contains("saas");
|
||||
this.allowedRoots =
|
||||
normalizeAll(applicationProperties.getPolicies().getAllowedFolderRoots());
|
||||
this.protectedRoots = List.of(normalize(Path.of(InstallationPathConfig.getConfigPath())));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check that {@code dir} is a permitted folder location, returning its normalised absolute
|
||||
* form.
|
||||
*
|
||||
* @throws IllegalArgumentException if folder access is disabled (SaaS or no roots configured),
|
||||
* the path is inside a protected directory, or it falls outside every allowed root
|
||||
*/
|
||||
public Path requirePermitted(Path dir) {
|
||||
if (saasActive) {
|
||||
throw new IllegalArgumentException(
|
||||
"folder sources and outputs are not available in SaaS mode");
|
||||
}
|
||||
Path normalized = normalize(dir);
|
||||
for (Path protectedRoot : protectedRoots) {
|
||||
if (normalized.startsWith(protectedRoot)) {
|
||||
throw new IllegalArgumentException(
|
||||
"folder may not point inside a protected Stirling directory");
|
||||
}
|
||||
}
|
||||
if (allowedRoots.isEmpty()) {
|
||||
throw new IllegalArgumentException(
|
||||
"folder access is disabled; set policies.allowedFolderRoots to permit it");
|
||||
}
|
||||
boolean within = allowedRoots.stream().anyMatch(normalized::startsWith);
|
||||
if (!within) {
|
||||
throw new IllegalArgumentException(
|
||||
"folder '" + normalized + "' is outside the allowed folder roots");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/** Whether this policy reads from or writes to a folder, and so is subject to these rules. */
|
||||
public boolean usesFolderAccess(Policy policy) {
|
||||
boolean readsFolder =
|
||||
policy.sources().stream().anyMatch(spec -> FOLDER_TYPE.equals(spec.type()));
|
||||
boolean writesFolder =
|
||||
policy.output() != null && FOLDER_TYPE.equals(policy.output().type());
|
||||
return readsFolder || writesFolder;
|
||||
}
|
||||
|
||||
private static List<Path> normalizeAll(List<String> roots) {
|
||||
List<Path> result = new ArrayList<>();
|
||||
for (String root : roots) {
|
||||
if (root != null && !root.isBlank()) {
|
||||
result.add(normalize(Path.of(root)));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Path normalize(Path path) {
|
||||
return path.toAbsolutePath().normalize();
|
||||
}
|
||||
}
|
||||
+351
@@ -0,0 +1,351 @@
|
||||
package stirling.software.proprietary.policy.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.multipart.MultipartHttpServletRequest;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
import io.github.pixee.security.Filenames;
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.job.JobResponse;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunHandle;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.PolicyValidator;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
import stirling.software.proprietary.policy.model.PolicyRunStatus;
|
||||
import stirling.software.proprietary.policy.model.PolicyRunView;
|
||||
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.security.config.PremiumEndpoint;
|
||||
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Manages policies and runs pipelines. The premium backend entry point: CRUD for stored {@code
|
||||
* Policy} objects, running a stored policy by id, and running an ad-hoc pipeline (for AI/Automate
|
||||
* one-offs).
|
||||
*
|
||||
* <p>Runs execute asynchronously and return a run id immediately. Poll {@code GET /run/{runId}} for
|
||||
* status, and download outputs via the existing {@code GET /api/v1/general/files/{fileId}} using
|
||||
* the file ids in the run view.
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/policies")
|
||||
@Hidden
|
||||
@PremiumEndpoint
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Policies", description = "Run tool pipelines on the backend")
|
||||
public class PolicyController {
|
||||
|
||||
private final PolicyRunner policyRunner;
|
||||
private final PolicyRunRegistry runRegistry;
|
||||
private final PolicyStore policyStore;
|
||||
private final PolicyValidator policyValidator;
|
||||
private final FolderAccessGuard folderAccessGuard;
|
||||
private final UserServiceInterface userService;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
@PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(
|
||||
summary = "Run a tool pipeline",
|
||||
description =
|
||||
"Accepts the documents to process (multipart field 'fileInput'), any supporting"
|
||||
+ " files (each under a multipart field named as its asset key, e.g."
|
||||
+ " 'company-logo'), and a JSON pipeline definition ('json'). Runs the"
|
||||
+ " steps in order asynchronously and returns a run id. Poll the run"
|
||||
+ " status endpoint and download outputs via /api/v1/general/files/{id}.")
|
||||
public ResponseEntity<JobResponse<Void>> run(
|
||||
@RequestParam("json") String json, MultipartHttpServletRequest request)
|
||||
throws IOException {
|
||||
PipelineDefinition definition = parseDefinition(json);
|
||||
PolicyInputs inputs = collectInputs(request);
|
||||
String runId =
|
||||
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId();
|
||||
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
|
||||
}
|
||||
|
||||
@PostMapping(value = "/run/stream", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(
|
||||
summary = "Run a tool pipeline with live progress",
|
||||
description =
|
||||
"Same as /run, but returns Server-Sent Events: a 'step' event as each step"
|
||||
+ " starts and completes, then a terminal 'completed', 'failed',"
|
||||
+ " 'cancelled', or 'waiting' event carrying the final run view.")
|
||||
public SseEmitter runStream(
|
||||
@RequestParam("json") String json, MultipartHttpServletRequest request)
|
||||
throws IOException {
|
||||
PipelineDefinition definition = parseDefinition(json);
|
||||
PolicyInputs inputs = collectInputs(request);
|
||||
|
||||
SseEmitter emitter =
|
||||
new SseEmitter(applicationProperties.getPolicies().getStreamTimeoutMs());
|
||||
emitter.onError(e -> log.warn("Policy run SSE emitter error", e));
|
||||
|
||||
PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter));
|
||||
// Close the stream with a terminal event once the run finishes. whenComplete runs on the
|
||||
// engine's worker thread after the run is done, so this never races the step events.
|
||||
handle.completion()
|
||||
.whenComplete(
|
||||
(run, throwable) -> {
|
||||
if (throwable != null) {
|
||||
sendEvent(
|
||||
emitter,
|
||||
"failed",
|
||||
Map.of("message", throwable.getMessage()));
|
||||
} else {
|
||||
sendEvent(emitter, terminalEventName(run), PolicyRunView.of(run));
|
||||
}
|
||||
emitter.complete();
|
||||
});
|
||||
return emitter;
|
||||
}
|
||||
|
||||
@GetMapping("/run/{runId}")
|
||||
@Operation(
|
||||
summary = "Get pipeline run status",
|
||||
description = "Returns the current status, step cursor, and output files of a run.")
|
||||
public ResponseEntity<PolicyRunView> status(@PathVariable String runId) {
|
||||
PolicyRun run = runRegistry.get(runId);
|
||||
if (run == null) {
|
||||
return ResponseEntity.notFound().build();
|
||||
}
|
||||
return ResponseEntity.ok(PolicyRunView.of(run));
|
||||
}
|
||||
|
||||
// --- Policy management ---
|
||||
|
||||
@PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
summary = "Create or update a policy",
|
||||
description =
|
||||
"Stores a policy (trigger config + steps + output + metadata). A blank id is"
|
||||
+ " assigned; returns the stored policy with its id.")
|
||||
public ResponseEntity<Policy> savePolicy(@RequestBody String json) {
|
||||
Policy policy = parsePolicy(json);
|
||||
requireAuthorizedForFolderAccess(policy);
|
||||
try {
|
||||
policyValidator.validate(policy);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
return ResponseEntity.ok(policyStore.save(policy));
|
||||
}
|
||||
|
||||
/**
|
||||
* A policy that reads from or writes to a server folder grants whoever saves it access to that
|
||||
* path, so restrict it to administrators on multi-user deployments. Single-user deployments
|
||||
* (login disabled, e.g. desktop) trust the local operator. The {@link FolderAccessGuard} still
|
||||
* enforces SaaS-off and the path allowlist during validation regardless of who saves.
|
||||
*/
|
||||
private void requireAuthorizedForFolderAccess(Policy policy) {
|
||||
if (!folderAccessGuard.usesFolderAccess(policy)) {
|
||||
return;
|
||||
}
|
||||
if (!applicationProperties.getSecurity().isEnableLogin()) {
|
||||
return;
|
||||
}
|
||||
if (!userService.isCurrentUserAdmin()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.FORBIDDEN,
|
||||
"Folder sources and outputs may only be configured by an administrator");
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping
|
||||
@Operation(summary = "List policies")
|
||||
public List<Policy> listPolicies() {
|
||||
return policyStore.all();
|
||||
}
|
||||
|
||||
@GetMapping("/{policyId}")
|
||||
@Operation(summary = "Get a policy by id")
|
||||
public ResponseEntity<Policy> getPolicy(@PathVariable String policyId) {
|
||||
return policyStore
|
||||
.get(policyId)
|
||||
.map(ResponseEntity::ok)
|
||||
.orElseGet(() -> ResponseEntity.notFound().build());
|
||||
}
|
||||
|
||||
@DeleteMapping("/{policyId}")
|
||||
@Operation(summary = "Delete a policy by id")
|
||||
public ResponseEntity<Void> deletePolicy(@PathVariable String policyId) {
|
||||
return policyStore.delete(policyId)
|
||||
? ResponseEntity.noContent().build()
|
||||
: ResponseEntity.notFound().build();
|
||||
}
|
||||
|
||||
@PostMapping(value = "/{policyId}/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(
|
||||
summary = "Run a stored policy",
|
||||
description =
|
||||
"Runs the stored policy's pipeline on the supplied files (primary documents"
|
||||
+ " under 'fileInput', supporting files under their asset-key fields)."
|
||||
+ " Runs regardless of the policy's enabled flag, which only gates"
|
||||
+ " automatic triggering. Returns a run id.")
|
||||
public ResponseEntity<JobResponse<Void>> runStoredPolicy(
|
||||
@PathVariable String policyId, MultipartHttpServletRequest request) throws IOException {
|
||||
Policy policy =
|
||||
policyStore
|
||||
.get(policyId)
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new ResponseStatusException(
|
||||
HttpStatus.NOT_FOUND, "No policy: " + policyId));
|
||||
PolicyInputs inputs = collectInputs(request);
|
||||
String runId = policyRunner.runWith(policy, inputs, PolicyProgressListener.NOOP).runId();
|
||||
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
|
||||
}
|
||||
|
||||
private Policy parsePolicy(String json) {
|
||||
try {
|
||||
return objectMapper.readValue(json, Policy.class);
|
||||
} catch (JacksonException e) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid policy JSON");
|
||||
}
|
||||
}
|
||||
|
||||
private PipelineDefinition parseDefinition(String json) {
|
||||
PipelineDefinition definition;
|
||||
try {
|
||||
definition = objectMapper.readValue(json, PipelineDefinition.class);
|
||||
} catch (JacksonException e) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Invalid pipeline definition JSON");
|
||||
}
|
||||
if (definition.steps().isEmpty()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Pipeline definition has no steps");
|
||||
}
|
||||
return definition;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split the multipart file parts into the primary document stream ("fileInput") and the named
|
||||
* supporting-file store: every other file field becomes an asset keyed by its field name, which
|
||||
* a step references from {@code fileParameters}.
|
||||
*/
|
||||
private PolicyInputs collectInputs(MultipartHttpServletRequest request) throws IOException {
|
||||
MultiValueMap<String, MultipartFile> fileMap = request.getMultiFileMap();
|
||||
List<Resource> primary = toResources(fileMap.get("fileInput"));
|
||||
Map<String, List<Resource>> supportingFiles = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, List<MultipartFile>> entry : fileMap.entrySet()) {
|
||||
if ("fileInput".equals(entry.getKey())) {
|
||||
continue;
|
||||
}
|
||||
List<Resource> assets = toResources(entry.getValue());
|
||||
if (!assets.isEmpty()) {
|
||||
supportingFiles.put(entry.getKey(), assets);
|
||||
}
|
||||
}
|
||||
return new PolicyInputs(primary, supportingFiles);
|
||||
}
|
||||
|
||||
/**
|
||||
* A progress listener that forwards each step transition to the SSE stream as a "step" event.
|
||||
*/
|
||||
private PolicyProgressListener streamListener(SseEmitter emitter) {
|
||||
return new PolicyProgressListener() {
|
||||
@Override
|
||||
public void onStepStart(int stepIndex, int stepCount, String operation) {
|
||||
sendEvent(emitter, "step", stepEvent("started", stepIndex, stepCount, operation));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onStepComplete(int stepIndex, int stepCount, String operation) {
|
||||
sendEvent(emitter, "step", stepEvent("completed", stepIndex, stepCount, operation));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static Map<String, Object> stepEvent(
|
||||
String phase, int stepIndex, int stepCount, String operation) {
|
||||
return Map.of(
|
||||
"phase", phase,
|
||||
"stepIndex", stepIndex,
|
||||
"stepCount", stepCount,
|
||||
"operation", operation);
|
||||
}
|
||||
|
||||
private static String terminalEventName(PolicyRun run) {
|
||||
PolicyRunStatus status = run.getStatus();
|
||||
return switch (status) {
|
||||
case COMPLETED -> "completed";
|
||||
case FAILED -> "failed";
|
||||
case CANCELLED -> "cancelled";
|
||||
case WAITING_FOR_INPUT -> "waiting";
|
||||
default -> "ended";
|
||||
};
|
||||
}
|
||||
|
||||
private void sendEvent(SseEmitter emitter, String name, Object data) {
|
||||
try {
|
||||
emitter.send(SseEmitter.event().name(name).data(data, MediaType.APPLICATION_JSON));
|
||||
} catch (IOException | IllegalStateException e) {
|
||||
// Client disconnected or the emitter already closed. The run continues and its results
|
||||
// remain downloadable via the job endpoints; nothing useful left to stream.
|
||||
log.debug("Dropping policy SSE event '{}': {}", name, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private List<Resource> toResources(List<MultipartFile> files) throws IOException {
|
||||
List<Resource> resources = new ArrayList<>();
|
||||
if (files == null) {
|
||||
return resources;
|
||||
}
|
||||
for (MultipartFile file : files) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
TempFile tempFile = tempFileManager.createManagedTempFile("policy-run");
|
||||
file.transferTo(tempFile.getPath());
|
||||
final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename());
|
||||
resources.add(
|
||||
new FileSystemResource(tempFile.getFile()) {
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return originalName;
|
||||
}
|
||||
});
|
||||
}
|
||||
return resources;
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
import stirling.software.common.service.FileStorage;
|
||||
import stirling.software.common.service.InternalApiTimeoutException;
|
||||
import stirling.software.common.service.JobOwnershipService;
|
||||
import stirling.software.common.service.JobQueue;
|
||||
import stirling.software.common.service.ResourceMonitor;
|
||||
import stirling.software.common.service.TaskManager;
|
||||
import stirling.software.common.util.ExecutorFactory;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
import stirling.software.proprietary.policy.model.WaitState;
|
||||
import stirling.software.proprietary.policy.output.PolicyOutputSink;
|
||||
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
|
||||
|
||||
/**
|
||||
* Runs pipelines asynchronously as tracked jobs.
|
||||
*
|
||||
* <p>Each run is the unit of async work: {@link #submit} returns a run id immediately and the
|
||||
* pipeline executes on a virtual thread, so a step blocking on a slow tool does not tie up a
|
||||
* platform thread. The run drives {@link PolicyExecutor} for the actual step loop, registers its
|
||||
* outputs and progress with {@link TaskManager} (so the existing job status/download endpoints work
|
||||
* unchanged), and keeps rich state in {@link PolicyRunRegistry}.
|
||||
*
|
||||
* <p>The engine deliberately manages its own virtual-thread execution rather than routing through
|
||||
* {@code JobExecutorService}: that path force-completes a job once its work returns, which is
|
||||
* incompatible with a run that suspends in {@code WAITING_FOR_INPUT}. It still applies the shared
|
||||
* {@link ResourceMonitor}/{@link JobQueue} admission control, so heavy runs queue under load
|
||||
* instead of oversubscribing.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PolicyEngine {
|
||||
|
||||
/**
|
||||
* Resource weight of a pipeline run for admission control. A run chains many tools and holds
|
||||
* intermediate files, so it is weighted as heavy work: the shared {@link ResourceMonitor}
|
||||
* should let it start while the system is healthy but hold it back under memory/CPU pressure.
|
||||
* See {@link ResourceMonitor#shouldQueueJob(int)} for how a weight maps to that decision.
|
||||
*/
|
||||
private static final int RUN_RESOURCE_WEIGHT = 50;
|
||||
|
||||
private final PolicyExecutor stepExecutor;
|
||||
private final TaskManager taskManager;
|
||||
private final PolicyRunRegistry registry;
|
||||
private final FileStorage fileStorage;
|
||||
private final JobOwnershipService jobOwnershipService;
|
||||
private final List<PolicyOutputSink> outputSinks;
|
||||
private final ResourceMonitor resourceMonitor;
|
||||
private final JobQueue jobQueue;
|
||||
|
||||
private final ExecutorService asyncExecutor = ExecutorFactory.newVirtualThreadExecutor();
|
||||
|
||||
/**
|
||||
* Submit a pipeline to run asynchronously. The returned handle's run id scopes a job in {@link
|
||||
* TaskManager}, so progress (notes), status, and result files are observable via the existing
|
||||
* job endpoints as well as via {@link #getRun(String)}; its completion future resolves when the
|
||||
* run reaches a terminal or paused state.
|
||||
*/
|
||||
public PolicyRunHandle submit(
|
||||
PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) {
|
||||
// Scope the run id to the current user (on this request thread) so the file-download
|
||||
// ownership check passes; NoOpJobOwnershipService returns the id unchanged when security
|
||||
// is off.
|
||||
String runId = jobOwnershipService.createScopedJobKey(UUID.randomUUID().toString());
|
||||
taskManager.createTask(runId);
|
||||
PolicyRun run = new PolicyRun(runId, definition);
|
||||
registry.register(run);
|
||||
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
|
||||
PolicyProgressListener tracking = trackingListener(runId, run, listener);
|
||||
Runnable task = () -> runToCompletion(run, inputs, tracking, completion);
|
||||
|
||||
// Each run is one admission unit; steps run synchronously within it, so this gates heavy
|
||||
// work under load without the pool-within-pool risk of queueing each tool call. Under
|
||||
// resource pressure the run waits in the shared JobQueue; otherwise it starts immediately.
|
||||
if (resourceMonitor.shouldQueueJob(RUN_RESOURCE_WEIGHT)) {
|
||||
log.debug("Queueing policy run {} under resource pressure", runId);
|
||||
jobQueue.queueJob(
|
||||
runId,
|
||||
RUN_RESOURCE_WEIGHT,
|
||||
() -> {
|
||||
task.run();
|
||||
return null;
|
||||
},
|
||||
0L)
|
||||
.exceptionally(ex -> failRejectedRun(run, completion, ex));
|
||||
} else {
|
||||
asyncExecutor.execute(task);
|
||||
}
|
||||
return new PolicyRunHandle(runId, completion);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a stored policy on demand. Builds the policy's pipeline and submits it. {@code enabled}
|
||||
* gates automatic triggering, not explicit runs, so this runs regardless of that flag.
|
||||
*/
|
||||
public PolicyRunHandle runPolicy(
|
||||
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
|
||||
return submit(policy.toDefinition(), inputs, listener);
|
||||
}
|
||||
|
||||
public PolicyRun getRun(String runId) {
|
||||
return registry.get(runId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Request cancellation of a run. Stage 1 marks the run cancelled in the registry if it has not
|
||||
* already finished; interrupting an in-flight tool call lands in a later stage.
|
||||
*/
|
||||
public boolean cancel(String runId) {
|
||||
PolicyRun run = registry.get(runId);
|
||||
if (run == null) {
|
||||
return false;
|
||||
}
|
||||
boolean cancelled = run.cancel();
|
||||
if (cancelled) {
|
||||
taskManager.addNote(runId, "Run cancelled by request");
|
||||
}
|
||||
return cancelled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume a run paused in {@code WAITING_FOR_INPUT}. Not yet implemented; the run shape and
|
||||
* {@link WaitState} snapshot are in place so this can be added without reworking the engine.
|
||||
*/
|
||||
public String resume(String runId, List<Resource> additionalInputs) {
|
||||
throw new UnsupportedOperationException("Pause/resume is not yet implemented");
|
||||
}
|
||||
|
||||
private void runToCompletion(
|
||||
PolicyRun run,
|
||||
PolicyInputs inputs,
|
||||
PolicyProgressListener listener,
|
||||
CompletableFuture<PolicyRun> completion) {
|
||||
String runId = run.getRunId();
|
||||
try {
|
||||
run.markRunning();
|
||||
PolicyExecutionResult result =
|
||||
stepExecutor.execute(run.getDefinition(), inputs, listener);
|
||||
OutputSpec output = run.getDefinition().output();
|
||||
List<ResultFile> outputs = sinkFor(output).deliver(runId, result.files(), output);
|
||||
taskManager.setMultipleFileResults(runId, outputs);
|
||||
taskManager.setComplete(runId);
|
||||
run.complete(outputs);
|
||||
} catch (PolicyInputRequiredException e) {
|
||||
// Designed-for path: suspend the run rather than fail it. Persist intermediates as
|
||||
// fileIds so the run can resume after this worker thread is gone.
|
||||
WaitState wait = suspend(e);
|
||||
run.waitForInput(wait);
|
||||
taskManager.addNote(runId, "Waiting for input: " + e.getMessage());
|
||||
} catch (InternalApiTimeoutException e) {
|
||||
String message = toolTimeoutMessage(e);
|
||||
log.error(
|
||||
"Policy run {} timed out on {}: {}",
|
||||
runId,
|
||||
e.getEndpointPath(),
|
||||
e.getMessage());
|
||||
run.fail(message);
|
||||
taskManager.setError(runId, message);
|
||||
} catch (Exception e) {
|
||||
String message = "Policy run failed: " + e.getMessage();
|
||||
log.error("Policy run {} failed", runId, e);
|
||||
run.fail(message);
|
||||
taskManager.setError(runId, message);
|
||||
} finally {
|
||||
// Always resolve the handle with the run's final state so stream/await callers unblock.
|
||||
completion.complete(run);
|
||||
}
|
||||
}
|
||||
|
||||
private ResponseEntity<?> failRejectedRun(
|
||||
PolicyRun run, CompletableFuture<PolicyRun> completion, Throwable ex) {
|
||||
// Only reached if the run never started (e.g. the queue was full). A run that started
|
||||
// always resolves its own completion in runToCompletion.
|
||||
if (!completion.isDone()) {
|
||||
String message = "Policy run could not be queued: " + ex.getMessage();
|
||||
log.error("Policy run {} was not admitted: {}", run.getRunId(), ex.getMessage());
|
||||
run.fail(message);
|
||||
taskManager.setError(run.getRunId(), message);
|
||||
completion.complete(run);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private WaitState suspend(PolicyInputRequiredException e) {
|
||||
List<String> fileIds = new ArrayList<>();
|
||||
for (Resource resource : e.getPendingFiles()) {
|
||||
String name = resource.getFilename() != null ? resource.getFilename() : "pending";
|
||||
try (InputStream is = resource.getInputStream()) {
|
||||
fileIds.add(fileStorage.storeInputStream(is, name).fileId());
|
||||
} catch (IOException io) {
|
||||
log.warn("Failed to persist pending file for paused run: {}", io.getMessage());
|
||||
}
|
||||
}
|
||||
return new WaitState(e.getMessage(), e.getResumeStepIndex(), fileIds);
|
||||
}
|
||||
|
||||
private PolicyProgressListener trackingListener(
|
||||
String runId, PolicyRun run, PolicyProgressListener delegate) {
|
||||
return new PolicyProgressListener() {
|
||||
@Override
|
||||
public void onStepStart(int stepIndex, int stepCount, String operation) {
|
||||
run.enterStep(stepIndex);
|
||||
taskManager.addNote(
|
||||
runId,
|
||||
"Step " + stepIndex + "/" + stepCount + ": " + operation + " started");
|
||||
delegate.onStepStart(stepIndex, stepCount, operation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onStepComplete(int stepIndex, int stepCount, String operation) {
|
||||
taskManager.addNote(
|
||||
runId,
|
||||
"Step " + stepIndex + "/" + stepCount + ": " + operation + " completed");
|
||||
delegate.onStepComplete(stepIndex, stepCount, operation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onHeartbeat() {
|
||||
delegate.onHeartbeat();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private PolicyOutputSink sinkFor(OutputSpec spec) {
|
||||
return outputSinks.stream()
|
||||
.filter(sink -> sink.supports(spec))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalStateException(
|
||||
"No output sink supports spec: "
|
||||
+ (spec == null ? "<null>" : spec.type())));
|
||||
}
|
||||
|
||||
private static String toolTimeoutMessage(InternalApiTimeoutException e) {
|
||||
return String.format(
|
||||
"The %s tool did not respond within %d seconds and was aborted.",
|
||||
e.getEndpointPath(), e.getReadTimeout().toSeconds());
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
/**
|
||||
* Result of running a pipeline through {@link PolicyExecutor}.
|
||||
*
|
||||
* <p>{@code files} are the final output resources (temp files, not yet stored to {@code
|
||||
* FileStorage}). {@code report} is the structured metadata payload captured from the last step that
|
||||
* produced one (a JSON body, or an {@code X-Stirling-Tool-Report} header), with {@code reportTool}
|
||||
* naming the step it came from; both are null when no step produced a report.
|
||||
*/
|
||||
public record PolicyExecutionResult(List<Resource> files, JsonNode report, String reportTool) {}
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.service.InternalApiTimeoutException;
|
||||
import stirling.software.common.service.ToolMetadataService;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.ZipExtractionUtils;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
|
||||
import stirling.software.proprietary.service.AiToolResponseHeaders;
|
||||
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Runs an ordered chain of tool steps, chaining each step's output files into the next step's
|
||||
* input.
|
||||
*
|
||||
* <p>This is the single execution loop for the proprietary surface (AI plans now;
|
||||
* manually-triggered runs and watched folders later). Each step is dispatched synchronously via
|
||||
* {@link InternalApiClient} loopback HTTP: the tool runs in its own handler and returns its file
|
||||
* inline. The caller decides how to run the executor itself (the AI turn loop calls it directly;
|
||||
* the engine runs it on a virtual thread for async runs). Files cross step boundaries as {@link
|
||||
* Resource} temp files; they are only persisted to durable storage at the run boundaries by the
|
||||
* caller.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PolicyExecutor {
|
||||
|
||||
private static final String FILTER_OPERATION_PREFIX = "/api/v1/filter/filter-";
|
||||
|
||||
private final InternalApiClient internalApiClient;
|
||||
private final ToolMetadataService toolMetadataService;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
/**
|
||||
* Internal value-class for tool responses. {@code files} holds any result files (typically one;
|
||||
* multiple for ZIP-response tools). {@code report} holds an optional structured metadata
|
||||
* payload the tool chose to surface alongside (or instead of) a file.
|
||||
*/
|
||||
private record ToolResult(List<Resource> files, JsonNode report) {}
|
||||
|
||||
/**
|
||||
* Execute every step in {@code definition} in order, feeding each step's output into the next.
|
||||
* Supporting files supplied in {@code inputs} are bound to steps' named file fields and never
|
||||
* enter the document stream.
|
||||
*
|
||||
* @param definition the pipeline to run (must have at least one step)
|
||||
* @param inputs the primary documents plus the named supporting-file store
|
||||
* @param listener receives per-step progress
|
||||
* @return the final output files plus the last structured report produced, if any
|
||||
* @throws InternalApiTimeoutException if a tool does not respond within its read timeout
|
||||
* @throws IOException if a tool returns a non-OK response, references a missing supporting
|
||||
* file, or a file cannot be read
|
||||
*/
|
||||
public PolicyExecutionResult execute(
|
||||
PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener)
|
||||
throws IOException {
|
||||
List<PipelineStep> steps = definition.steps();
|
||||
if (steps.isEmpty()) {
|
||||
throw new IllegalArgumentException("Pipeline definition has no steps");
|
||||
}
|
||||
|
||||
List<Resource> currentFiles = inputs.primary();
|
||||
Map<String, List<Resource>> supportingFiles = inputs.supportingFiles();
|
||||
// Propagate the *last* non-null report; the terminal step defines the output.
|
||||
JsonNode lastReport = null;
|
||||
String lastReportTool = null;
|
||||
|
||||
for (int i = 0; i < steps.size(); i++) {
|
||||
PipelineStep step = steps.get(i);
|
||||
String operation = step.operation();
|
||||
if (operation == null || operation.isBlank()) {
|
||||
throw new IllegalArgumentException(
|
||||
"Pipeline step " + (i + 1) + " has no operation");
|
||||
}
|
||||
listener.onStepStart(i + 1, steps.size(), operation);
|
||||
ToolResult stepResult = executeStep(step, currentFiles, supportingFiles);
|
||||
currentFiles = stepResult.files();
|
||||
if (stepResult.report() != null) {
|
||||
lastReport = stepResult.report();
|
||||
lastReportTool = operation;
|
||||
}
|
||||
listener.onStepComplete(i + 1, steps.size(), operation);
|
||||
}
|
||||
|
||||
return new PolicyExecutionResult(currentFiles, lastReport, lastReportTool);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single tool step. If the endpoint accepts multiple files, all files are sent in one
|
||||
* call. Otherwise, the endpoint is called once per file. ZIP responses are unpacked so each
|
||||
* inner file is treated as its own result (e.g. split outputs a ZIP of pages).
|
||||
*
|
||||
* <p>A structured {@code report} may be returned alongside (or instead of) files; see {@link
|
||||
* ToolResult}. For per-file dispatch (single-input endpoints called once per input), the first
|
||||
* non-null report wins.
|
||||
*/
|
||||
private ToolResult executeStep(
|
||||
PipelineStep step,
|
||||
List<Resource> inputFiles,
|
||||
Map<String, List<Resource>> supportingFiles)
|
||||
throws IOException {
|
||||
requireAcceptedTypes(step.operation(), inputFiles);
|
||||
List<Resource> files = new ArrayList<>();
|
||||
JsonNode report = null;
|
||||
if (toolMetadataService.isMultiInput(step.operation())) {
|
||||
ToolResult r = callEndpoint(step, inputFiles, supportingFiles);
|
||||
files.addAll(r.files());
|
||||
report = r.report();
|
||||
} else {
|
||||
for (Resource file : inputFiles) {
|
||||
ToolResult r = callEndpoint(step, List.of(file), supportingFiles);
|
||||
files.addAll(r.files());
|
||||
if (report == null) {
|
||||
report = r.report();
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ToolResult(files, report);
|
||||
}
|
||||
|
||||
/**
|
||||
* Call an endpoint and return its result files and optional report.
|
||||
*
|
||||
* <ul>
|
||||
* <li>JSON body (Content-Type: application/json): the entire body is the report, no files are
|
||||
* returned.
|
||||
* <li>File body (PDF etc.): the file is returned; if an {@link
|
||||
* AiToolResponseHeaders#TOOL_REPORT} header is present, its (minified JSON) value is
|
||||
* parsed as the report.
|
||||
* <li>ZIP responses declared by the tool metadata service are unpacked so callers always see
|
||||
* a flat list of result files.
|
||||
* </ul>
|
||||
*/
|
||||
private ToolResult callEndpoint(
|
||||
PipelineStep step, List<Resource> files, Map<String, List<Resource>> supportingFiles)
|
||||
throws IOException {
|
||||
String endpointPath = step.operation();
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
for (Resource file : files) {
|
||||
body.add("fileInput", file);
|
||||
}
|
||||
// Bind supporting files to their named tool fields (e.g. stampImage, overlayFiles). These
|
||||
// come from the run's named asset store, not the document stream.
|
||||
for (Map.Entry<String, String> binding : step.fileParameters().entrySet()) {
|
||||
String fieldName = binding.getKey();
|
||||
String assetKey = binding.getValue();
|
||||
List<Resource> assets = supportingFiles.get(assetKey);
|
||||
if (assets == null || assets.isEmpty()) {
|
||||
throw new IOException(
|
||||
"Step "
|
||||
+ endpointPath
|
||||
+ " references supporting file '"
|
||||
+ assetKey
|
||||
+ "' for field '"
|
||||
+ fieldName
|
||||
+ "' but no such file was provided");
|
||||
}
|
||||
for (Resource asset : assets) {
|
||||
body.add(fieldName, asset);
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, Object> entry : step.parameters().entrySet()) {
|
||||
if (entry.getValue() instanceof List<?> list) {
|
||||
if (containsStructuredElements(list)) {
|
||||
// Endpoints binding lists of structured objects (e.g. /security/redact's
|
||||
// redactions, /general/edit-text's edits) parse a single JSON string field via
|
||||
// a property editor. Pre-serialize the whole list so binding succeeds.
|
||||
body.add(entry.getKey(), objectMapper.writeValueAsString(list));
|
||||
} else {
|
||||
for (Object item : list) {
|
||||
body.add(entry.getKey(), item);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
body.add(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
ResponseEntity<Resource> response = internalApiClient.post(endpointPath, body);
|
||||
if (!HttpStatus.OK.equals(response.getStatusCode()) || response.getBody() == null) {
|
||||
throw new IOException(
|
||||
"Tool returned HTTP " + response.getStatusCode() + " for " + endpointPath);
|
||||
}
|
||||
Resource resource = response.getBody();
|
||||
|
||||
// Filter operations return an empty body to signal the file was filtered out: drop it
|
||||
// rather than forwarding a zero-byte document.
|
||||
if (isFilterOperation(endpointPath) && isEmpty(resource)) {
|
||||
return new ToolResult(List.of(), null);
|
||||
}
|
||||
|
||||
HttpHeaders headers = response.getHeaders();
|
||||
MediaType contentType = headers.getContentType();
|
||||
|
||||
// JSON-only response: the whole body is the structured report, no result file.
|
||||
if (contentType != null && MediaType.APPLICATION_JSON.isCompatibleWith(contentType)) {
|
||||
try (InputStream is = resource.getInputStream()) {
|
||||
JsonNode report = objectMapper.readTree(is);
|
||||
return new ToolResult(List.of(), report);
|
||||
}
|
||||
}
|
||||
|
||||
JsonNode report = parseReportHeader(headers, endpointPath);
|
||||
if (toolMetadataService.shouldUnpackZipResponse(endpointPath)) {
|
||||
return new ToolResult(ZipExtractionUtils.extractZip(resource, tempFileManager), report);
|
||||
}
|
||||
return new ToolResult(List.of(resource), report);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the optional {@link AiToolResponseHeaders#TOOL_REPORT} header into a {@link JsonNode},
|
||||
* or return null.
|
||||
*/
|
||||
private JsonNode parseReportHeader(HttpHeaders headers, String endpointPath) {
|
||||
String raw = headers.getFirst(AiToolResponseHeaders.TOOL_REPORT);
|
||||
if (raw == null || raw.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return objectMapper.readTree(raw);
|
||||
} catch (JacksonException e) {
|
||||
log.warn(
|
||||
"Ignoring malformed {} header from {}: {}",
|
||||
AiToolResponseHeaders.TOOL_REPORT,
|
||||
endpointPath,
|
||||
e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean containsStructuredElements(List<?> list) {
|
||||
for (Object item : list) {
|
||||
if (item instanceof Map<?, ?> || item instanceof List<?>) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail the run if any document in the primary stream is not a file type the step accepts. An
|
||||
* endpoint that declares no specific input type accepts anything.
|
||||
*/
|
||||
private void requireAcceptedTypes(String operation, List<Resource> files) throws IOException {
|
||||
List<String> accepted = toolMetadataService.getExtensionTypes(false, operation);
|
||||
if (accepted == null || accepted.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (Resource file : files) {
|
||||
if (!matchesType(file, accepted)) {
|
||||
throw new IOException(
|
||||
"Step "
|
||||
+ operation
|
||||
+ " accepts "
|
||||
+ accepted
|
||||
+ " but received '"
|
||||
+ file.getFilename()
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean matchesType(Resource file, List<String> acceptedExtensions) {
|
||||
String filename = file.getFilename();
|
||||
if (filename == null) {
|
||||
return false;
|
||||
}
|
||||
int dot = filename.lastIndexOf('.');
|
||||
if (dot < 0 || dot == filename.length() - 1) {
|
||||
return false;
|
||||
}
|
||||
return acceptedExtensions.contains(filename.substring(dot + 1).toLowerCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
private static boolean isFilterOperation(String operation) {
|
||||
return operation.startsWith(FILTER_OPERATION_PREFIX);
|
||||
}
|
||||
|
||||
private static boolean isEmpty(Resource resource) {
|
||||
try {
|
||||
return resource.contentLength() == 0;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* Thrown by a step to signal that the run cannot proceed without further user input, pausing the
|
||||
* run in {@code WAITING_FOR_INPUT} rather than failing it.
|
||||
*
|
||||
* <p>Carries everything needed to resume: a human-readable reason, the 0-based index of the step to
|
||||
* resume from, and the intermediate files produced so far. The engine persists those files and
|
||||
* suspends the run.
|
||||
*
|
||||
* <p>Defined now to fix the run shape; no step throws it yet, and the resume handshake is
|
||||
* implemented in a later stage.
|
||||
*/
|
||||
@Getter
|
||||
public class PolicyInputRequiredException extends RuntimeException {
|
||||
|
||||
private final transient List<Resource> pendingFiles;
|
||||
private final int resumeStepIndex;
|
||||
|
||||
public PolicyInputRequiredException(
|
||||
String reason, int resumeStepIndex, List<Resource> pendingFiles) {
|
||||
super(reason);
|
||||
this.resumeStepIndex = resumeStepIndex;
|
||||
this.pendingFiles = pendingFiles == null ? List.of() : pendingFiles;
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
|
||||
/**
|
||||
* Returned by {@link PolicyEngine#submit}: the run id (for status polling and result download) plus
|
||||
* a future that resolves when the run reaches a terminal or paused state.
|
||||
*
|
||||
* <p>The completion future lets callers react to the end of a run (e.g. an SSE endpoint sending a
|
||||
* final event and closing the stream) without polling. It carries the {@link PolicyRun} whose
|
||||
* status describes the outcome (completed, failed, cancelled, or waiting for input); it does not
|
||||
* complete exceptionally for ordinary run failures.
|
||||
*/
|
||||
public record PolicyRunHandle(String runId, CompletableFuture<PolicyRun> completion) {}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
|
||||
/**
|
||||
* In-memory store of live {@link PolicyRun} state, keyed by runId. Holds the authoritative run
|
||||
* state machine; durable status/files for download are projected separately into {@code
|
||||
* TaskManager}.
|
||||
*
|
||||
* <p>Finished runs are evicted on a fixed interval once they age past {@code
|
||||
* policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a run's
|
||||
* rich in-memory state does not outlive the process. Only terminal runs are evicted; active and
|
||||
* paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not
|
||||
* touched here: a run shares its runId with a {@code TaskManager} job, which owns file-lifecycle
|
||||
* cleanup, so eviction only frees this map's entry.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class PolicyRunRegistry {
|
||||
|
||||
private final Map<String, PolicyRun> runs = new ConcurrentHashMap<>();
|
||||
|
||||
private final Duration runExpiry;
|
||||
private final ScheduledExecutorService cleanupExecutor =
|
||||
Executors.newSingleThreadScheduledExecutor(
|
||||
Thread.ofVirtual().name("policy-run-cleanup-", 0).factory());
|
||||
|
||||
public PolicyRunRegistry(ApplicationProperties applicationProperties) {
|
||||
int runExpiryMinutes = applicationProperties.getPolicies().getRunExpiryMinutes();
|
||||
this.runExpiry = Duration.ofMinutes(runExpiryMinutes);
|
||||
cleanupExecutor.scheduleAtFixedRate(this::evictExpiredRuns, 10, 10, TimeUnit.MINUTES);
|
||||
log.debug(
|
||||
"Policy run registry initialized with run expiry of {} minutes", runExpiryMinutes);
|
||||
}
|
||||
|
||||
public void register(PolicyRun run) {
|
||||
runs.put(run.getRunId(), run);
|
||||
}
|
||||
|
||||
public PolicyRun get(String runId) {
|
||||
return runs.get(runId);
|
||||
}
|
||||
|
||||
public Collection<PolicyRun> all() {
|
||||
return runs.values();
|
||||
}
|
||||
|
||||
/** Scheduled hook: evict terminal runs that finished before the expiry window. */
|
||||
private void evictExpiredRuns() {
|
||||
try {
|
||||
evictExpired(Instant.now().minus(runExpiry));
|
||||
} catch (Exception e) {
|
||||
log.error("Error during policy run cleanup: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove every terminal run last updated before {@code cutoff}; active and paused runs are kept
|
||||
* regardless of age. Returns the number evicted. Package-visible so the scheduled sweep and
|
||||
* tests exercise the same path with an explicit cutoff.
|
||||
*/
|
||||
int evictExpired(Instant cutoff) {
|
||||
int removed = 0;
|
||||
for (Map.Entry<String, PolicyRun> entry : runs.entrySet()) {
|
||||
PolicyRun run = entry.getValue();
|
||||
if (run.getStatus().isTerminal() && run.getUpdatedAt().isBefore(cutoff)) {
|
||||
runs.remove(entry.getKey());
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
if (removed > 0) {
|
||||
log.info("Evicted {} expired policy runs", removed);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
void shutdown() {
|
||||
cleanupExecutor.shutdownNow();
|
||||
}
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.input.ResolvedInput;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
import stirling.software.proprietary.policy.model.PolicyRunStatus;
|
||||
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
|
||||
|
||||
/**
|
||||
* Runs policies, and is the one place that knows how to turn a policy's configured {@link InputSpec
|
||||
* sources} into actual runs. Triggers (schedule, and future webhook/folder-watch) decide
|
||||
* <em>when</em> to run and call {@link #run(Policy)}; they never touch sources themselves. The
|
||||
* controller uses the supplied-input and ad-hoc entry points for on-demand work.
|
||||
*
|
||||
* <p>This is the seam that keeps triggers and sources independent: a trigger depends on the runner,
|
||||
* the runner depends on the {@link InputSource} beans, and a source depends on neither - it just
|
||||
* yields {@link ResolvedInput units of work}, each carrying its own completion hook.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PolicyRunner {
|
||||
|
||||
private final PolicyEngine policyEngine;
|
||||
private final List<InputSource> inputSources;
|
||||
|
||||
/**
|
||||
* Run a policy by pulling from every source it configures: each source yields zero or more
|
||||
* units of work, and each unit becomes its own run so one failure does not affect the others. A
|
||||
* policy with no sources runs once with no input files (a generator pipeline). Used by
|
||||
* automatic triggers.
|
||||
*/
|
||||
public void run(Policy policy) {
|
||||
List<InputSpec> sources = policy.sources();
|
||||
if (sources.isEmpty()) {
|
||||
startRun(policy, PolicyInputs.of(List.of()), unused -> {});
|
||||
return;
|
||||
}
|
||||
for (InputSpec spec : sources) {
|
||||
pullAndRun(policy, spec);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a stored policy on files supplied directly by the caller (e.g. a manual run with
|
||||
* uploads), bypassing its configured sources. Returns the run handle so callers can stream
|
||||
* progress.
|
||||
*/
|
||||
public PolicyRunHandle runWith(
|
||||
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
|
||||
return policyEngine.runPolicy(policy, inputs, listener);
|
||||
}
|
||||
|
||||
/** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */
|
||||
public PolicyRunHandle runAdHoc(
|
||||
PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) {
|
||||
return policyEngine.submit(definition, inputs, listener);
|
||||
}
|
||||
|
||||
private void pullAndRun(Policy policy, InputSpec spec) {
|
||||
InputSource source = sourceFor(spec);
|
||||
if (source == null) {
|
||||
log.warn(
|
||||
"No input source for type '{}' (policy {}); skipping",
|
||||
spec.type(),
|
||||
policy.id());
|
||||
return;
|
||||
}
|
||||
List<ResolvedInput> work;
|
||||
try {
|
||||
work = source.resolve(spec);
|
||||
} catch (IOException | RuntimeException e) {
|
||||
log.warn(
|
||||
"Failed to resolve source '{}' for policy {}: {}",
|
||||
spec.type(),
|
||||
policy.id(),
|
||||
e.getMessage());
|
||||
return;
|
||||
}
|
||||
for (ResolvedInput unit : work) {
|
||||
startRun(policy, unit.inputs(), unit.onComplete());
|
||||
}
|
||||
}
|
||||
|
||||
private void startRun(Policy policy, PolicyInputs inputs, Consumer<Boolean> onComplete) {
|
||||
log.info("Running policy {} ({})", policy.id(), policy.name());
|
||||
PolicyRunHandle handle =
|
||||
policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP);
|
||||
handle.completion()
|
||||
.whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable)));
|
||||
}
|
||||
|
||||
private static boolean succeeded(PolicyRun run, Throwable throwable) {
|
||||
return throwable == null && run != null && run.getStatus() == PolicyRunStatus.COMPLETED;
|
||||
}
|
||||
|
||||
private InputSource sourceFor(InputSpec spec) {
|
||||
return inputSources.stream()
|
||||
.filter(source -> source.supports(spec))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.TriggerConfig;
|
||||
import stirling.software.proprietary.policy.output.PolicyOutputSink;
|
||||
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
|
||||
|
||||
/**
|
||||
* Validates a policy's trigger, sources, and output configuration by delegating each facet to the
|
||||
* bean that handles its type. Called when a policy is saved so a misconfigured schedule, missing
|
||||
* folder directory, or unknown type fails fast instead of silently misbehaving at run time.
|
||||
*
|
||||
* <p>The trigger is optional (a {@code null} trigger is a manual-only policy and needs no
|
||||
* validation); every configured source is validated.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PolicyValidator {
|
||||
|
||||
private final List<PolicyTrigger> triggers;
|
||||
private final List<InputSource> inputSources;
|
||||
private final List<PolicyOutputSink> outputSinks;
|
||||
|
||||
/**
|
||||
* @throws IllegalArgumentException if any facet's type is unknown or its configuration is
|
||||
* invalid
|
||||
*/
|
||||
public void validate(Policy policy) {
|
||||
if (policy.trigger() != null) {
|
||||
triggerFor(policy.trigger()).validate(policy);
|
||||
}
|
||||
for (InputSpec source : policy.sources()) {
|
||||
inputSourceFor(source).validate(source);
|
||||
}
|
||||
outputSinkFor(policy.output()).validate(policy.output());
|
||||
}
|
||||
|
||||
private PolicyTrigger triggerFor(TriggerConfig config) {
|
||||
return triggers.stream()
|
||||
.filter(trigger -> trigger.type().equals(config.type()))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalArgumentException(
|
||||
"unknown trigger type: " + config.type()));
|
||||
}
|
||||
|
||||
private InputSource inputSourceFor(InputSpec spec) {
|
||||
return inputSources.stream()
|
||||
.filter(source -> source.supports(spec))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalArgumentException(
|
||||
"unknown input source type: " + spec.type()));
|
||||
}
|
||||
|
||||
private PolicyOutputSink outputSinkFor(OutputSpec spec) {
|
||||
return outputSinks.stream()
|
||||
.filter(sink -> sink.supports(spec))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException("unknown output type: " + spec.type()));
|
||||
}
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.util.FileReadinessChecker;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
|
||||
/**
|
||||
* Reads input files from a directory. Each ready file becomes its own unit of work (one run per
|
||||
* file) so a failure on one file does not affect the others.
|
||||
*
|
||||
* <p>Two modes via the {@code mode} option:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code "consume"} (default) - claim each file by moving it into {@code
|
||||
* .stirling/processing}, then route it to {@code .stirling/done} or {@code .stirling/error}
|
||||
* when its run finishes. Each file is processed once; right for "process new arrivals" (and
|
||||
* the basis of watched folders).
|
||||
* <li>{@code "snapshot"} - read the directory's current files without moving them; every run sees
|
||||
* the full set again. Right for "always regenerate from a fixed input set".
|
||||
* </ul>
|
||||
*
|
||||
* Readiness is checked first (via {@link FileReadinessChecker}) so files mid-write are skipped.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class FolderInputSource implements InputSource {
|
||||
|
||||
private static final String TYPE = FolderAccessGuard.FOLDER_TYPE;
|
||||
// Bookkeeping lives under one hidden namespace dir so the watched folder stays tidy.
|
||||
private static final String WORK_SUBDIR = ".stirling";
|
||||
private static final String PROCESSING_SUBDIR = "processing";
|
||||
private static final String DONE_SUBDIR = "done";
|
||||
private static final String ERROR_SUBDIR = "error";
|
||||
|
||||
private final FileReadinessChecker readinessChecker;
|
||||
private final FolderAccessGuard accessGuard;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supports(InputSpec spec) {
|
||||
return spec != null && TYPE.equals(spec.type());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(InputSpec spec) {
|
||||
accessGuard.requirePermitted(FolderConfig.from(spec.options()).directory());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Path> watchTargets(InputSpec spec) {
|
||||
return List.of(FolderConfig.from(spec.options()).directory());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResolvedInput> resolve(InputSpec spec) throws IOException {
|
||||
FolderConfig config = FolderConfig.from(spec.options());
|
||||
Path inputDir = accessGuard.requirePermitted(config.directory());
|
||||
if (!Files.isDirectory(inputDir)) {
|
||||
log.debug("Folder input dir does not exist: {}", inputDir);
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<Path> ready = new ArrayList<>();
|
||||
try (Stream<Path> entries = Files.list(inputDir)) {
|
||||
entries.filter(Files::isRegularFile)
|
||||
.filter(readinessChecker::isReady)
|
||||
.forEach(ready::add);
|
||||
}
|
||||
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
for (Path file : ready) {
|
||||
if (config.snapshot()) {
|
||||
work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file)))));
|
||||
} else {
|
||||
Path claimed = claim(inputDir, file);
|
||||
if (claimed == null) {
|
||||
continue; // another sweep/process grabbed it
|
||||
}
|
||||
work.add(
|
||||
new ResolvedInput(
|
||||
PolicyInputs.of(List.of(fileResource(claimed))),
|
||||
success -> route(inputDir, claimed, success)));
|
||||
}
|
||||
}
|
||||
return work;
|
||||
}
|
||||
|
||||
private Path claim(Path inputDir, Path file) {
|
||||
try {
|
||||
Path processingDir = workDir(inputDir, PROCESSING_SUBDIR);
|
||||
Files.createDirectories(processingDir);
|
||||
Path claimed = uniqueTarget(processingDir, file.getFileName().toString());
|
||||
Files.move(file, claimed, StandardCopyOption.ATOMIC_MOVE);
|
||||
return claimed;
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not claim {}: {}", file, e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void route(Path inputDir, Path claimed, boolean success) {
|
||||
String subdir = success ? DONE_SUBDIR : ERROR_SUBDIR;
|
||||
try {
|
||||
Path destDir = workDir(inputDir, subdir);
|
||||
Files.createDirectories(destDir);
|
||||
Files.move(
|
||||
claimed,
|
||||
uniqueTarget(destDir, claimed.getFileName().toString()),
|
||||
StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (IOException e) {
|
||||
log.warn(
|
||||
"Could not move processed input {} to {}: {}", claimed, subdir, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/** A bookkeeping subdirectory under the watched folder's {@code .stirling} namespace. */
|
||||
private static Path workDir(Path inputDir, String subdir) {
|
||||
return inputDir.resolve(WORK_SUBDIR).resolve(subdir);
|
||||
}
|
||||
|
||||
private static Resource fileResource(Path path) {
|
||||
String name = path.getFileName().toString();
|
||||
return new FileSystemResource(path.toFile()) {
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return name;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static Path uniqueTarget(Path dir, String filename) {
|
||||
Path candidate = dir.resolve(filename);
|
||||
if (!Files.exists(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
int dot = filename.lastIndexOf('.');
|
||||
String base = dot < 0 ? filename : filename.substring(0, dot);
|
||||
String ext = dot < 0 ? "" : filename.substring(dot);
|
||||
for (int n = 1; ; n++) {
|
||||
Path next = dir.resolve(base + " (" + n + ")" + ext);
|
||||
if (!Files.exists(next)) {
|
||||
return next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The typed, validated form of a folder source's options: the directory and dedup mode. */
|
||||
record FolderConfig(Path directory, boolean snapshot) {
|
||||
|
||||
private static final String DIRECTORY_OPTION = "directory";
|
||||
private static final String MODE_OPTION = "mode";
|
||||
private static final String MODE_SNAPSHOT = "snapshot";
|
||||
|
||||
static FolderConfig from(Map<String, Object> options) {
|
||||
Object directory = options.get(DIRECTORY_OPTION);
|
||||
if (directory == null || directory.toString().isBlank()) {
|
||||
throw new IllegalArgumentException("folder input requires a 'directory' option");
|
||||
}
|
||||
Object mode = options.get(MODE_OPTION);
|
||||
boolean snapshot = mode != null && MODE_SNAPSHOT.equals(mode.toString());
|
||||
return new FolderConfig(Path.of(directory.toString()), snapshot);
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
|
||||
/**
|
||||
* Resolves one of a policy's {@link InputSpec sources} into the files to run on - answering
|
||||
* <em>where</em> a run's files come from, independent of <em>when</em> it runs. The counterpart of
|
||||
* {@code PolicyOutputSink}: implementations are beans selected by {@link #supports(InputSpec)}, so
|
||||
* a new source kind (folder, S3) is just a new bean.
|
||||
*
|
||||
* <p>Driven by the {@code PolicyRunner}, which a trigger calls when a policy is due; a source is
|
||||
* passive and knows nothing about what triggered the run. A manual run may instead supply files
|
||||
* directly and bypass sources entirely.
|
||||
*/
|
||||
public interface InputSource {
|
||||
|
||||
/** Stable identifier for this source, matching {@code InputSpec.type()} (e.g. "folder"). */
|
||||
String type();
|
||||
|
||||
/** Whether this source can handle the given spec. */
|
||||
boolean supports(InputSpec spec);
|
||||
|
||||
/**
|
||||
* Check that an input spec is usable, throwing {@link IllegalArgumentException} if not. Called
|
||||
* when a policy is saved so misconfiguration fails fast rather than at run time.
|
||||
*/
|
||||
default void validate(InputSpec spec) {}
|
||||
|
||||
/**
|
||||
* Resolve the spec into zero or more units of work, each carrying the files for one run and a
|
||||
* completion hook. Returning an empty list means there is nothing to run right now.
|
||||
*/
|
||||
List<ResolvedInput> resolve(InputSpec spec) throws IOException;
|
||||
|
||||
/**
|
||||
* The local filesystem directories this source draws from, if any, for triggers that want to
|
||||
* react to changes there (the folder-watch trigger) rather than poll. Advisory only: it merely
|
||||
* tells a trigger <em>where</em> to watch; resolving the spec into files is still this source's
|
||||
* job via {@link #resolve}. Non-filesystem sources (S3, ...) return an empty list and so are
|
||||
* simply not watchable. Default: nothing to watch.
|
||||
*/
|
||||
default List<Path> watchTargets(InputSpec spec) {
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
|
||||
/**
|
||||
* One unit of work produced by an {@link InputSource}: the files to run plus a completion callback
|
||||
* invoked with the run's success once it finishes (e.g. a folder source routes the input to {@code
|
||||
* .stirling/done} or {@code .stirling/error}). A source may return several of these (e.g. one per
|
||||
* file).
|
||||
*/
|
||||
public record ResolvedInput(PolicyInputs inputs, Consumer<Boolean> onComplete) {
|
||||
|
||||
public ResolvedInput {
|
||||
onComplete = onComplete == null ? success -> {} : onComplete;
|
||||
}
|
||||
|
||||
/** A unit of work with no completion side effect. */
|
||||
public static ResolvedInput of(PolicyInputs inputs) {
|
||||
return new ResolvedInput(inputs, success -> {});
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user