Compare commits

..
360 changed files with 7666 additions and 29958 deletions
+9 -9
View File
@@ -1,20 +1,20 @@
###############################################################################
# Stirling-PDF SaaS environment defaults.
# Stirling-PDF SaaS local environment template.
#
# This file is committed and provides non-secret defaults loaded by
# `task backend:dev:saas`. Put real values for secrets (passwords, project
# refs, edge function secrets) in `.env.saas.local` - any variable set there
# takes precedence over what's defined here.
# Copy this file to `.env.saas.local` (gitignored) and fill in real values.
# Loaded by `task backend:dev:saas` via Taskfile's `dotenv:` directive, then
# read by Spring Boot's `${...}` placeholders in application-saas.properties
# and application-dev.properties.
#
# DO NOT commit `.env.saas.local`. Only `.env.saas` is checked in.
# DO NOT commit `.env.saas.local`. Only `.env.saas.example` is checked in.
###############################################################################
# ---------- Supabase project ----------
# Project reference (the subdomain part of <ref>.supabase.co). Required.
# Set in .env.saas.local.
# Example dev project:
SAAS_DB_PROJECT_REF=
# Edge function secret used by billing/license rollup calls. Set in .env.saas.local.
# Edge function secret used by billing/license rollup calls.
SUPABASE_EDGE_FUNCTION_SECRET=
# ---------- Database (saas profile) ----------
@@ -28,7 +28,7 @@ SAAS_DB_PASSWORD=
# ---------- Database (dev profile overrides) ----------
# Used when `--spring.profiles.include=dev` is active. The dev profile
# defaults the URL/username to the shared dev Supabase project, but the
# password must still be provided in .env.saas.local.
# password must still be provided here.
SAAS_DEV_DB_URL=
SAAS_DEV_DB_USERNAME=postgres
SAAS_DEV_DB_PASSWORD=
+1 -1
View File
@@ -24,7 +24,7 @@ runs:
id: generate-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
client-id: ${{ inputs.app-id }}
app-id: ${{ inputs.app-id }}
private-key: ${{ inputs.private-key }}
- name: Configure Git
run: |
-2
View File
@@ -38,8 +38,6 @@ project: &project
- frontend/**
- docker/**
- scripts/RestartHelper.java
- scripts/db-migration/**
- .github/workflows/db-migration-test.yml
frontend: &frontend
- frontend/**
+3 -4
View File
@@ -13,7 +13,7 @@ Usage:
"""
# Sample for Windows:
# python .github/scripts/check_language_toml.py --reference-file frontend/editor/public/locales/en-GB/translation.toml --branch "" --files frontend/editor/public/locales/de-DE/translation.toml frontend/editor/public/locales/fr-FR/translation.toml
# python .github/scripts/check_language_toml.py --reference-file frontend/public/locales/en-GB/translation.toml --branch "" --files frontend/public/locales/de-DE/translation.toml frontend/public/locales/fr-FR/translation.toml
import argparse
import glob
@@ -184,7 +184,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
if len(file_list) == 1:
file_arr = file_list[0].split()
base_dir = Path.cwd() / "frontend" / "editor" / "public" / "locales"
base_dir = Path.cwd() / "frontend" / "public" / "locales"
for file_path in file_arr:
file_path = Path(file_path)
@@ -308,7 +308,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
report.append("## ❌ Overall Check Status: **_Failed_**")
report.append("")
report.append(
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [en-GB/translation.toml](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/frontend/editor/public/locales/en-GB/translation.toml)"
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [en-GB/translation.toml](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/frontend/public/locales/en-GB/translation.toml)"
)
else:
report.append("## ✅ Overall Check Status: **_Success_**")
@@ -372,7 +372,6 @@ if __name__ == "__main__":
os.path.join(
os.getcwd(),
"frontend",
"editor",
"public",
"locales",
"*",
+1 -2
View File
@@ -287,7 +287,6 @@ jobs:
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/data:/usr/share/tessdata:rw
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/config:/configs:rw
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/logs:/logs:rw
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/storage:/storage:rw
environment:
DISABLE_ADDITIONAL_FEATURES: "false"
SECURITY_ENABLELOGIN: "true"
@@ -310,7 +309,7 @@ jobs:
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << ENDSSH
# Create V2 PR-specific directories
mkdir -p /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/{data,config,logs,storage}
mkdir -p /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/{data,config,logs}
# Move docker-compose file to correct location
mv /tmp/docker-compose-v2.yml /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/docker-compose.yml
+27 -124
View File
@@ -1,9 +1,8 @@
name: AI Engine CI
# Validates the Python AI engine: regenerates tool models and runs the
# engine quality gate (lint, type-check, format-check, tests). Called from
# build.yml on PRs and merge_group; also runs directly on push to main as
# a post-merge safety net.
# Validates the Python AI engine: regenerates tool models, runs fixers,
# lint, type-check, and tests. Called from build.yml on PRs and merge_group;
# also runs directly on push to main as a post-merge safety net.
on:
workflow_call:
push:
@@ -52,95 +51,27 @@ jobs:
run: task engine:tool-models
- name: Verify tool models are up to date
id: tool-models-check
continue-on-error: true
run: git diff --exit-code engine/src/stirling/models/tool_models.py
- name: Comment on tool models check failure
# Only post a comment on PRs. github-script's PR helpers need an
# issue/PR number, which doesn't exist on merge_group runs.
if: steps.tool-models-check.outcome == 'failure' && github.event_name == 'pull_request'
continue-on-error: true
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const marker = '<!-- tool-models-check -->';
const body = [
marker,
'### Tool Models Check Failed',
'',
'The generated `engine/src/stirling/models/tool_models.py` is out of date with the Java OpenAPI spec and will need to be regenerated before it can be merged in.',
'',
'Run `task engine:tool-models` to regenerate, then commit the updated file.',
].join('\n');
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body,
});
}
- name: Fail if tool models check failed
if: steps.tool-models-check.outcome == 'failure'
run: |
echo "============================================"
echo " Tool Models Check Failed"
echo "============================================"
echo ""
echo "The generated engine/src/stirling/models/tool_models.py"
echo "is out of date with the Java OpenAPI spec and will"
echo "need to be regenerated before it can be merged in."
echo ""
echo "Run 'task engine:tool-models' to regenerate, then"
echo "commit the updated file."
echo "============================================"
exit 1
if ! git diff --exit-code engine/src/stirling/models/tool_models.py; then
echo "tool_models.py is out of date."
echo "Run 'task engine:tool-models' locally and commit the updated file."
exit 1
fi
- name: Remove tool models check comment on success
if: steps.tool-models-check.outcome == 'success' && github.event_name == 'pull_request'
continue-on-error: true
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const marker = '<!-- tool-models-check -->';
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
});
}
- name: Run fixers
run: task engine:fix
- name: Quality-check engine
id: engine-check
run: task engine:check
continue-on-error: true
- name: Verify fixes are committed
id: fixer_changes
run: |
if ! git diff --quiet; then
git --no-pager diff --stat
echo "::error::There are issues with your Python code that will need to be fixed before they can be merged in. Run 'task engine:fix' to auto-fix what can be fixed automatically, then run 'task engine:check' to see what still needs fixing manually."
exit 1
fi
- name: Comment on engine check failure
# Only post a comment on PRs. github-script's PR helpers need an
# issue/PR number, which doesn't exist on merge_group runs.
if: steps.engine-check.outcome == 'failure' && github.event_name == 'pull_request'
- name: Comment on fixer failures
if: steps.fixer_changes.outcome == 'failure' && github.event_name == 'pull_request'
continue-on-error: true
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
@@ -176,39 +107,11 @@ jobs:
});
}
- name: Fail if engine check failed
if: steps.engine-check.outcome == 'failure'
run: |
echo "============================================"
echo " Engine Check Failed"
echo "============================================"
echo ""
echo "There are issues with your Python code that"
echo "will need to be fixed before they can be merged in."
echo ""
echo "Run 'task engine:fix' to auto-fix what can be"
echo "fixed automatically, then run 'task engine:check'"
echo "to see what still needs fixing manually."
echo "============================================"
exit 1
- name: Run linting
run: task engine:lint
- name: Remove engine check comment on success
if: steps.engine-check.outcome == 'success' && github.event_name == 'pull_request'
continue-on-error: true
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const marker = '<!-- engine-check -->';
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
});
}
- name: Run type checking
run: task engine:typecheck
- name: Run tests
run: task engine:test
+15 -32
View File
@@ -67,7 +67,7 @@ jobs:
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
- name: Comment on backend format check failure
- name: Comment on Java formatting failure
# Only post a comment on PRs. github-script's PR helpers need an
# issue/PR number, which doesn't exist on merge_group runs.
if: steps.spotless-check.outcome == 'failure' && github.event_name == 'pull_request'
@@ -78,11 +78,15 @@ jobs:
const marker = '<!-- java-formatting-check -->';
const body = [
marker,
'### Backend Format Check Failed',
'### Java Formatting Check Failed',
'',
'There are formatting issues in your Java code that will need to be fixed before they can be merged in.',
'Your code has formatting issues. Run the following command to fix them:',
'',
'Run `task backend:format` to auto-fix, then commit and push the changes.',
'```bash',
'task backend:format',
'```',
'',
'Then commit and push the changes.',
].join('\n');
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
@@ -106,43 +110,22 @@ jobs:
});
}
- name: Fail if backend format check failed
- name: Fail if Java formatting issues found
if: steps.spotless-check.outcome == 'failure'
run: |
echo "============================================"
echo " Backend Format Check Failed"
echo " Java Formatting Check Failed"
echo "============================================"
echo ""
echo "There are formatting issues in your Java code"
echo "that will need to be fixed before they can be"
echo "merged in."
echo "Your code has formatting issues."
echo "Run the following command to fix them:"
echo ""
echo "Run 'task backend:format' to auto-fix, then"
echo "commit and push the changes."
echo " task backend:format"
echo ""
echo "Then commit and push the changes."
echo "============================================"
exit 1
- name: Remove backend format check comment on success
if: steps.spotless-check.outcome == 'success' && github.event_name == 'pull_request'
continue-on-error: true
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const marker = '<!-- java-formatting-check -->';
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
});
}
- name: Build with Gradle and spring security ${{ matrix.spring-security }}
run: task backend:build:ci
env:
-14
View File
@@ -68,18 +68,6 @@ jobs:
uses: ./.github/workflows/backend-build.yml
secrets: inherit
db-migration-test:
# Boots the current bootJar against H2 fixtures captured from past
# releases (v2.0.0 / v2.5.0 / v2.10.0) and verifies admin login still
# works after Hibernate's ddl-auto=update migrates the schema. Gated on
# the `project` filter so doc-only PRs skip this ~5-minute job.
if: needs.files-changed.outputs.project == 'true'
needs: [files-changed]
permissions:
contents: read
uses: ./.github/workflows/db-migration-test.yml
secrets: inherit
check-generateOpenApiDocs:
if: needs.files-changed.outputs.openapi == 'true'
needs: [files-changed]
@@ -196,7 +184,6 @@ jobs:
needs:
- files-changed
- build
- db-migration-test
- check-generateOpenApiDocs
- frontend-validation
- playwright-e2e
@@ -221,7 +208,6 @@ jobs:
RESULTS: |
files-changed=${{ needs.files-changed.result }}
build=${{ needs.build.result }}
db-migration-test=${{ needs.db-migration-test.result }}
check-generateOpenApiDocs=${{ needs.check-generateOpenApiDocs.result }}
frontend-validation=${{ needs.frontend-validation.result }}
playwright-e2e=${{ needs.playwright-e2e.result }}
-93
View File
@@ -1,93 +0,0 @@
name: DB migration smoke test
# Boots the current Stirling-PDF JAR against H2 fixtures captured from past
# releases (v2.0.0 / v2.5.0 / v2.10.0) and verifies admin login still works.
# Catches schema changes that would break existing user databases under
# Hibernate's `ddl-auto=update` upgrade path.
on:
workflow_call:
permissions:
contents: read
jobs:
pick:
uses: ./.github/workflows/_runner-pick.yml
migration-test:
needs: pick
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-8' }}
timeout-minutes: 30
env:
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up JDK 25
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
with:
java-version: 25
distribution: temurin
- name: Cache Gradle dependency artifacts
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
~/.gradle/caches/modules-2/files-2.1
~/.gradle/caches/modules-2/metadata-2.*
key: gradle-deps-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }}
- name: Setup Gradle
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
with:
gradle-version: 9.3.1
cache-disabled: true
# No `-PnoSpotless` here yet because the upstream cache layer matches the
# backend build's; reuse keeps cold-cache cost identical.
- name: Build Stirling-PDF JAR
env:
MAVEN_USER: ${{ secrets.MAVEN_USER }}
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
run: ./gradlew :stirling-pdf:bootJar -PnoSpotless --no-daemon
- name: Locate built JAR
id: jar
run: |
jar=$(find app/core/build/libs -maxdepth 1 -name 'Stirling-PDF*.jar' -o -name 'stirling-pdf*.jar' 2>/dev/null \
| grep -vE '(-plain|-sources)\.jar$' | head -n 1)
if [[ -z "$jar" ]]; then
echo "::error::No JAR under app/core/build/libs"
ls -lah app/core/build/libs || true
exit 1
fi
# Absolute path - the migration script pushd's into a temp workdir
# before invoking java, which would dangle a relative path.
jar=$(realpath "$jar")
echo "path=$jar" >> "$GITHUB_OUTPUT"
echo "Built JAR: $jar"
- name: Run migration smoke test
env:
STIRLING_JAR: ${{ steps.jar.outputs.path }}
run: bash scripts/db-migration/run-migration-test.sh
- name: Upload app logs on failure
if: failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: db-migration-app-logs
# Path matches the preserved workdir in run-migration-test.sh -
# only failing fixtures leave a directory behind.
path: /tmp/stirling-migration-failed-*/app.log
retention-days: 7
if-no-files-found: warn
+7 -9
View File
@@ -586,23 +586,21 @@ jobs:
if: always() && steps.digicert-setup.conclusion != 'failure'
shell: bash
run: |
# Absolute dist path so the cd below can't break the copy targets.
DIST="$GITHUB_WORKSPACE/dist"
mkdir -p "$DIST"
mkdir -p ./dist
cd ./frontend/editor/src-tauri/target
# Find and rename artifacts based on platform
if [ "${{ matrix.platform }}" = "windows-latest" ]; then
# Only ship the MSI installer on Windows. The loose exe and WiX toolset exes
# are not the user-facing installer - the MSI contains the signed inner exe.
find . -name "*.msi" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.msi" \;
find . -name "*.msi" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.msi" \;
elif [ "${{ matrix.platform }}" = "macos-15" ]; then
find . -name "*.dmg" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.dmg" \;
find . -name "*.app" -exec cp -r {} "$DIST/Stirling-PDF-${{ matrix.name }}.app" \;
find . -name "*.dmg" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.dmg" \;
find . -name "*.app" -exec cp -r {} "../../../dist/Stirling-PDF-${{ matrix.name }}.app" \;
else
find . -name "*.deb" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.deb" \;
find . -name "*.rpm" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.rpm" \;
find . -name "*.AppImage" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.AppImage" \;
find . -name "*.deb" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.deb" \;
find . -name "*.rpm" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.rpm" \;
find . -name "*.AppImage" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.AppImage" \;
fi
- name: Upload build artifacts
+6 -11
View File
@@ -157,9 +157,6 @@ jobs:
JPDFIUM_PLATFORMS: ${{ matrix.jpdfium_platforms }}
run: task desktop:prepare
- name: Run Tauri/Cargo tests
run: task desktop:test
# DigiCert KeyLocker Setup (Cloud HSM)
- name: Setup DigiCert KeyLocker
id: digicert-setup
@@ -420,22 +417,20 @@ jobs:
- name: Rename artifacts
shell: bash
run: |
# Absolute dist path so the cd below can't break the copy targets.
DIST="$GITHUB_WORKSPACE/dist"
mkdir -p "$DIST"
mkdir -p ./dist
cd ./frontend/editor/src-tauri/target
# Find and rename artifacts based on platform
if [ "${{ matrix.platform }}" = "windows-latest" ]; then
# Only ship the MSI installer. The loose exe and WiX toolset exes
# are not the user-facing installer - the MSI contains the signed inner exe.
find . -name "*.msi" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.msi" \;
find . -name "*.msi" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.msi" \;
elif [ "${{ matrix.platform }}" = "macos-15" ]; then
find . -name "*.dmg" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.dmg" \;
find . -name "*.dmg" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.dmg" \;
else
find . -name "*.deb" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.deb" \;
find . -name "*.rpm" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.rpm" \;
find . -name "*.AppImage" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.AppImage" \;
find . -name "*.deb" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.deb" \;
find . -name "*.rpm" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.rpm" \;
find . -name "*.AppImage" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.AppImage" \;
fi
# Verify the MSI AND the inner exe extracted from it are signed.
+1 -7
View File
@@ -23,10 +23,6 @@ customFiles/
configs/
watchedFolders/
clientWebUI/
# Scratch dir used by local fixture-regeneration runs (see
# app/proprietary/src/test/resources/db-migration-fixtures/README.md).
# Holds downloaded JARs and disposable workdirs. Never committed.
.alpha-local/
!cucumber/
!cucumber/exampleFiles/
!cucumber/exampleFiles/example_html.zip
@@ -178,6 +174,7 @@ venv.bak/
# Env files (secrets / local overrides). Subproject .gitignore files whitelist any committed defaults.
.env*
!.env.saas.example
# VS Code
/.vscode/**/*
@@ -277,6 +274,3 @@ docs/type3/signatures/
# Playwright MCP screenshots / traces
.playwright-mcp/
*.playwright-mcp.png
# Local screenshot artifacts from *-screenshots.spec.ts
frontend/screenshots/
+5 -6
View File
@@ -22,13 +22,12 @@ tasks:
vars:
PORT: '{{.PORT | default "8080"}}'
AIENGINE_URL: '{{.AIENGINE_URL | default ""}}'
AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS | default "120"}}'
env:
SERVER_PORT: '{{.PORT}}'
cmds:
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}cmd /c ".\gradlew.bat :stirling-pdf:bootRun"'
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}cmd /c ".\gradlew.bat :stirling-pdf:bootRun"'
platforms: [windows]
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}./gradlew :stirling-pdf:bootRun'
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}./gradlew :stirling-pdf:bootRun'
platforms: [linux, darwin]
dev:bundled:
@@ -41,10 +40,10 @@ tasks:
platforms: [linux, darwin]
dev:saas:
desc: "Start backend in SaaS flavor against Supabase"
desc: "Start backend in SaaS flavor against Supabase (loads .env.saas.local)"
# `dotenv:` reads from the root Taskfile's directory (".") because this
# subtaskfile is included with `dir: .`.
dotenv: ['app/.env.saas.local', 'app/.env.saas']
# subtaskfile is included with `dir: .`. Drop the file at the repo root.
dotenv: ['.env.saas.local']
ignore_error: true
vars:
PORT: '{{.PORT | default "8080"}}'
+1 -29
View File
@@ -78,13 +78,6 @@ tasks:
cmds:
- npx tauri build --bundles appimage
test:
desc: "Run Tauri/Cargo tests"
deps: [prepare]
dir: editor/src-tauri
cmds:
- cargo test
clean:
desc: "Clean Tauri/Cargo build artifacts"
dir: editor
@@ -133,29 +126,8 @@ tasks:
--no-header-files
--no-man-pages
--output runtime/jre
# jlink emits its files mode 444 (read-only). Tauri's build-script
# resource copier preserves source permissions when staging
# `runtime/jre/**/*` into `target/<profile>/runtime/jre/...`, so the
# staged copies are read-only too. On any subsequent incremental
# build the copier tries to overwrite them and fails with a bare
# `Permission denied (os error 13)` (Rust's io::Error Display drops
# the path, so the failure is opaque). Make the source writable here
# so the staged destinations are writable and can be overwritten.
#
# Trade-off: this task runs for both `task desktop:dev` and
# `task desktop:build`, so production bundles also ship mode-644
# JRE files instead of 444. Functionally harmless on POSIX (the
# `other` bit is `r--` either way, and on macOS code signing is the
# real integrity check) and on Windows the DOS read-only attribute
# isn't load-bearing for the bundled JDK. If we ever need strict
# 444 in production, split the chmod into a dev-only step and have
# `desktop:build` run `jlink:clean` first to force a fresh build.
- cmd: chmod -R u+w runtime/jre
platforms: [linux, darwin]
- cmd: powershell -NoProfile -Command "Get-ChildItem -Recurse runtime/jre | ForEach-Object { $_.IsReadOnly = $false }"
platforms: [windows]
status:
- test -f runtime/jre/release
- test -d editor/src-tauri/runtime/jre
jlink:clean:
desc: "Remove JLink runtime and bundled JARs"
-4
View File
@@ -10,10 +10,6 @@ if that directory exists, is licensed under the license defined in "app/propriet
if that directory exists, is licensed under the license defined in "app/saas/LICENSE".
* All content that resides under the "engine/" directory of this repository,
if that directory exists, is licensed under the license defined in "engine/LICENSE".
* "scripts/pymupdf_convert.py", if that file exists, is licensed under the GNU Affero
General Public License v3.0 (or later) as declared in its file header. It is a separate
program invoked as an OS subprocess; its license does not extend to other content in
this repository.
* All content that resides under the "frontend/src/proprietary/" directory of this repository,
if that directory exists, is licensed under the license defined in "frontend/src/proprietary/LICENSE".
* All content that resides under the "frontend/src/desktop/" directory of this repository,
+1 -18
View File
@@ -58,23 +58,6 @@ tasks:
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
OPEN: "true"
dev:saas:
desc: "Start SaaS backend + frontend concurrently on free ports"
vars:
PORTS:
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 5173{{else}}{{.FIND_FREE_PORT_SH}} 8080 5173{{end}}'
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
FRONTEND_PORT: '{{index (splitList "\n" .PORTS) 1}}'
deps:
- task: backend:dev:saas
vars:
PORT: '{{.BACKEND_PORT}}'
- task: frontend:dev:saas
vars:
PORT: '{{.FRONTEND_PORT}}'
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
OPEN: "true"
dev:all:
desc: "Start backend + frontend + engine concurrently on free ports"
vars:
@@ -91,7 +74,7 @@ tasks:
vars:
PORT: '{{.BACKEND_PORT}}'
AIENGINE_URL: 'http://localhost:{{.ENGINE_PORT}}'
- task: frontend:dev
- task: frontend:dev:prototypes
vars:
PORT: '{{.FRONTEND_PORT}}'
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
-3
View File
@@ -1,3 +0,0 @@
# Whitelist committed env defaults. `.env.saas.local` (and any other .env*)
# stays ignored via the root .gitignore.
!.env.saas
-4
View File
@@ -44,10 +44,6 @@
"moduleName": ".*",
"moduleLicense": "The MIT License"
},
{
"moduleName": ".*",
"moduleLicense": "MIT-0"
},
{
"moduleName": "com.github.jai-imageio:jai-imageio-core",
"moduleLicense": "LICENSE.txt"
@@ -77,10 +77,6 @@ public @interface AutoJobPostMapping {
/**
* Relative resource weight (1-100). See {@link
* stirling.software.common.enumeration.ResourceWeight} for the standard tiers.
*
* <p>The default is a sentinel ({@link Integer#MIN_VALUE}); {@code
* AutoJobPostMappingWeightTest} fails the build if any endpoint leaves it unset. Runtime
* readers clamp the value into {@code [1, 100]}.
*/
int resourceWeight() default Integer.MIN_VALUE;
int resourceWeight() default 1;
}
@@ -2,7 +2,15 @@ package stirling.software.common.cluster;
import java.time.Duration;
/** Token-bucket rate limiting backed by the cluster backplane. */
/**
* Token-bucket rate limiting backed by the cluster backplane.
*
* <p>In-process implementations enforce a per-JVM limit (identical to today's behaviour).
* Distributed implementations enforce a single global limit across every node.
*
* <p>Both implementations use a Bucket4j greedy-refill token bucket so semantics match across
* single-node and cluster deployments (no fixed-window boundary doubling).
*/
public interface RateLimitStore {
/**
@@ -80,7 +80,6 @@ public class ConfigInitializer {
YamlHelper settingsFile = new YamlHelper(settingTempPath);
migrateEnterpriseEditionToPremium(settingsFile, settingsTemplateFile);
migrateProFeaturesKeyCasing(settingsFile, settingsTemplateFile);
boolean changesMade =
settingsTemplateFile.updateValuesFromYaml(settingsFile, settingsTemplateFile);
@@ -117,52 +116,31 @@ public class ConfigInitializer {
}
if (yaml.getValueByExactKeyPath("enterpriseEdition", "SSOAutoLogin") != null) {
template.updateValue(
List.of("premium", "proFeatures", "ssoAutoLogin"),
List.of("premium", "proFeatures", "SSOAutoLogin"),
yaml.getValueByExactKeyPath("enterpriseEdition", "SSOAutoLogin"));
}
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "autoUpdateMetadata")
!= null) {
template.updateValue(
List.of("premium", "proFeatures", "customMetadata", "autoUpdateMetadata"),
List.of("premium", "proFeatures", "CustomMetadata", "autoUpdateMetadata"),
yaml.getValueByExactKeyPath(
"enterpriseEdition", "CustomMetadata", "autoUpdateMetadata"));
}
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "author") != null) {
template.updateValue(
List.of("premium", "proFeatures", "customMetadata", "author"),
List.of("premium", "proFeatures", "CustomMetadata", "author"),
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "author"));
}
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "creator") != null) {
template.updateValue(
List.of("premium", "proFeatures", "customMetadata", "creator"),
List.of("premium", "proFeatures", "CustomMetadata", "creator"),
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "creator"));
}
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "producer")
!= null) {
template.updateValue(
List.of("premium", "proFeatures", "customMetadata", "producer"),
List.of("premium", "proFeatures", "CustomMetadata", "producer"),
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "producer"));
}
}
// TODO: Remove post migration
// settings.yml.template renamed the two non-camelCase proFeatures keys
// ("SSOAutoLogin" -> "ssoAutoLogin", "CustomMetadata" -> "customMetadata") so the whole
// settings pipeline is consistent camelCase. The save path (YamlHelper.updateValue) matches
// keys case-sensitively, so without this carry-forward an existing install's values written
// under the old PascalCase keys would be dropped on upgrade and reset to template defaults.
void migrateProFeaturesKeyCasing(YamlHelper yaml, YamlHelper template) {
Object ssoAutoLogin = yaml.getValueByExactKeyPath("premium", "proFeatures", "SSOAutoLogin");
if (ssoAutoLogin != null) {
template.updateValue(List.of("premium", "proFeatures", "ssoAutoLogin"), ssoAutoLogin);
}
for (String field : List.of("autoUpdateMetadata", "author", "creator", "producer")) {
Object value =
yaml.getValueByExactKeyPath("premium", "proFeatures", "CustomMetadata", field);
if (value != null) {
template.updateValue(
List.of("premium", "proFeatures", "customMetadata", field), value);
}
}
}
}
@@ -528,16 +528,6 @@ public class ApplicationProperties {
private String provider;
private Client client = new Client();
/**
* When true, the OAuth2/OIDC login flow logs the full set of ID token and UserInfo
* claims at INFO level (and again at ERROR level if the username attribute cannot be
* resolved). Used to diagnose provider misconfiguration (for example ADFS not returning
* an {@code email} claim). WARNING: writes PII (sub, email, name) to application logs.
* Leave disabled in production; enable only while actively troubleshooting and disable
* again afterwards.
*/
private Boolean debugLogging = false;
public void setScopes(String scopes) {
List<String> scopesList =
Arrays.stream(scopes.split(",")).map(String::trim).toList();
@@ -788,7 +778,6 @@ public class ApplicationProperties {
private boolean enabled = false;
private String provider = "local";
private Local local = new Local();
private S3 s3 = new S3();
private Quotas quotas = new Quotas();
private Sharing sharing = new Sharing();
private Signing signing = new Signing();
@@ -798,57 +787,6 @@ public class ApplicationProperties {
private String basePath = InstallationPathConfig.getPath() + "storage";
}
@Data
public static class S3 {
/**
* Optional custom endpoint (e.g. {@code https://<account>.r2.cloudflarestorage.com},
* {@code https://<project>.supabase.co/storage/v1/s3}, or {@code http://localhost:9000}
* for MinIO). Blank = use AWS regional default.
*/
private String endpoint = "";
private String bucket = "";
private String region = "us-east-1";
private String accessKey = "";
private String secretKey = "";
/**
* When {@code true} use path-style URLs ({@code <endpoint>/<bucket>/<key>}) instead of
* virtual-hosted ({@code <bucket>.<endpoint>/<key>}). MinIO and most S3-compatible
* gateways require path-style; AWS S3 prefers virtual-hosted.
*/
private boolean pathStyleAccess = false;
/**
* When {@code false} (default), {@code endpoint} hostnames that resolve to private,
* loopback, or link-local addresses are rejected at startup to block SSRF attacks via
* the cloud metadata service (e.g. {@code http://169.254.169.254/}). Set to {@code
* true} to opt in for MinIO / in-cluster S3 endpoints on private networks.
*/
private boolean allowPrivateEndpoints = false;
/**
* Controls when the SDK adds an {@code x-amz-checksum-*} header on PUT/UploadPart.
* Default {@code WHEN_SUPPORTED} (the SDK default since 2.30) makes the SDK send a
* CRC32 checksum on every upload - this works on AWS S3, MinIO, current Supabase,
* Backblaze B2 (post-July-2025), and modern R2. Set to {@code WHEN_REQUIRED} to
* suppress the auto-checksum on vendors that reject unknown {@code x-amz-checksum-*}
* headers (older Backblaze B2, some R2 corner cases, GCS S3 endpoint). Invalid values
* fall back to {@code WHEN_SUPPORTED}.
*/
private String requestChecksumCalculation = "WHEN_SUPPORTED";
/**
* Controls when the SDK validates returned {@code x-amz-checksum-*} headers on GET
* responses. Default {@code WHEN_SUPPORTED}. Set to {@code WHEN_REQUIRED} if your
* vendor never returns these headers and you see false-positive checksum-mismatch
* errors. Invalid values fall back to {@code WHEN_SUPPORTED}.
*/
private String responseChecksumValidation = "WHEN_SUPPORTED";
}
@Data
public static class Sharing {
private boolean enabled = false;
@@ -1350,10 +1288,6 @@ public class ApplicationProperties {
public int getFfmpegSessionLimit() {
return ffmpegSessionLimit > 0 ? ffmpegSessionLimit : 2;
}
public int getPyMuPdfConvertSessionLimit() {
return 2;
}
}
@Data
@@ -1431,10 +1365,6 @@ public class ApplicationProperties {
public long getFfmpegTimeoutMinutes() {
return ffmpegTimeoutMinutes > 0 ? ffmpegTimeoutMinutes : 30;
}
public long getPyMuPdfConvertTimeoutMinutes() {
return 10;
}
}
}
}
@@ -60,7 +60,7 @@ public class CustomPDFDocumentFactory {
this(pdfMetadataService, null);
}
/** Documents ≤ this size are loaded entirely into heap no temp files needed. */
/** Documents ≤ this size are loaded entirely into heap - no temp files needed. */
public static final long SMALL_FILE_THRESHOLD = 10L * 1024 * 1024; // 10 MB
/** Upper boundary of the "mixed" memory+file zone; above this always file-backed. */
@@ -130,7 +130,6 @@ public class CustomPDFDocumentFactory {
MemorySnapshot mem = MemorySnapshot.capture();
// Use the overridable method so that test spies (SpyPDFDocumentFactory) can intercept.
StreamCacheCreateFunction cache = getStreamCacheFunction(size, mem);
// Non-destructive — caller's file is never deleted
RandomAccessReadBufferedFile raf = new RandomAccessReadBufferedFile(file);
PDDocument doc;
try {
@@ -112,25 +112,12 @@ public class JobExecutorService {
log.debug("Generated jobId: {} (base: {})", scopedJobKey, baseJobId);
// Store the scoped job ID in the request for potential use by other components
// Store the scoped job ID in the request for potential use by other components.
// Ownership lives in the scoped key itself (userId:jobId) plus the cluster-visible
// JobStore entry, so we no longer mirror it into the HTTP session - that did not
// survive a node hop in cluster mode.
if (request != null) {
request.setAttribute("jobId", scopedJobKey);
// Also track this job ID in the user's session for authorization purposes
// This ensures users can only cancel their own jobs
if (request.getSession() != null) {
@SuppressWarnings("unchecked")
java.util.Set<String> userJobIds =
(java.util.Set<String>) request.getSession().getAttribute("userJobIds");
if (userJobIds == null) {
userJobIds = new java.util.concurrent.ConcurrentSkipListSet<>();
request.getSession().setAttribute("userJobIds", userJobIds);
}
userJobIds.add(scopedJobKey);
log.debug("Added scoped job ID {} to user session", scopedJobKey);
}
}
String jobId = scopedJobKey;
@@ -1,91 +0,0 @@
package stirling.software.common.service;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.util.ProcessExecutor;
/**
* Converts PDFs to Markdown by invoking the {@code pymupdf-convert} CLI tool as a separate
* subprocess.
*/
@Slf4j
@Service
public class PyMuPdfConverter {
private boolean available;
@PostConstruct
void init() {
available = probe();
if (available) {
log.info("pymupdf-convert found — PyMuPDF Markdown conversion enabled.");
} else {
log.info("pymupdf-convert not found — PyMuPDF Markdown conversion disabled.");
}
}
public boolean isAvailable() {
return available;
}
private boolean probe() {
boolean isWindows =
System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("windows");
List<String> cmd =
isWindows
? List.of("where", "pymupdf-convert")
: List.of("which", "pymupdf-convert");
try {
Process p = new ProcessBuilder(cmd).redirectErrorStream(true).start();
boolean done = p.waitFor(5, TimeUnit.SECONDS);
return done && p.exitValue() == 0;
} catch (Exception e) {
log.debug("pymupdf-convert availability check failed: {}", e.getMessage());
return false;
}
}
/**
* Convert a PDF to Markdown by invoking {@code pymupdf-convert} as a subprocess.
*
* @throws IOException on process failure or if the tool is not installed
*/
public String convertToMarkdown(byte[] pdfBytes, String filename) throws IOException {
String safeName =
(filename == null || filename.isBlank())
? "document.pdf"
: filename.replace("\"", "");
Path tempDir = Files.createTempDirectory("stirling-pymupdf-");
Path inputPdf = tempDir.resolve(safeName);
Path outputMd = tempDir.resolve("output.md");
try {
Files.write(inputPdf, pdfBytes);
ProcessExecutor.getInstance(ProcessExecutor.Processes.PYMUPDF_CONVERT)
.runCommandWithOutputHandling(
List.of(
"pymupdf-convert",
inputPdf.toAbsolutePath().toString(),
outputMd.toAbsolutePath().toString()));
return Files.readString(outputMd, StandardCharsets.UTF_8);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("PyMuPDF conversion interrupted", e);
} finally {
Files.deleteIfExists(inputPdf);
Files.deleteIfExists(outputMd);
Files.deleteIfExists(tempDir);
}
}
}
@@ -4,7 +4,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -31,7 +30,6 @@ import io.github.pixee.security.Filenames;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.configuration.RuntimePathConfig;
import stirling.software.common.service.PyMuPdfConverter;
import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult;
@Slf4j
@@ -161,57 +159,6 @@ public class PDFToFile {
finalOut, fileName, MediaType.APPLICATION_OCTET_STREAM);
}
/**
* PDF-&gt;Markdown with optional PyMuPDF acceleration.
*
* <p>When {@code pymupdf-convert} is installed and on PATH, conversion is delegated to it as a
* subprocess. On any failure — or when the tool is absent — this transparently falls back to
* the bundled {@code pdftohtml}-based converter.
*/
public ResponseEntity<Resource> processPdfToMarkdown(
MultipartFile inputFile, PyMuPdfConverter pyMuPdfConverter)
throws IOException, InterruptedException {
if (!MediaType.APPLICATION_PDF_VALUE.equals(inputFile.getContentType())) {
return new ResponseEntity<>(HttpStatus.BAD_REQUEST);
}
if (pyMuPdfConverter != null && pyMuPdfConverter.isAvailable()) {
try {
String originalName = Filenames.toSimpleFileName(inputFile.getOriginalFilename());
String baseName = originalName;
if (originalName != null && originalName.contains(".")) {
baseName = originalName.substring(0, originalName.lastIndexOf('.'));
}
String markdown =
pyMuPdfConverter.convertToMarkdown(inputFile.getBytes(), originalName);
return buildMarkdownZipResponse(markdown, baseName);
} catch (IOException e) {
log.warn(
"PyMuPDF conversion failed; falling back to pdftohtml converter: {}",
e.getMessage());
}
}
return processPdfToMarkdown(inputFile);
}
private ResponseEntity<Resource> buildMarkdownZipResponse(String markdown, String pdfBaseName)
throws IOException {
String fileName = pdfBaseName + "ToMarkdown.zip";
TempFile finalOut = tempFileManager.createManagedTempFile(".zip");
try (OutputStream fos = Files.newOutputStream(finalOut.getPath());
ZipOutputStream zipOutputStream = new ZipOutputStream(fos)) {
ZipEntry mdEntry = new ZipEntry(pdfBaseName + ".md");
zipOutputStream.putNextEntry(mdEntry);
zipOutputStream.write(markdown.getBytes(StandardCharsets.UTF_8));
zipOutputStream.closeEntry();
} catch (Exception e) {
finalOut.close();
throw e;
}
return WebResponseUtils.fileToWebResponse(
finalOut, fileName, MediaType.APPLICATION_OCTET_STREAM);
}
/**
* Updates image references in markdown to point to the images/ folder. Matches patterns like
* ![alt](filename.png) and converts to ![alt](images/filename.png)
@@ -115,11 +115,6 @@ public class ProcessExecutor {
.getProcessExecutor()
.getSessionLimit()
.getFfmpegSessionLimit();
case PYMUPDF_CONVERT ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getPyMuPdfConvertSessionLimit();
};
long timeoutMinutes =
@@ -185,11 +180,6 @@ public class ProcessExecutor {
.getProcessExecutor()
.getTimeoutMinutes()
.getFfmpegTimeoutMinutes();
case PYMUPDF_CONVERT ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getPyMuPdfConvertTimeoutMinutes();
};
return new ProcessExecutor(
processType, semaphoreLimit, liveUpdates, timeoutMinutes);
@@ -560,8 +550,7 @@ public class ProcessExecutor {
GHOSTSCRIPT,
OCR_MY_PDF,
CFF_CONVERTER,
FFMPEG,
PYMUPDF_CONVERT
FFMPEG
}
@Setter
@@ -83,16 +83,7 @@ public class RequestUriUtils {
return false;
}
// Blocklist of backend/non-frontend paths that should still go through filters.
//
// `/files` was historically a backend route; it is now a frontend route
// owned by HomePage / FileManagerView. Direct-nav or refresh on /files
// (or /files/<folder-uuid>) was returning the Spring auth filter's 401
// JSON instead of serving index.html, so the SPA never got a chance to
// mount and the user saw a raw error response. There are no `/files`
// backend mappings at the servlet root - the real storage endpoints
// live under `/api/v1/storage/files`, which is filtered out a few lines
// up by the `startsWith("/api/")` guard.
// Blocklist of backend/non-frontend paths that should still go through filters
String[] backendOnlyPrefixes = {
"/register",
"/pipeline",
@@ -100,6 +91,7 @@ public class RequestUriUtils {
"/pdfjs-legacy",
"/fonts",
"/images",
"/files",
"/css",
"/js",
"/swagger",
@@ -189,7 +181,7 @@ public class RequestUriUtils {
|| trimmedUri.startsWith(
"/api/v1/mobile-scanner/") // Mobile scanner endpoints (no auth)
|| trimmedUri.startsWith("/v1/api-docs")
// Workflow participant endpoints - access controlled by share tokens, not login
// Workflow participant endpoints access controlled by share tokens, not login
|| trimmedUri.startsWith("/api/v1/workflow/participant/")
// Share-link SPA bootstrap; data APIs remain protected
|| trimmedUri.matches("^/share/[^/]+/?$");
@@ -24,7 +24,9 @@ class InProcessDistributedLockTest {
}
@Test
void reentryFromSameThreadFails() {
void reentryFromSameThreadFails_parityWithValkey() {
// The Valkey impl refuses reentry (SET NX semantics); the in-process impl must match,
// otherwise code working in single-instance silently breaks in cluster mode.
DistributedLock lock = new InProcessDistributedLock();
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
Optional<DistributedLock.LockHandle> reentry = lock.tryAcquire("k", Duration.ofSeconds(30));
@@ -1,95 +0,0 @@
package stirling.software.common.configuration;
import static org.junit.jupiter.api.Assertions.*;
import org.junit.jupiter.api.Test;
import org.snakeyaml.engine.v2.api.LoadSettings;
import stirling.software.common.util.YamlHelper;
class ConfigInitializerTest {
private static final LoadSettings LOAD_SETTINGS =
LoadSettings.builder()
.setUseMarks(true)
.setMaxAliasesForCollections(Integer.MAX_VALUE)
.setAllowRecursiveKeys(true)
.setParseComments(true)
.build();
// Mirrors the proFeatures block of settings.yml.template after the camelCase rename.
private static final String CAMEL_CASE_TEMPLATE =
"""
premium:
proFeatures:
ssoAutoLogin: false
customMetadata:
autoUpdateMetadata: false
author: username
creator: Stirling-PDF
producer: Stirling-PDF
""";
@Test
void migrateProFeaturesKeyCasing_carriesForwardLegacyPascalCaseValues() {
// An existing install whose settings.yml still uses the old PascalCase keys.
String legacy =
"""
premium:
proFeatures:
SSOAutoLogin: true
CustomMetadata:
autoUpdateMetadata: true
author: alice
creator: bob
producer: carol
""";
YamlHelper template = new YamlHelper(LOAD_SETTINGS, CAMEL_CASE_TEMPLATE);
YamlHelper existing = new YamlHelper(LOAD_SETTINGS, legacy);
new ConfigInitializer().migrateProFeaturesKeyCasing(existing, template);
assertEquals(
"true", template.getValueByExactKeyPath("premium", "proFeatures", "ssoAutoLogin"));
assertEquals(
"true",
template.getValueByExactKeyPath(
"premium", "proFeatures", "customMetadata", "autoUpdateMetadata"));
assertEquals(
"alice",
template.getValueByExactKeyPath(
"premium", "proFeatures", "customMetadata", "author"));
assertEquals(
"bob",
template.getValueByExactKeyPath(
"premium", "proFeatures", "customMetadata", "creator"));
assertEquals(
"carol",
template.getValueByExactKeyPath(
"premium", "proFeatures", "customMetadata", "producer"));
}
@Test
void migrateProFeaturesKeyCasing_withoutLegacyKeys_keepsTemplateDefaults() {
// No PascalCase keys present -> this migration step must be a no-op.
String alreadyCamel =
"""
premium:
proFeatures:
ssoAutoLogin: true
customMetadata:
author: dave
""";
YamlHelper template = new YamlHelper(LOAD_SETTINGS, CAMEL_CASE_TEMPLATE);
YamlHelper existing = new YamlHelper(LOAD_SETTINGS, alreadyCamel);
new ConfigInitializer().migrateProFeaturesKeyCasing(existing, template);
assertEquals(
"false", template.getValueByExactKeyPath("premium", "proFeatures", "ssoAutoLogin"));
assertEquals(
"username",
template.getValueByExactKeyPath(
"premium", "proFeatures", "customMetadata", "author"));
}
}
@@ -86,6 +86,8 @@ class TaskManagerJobStoreDelegationTest {
@Override
public boolean shouldRunLocalCleanup() {
// Distributed backplanes own job TTL eviction themselves; this mock
// mirrors the real ValkeyClusterBackplane override of the default true.
return false;
}
};
@@ -2,8 +2,6 @@ package stirling.software.common.util;
import static org.junit.jupiter.api.Assertions.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -14,47 +12,9 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import stirling.software.common.configuration.InstallationPathConfig;
public class GeneralUtilsTest {
// Regression guard for the SSO auto-login persistence bug: the admin UI writes camelCase
// proFeatures keys, so saveKeyToSettings must match (and persist) them against the camelCase
// settings.yml.template. A case mismatch makes YamlHelper.updateValue silently no-op.
@Test
void saveKeyToSettings_persistsCamelCaseProFeatureKeys(@TempDir Path tempDir) throws Exception {
Path settings = tempDir.resolve("settings.yml");
Files.writeString(
settings,
"""
premium:
proFeatures:
ssoAutoLogin: false
customMetadata:
author: username
""");
try (MockedStatic<InstallationPathConfig> mocked =
Mockito.mockStatic(InstallationPathConfig.class)) {
mocked.when(InstallationPathConfig::getSettingsPath).thenReturn(settings.toString());
GeneralUtils.saveKeyToSettings("premium.proFeatures.ssoAutoLogin", true);
GeneralUtils.saveKeyToSettings("premium.proFeatures.customMetadata.author", "alice");
}
YamlHelper reloaded = new YamlHelper(settings);
assertEquals(
"true", reloaded.getValueByExactKeyPath("premium", "proFeatures", "ssoAutoLogin"));
assertEquals(
"alice",
reloaded.getValueByExactKeyPath(
"premium", "proFeatures", "customMetadata", "author"));
}
@Test
void testParsePageListWithAll() {
List<Integer> result = GeneralUtils.parsePageList(new String[] {"all"}, 5, false);
@@ -98,17 +98,6 @@ class RequestUriUtilsTest {
assertTrue(RequestUriUtils.isFrontendRoute("", "/split-pdf"));
}
@Test
void testIsFrontendRoute_filesRouteOwnedByFrontend() {
// /files and /files/<folder-uuid> are FileManagerView routes - they
// must fall through to the SPA index.html, not get blocked by the
// backend auth filter. Regression test for direct-nav/refresh on
// the file manager returning a 401 JSON.
assertTrue(RequestUriUtils.isFrontendRoute("", "/files"));
assertTrue(
RequestUriUtils.isFrontendRoute("", "/files/3331910a-4155-4f71-8111-e38c896bc458"));
}
@Test
void testIsFrontendRoute_pathWithExtension() {
assertFalse(RequestUriUtils.isFrontendRoute("", "/some/file.pdf"));
@@ -194,7 +183,7 @@ class RequestUriUtilsTest {
@Test
void testIsPublicAuthEndpoint_shareRootNotPublic() {
// Avoid matching bare "/share" or "/share/" - must have a token segment
// Avoid matching bare "/share" or "/share/" must have a token segment
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/share", ""));
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/share/", ""));
}
@@ -208,7 +197,7 @@ class RequestUriUtilsTest {
@Test
void testIsPublicAuthEndpoint_shareApiStillProtected() {
// Share-link data APIs must NOT be public - they enforce auth + access checks
// Share-link data APIs must NOT be public they enforce auth + access checks
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/api/v1/storage/share-links/abc123", ""));
assertFalse(
RequestUriUtils.isPublicAuthEndpoint(
@@ -21,13 +21,6 @@ public class EndpointInterceptor implements HandlerInterceptor {
HttpServletRequest request, HttpServletResponse response, Object handler)
throws Exception {
String requestURI = request.getRequestURI();
// Prevent API responses from being stored by browsers or intermediary caches by default
String servletPath = request.getServletPath();
if (servletPath != null && servletPath.startsWith("/api/")) {
response.setHeader("Cache-Control", "private, no-store");
}
boolean isEnabled = endpointConfiguration.isEndpointEnabledForUri(requestURI);
if (!isEnabled) {
response.sendError(HttpServletResponse.SC_FORBIDDEN, "This endpoint is disabled");
@@ -101,9 +101,6 @@ public class ExternalAppDepConfig {
// Python / OpenCV special handling
checkPythonAndOpenCV();
// PyMuPDF optional acceleration
checkPyMuPdf();
dependenciesChecked = true;
} finally {
endpointConfiguration.logDisabledEndpointsSummary();
@@ -239,15 +236,6 @@ public class ExternalAppDepConfig {
}
}
private void checkPyMuPdf() {
if (isCommandAvailable("pymupdf-convert")) {
log.warn("pymupdf-convert detected — PDF->Markdown will use PyMuPDF acceleration.");
} else {
log.info(
"pymupdf-convert not found — PDF->Markdown will use the bundled pdftohtml converter.");
}
}
private void disablePythonAndOpenCV(String reason) {
List<String> pythonFeatures = getAffectedFeatures("Python");
List<String> openCVFeatures = getAffectedFeatures("OpenCV");
@@ -1,8 +1,5 @@
package stirling.software.SPDF.config;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
@@ -27,10 +24,6 @@ public class WebMvcConfig implements WebMvcConfigurer {
private static final Logger logger = LoggerFactory.getLogger(WebMvcConfig.class);
private static final CacheControl NO_CACHE = CacheControl.noCache();
private static final CacheControl IMMUTABLE_ONE_YEAR =
CacheControl.maxAge(365, TimeUnit.DAYS).cachePublic().immutable();
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(endpointInterceptor);
@@ -38,95 +31,37 @@ public class WebMvcConfig implements WebMvcConfigurer {
@Override
public void addResourceHandlers(ResourceHandlerRegistry registry) {
String staticPath =
"file:"
+ stirling.software.common.configuration.InstallationPathConfig
.getStaticPath();
// 1. Service worker and PWA metadata (never store)
// Browsers revalidate SW bytes anyway; no-store is the safest for atomic updates.
registry.addResourceHandler(
"/sw.js", "/manifest.json", "/site.webmanifest", "/browserconfig.xml")
.addResourceLocations(staticPath, "classpath:/static/")
.setCacheControl(CacheControl.noStore())
.resourceChain(true);
// 2. Vite fingerprinted assets (immutable)
// These already have content hashes in filenames (e.g. index-ChAS4tCC.js)
// Cache hashed assets (JS/CSS with content hashes) for 1 year
// These files have names like index-ChAS4tCC.js that change when content changes
// Check customFiles/static first, then fall back to classpath
registry.addResourceHandler("/assets/**")
.addResourceLocations(staticPath + "assets/", "classpath:/static/assets/")
.setCacheControl(IMMUTABLE_ONE_YEAR)
.resourceChain(true);
// 3. Media and fonts (immutable)
registry.addResourceHandler("/images/**", "/fonts/**")
.addResourceLocations(
staticPath + "images/",
"classpath:/static/images/",
staticPath + "fonts/",
"classpath:/static/fonts/")
.setCacheControl(IMMUTABLE_ONE_YEAR)
.resourceChain(true);
"file:"
+ stirling.software.common.configuration.InstallationPathConfig
.getStaticPath()
+ "assets/",
"classpath:/static/assets/")
.setCacheControl(CacheControl.maxAge(365, TimeUnit.DAYS).cachePublic());
// 4. Branding and stable non-fingerprinted assets (1 day + SWR)
// Use stale-while-revalidate to improve perceived performance.
registry.addResourceHandler(
"/favicon.*",
"/apple-touch-icon.png",
"/android-chrome-*.png",
"/mstile-*.png",
"/safari-pinned-tab.svg",
"/icons/**",
"/modern-logo/**",
"/classic-logo/**",
"/robots.txt",
"/3rdPartyLicenses.json",
"/pdfjs/**",
"/pdfjs-legacy/**",
"/pdfium/**",
"/locales/**",
"/css/**",
"/js/**",
"/vendor/**",
"/samples/**",
"/og_images/**",
"/Login/**",
"/manifest-classic.json")
// Don't cache index.html - it needs to be fresh to reference latest hashed assets
// Note: index.html is handled by ReactRoutingController for dynamic processing
registry.addResourceHandler("/index.html")
.addResourceLocations(
staticPath,
"classpath:/static/",
staticPath + "pdfjs/",
"classpath:/static/pdfjs/",
staticPath + "pdfjs-legacy/",
"classpath:/static/pdfjs-legacy/",
staticPath + "pdfium/",
"classpath:/static/pdfium/",
staticPath + "locales/",
"classpath:/static/locales/",
staticPath + "css/",
"classpath:/static/css/",
staticPath + "js/",
"classpath:/static/js/",
staticPath + "vendor/",
"classpath:/static/vendor/",
staticPath + "samples/",
"classpath:/static/samples/",
staticPath + "og_images/",
"classpath:/static/og_images/",
staticPath + "Login/",
"classpath:/static/Login/")
.setCacheControl(
CacheControl.maxAge(Duration.ofDays(1))
.cachePublic()
.staleWhileRevalidate(Duration.ofDays(7)))
.resourceChain(true);
"file:"
+ stirling.software.common.configuration.InstallationPathConfig
.getStaticPath(),
"classpath:/static/")
.setCacheControl(CacheControl.noCache().mustRevalidate());
// 5. Catch-all (SPA fallback)
// Must check with server to ensure index.html is always fresh.
// Handle all other static resources (js, css, images, fonts, etc.)
// Check customFiles/static first for user overrides
registry.addResourceHandler("/**")
.addResourceLocations(staticPath, "classpath:/static/")
.setCacheControl(NO_CACHE)
.resourceChain(true);
.addResourceLocations(
"file:"
+ stirling.software.common.configuration.InstallationPathConfig
.getStaticPath(),
"classpath:/static/")
.setCacheControl(CacheControl.maxAge(1, TimeUnit.HOURS));
}
@Override
@@ -180,8 +115,9 @@ public class WebMvcConfig implements WebMvcConfigurer {
applicationProperties.getSystem().getCorsAllowedOrigins());
// Combine user-configured origins with Tauri origins
List<String> allOrigins =
new ArrayList<>(applicationProperties.getSystem().getCorsAllowedOrigins());
java.util.List<String> allOrigins =
new java.util.ArrayList<>(
applicationProperties.getSystem().getCorsAllowedOrigins());
// Always include Tauri origins for desktop app compatibility
// Tauri v1 uses tauri://localhost, v2 uses http(s)://tauri.localhost
@@ -222,8 +158,7 @@ public class WebMvcConfig implements WebMvcConfigurer {
} else {
// Default to allowing all origins when nothing is configured
logger.debug(
"No CORS allowed origins configured in settings.yml"
+ " (system.corsAllowedOrigins); WebMvcConfig allowing all origins.");
"No CORS allowed origins configured in settings.yml (system.corsAllowedOrigins); WebMvcConfig allowing all origins.");
registry.addMapping("/**")
.allowedOriginPatterns("*")
.allowedMethods("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS")
@@ -32,7 +32,6 @@ import stirling.software.SPDF.model.json.PdfJsonTextElement;
import stirling.software.SPDF.service.PdfJsonConversionService;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.enumeration.ResourceWeight;
import stirling.software.common.model.api.general.EditTextOperation;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.GeneralUtils;
@@ -76,10 +75,7 @@ public class EditTextController {
new StringToArrayListPropertyEditor<>(EditTextOperation.class));
}
@AutoJobPostMapping(
consumes = "multipart/form-data",
value = "/edit-text",
resourceWeight = ResourceWeight.LARGE_WEIGHT)
@AutoJobPostMapping(consumes = "multipart/form-data", value = "/edit-text")
@StandardPdfResponse
@Operation(
summary = "Edit text in a PDF via find and replace",
@@ -40,8 +40,7 @@ public class ScalePagesController {
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final TempFileManager tempFileManager;
private static PDRectangle getTargetSize(
String targetPDRectangle, String orientation, PDDocument sourceDocument) {
private static PDRectangle getTargetSize(String targetPDRectangle, PDDocument sourceDocument) {
if ("KEEP".equals(targetPDRectangle)) {
if (sourceDocument.getNumberOfPages() == 0) {
throw ExceptionUtils.createInvalidPageSizeException("KEEP");
@@ -58,19 +57,18 @@ public class ScalePagesController {
}
Map<String, PDRectangle> sizeMap = getSizeMap();
PDRectangle base = sizeMap.get(targetPDRectangle);
if (base == null) {
throw ExceptionUtils.createInvalidPageSizeException(targetPDRectangle);
if (sizeMap.containsKey(targetPDRectangle)) {
return sizeMap.get(targetPDRectangle);
}
if ("LANDSCAPE".equalsIgnoreCase(orientation)) {
return new PDRectangle(base.getHeight(), base.getWidth());
}
return base;
throw ExceptionUtils.createInvalidPageSizeException(targetPDRectangle);
}
private static Map<String, PDRectangle> getSizeMap() {
Map<String, PDRectangle> sizeMap = new HashMap<>();
// Portrait sizes (A0-A6)
sizeMap.put("A0", PDRectangle.A0);
sizeMap.put("A1", PDRectangle.A1);
sizeMap.put("A2", PDRectangle.A2);
@@ -78,8 +76,42 @@ public class ScalePagesController {
sizeMap.put("A4", PDRectangle.A4);
sizeMap.put("A5", PDRectangle.A5);
sizeMap.put("A6", PDRectangle.A6);
// Landscape sizes (A0-A6)
sizeMap.put(
"A0_LANDSCAPE",
new PDRectangle(PDRectangle.A0.getHeight(), PDRectangle.A0.getWidth()));
sizeMap.put(
"A1_LANDSCAPE",
new PDRectangle(PDRectangle.A1.getHeight(), PDRectangle.A1.getWidth()));
sizeMap.put(
"A2_LANDSCAPE",
new PDRectangle(PDRectangle.A2.getHeight(), PDRectangle.A2.getWidth()));
sizeMap.put(
"A3_LANDSCAPE",
new PDRectangle(PDRectangle.A3.getHeight(), PDRectangle.A3.getWidth()));
sizeMap.put(
"A4_LANDSCAPE",
new PDRectangle(PDRectangle.A4.getHeight(), PDRectangle.A4.getWidth()));
sizeMap.put(
"A5_LANDSCAPE",
new PDRectangle(PDRectangle.A5.getHeight(), PDRectangle.A5.getWidth()));
sizeMap.put(
"A6_LANDSCAPE",
new PDRectangle(PDRectangle.A6.getHeight(), PDRectangle.A6.getWidth()));
// Portrait US sizes
sizeMap.put("LETTER", PDRectangle.LETTER);
sizeMap.put("LEGAL", PDRectangle.LEGAL);
// Landscape US sizes
sizeMap.put(
"LETTER_LANDSCAPE",
new PDRectangle(PDRectangle.LETTER.getHeight(), PDRectangle.LETTER.getWidth()));
sizeMap.put(
"LEGAL_LANDSCAPE",
new PDRectangle(PDRectangle.LEGAL.getHeight(), PDRectangle.LEGAL.getWidth()));
return sizeMap;
}
@@ -96,14 +128,13 @@ public class ScalePagesController {
throws IOException {
MultipartFile file = request.getFileInput();
String targetPDRectangle = request.getPageSize();
String orientation = request.getOrientation();
float scaleFactor = request.getScaleFactor();
try (PDDocument sourceDocument = pdfDocumentFactory.load(file);
PDDocument outputDocument =
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument)) {
PDRectangle targetSize = getTargetSize(targetPDRectangle, orientation, sourceDocument);
PDRectangle targetSize = getTargetSize(targetPDRectangle, sourceDocument);
// Create LayerUtility once outside the loop for better performance
LayerUtility layerUtility = new LayerUtility(outputDocument);
@@ -275,10 +275,7 @@ public class ConvertImgPDFController {
GeneralUtils.generateFilename(file[0].getOriginalFilename(), "_converted.pdf"));
}
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/cbz/pdf",
resourceWeight = ResourceWeight.MEDIUM_WEIGHT)
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/cbz/pdf")
@Operation(
summary = "Convert CBZ comic book archive to PDF",
description =
@@ -304,10 +301,7 @@ public class ConvertImgPDFController {
return WebResponseUtils.pdfFileToWebResponse(pdfFile, filename);
}
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/pdf/cbz",
resourceWeight = ResourceWeight.LARGE_WEIGHT)
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/pdf/cbz")
@Operation(
summary = "Convert PDF to CBZ comic book archive",
description =
@@ -330,10 +324,7 @@ public class ConvertImgPDFController {
return WebResponseUtils.zipFileToWebResponse(cbzFile, filename);
}
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/cbr/pdf",
resourceWeight = ResourceWeight.MEDIUM_WEIGHT)
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/cbr/pdf")
@Operation(
summary = "Convert CBR comic book archive to PDF",
description =
@@ -359,10 +350,7 @@ public class ConvertImgPDFController {
return WebResponseUtils.bytesToWebResponse(pdfBytes, filename);
}
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/pdf/cbr",
resourceWeight = ResourceWeight.LARGE_WEIGHT)
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/pdf/cbr")
@Operation(
summary = "Convert PDF to CBR comic book archive",
description =
@@ -141,8 +141,7 @@ public class AttachmentController {
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/extract-attachments",
resourceWeight = ResourceWeight.SMALL_WEIGHT)
value = "/extract-attachments")
@Operation(
summary = "Extract attachments from PDF",
description =
@@ -177,10 +176,7 @@ public class AttachmentController {
}
}
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/list-attachments",
resourceWeight = ResourceWeight.SMALL_WEIGHT)
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/list-attachments")
@Operation(
summary = "List attachments in PDF",
description =
@@ -197,8 +193,7 @@ public class AttachmentController {
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/rename-attachment",
resourceWeight = ResourceWeight.SMALL_WEIGHT)
value = "/rename-attachment")
@StandardPdfResponse
@Operation(
summary = "Rename attachment in PDF",
@@ -233,8 +228,7 @@ public class AttachmentController {
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/delete-attachment",
resourceWeight = ResourceWeight.SMALL_WEIGHT)
value = "/delete-attachment")
@StandardPdfResponse
@Operation(
summary = "Delete attachment from PDF",
@@ -12,8 +12,6 @@ import org.springframework.web.bind.annotation.RequestParam;
import io.swagger.v3.oas.annotations.Hidden;
import jakarta.servlet.http.HttpServletRequest;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.config.EndpointConfiguration;
@@ -93,63 +91,6 @@ public class ConfigController {
return null;
}
/**
* Resolve the frontend URL the client should advertise to phones / share-link recipients.
* Priority: explicit system.frontendUrl, then the Host the user is already using to reach this
* server (works for Docker, reverse proxies, and bare-metal LANs), then a detected site-local
* IPv4, then empty.
*/
// visible for testing
String resolveFrontendUrl(HttpServletRequest request, AppConfig appConfig) {
String configured = applicationProperties.getSystem().getFrontendUrl();
if (configured != null && !configured.isBlank()) {
return configured;
}
if (request != null) {
String host = request.getServerName();
if (host != null && !host.isBlank() && !isLoopbackHost(host)) {
String scheme = request.getScheme();
int port = request.getServerPort();
boolean defaultPort =
("http".equals(scheme) && port == 80)
|| ("https".equals(scheme) && port == 443);
return defaultPort ? scheme + "://" + host : scheme + "://" + host + ":" + port;
}
}
String localIp = GeneralUtils.getLocalNetworkIp();
if (localIp != null) {
String scheme = appConfig.getBackendUrl().startsWith("https") ? "https" : "http";
return scheme + "://" + localIp + ":" + resolveEffectiveServerPort(appConfig);
}
return "";
}
/**
* The port the embedded server is actually listening on. With {@code server.port=0} (an
* ephemeral port, which the desktop bundle uses to dodge port clashes) the configured value
* stays {@code "0"} while Spring publishes the real bound port as {@code local.server.port}
* once the server is up. Advertised URLs (the mobile-scanner QR, share links) must carry the
* real port - a literal {@code :0} is unreachable and browsers reject it as ERR_UNSAFE_PORT.
*/
// visible for testing
String resolveEffectiveServerPort(AppConfig appConfig) {
String configured = appConfig.getServerPort();
if (configured == null || "0".equals(configured.trim())) {
String actual = applicationContext.getEnvironment().getProperty("local.server.port");
if (actual != null && !actual.isBlank()) {
return actual;
}
}
return configured;
}
private static boolean isLoopbackHost(String host) {
return "localhost".equalsIgnoreCase(host)
|| "127.0.0.1".equals(host)
|| "::1".equals(host)
|| "0:0:0:0:0:0:0:1".equals(host);
}
/** Check if running Enterprise edition dynamically. */
private Boolean isRunningEE() {
// Use LicenseService for fresh license status if available
@@ -166,7 +107,7 @@ public class ConfigController {
}
@GetMapping("/app-config")
public ResponseEntity<Map<String, Object>> getAppConfig(HttpServletRequest request) {
public ResponseEntity<Map<String, Object>> getAppConfig() {
Map<String, Object> configData = new HashMap<>();
try {
@@ -180,10 +121,20 @@ public class ConfigController {
// Note: Frontend expects "baseUrl" field name for compatibility
configData.put("baseUrl", appConfig.getBackendUrl());
configData.put("contextPath", appConfig.getContextPath());
configData.put("serverPort", resolveEffectiveServerPort(appConfig));
configData.put("serverPort", appConfig.getServerPort());
String frontendUrl = applicationProperties.getSystem().getFrontendUrl();
configData.put("frontendUrl", resolveFrontendUrl(request, appConfig));
if ((frontendUrl == null || frontendUrl.isBlank())
&& Boolean.parseBoolean(
System.getProperty("STIRLING_PDF_TAURI_MODE", "false"))) {
String localIp = GeneralUtils.getLocalNetworkIp();
if (localIp != null) {
String scheme =
appConfig.getBackendUrl().startsWith("https") ? "https" : "http";
frontendUrl = scheme + "://" + localIp + ":" + appConfig.getServerPort();
}
}
configData.put("frontendUrl", frontendUrl != null ? frontendUrl : "");
// Add mobile scanner settings
configData.put(
@@ -326,9 +277,6 @@ public class ConfigController {
// Premium/Enterprise settings
configData.put("premiumEnabled", applicationProperties.getPremium().isEnabled());
// AI Engine settings
configData.put("aiEngineEnabled", applicationProperties.getAiEngine().isEnabled());
// Timestamp TSA settings single source of truth for presets + admin URLs
ApplicationProperties.Security.Timestamp tsConfig =
applicationProperties.getSecurity().getTimestamp();
@@ -12,7 +12,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.CacheControl;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
@@ -135,22 +134,13 @@ public class ReactRoutingController {
public ResponseEntity<String> serveIndexHtml(HttpServletRequest request) {
try {
if (indexHtmlExists && cachedIndexHtml != null) {
return ResponseEntity.ok()
.cacheControl(CacheControl.noCache().mustRevalidate())
.contentType(MediaType.TEXT_HTML)
.body(cachedIndexHtml);
return ResponseEntity.ok().contentType(MediaType.TEXT_HTML).body(cachedIndexHtml);
}
// Fallback: process on each request (dev mode or cache failed)
return ResponseEntity.ok()
.cacheControl(CacheControl.noCache().mustRevalidate())
.contentType(MediaType.TEXT_HTML)
.body(processIndexHtml());
return ResponseEntity.ok().contentType(MediaType.TEXT_HTML).body(processIndexHtml());
} catch (Exception ex) {
log.error("Failed to serve index.html, returning fallback", ex);
return ResponseEntity.ok()
.cacheControl(CacheControl.noCache().mustRevalidate())
.contentType(MediaType.TEXT_HTML)
.body(buildFallbackHtml());
return ResponseEntity.ok().contentType(MediaType.TEXT_HTML).body(buildFallbackHtml());
}
}
@@ -170,19 +160,14 @@ public class ReactRoutingController {
return ResponseEntity.ok().contentType(MediaType.TEXT_HTML).body(cachedCallbackHtml);
}
// `files` was historically a backend static-asset directory and was therefore
// in the exclusion list - removing it lets /files and /files/<folder-uuid>
// forward to the SPA index.html, which is what FileManagerView expects.
// (Real storage endpoints live under /api/v1/storage/files, already
// excluded by the leading `api` token in the same regex.)
@GetMapping(
"/{path:^(?!api|static|robots\\.txt|favicon\\.ico|manifest.*\\.json|pipeline|pdfjs|pdfjs-legacy|pdfium|vendor|fonts|images|css|js|assets|locales|modern-logo|classic-logo|Login|og_images|samples)[^\\.]*$}")
"/{path:^(?!api|static|robots\\.txt|favicon\\.ico|manifest.*\\.json|pipeline|pdfjs|pdfjs-legacy|pdfium|vendor|fonts|images|files|css|js|assets|locales|modern-logo|classic-logo|Login|og_images|samples)[^\\.]*$}")
public ResponseEntity<String> forwardRootPaths(HttpServletRequest request) throws IOException {
return serveIndexHtml(request);
}
@GetMapping(
"/{path:^(?!api|static|pipeline|pdfjs|pdfjs-legacy|pdfium|vendor|fonts|images|css|js|assets|locales|modern-logo|classic-logo|Login|og_images|samples)[^\\.]*}/{subpath:^(?!.*\\.).*$}")
"/{path:^(?!api|static|pipeline|pdfjs|pdfjs-legacy|pdfium|vendor|fonts|images|files|css|js|assets|locales|modern-logo|classic-logo|Login|og_images|samples)[^\\.]*}/{subpath:^(?!.*\\.).*$}")
public ResponseEntity<String> forwardNestedPaths(HttpServletRequest request)
throws IOException {
return serveIndexHtml(request);
@@ -22,7 +22,6 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.multipart.MaxUploadSizeExceededException;
import org.springframework.web.multipart.support.MissingServletRequestPartException;
import org.springframework.web.server.ResponseStatusException;
import org.springframework.web.servlet.NoHandlerFoundException;
import jakarta.servlet.http.HttpServletRequest;
@@ -197,12 +196,12 @@ public class GlobalExceptionHandler {
/**
* Checks whether the given IOException indicates that the client disconnected before the
* response could be written (broken pipe, connection reset, etc.). When this happens there is
* no point in serialising a {@link ProblemDetail} body because the socket is already closed -
* no point in serialising a {@link ProblemDetail} body because the socket is already closed
* and attempting to do so may trigger a secondary {@code HttpMessageNotWritableException} if
* the response Content-Type was already committed as a non-JSON type (e.g. image/png).
*/
private static boolean isClientDisconnectException(IOException ex) {
// Walk the causal chain - Jetty/Tomcat may wrap the low-level SocketException
// Walk the causal chain Jetty/Tomcat may wrap the low-level SocketException
Throwable current = ex;
while (current != null) {
String msg = current.getMessage();
@@ -1041,43 +1040,6 @@ public class GlobalExceptionHandler {
* @param request the HTTP servlet request
* @return ProblemDetail with appropriate HTTP status
*/
/**
* Handle ResponseStatusException explicitly so its embedded HTTP status reaches the client
* instead of being swallowed by the {@code RuntimeException} catch-all (which would downgrade
* every controller-thrown 400/404/409 to a generic 500). Folder/file storage controllers and
* any other code that throws {@code ResponseStatusException} relies on this handler taking
* precedence.
*/
@ExceptionHandler(ResponseStatusException.class)
public ResponseEntity<ProblemDetail> handleResponseStatusException(
ResponseStatusException ex, HttpServletRequest request) {
HttpStatus status =
HttpStatus.resolve(ex.getStatusCode().value()) != null
? HttpStatus.valueOf(ex.getStatusCode().value())
: HttpStatus.INTERNAL_SERVER_ERROR;
String reason = ex.getReason() != null ? ex.getReason() : status.getReasonPhrase();
ProblemDetail problemDetail = createBaseProblemDetail(status, reason, request);
problemDetail.setType(URI.create("/errors/" + status.value()));
problemDetail.setTitle(status.getReasonPhrase());
problemDetail.setProperty("title", status.getReasonPhrase());
// 5xx is operator-relevant; 4xx is a normal client-rejection - log at the right level.
if (status.is5xxServerError()) {
log.error(
"ResponseStatusException {} at {}: {}",
status.value(),
request.getRequestURI(),
reason,
ex);
} else {
log.debug(
"ResponseStatusException {} at {}: {}",
status.value(),
request.getRequestURI(),
reason);
}
return ResponseEntity.status(status).contentType(PROBLEM_JSON).body(problemDetail);
}
@ExceptionHandler(RuntimeException.class)
public ResponseEntity<ProblemDetail> handleRuntimeException(
RuntimeException ex, HttpServletRequest request) {
@@ -18,11 +18,4 @@ public class PDFWithPageSize extends PDFFile {
requiredMode = Schema.RequiredMode.REQUIRED,
allowableValues = {"A0", "A1", "A2", "A3", "A4", "A5", "A6", "LETTER", "LEGAL", "KEEP"})
private String pageSize;
@Schema(
description =
"Orientation to apply to the target page size. Ignored when pageSize is KEEP.",
defaultValue = "PORTRAIT",
allowableValues = {"PORTRAIT", "LANDSCAPE"})
private String orientation = "PORTRAIT";
}
@@ -13,9 +13,7 @@ import lombok.RequiredArgsConstructor;
import stirling.software.SPDF.config.swagger.MarkdownConversionResponse;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.ConvertApi;
import stirling.software.common.enumeration.ResourceWeight;
import stirling.software.common.model.api.PDFFile;
import stirling.software.common.service.PyMuPdfConverter;
import stirling.software.common.util.PDFToFile;
import stirling.software.common.util.TempFileManager;
@@ -24,12 +22,8 @@ import stirling.software.common.util.TempFileManager;
public class ConvertPDFToMarkdown {
private final TempFileManager tempFileManager;
private final PyMuPdfConverter pyMuPdfConverter;
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/pdf/markdown",
resourceWeight = ResourceWeight.MEDIUM_WEIGHT)
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/pdf/markdown")
@MarkdownConversionResponse
@Operation(
summary = "Convert PDF to Markdown",
@@ -39,6 +33,6 @@ public class ConvertPDFToMarkdown {
throws Exception {
MultipartFile inputFile = file.getFileInput();
PDFToFile pdfToFile = new PDFToFile(tempFileManager);
return pdfToFile.processPdfToMarkdown(inputFile, pyMuPdfConverter);
return pdfToFile.processPdfToMarkdown(inputFile);
}
}
@@ -4,6 +4,7 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
@@ -22,6 +23,10 @@ import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.cluster.JobStoreEntry;
import stirling.software.common.cluster.StickyMissRecorder;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.model.job.ResultFile;
import stirling.software.common.service.FileStorage;
@@ -42,10 +47,22 @@ public class JobController {
private final FileStorage fileStorage;
private final JobQueue jobQueue;
private final HttpServletRequest request;
private final ClusterBackplane clusterBackplane;
private final JobStore jobStore;
/**
* Process-local short-TTL cache fronting {@link JobStore#get(String)} on the sticky-410 path.
* Without this every result download / status poll fires a Valkey HGETALL which doubles RTT on
* the hot path when the same client re-requests the same job within seconds.
*/
private final JobOwnershipCache ownershipCache = new JobOwnershipCache();
@Autowired(required = false)
private JobOwnershipService jobOwnershipService;
@Autowired(required = false)
private StickyMissRecorder stickyMissRecorder;
/**
* Get the status of a job
*
@@ -55,6 +72,14 @@ public class JobController {
@GetMapping("/job/{jobId}")
@Operation(summary = "Get job status")
public ResponseEntity<?> getJobStatus(@PathVariable("jobId") String jobId) {
// Sticky-410 must precede user-auth (403): a non-owner node has no way to verify
// ownership for a job it doesn't own, and a 403 here would leak job existence to
// unauthorized callers. Return 410 first so the LB re-routes to the owner.
Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
if (peerOwned.isPresent()) {
return peerOwned.get();
}
// Validate job ownership
if (!validateJobAccess(jobId)) {
log.warn("Unauthorized attempt to access job status: {}", jobId);
@@ -91,6 +116,14 @@ public class JobController {
@GetMapping("/job/{jobId}/result")
@Operation(summary = "Get job result")
public ResponseEntity<?> getJobResult(@PathVariable("jobId") String jobId) {
// Sticky-410 must precede user-auth (403): a non-owner node has no way to verify
// ownership for a job it doesn't own, and a 403 here would leak job existence to
// unauthorized callers. Return 410 first so the LB re-routes to the owner.
Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
if (peerOwned.isPresent()) {
return peerOwned.get();
}
// Validate job ownership
if (!validateJobAccess(jobId)) {
log.warn("Unauthorized attempt to access job result: {}", jobId);
@@ -125,11 +158,14 @@ public class JobController {
result.getAllResultFiles()));
}
// Handle single file (download directly)
// Handle single file (download directly). Cross-node ownership was already resolved
// at the top of this method, so reaching here means we ARE the owner (or single-node)
// and the bytes live on our local disk.
if (result.hasFiles() && !result.hasMultipleFiles()) {
try {
List<ResultFile> files = result.getAllResultFiles();
ResultFile singleFile = files.get(0);
byte[] fileContent = fileStorage.retrieveBytes(singleFile.getFileId());
return ResponseEntity.ok()
.header("Content-Type", singleFile.getContentType())
@@ -163,6 +199,15 @@ public class JobController {
public ResponseEntity<?> cancelJob(@PathVariable("jobId") String jobId) {
log.debug("Request to cancel job: {}", jobId);
// Sticky-410 must precede user-auth (403): a non-owner node has no way to verify
// ownership for a job it doesn't own, and a 403 here would leak job existence to
// unauthorized callers. Return 410 first so the LB re-routes to the owner who can
// actually cancel.
Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
if (peerOwned.isPresent()) {
return peerOwned.get();
}
// Validate job ownership
if (!validateJobAccess(jobId)) {
log.warn("Unauthorized attempt to cancel job: {}", jobId);
@@ -201,7 +246,9 @@ public class JobController {
"queuePosition",
queuePosition >= 0 ? queuePosition : "n/a"));
} else {
// Job not found or already complete
// Job not found or already complete. Cross-node ownership was already resolved at
// the top of this method (sticky-410 precedes user-auth), so any peer-owned case
// has been returned already; reaching here means we ARE the owner (or single-node).
JobResult result = taskManager.getJobResult(jobId);
if (result == null) {
return ResponseEntity.notFound().build();
@@ -224,6 +271,14 @@ public class JobController {
@GetMapping("/job/{jobId}/result/files")
@Operation(summary = "Get job result files")
public ResponseEntity<?> getJobFiles(@PathVariable("jobId") String jobId) {
// Sticky-410 must precede user-auth (403): a non-owner node has no way to verify
// ownership for a job it doesn't own, and a 403 here would leak job existence to
// unauthorized callers. Return 410 first so the LB re-routes to the owner.
Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
if (peerOwned.isPresent()) {
return peerOwned.get();
}
// Validate job ownership
if (!validateJobAccess(jobId)) {
log.warn("Unauthorized attempt to access job files: {}", jobId);
@@ -267,6 +322,14 @@ public class JobController {
return ResponseEntity.notFound().build();
}
// Sticky-410 must precede user-auth (403): a non-owner node has no way to verify
// ownership for a job it doesn't own, and a 403 here would leak file existence to
// unauthorized callers. Return 410 first so the LB re-routes to the owner.
Optional<ResponseEntity<?>> notOwner = guardNonOwner(jobKey);
if (notOwner.isPresent()) {
return notOwner.get();
}
if (!validateJobAccess(jobKey)) {
log.warn("Unauthorized attempt to access file metadata: {}", fileId);
return ResponseEntity.status(403)
@@ -323,15 +386,21 @@ public class JobController {
return ResponseEntity.notFound().build();
}
// Sticky-410 must precede the user-auth (403) check: a non-owner node has no way to
// verify ownership for a job it doesn't own, and a 403 here would leak file existence
// to unauthorized callers. Return 410 first so the LB re-routes to the owner where
// the real auth check can run.
Optional<ResponseEntity<?>> notOwner = guardNonOwner(jobKey);
if (notOwner.isPresent()) {
return notOwner.get();
}
if (!validateJobAccess(jobKey)) {
log.warn("Unauthorized attempt to download file: {}", fileId);
return ResponseEntity.status(403)
.body(Map.of("message", "You are not authorized to access this file"));
}
// Retrieve file content
byte[] fileContent = fileStorage.retrieveBytes(fileId);
// Find the file metadata from any job that contains this file
// This is for getting the original filename and content type
ResultFile resultFile = taskManager.findResultFileByFileId(fileId);
@@ -342,6 +411,9 @@ public class JobController {
? resultFile.getContentType()
: MediaType.APPLICATION_OCTET_STREAM_VALUE;
// Retrieve file content from local disk
byte[] fileContent = fileStorage.retrieveBytes(fileId);
return ResponseEntity.ok()
.header("Content-Type", contentType)
.header("Content-Disposition", createContentDispositionHeader(fileName))
@@ -356,6 +428,76 @@ public class JobController {
return jobOwnershipService != null;
}
/**
* Returns {@code 410 Gone} with {@code {message, ownedBy, currentNode}} and {@code Retry-After:
* 0} when the job is owned by a peer node. Returns {@link Optional#empty()} when we are the
* owner, when cluster mode is off / JobStore has no entry, or when {@code owningNodeId} is
* blank (caller proceeds with its normal not-found / 200 path).
*
* <p>Wraps the {@link JobStore#get(String)} call in a short-TTL local cache and a defensive
* try/catch so that Valkey RTT cost is not multiplied by every download retry and so that a
* Valkey timeout falls through to the local-disk path instead of surfacing as 500.
*/
private Optional<ResponseEntity<?>> guardNonOwner(String jobId) {
if (clusterBackplane == null || jobStore == null) {
return Optional.empty();
}
Optional<JobStoreEntry> entry;
Optional<Optional<JobStoreEntry>> cached = ownershipCache.get(jobId);
if (cached.isPresent()) {
entry = cached.get();
} else {
try {
entry = jobStore.get(jobId);
} catch (RuntimeException ex) {
// Valkey unavailable / timeout: treat as "no cluster-visible entry" so the request
// can proceed to the local-disk path. Surfacing a 500 here would break every
// download attempt during a brief Valkey blip; the worst case if we miss a real
// peer-owned entry is one wasted round trip + a 404 from the local node.
log.warn(
"JobStore lookup failed for jobId={} - treating as not-found and falling"
+ " through to local path: {}",
jobId,
ex.getMessage());
return Optional.empty();
}
ownershipCache.put(jobId, entry);
}
if (entry.isEmpty()) {
return Optional.empty();
}
String owner = entry.get().owningNodeId();
if (owner == null || owner.isBlank()) {
return Optional.empty();
}
String localId = clusterBackplane.localNodeId();
if (owner.equals(localId)) {
return Optional.empty();
}
log.info(
"Sticky-session miss for jobId={} (owner={}, local={}); returning 410 so client"
+ " retries via LB affinity",
jobId,
owner,
localId);
if (stickyMissRecorder != null) {
stickyMissRecorder.recordStickyMiss();
}
return Optional.of(
ResponseEntity.status(410)
.header("Retry-After", "0")
.body(
Map.of(
"message",
"Result lives on another node. Retry to be routed there"
+ " by the load balancer's sticky-session"
+ " affinity, or re-run the job.",
"ownedBy",
owner,
"currentNode",
localId == null ? "" : localId)));
}
/**
* Create Content-Disposition header with UTF-8 filename support
*
@@ -0,0 +1,51 @@
package stirling.software.common.controller;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import stirling.software.common.cluster.JobStoreEntry;
/**
* Process-local TTL cache for {@link JobStoreEntry} lookups to suppress redundant Valkey HGETALL
* round-trips on the hot result-download path (sticky-410 ownership check).
*
* <p>5 second TTL is short enough that a job's lifecycle transitions (RUNNING -> COMPLETE -> TTL
* expiry) propagate to all nodes within the LB's sticky-session window, and short enough that a
* mistakenly-cached "not found" recovers quickly when an entry actually shows up. Cap the map at
* 2048 entries to bound memory; eviction is best-effort (clear-and-restart) since the cache is
* advisory.
*/
final class JobOwnershipCache {
private static final long TTL_NANOS = 5L * 1_000_000_000L; // 5 s
private static final int MAX_ENTRIES = 2048;
private final ConcurrentMap<String, Entry> entries = new ConcurrentHashMap<>();
Optional<Optional<JobStoreEntry>> get(String jobId) {
Entry e = entries.get(jobId);
if (e == null) {
return Optional.empty();
}
if (System.nanoTime() - e.storedAtNanos > TTL_NANOS) {
entries.remove(jobId, e);
return Optional.empty();
}
return Optional.of(e.value);
}
void put(String jobId, Optional<JobStoreEntry> value) {
if (entries.size() >= MAX_ENTRIES) {
// Best-effort eviction; under burst the cache simply rebuilds.
entries.clear();
}
entries.put(jobId, new Entry(value, System.nanoTime()));
}
void invalidate(String jobId) {
entries.remove(jobId);
}
private record Entry(Optional<JobStoreEntry> value, long storedAtNanos) {}
}
@@ -19,6 +19,10 @@ logging.level.stirling.software.common.service.TaskManager=INFO
spring.jpa.open-in-view=false
server.forward-headers-strategy=NATIVE
# Prevent Spring Boot auto-configuring a session repository from spring-session-data-redis being on
# the classpath. ClusterSessionConfiguration enables Redis-backed sessions when cluster.enabled=true.
spring.session.store-type=none
# Enable HTTP/2 for improved performance (multiplexed streams, header compression)
server.http2.enabled=true
@@ -33,7 +37,7 @@ spring.security.filter.dispatcher-types=REQUEST,ERROR
# Response compression
server.compression.enabled=true
server.compression.min-response-size=1024
server.compression.mime-types=application/json,application/xml,text/html,text/plain,text/css,application/javascript,image/svg+xml,application/x-font-ttf,font/opentype,application/vnd.ms-fontobject,font/woff,font/woff2,application/font-woff,application/font-woff2
server.compression.mime-types=application/json,application/xml,text/html,text/plain,text/css,application/javascript
spring.web.error.path=/error
spring.web.error.whitelabel.enabled=false
@@ -20,7 +20,6 @@ security:
password: "" # initial password for the first login
oauth2:
enabled: false # set to 'true' to enable login (Note: enableLogin must also be 'true' for this to work)
debugLogging: false # set to 'true' to log full ID token and UserInfo claims during OAuth2/OIDC login. Use this to diagnose claim issues (e.g. "Attribute value for 'email' cannot be null" with ADFS). WARNING: writes PII (sub, email, name) to logs; disable after troubleshooting.
client:
keycloak:
issuer: "" # URL of the Keycloak realm's OpenID Connect Discovery endpoint
@@ -94,8 +93,8 @@ premium:
key: 00000000-0000-0000-0000-000000000000
enabled: false # Enable license key checks for pro/enterprise features
proFeatures:
ssoAutoLogin: false
customMetadata:
SSOAutoLogin: false
CustomMetadata:
autoUpdateMetadata: false
author: username
creator: Stirling-PDF
@@ -246,39 +245,6 @@ storage:
provider: local # storage provider: 'local' for filesystem storage, 'database' for DB-backed storage
local:
basePath: './storage' # base directory for stored files
# ====================================================================================
# S3-COMPATIBLE OBJECT STORAGE - PRO / ENTERPRISE LICENSE REQUIRED
# storage.provider=s3, storage.provider=database, and cluster.artifactStore=s3 all
# require a valid Pro or Enterprise license.
# ====================================================================================
# Used when provider=s3 (persistent user uploads) and/or cluster.artifactStore=s3
# (transient cluster artifacts). The two consumers share this block.
# Vendor cheat sheet (set the highlighted flags to taste):
# AWS S3 -> endpoint='' region='<your-region>' pathStyleAccess=false
# Cloudflare R2 -> endpoint='https://<acct>.r2.cloudflarestorage.com' region='auto'
# pathStyleAccess=false; if uploads fail with 'unsupported header
# x-amz-checksum-*' set requestChecksumCalculation=WHEN_REQUIRED
# Supabase Storage -> endpoint='https://<project>.supabase.co/storage/v1/s3'
# region='<project-region>' pathStyleAccess=true
# (filenames with non-ASCII display fine - the storage key is opaque)
# MinIO (in-cluster) -> endpoint='http://minio:9000' region='us-east-1'
# pathStyleAccess=true allowPrivateEndpoints=true
# Backblaze B2 -> endpoint='https://s3.<region>.backblazeb2.com'
# If on a B2 deployment older than July-2025 and uploads return
# 'Unsupported header x-amz-checksum-crc32', set
# requestChecksumCalculation=WHEN_REQUIRED
# DigitalOcean Spaces -> endpoint='https://<region>.digitaloceanspaces.com'
# Note: 5GB per-object cap (regardless of multipart)
s3:
endpoint: "" # blank = use AWS regional default; otherwise full URL incl. https://
bucket: "" # required when provider=s3 or cluster.artifactStore=s3
region: us-east-1
accessKey: "" # blank = fall back to AWS DefaultCredentialsProvider (env / profile / IMDS)
secretKey: ""
pathStyleAccess: false # true for MinIO and Supabase; false for AWS/R2/most CDNs
allowPrivateEndpoints: false # true required when endpoint resolves to a private/loopback IP (e.g. in-cluster MinIO). SSRF guard - leave false for any internet-facing vendor.
requestChecksumCalculation: WHEN_SUPPORTED # WHEN_SUPPORTED|WHEN_REQUIRED|DISABLED. Set WHEN_REQUIRED if your vendor rejects auto-added x-amz-checksum-* headers (older Backblaze B2, some R2 corner cases).
responseChecksumValidation: WHEN_SUPPORTED # WHEN_SUPPORTED|WHEN_REQUIRED|DISABLED. Set WHEN_REQUIRED if you see false-positive checksum-mismatch errors on GET from a vendor that never returns checksum headers.
quotas:
maxStorageMbPerUser: -1 # Max storage per user in MB; -1 disables per-user cap
maxStorageMbTotal: -1 # Max storage across all users in MB; -1 disables total cap
@@ -369,8 +335,6 @@ cluster:
enabled: false # Master switch. 'false' (default) wires the in-process backplane and skips all cluster checks. Single-instance installs do not need to change anything here.
backplane: inprocess # Backplane implementation: 'inprocess' (single JVM only) or 'valkey' (multi-node via Valkey/Redis)
artifactStore: local # Transient cluster job-artifact backend: 'local' (per-node disk; single-node only) or 's3' (shared object store; required for multi-node). Distinct from 'storage.provider' which controls persistent user uploads - when both are 's3' they share the storage.s3.* credentials block. Multi-node deployments MUST set this to 's3'.
s3:
keyPrefix: transient/ # Bucket key prefix used by the cluster artifact store when artifactStore=s3. Trailing slash recommended. Lets a single bucket host both persistent uploads (storage.s3.*) and transient job artifacts under separate prefixes.
valkey:
url: "" # Valkey/Redis URL, e.g. 'redis://valkey:6379' or 'rediss://...' for TLS. Required when enabled=true and backplane=valkey.
tls:
@@ -1,132 +0,0 @@
package stirling.software.SPDF.config;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.core.io.support.ResourcePatternResolver;
import org.springframework.core.type.classreading.CachingMetadataReaderFactory;
import org.springframework.core.type.classreading.MetadataReader;
import org.springframework.core.type.classreading.MetadataReaderFactory;
import org.springframework.core.type.filter.TypeFilter;
import stirling.software.common.annotations.AutoJobPostMapping;
/**
* Build-time guardrail: every {@link AutoJobPostMapping} method must declare an explicit {@code
* resourceWeight}.
*
* <p>The credits interceptor multiplies {@code resourceWeight} into the per-call charge. An
* endpoint that falls through to the annotation default produces a charge derived from a value
* nobody chose silently under- or over-billing depending on the endpoint's true cost. Forcing
* each method to pick a value from {@link stirling.software.common.enumeration.ResourceWeight}
* keeps the choice deliberate.
*
* <p>The annotation's default is {@link Integer#MIN_VALUE} (a sentinel). Runtime readers clamp the
* value into {@code [1, 100]}, so a missed declaration can't crash production this test is the
* contract, the clamp is the safety net.
*
* <p>Lives in {@code :stirling-pdf} (core) because that's the module whose compile classpath
* transitively sees every other module's controllers ({@code :common}, {@code :proprietary}, and
* {@code :saas} when enabled).
*/
class AutoJobPostMappingWeightTest {
private static final String SCAN_BASE_PACKAGE = "stirling.software";
@Test
void everyAutoJobPostMappingDeclaresExplicitResourceWeight() throws Exception {
List<String> offenders = findOffendingMethods();
assertTrue(
offenders.isEmpty(),
() ->
"The following @AutoJobPostMapping methods do not declare an explicit"
+ " resourceWeight. Pick a value from"
+ " stirling.software.common.enumeration.ResourceWeight (SMALL,"
+ " MEDIUM, LARGE, XLARGE) and add it to the annotation:\n - "
+ String.join("\n - ", offenders));
}
private List<String> findOffendingMethods() throws IOException, ClassNotFoundException {
List<String> offenders = new ArrayList<>();
for (Class<?> candidate : scanForCandidateClasses()) {
for (Method method : candidate.getDeclaredMethods()) {
AutoJobPostMapping annotation = method.getAnnotation(AutoJobPostMapping.class);
if (annotation == null) {
continue;
}
if (annotation.resourceWeight() == Integer.MIN_VALUE) {
offenders.add(candidate.getName() + "#" + method.getName());
}
}
}
return offenders;
}
/**
* Returns every class under {@link #SCAN_BASE_PACKAGE} that has an @AutoJobPostMapping method.
*/
private List<Class<?>> scanForCandidateClasses() throws IOException, ClassNotFoundException {
ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
MetadataReaderFactory metadataReaderFactory = new CachingMetadataReaderFactory(resolver);
String pattern = "classpath*:" + SCAN_BASE_PACKAGE.replace('.', '/') + "/**/*.class";
Resource[] resources = resolver.getResources(pattern);
// Pre-filter by reading annotation metadata from the class file so we don't have to load
// every class on the test classpath just to find the few that are annotated.
TypeFilter mentionsAutoJobPostMapping =
(reader, factory) ->
reader.getAnnotationMetadata()
.getAnnotatedMethods(AutoJobPostMapping.class.getName())
.size()
> 0;
List<Class<?>> matches = new ArrayList<>();
for (Resource resource : resources) {
if (!resource.isReadable()) {
continue;
}
MetadataReader reader = metadataReaderFactory.getMetadataReader(resource);
if (!mentionsAutoJobPostMapping.match(reader, metadataReaderFactory)) {
continue;
}
matches.add(Class.forName(reader.getClassMetadata().getClassName()));
}
return matches;
}
/**
* Sanity check that the classpath scan returns non-empty; otherwise the main test passes
* vacuously.
*/
@Test
void scannerFindsAtLeastOneAutoJobPostMapping() throws Exception {
long count =
scanForCandidateClasses().stream()
.flatMap(c -> java.util.Arrays.stream(c.getDeclaredMethods()))
.filter(m -> m.isAnnotationPresent(AutoJobPostMapping.class))
.count();
assertTrue(
count > 10,
() ->
"Expected the classpath scan to find many @AutoJobPostMapping methods but"
+ " found only "
+ count
+ ". Scanner regression?");
}
@SuppressWarnings("unused")
private static String describeCandidates(List<Class<?>> candidates) {
return candidates.stream().map(Class::getName).collect(Collectors.joining(", "));
}
}
@@ -237,8 +237,7 @@ class ScalePagesControllerTest {
ScalePagesRequest request = new ScalePagesRequest();
request.setFileInput(file);
request.setPageSize("A4");
request.setOrientation("LANDSCAPE");
request.setPageSize("A4_LANDSCAPE");
request.setScaleFactor(1.0f);
setupFactory();
@@ -15,14 +15,10 @@ import org.springframework.context.ApplicationContext;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.SPDF.config.EndpointConfiguration;
import stirling.software.SPDF.config.EndpointConfiguration.DisableReason;
import stirling.software.SPDF.config.EndpointConfiguration.EndpointAvailability;
import stirling.software.common.configuration.AppConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.ApplicationProperties.System;
import stirling.software.common.service.LicenseServiceInterface;
import stirling.software.common.service.ServerCertificateServiceInterface;
import stirling.software.common.service.UserServiceInterface;
@@ -177,119 +173,4 @@ class ConfigControllerTest {
assertEquals(HttpStatus.OK, response.getStatusCode());
verify(endpointConfiguration).getAllEndpoints();
}
@Test
void resolveFrontendUrl_prefersExplicitConfiguredValue() {
System sys = mock(System.class);
when(applicationProperties.getSystem()).thenReturn(sys);
when(sys.getFrontendUrl()).thenReturn("https://pdf.example.com");
// Request would say something else, but configured wins.
HttpServletRequest req = mock(HttpServletRequest.class);
AppConfig appConfig = mock(AppConfig.class);
assertEquals(
"https://pdf.example.com", configController.resolveFrontendUrl(req, appConfig));
}
@Test
void resolveFrontendUrl_usesRequestHostWhenNotConfigured() {
System sys = mock(System.class);
when(applicationProperties.getSystem()).thenReturn(sys);
when(sys.getFrontendUrl()).thenReturn(null);
HttpServletRequest req = mock(HttpServletRequest.class);
when(req.getServerName()).thenReturn("192.168.1.100");
when(req.getScheme()).thenReturn("http");
when(req.getServerPort()).thenReturn(8080);
assertEquals(
"http://192.168.1.100:8080",
configController.resolveFrontendUrl(req, mock(AppConfig.class)));
}
@Test
void resolveFrontendUrl_elidesDefaultHttpsPort() {
System sys = mock(System.class);
when(applicationProperties.getSystem()).thenReturn(sys);
when(sys.getFrontendUrl()).thenReturn("");
HttpServletRequest req = mock(HttpServletRequest.class);
when(req.getServerName()).thenReturn("pdf.example.com");
when(req.getScheme()).thenReturn("https");
when(req.getServerPort()).thenReturn(443);
assertEquals(
"https://pdf.example.com",
configController.resolveFrontendUrl(req, mock(AppConfig.class)));
}
@Test
void resolveFrontendUrl_fallsThroughOnLoopbackHost() {
System sys = mock(System.class);
when(applicationProperties.getSystem()).thenReturn(sys);
when(sys.getFrontendUrl()).thenReturn(null);
HttpServletRequest req = mock(HttpServletRequest.class);
when(req.getServerName()).thenReturn("localhost");
AppConfig appConfig = mock(AppConfig.class);
when(appConfig.getBackendUrl()).thenReturn("http://localhost:8080");
when(appConfig.getServerPort()).thenReturn("8080");
// Detected IP (if any) wins over loopback request host. We can't assert the
// exact value (depends on the host running the test) but we can assert it
// never returns "localhost".
String result = configController.resolveFrontendUrl(req, appConfig);
assertNotNull(result);
assertFalse(result.contains("localhost"));
}
@Test
void resolveFrontendUrl_usesActualPortWhenServerPortIsEphemeral() {
System sys = mock(System.class);
when(applicationProperties.getSystem()).thenReturn(sys);
when(sys.getFrontendUrl()).thenReturn(null);
// Loopback host forces the detected-LAN-IP branch, which is where an
// ephemeral server.port=0 would otherwise leak through as ":0".
HttpServletRequest req = mock(HttpServletRequest.class);
when(req.getServerName()).thenReturn("localhost");
AppConfig appConfig = mock(AppConfig.class);
when(appConfig.getBackendUrl()).thenReturn("http://localhost");
when(appConfig.getServerPort()).thenReturn("0");
org.springframework.core.env.Environment environment =
mock(org.springframework.core.env.Environment.class);
when(applicationContext.getEnvironment()).thenReturn(environment);
when(environment.getProperty("local.server.port")).thenReturn("54321");
String result = configController.resolveFrontendUrl(req, appConfig);
assertNotNull(result);
assertTrue(result.endsWith(":54321"));
assertFalse(result.contains(":0"));
}
@Test
void resolveEffectiveServerPort_prefersActualBoundPortWhenConfiguredZero() {
AppConfig appConfig = mock(AppConfig.class);
when(appConfig.getServerPort()).thenReturn("0");
org.springframework.core.env.Environment environment =
mock(org.springframework.core.env.Environment.class);
when(applicationContext.getEnvironment()).thenReturn(environment);
when(environment.getProperty("local.server.port")).thenReturn("54321");
assertEquals("54321", configController.resolveEffectiveServerPort(appConfig));
}
@Test
void resolveEffectiveServerPort_keepsConfiguredNonZeroPort() {
AppConfig appConfig = mock(AppConfig.class);
when(appConfig.getServerPort()).thenReturn("8080");
// Non-zero configured port is authoritative; the runtime env is never consulted.
assertEquals("8080", configController.resolveEffectiveServerPort(appConfig));
}
}
@@ -2,7 +2,6 @@ package stirling.software.SPDF.model.api.converters;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
@@ -24,13 +23,12 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.common.service.PyMuPdfConverter;
import stirling.software.common.util.PDFToFile;
class ConvertPDFToMarkdownTest {
private MockMvc mockMvc() {
return MockMvcBuilders.standaloneSetup(new ConvertPDFToMarkdown(null, null))
return MockMvcBuilders.standaloneSetup(new ConvertPDFToMarkdown(null))
.setControllerAdvice(new GlobalErrorHandler())
.build();
}
@@ -54,9 +52,7 @@ class ConvertPDFToMarkdownTest {
Mockito.mockConstruction(
PDFToFile.class,
(mock, ctx) -> {
when(mock.processPdfToMarkdown(
any(MultipartFile.class),
nullable(PyMuPdfConverter.class)))
when(mock.processPdfToMarkdown(any(MultipartFile.class)))
.thenAnswer(
inv ->
ResponseEntity.ok()
@@ -87,8 +83,7 @@ class ConvertPDFToMarkdownTest {
// And that the uploaded file was passed to processPdfToMarkdown()
PDFToFile created = construction.constructed().get(0);
ArgumentCaptor<MultipartFile> captor = ArgumentCaptor.forClass(MultipartFile.class);
verify(created, times(1))
.processPdfToMarkdown(captor.capture(), nullable(PyMuPdfConverter.class));
verify(created, times(1)).processPdfToMarkdown(captor.capture());
MultipartFile passed = captor.getValue();
// Minimal plausibility checks
@@ -103,9 +98,7 @@ class ConvertPDFToMarkdownTest {
Mockito.mockConstruction(
PDFToFile.class,
(mock, ctx) -> {
when(mock.processPdfToMarkdown(
any(MultipartFile.class),
nullable(PyMuPdfConverter.class)))
when(mock.processPdfToMarkdown(any(MultipartFile.class)))
.thenThrow(new RuntimeException("boom"));
})) {
@@ -1,93 +0,0 @@
package stirling.software.common.configuration;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.mockStatic;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.MockedStatic;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.YamlHelper;
/**
* End-to-end check of the container-restart path. {@link ConfigInitializer#ensureConfigExists()} is
* what runs on every startup, merging the on-disk settings.yml with the bundled
* settings.yml.template. These tests exercise it against the real template on the classpath to
* prove admin-saved proFeatures values survive a restart - the bug behind "the SSO auto-login
* button resets every time the container resets".
*/
class ConfigInitializerRestartTest {
private static String read(Path settings, String... keyPath) throws IOException {
return String.valueOf(new YamlHelper(settings).getValueByExactKeyPath(keyPath));
}
@Test
void ssoAutoLoginAndCustomMetadata_persistAcrossRestart(@TempDir Path tmp) throws Exception {
Path settings = tmp.resolve("settings.yml");
Path custom = tmp.resolve("custom_settings.yml");
try (MockedStatic<InstallationPathConfig> paths =
mockStatic(InstallationPathConfig.class)) {
paths.when(InstallationPathConfig::getSettingsPath).thenReturn(settings.toString());
paths.when(InstallationPathConfig::getCustomSettingsPath).thenReturn(custom.toString());
ConfigInitializer init = new ConfigInitializer();
// First boot: settings.yml created from the bundled template (camelCase, default off).
init.ensureConfigExists();
assertEquals("false", read(settings, "premium", "proFeatures", "ssoAutoLogin"));
// Admin enables SSO auto-login and edits custom metadata via the exact save path the
// admin settings controller uses.
GeneralUtils.saveKeyToSettings("premium.proFeatures.ssoAutoLogin", true);
GeneralUtils.saveKeyToSettings("premium.proFeatures.customMetadata.author", "acme");
// Container restart: ensureConfigExists merges the saved file with the template again.
init.ensureConfigExists();
assertEquals("true", read(settings, "premium", "proFeatures", "ssoAutoLogin"));
assertEquals(
"acme", read(settings, "premium", "proFeatures", "customMetadata", "author"));
}
}
@Test
void legacyPascalCaseConfig_isMigratedAndPreservedOnRestart(@TempDir Path tmp)
throws Exception {
Path settings = tmp.resolve("settings.yml");
Path custom = tmp.resolve("custom_settings.yml");
try (MockedStatic<InstallationPathConfig> paths =
mockStatic(InstallationPathConfig.class)) {
paths.when(InstallationPathConfig::getSettingsPath).thenReturn(settings.toString());
paths.when(InstallationPathConfig::getCustomSettingsPath).thenReturn(custom.toString());
ConfigInitializer init = new ConfigInitializer();
// Seed a full settings.yml as an OLD install would have written it: PascalCase keys
// with
// SSO auto-login enabled.
init.ensureConfigExists();
String legacy =
Files.readString(settings)
.replace("ssoAutoLogin: false", "SSOAutoLogin: true")
.replace("customMetadata:", "CustomMetadata:");
Files.writeString(settings, legacy);
// Upgrade restart.
init.ensureConfigExists();
// Value carried forward onto the new camelCase key; the legacy PascalCase key is gone.
assertEquals("true", read(settings, "premium", "proFeatures", "ssoAutoLogin"));
assertNull(
new YamlHelper(settings)
.getValueByExactKeyPath("premium", "proFeatures", "SSOAutoLogin"));
}
}
}
@@ -0,0 +1,502 @@
package stirling.software.common.controller;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.util.ReflectionTestUtils;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.cluster.JobStoreEntry;
import stirling.software.common.cluster.StickyMissRecorder;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.JobOwnershipService;
import stirling.software.common.service.JobQueue;
import stirling.software.common.service.TaskManager;
/**
* Sticky-session ownership behavior for {@link JobController}.
*
* <p>Result PDFs live on the local disk of whichever node ran the job. When the load balancer's
* cookie/IP affinity *misses* and routes a download to a non-owner node, the controller must return
* {@code 410 Gone} with a structured payload that tells the client to retry (the LB will usually
* route them to the owner on the second attempt).
*
* <p>Contract verified here:
*
* <ul>
* <li>Owner == this node file is read from local disk (normal 200).
* <li>Owner == another node {@code 410 Gone} with {@code ownedBy} + {@code currentNode} fields,
* and {@code FileStorage} is <b>never</b> touched.
* <li>JobStore has no entry behave as single-instance (no 410).
* <li>{@code owningNodeId} blank behave as single-instance (no 410).
* <li>Single-instance install (no JobStore / no ClusterBackplane) no 410, no NPE.
* </ul>
*
* <p>Manual mock construction (no {@code MockitoExtension}) so each test can wire its own
* controller with a different {@code ClusterBackplane} / {@code JobStore} combo without setUp stubs
* leaking across cases.
*/
class JobControllerOwnershipTest {
private TaskManager taskManager;
private FileStorage fileStorage;
private JobQueue jobQueue;
private HttpServletRequest request;
private JobOwnershipService jobOwnershipService;
private ClusterBackplane clusterBackplane;
private JobStore jobStore;
private StickyMissRecorder stickyMissRecorder;
private static final String JOB_ID = "job-42";
private static final String FILE_ID = "file-abc";
private static final String LOCAL_NODE = "node-self";
private static final String PEER_NODE = "node-peer";
@BeforeEach
void setUp() {
taskManager = mock(TaskManager.class);
fileStorage = mock(FileStorage.class);
jobQueue = mock(JobQueue.class);
request = mock(HttpServletRequest.class);
jobOwnershipService = mock(JobOwnershipService.class);
clusterBackplane = mock(ClusterBackplane.class);
jobStore = mock(JobStore.class);
stickyMissRecorder = mock(StickyMissRecorder.class);
}
private JobController makeController(ClusterBackplane backplane, JobStore store) {
JobController c =
new JobController(taskManager, fileStorage, jobQueue, request, backplane, store);
// jobOwnershipService is @Autowired(required=false) - field-injected. When non-null,
// validateJobAccess delegates to it. We leave it null by default so the security
// check is a no-op (backwards compat path) and the test focuses on sticky-410.
// stickyMissRecorder is also field-injected; wire by default so the metric assertions
// work without per-test setup.
ReflectionTestUtils.setField(c, "stickyMissRecorder", stickyMissRecorder);
return c;
}
private JobController makeController() {
return makeController(clusterBackplane, jobStore);
}
private JobStoreEntry entryOwnedBy(String ownerNodeId) {
return new JobStoreEntry(
JOB_ID,
JobStoreEntry.JobState.COMPLETE,
ownerNodeId,
Instant.now(),
Instant.now(),
null,
List.of(FILE_ID),
Map.of());
}
private JobResult completedJobWithFile() {
JobResult result = new JobResult();
result.setJobId(JOB_ID);
// completeWithSingleFile populates the resultFiles list, sets complete=true,
// and sets completedAt - all required for the getJobResult single-file branch.
result.completeWithSingleFile(FILE_ID, "out.pdf", "application/pdf", 7L);
return result;
}
/**
* Full sticky-410 contract for {@code downloadFile} when the requested job is owned by a peer.
* Asserts everything in one place (status, Retry-After header, payload shape, no
* implementation-detail leak, metric incremented, storage never touched).
*
* <p>Other tests cover the edge cases independently (locally-owned, no JobStore entry, blank
* owner, etc.) so a single failure here points at exactly one missing or broken contract
* property.
*/
@Test
@DisplayName(
"downloadFile peer-owned → full sticky-410 contract"
+ " (status + Retry-After + payload + metric + storage untouched)")
void downloadFile_peerOwned_fullStickyContract() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
// 1. Status + Retry-After header (immediate-retry hint).
assertEquals(HttpStatus.GONE, response.getStatusCode());
assertEquals("0", response.getHeaders().getFirst("Retry-After"));
// 2. Payload shape: exactly { message, ownedBy, currentNode }, with no leaked secrets.
assertInstanceOf(Map.class, response.getBody());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(3, body.size(), "exactly: message, ownedBy, currentNode");
assertEquals(PEER_NODE, body.get("ownedBy"));
assertEquals(LOCAL_NODE, body.get("currentNode"));
assertNotNull(body.get("message"));
assertTrue(((String) body.get("message")).toLowerCase().contains("retry"));
assertNull(body.get("internalSecret"));
assertNull(body.get("filePath"));
// 3. Operator-alert metric incremented exactly once.
verify(stickyMissRecorder).recordStickyMiss();
// 4. Storage layer NEVER touched - bytes don't live here, so reading would be wrong.
verify(fileStorage, never()).retrieveBytes(FILE_ID);
}
// --------------------------------------------------------------------------------------
// Happy-path ownership matrix: any non-peer signal (local owner, no entry, blank owner)
// must produce a 200 from FileStorage with NO sticky-miss metric increment.
// --------------------------------------------------------------------------------------
private static Stream<Arguments> downloadHappyPathScenarios() {
return Stream.of(
Arguments.of("locallyOwned", LOCAL_NODE, true),
Arguments.of("noJobStoreEntry", null, false),
Arguments.of("blankOwningNodeId", "", true));
}
@ParameterizedTest(name = "downloadFile {0} -> 200, no sticky-miss")
@MethodSource("downloadHappyPathScenarios")
void downloadFile_happyPath_returnsOkAndNoMetric(
String scenario, String ownerNodeId, boolean entryPresent) throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID))
.thenReturn(
entryPresent ? Optional.of(entryOwnedBy(ownerNodeId)) : Optional.empty());
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode(), scenario);
verify(fileStorage).retrieveBytes(FILE_ID);
verify(stickyMissRecorder, never()).recordStickyMiss();
}
@Test
@DisplayName("getJobResult: locally-owned single-file result → reads from FileStorage, 200 OK")
void getJobResult_singleFile_locallyOwned_readsFromStorage() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController().getJobResult(JOB_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
}
// --------------------------------------------------------------------------------------
// Peer-owned 410 matrix across endpoints. Cross-cutting contract:
// - status = 410, Retry-After = 0
// - body.ownedBy = peer node, body.currentNode = local node
// - sticky-miss metric incremented exactly once per request
// The contract test above covers ALL of these properties for downloadFile in detail; this
// parameterized matrix asserts the same status + ownedBy + metric signals on every endpoint.
// --------------------------------------------------------------------------------------
private enum Endpoint {
DOWNLOAD_FILE,
GET_JOB_RESULT,
GET_JOB_STATUS,
CANCEL_JOB
}
private static Stream<Arguments> peerOwned410Scenarios() {
return Stream.of(
Arguments.of(Endpoint.DOWNLOAD_FILE),
Arguments.of(Endpoint.GET_JOB_RESULT),
Arguments.of(Endpoint.GET_JOB_STATUS),
Arguments.of(Endpoint.CANCEL_JOB));
}
@ParameterizedTest(name = "{0} peer-owned -> 410, ownedBy=peer, metric++")
@MethodSource("peerOwned410Scenarios")
void endpoint_peerOwned_returns410(Endpoint endpoint) throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
// Endpoint-specific wiring: the per-endpoint code path needs different mock setup
// before it reaches the sticky-410 guard.
switch (endpoint) {
case DOWNLOAD_FILE -> when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
case GET_JOB_RESULT ->
when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
case GET_JOB_STATUS -> when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
case CANCEL_JOB -> {
when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
}
}
ResponseEntity<?> response =
switch (endpoint) {
case DOWNLOAD_FILE -> makeController().downloadFile(FILE_ID);
case GET_JOB_RESULT -> makeController().getJobResult(JOB_ID);
case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID);
case CANCEL_JOB -> makeController().cancelJob(JOB_ID);
};
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(PEER_NODE, body.get("ownedBy"));
assertEquals(LOCAL_NODE, body.get("currentNode"));
verify(stickyMissRecorder).recordStickyMiss();
// Storage / mutation must never be touched by a peer-routed request.
verify(fileStorage, never()).retrieveBytes(FILE_ID);
if (endpoint == Endpoint.CANCEL_JOB) {
verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user");
}
}
// --------------------------------------------------------------------------------------
// Unknown-job 404 matrix: when neither TaskManager nor JobStore knows the jobId, the
// controller must return 404 (not 410) and must NOT count it as a sticky miss.
// --------------------------------------------------------------------------------------
private static Stream<Arguments> unknownJob404Scenarios() {
return Stream.of(Arguments.of(Endpoint.GET_JOB_STATUS), Arguments.of(Endpoint.CANCEL_JOB));
}
@ParameterizedTest(name = "{0} unknown jobId -> 404 (not 410), no metric")
@MethodSource("unknownJob404Scenarios")
void endpoint_unknownJob_returns404(Endpoint endpoint) {
when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
when(jobStore.get(JOB_ID)).thenReturn(Optional.empty());
if (endpoint == Endpoint.CANCEL_JOB) {
when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
}
ResponseEntity<?> response =
switch (endpoint) {
case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID);
case CANCEL_JOB -> makeController().cancelJob(JOB_ID);
default -> throw new IllegalArgumentException(endpoint.name());
};
assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode());
verify(stickyMissRecorder, never()).recordStickyMiss();
}
// --------------------------------------------------------------------------------------
// Single-instance / null-bean wiring: no NPE, no 410, no metric. These test SPECIFIC
// wiring permutations and so stay as discrete tests rather than rows.
// --------------------------------------------------------------------------------------
@Test
@DisplayName("Single-instance install (no ClusterBackplane bean): no 410, no NPE")
void singleInstance_noClusterBackplane_noGoneResponse() throws Exception {
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController(null, jobStore).downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
verify(fileStorage).retrieveBytes(FILE_ID);
}
@Test
@DisplayName("Single-instance install (no JobStore bean): no 410, no NPE")
void singleInstance_noJobStore_noGoneResponse() throws Exception {
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController(clusterBackplane, null).downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
verify(fileStorage).retrieveBytes(FILE_ID);
}
@Test
@DisplayName("Single-instance (no StickyMissRecorder bean) → no NPE, still 200 OK")
void noStickyMissRecorder_works() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
JobController c = makeController();
ReflectionTestUtils.setField(c, "stickyMissRecorder", null);
ResponseEntity<?> response = c.downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
}
@Test
@DisplayName(
"cluster-mode but localNodeId is null → no NPE; 410 because owner is set and"
+ " differs from blank")
void clusterBackplanePresent_butLocalNodeIdNull_falsBackGracefully() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(null);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
// We still 410: owner is "node-peer", local is null they don't match. Rather than
// silently 200-from-wrong-disk (which would serve garbage), we surface the mismatch.
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals("", body.get("currentNode"), "blank when localNodeId is null");
assertEquals(PEER_NODE, body.get("ownedBy"));
}
// --------------------------------------------------------------------------------------
// Ownership-service interaction: sticky-410 must take precedence over per-user auth so
// we never 403 on a peer-owned resource (which would leak existence + defeat the redirect).
// --------------------------------------------------------------------------------------
@Test
@DisplayName("Owner returns 410 even when JobOwnershipService allows access (orthogonal)")
void ownershipService_passes_butStickyStillReturns410() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(true);
JobController c = makeController();
ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
ResponseEntity<?> response = c.downloadFile(FILE_ID);
// OwnershipService is about *user* auth; sticky-410 is about *node* topology.
// Both must pass for a 200, and node-ownership is checked first so a non-owner
// never even runs the user-auth check.
assertEquals(HttpStatus.GONE, response.getStatusCode());
}
@Test
@DisplayName(
"downloadFile: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak"
+ " file existence")
void downloadFile_peerOwned_ownershipDenied_returns410NotForbidden() throws Exception {
// Guard ordering: sticky-410 must run before user-auth. If user-auth ran first, a
// peer-owned-file request on the wrong node would fail user-auth (this node cannot
// verify access to a job it does not own) and return 403, which leaks file existence
// AND defeats the sticky-410 design (the frontend can't retry-with-affinity off a 403).
// Guard first so the user is redirected to the owner where the real auth check happens.
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
JobController c = makeController();
ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
ResponseEntity<?> response = c.downloadFile(FILE_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(PEER_NODE, body.get("ownedBy"));
// Crucially: never reached fileStorage, never returned 403.
verify(fileStorage, never()).retrieveBytes(FILE_ID);
}
@Test
@DisplayName(
"getJobStatus: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak"
+ " job existence")
void getJobStatus_peerOwned_ownershipDenied_returns410NotForbidden() {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
JobController c = makeController();
ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
ResponseEntity<?> response = c.getJobStatus(JOB_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(PEER_NODE, body.get("ownedBy"));
}
@Test
@DisplayName(
"cancelJob: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak job"
+ " existence")
void cancelJob_peerOwned_ownershipDenied_returns410NotForbidden() {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
JobController c = makeController();
ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
ResponseEntity<?> response = c.cancelJob(JOB_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(PEER_NODE, body.get("ownedBy"));
verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user");
}
// --------------------------------------------------------------------------------------
// S2: JobStore lookup hardening (local cache + Valkey-fault graceful-degrade)
// --------------------------------------------------------------------------------------
@Test
@DisplayName(
"guardNonOwner caches JobStore.get within TTL window: second call same jobId hits"
+ " cache, not Valkey")
void guardNonOwner_cachesJobStoreLookupWithinTtl() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
JobController c = makeController();
c.downloadFile(FILE_ID);
c.downloadFile(FILE_ID);
c.downloadFile(FILE_ID);
// Three downloads of the same fileId == one HGETALL. Without the cache this would have
// been three Valkey round-trips on the hot download path.
verify(jobStore, times(1)).get(JOB_ID);
}
@Test
@DisplayName(
"guardNonOwner: JobStore.get throws (Valkey timeout) → falls through to local-disk"
+ " path, no 500 leaks to caller")
void guardNonOwner_jobStoreException_fallsThroughToLocalPath() throws Exception {
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
// Simulate a Valkey timeout. spring-data-redis surfaces these as runtime exceptions
// wrapping Lettuce errors; the controller must not let any RuntimeException leak out.
when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
// The download must succeed via the local-disk path. A brief Valkey blip cannot break
// every download attempt with 500 - we cleanly degrade to single-node behavior.
assertEquals(HttpStatus.OK, response.getStatusCode());
verify(fileStorage).retrieveBytes(FILE_ID);
// The exception did NOT count as a sticky-miss (it wasn't one; we just couldn't see).
verify(stickyMissRecorder, never()).recordStickyMiss();
}
}
@@ -19,6 +19,8 @@ import org.springframework.test.util.ReflectionTestUtils;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.JobOwnershipService;
@@ -37,6 +39,10 @@ class JobControllerTest {
@Mock private JobOwnershipService jobOwnershipService;
@Mock private ClusterBackplane clusterBackplane;
@Mock private JobStore jobStore;
private MockHttpSession session;
@InjectMocks private JobController controller;
-2
View File
@@ -123,8 +123,6 @@ SwaggerDoc.json
*.tar.gz
*.rar
*.db
# Whitelist the H2 fixtures that feed the version-migration CI smoke test.
!src/test/resources/db-migration-fixtures/*.mv.db
/build
/app/proprietary/build/
+11 -9
View File
@@ -5,8 +5,6 @@ repositories {
ext {
jwtVersion = '0.13.0'
awsSdkVersion = '2.44.12'
testcontainersMinioVersion = '1.21.4'
}
bootRun {
@@ -46,6 +44,7 @@ dependencies {
api 'org.springframework:spring-jdbc'
api 'org.springframework:spring-webmvc'
api 'org.springframework.session:spring-session-core'
implementation 'org.springframework.session:spring-session-data-redis'
api "org.springframework.security:spring-security-core:$springSecuritySamlVersion"
api "org.springframework.security:spring-security-saml2-service-provider:$springSecuritySamlVersion"
api 'org.springframework.boot:spring-boot-starter-jetty'
@@ -55,8 +54,13 @@ dependencies {
api 'org.springframework.boot:spring-boot-starter-mail'
api 'org.springframework.boot:spring-boot-starter-cache'
api 'com.github.ben-manes.caffeine:caffeine'
implementation 'org.springframework.boot:spring-boot-starter-data-redis'
api 'io.swagger.core.v3:swagger-core-jakarta:2.2.46'
implementation 'com.bucket4j:bucket4j_jdk17-core:8.18.0'
implementation 'com.bucket4j:bucket4j_jdk17-core:8.19.0'
// Lettuce-backed Bucket4j ProxyManager used by ValkeyRateLimitStore for cluster-wide
// token-bucket rate limiting (parity with in-process Bucket4j semantics; no fixed-window
// boundary doubling).
implementation 'com.bucket4j:bucket4j_jdk17-lettuce:8.19.0'
// https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17
implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
@@ -74,12 +78,10 @@ dependencies {
exclude group: 'org.opensaml', module: 'opensaml-core'
}
implementation "software.amazon.awssdk:s3:$awsSdkVersion"
implementation "software.amazon.awssdk:url-connection-client:$awsSdkVersion"
testImplementation "org.testcontainers:minio:$testcontainersMinioVersion"
testImplementation "org.testcontainers:junit-jupiter:$testcontainersMinioVersion"
testImplementation "org.testcontainers:localstack:$testcontainersMinioVersion"
// Testcontainers: spins up a real Valkey for LiveValkeyIntegrationTest in CI without
// needing a manually-started instance. Tests skip cleanly when Docker is unavailable.
testImplementation 'org.testcontainers:testcontainers:1.21.4'
testImplementation 'org.testcontainers:junit-jupiter:1.21.4'
}
tasks.register('prepareKotlinBuildScriptModel') {}
@@ -0,0 +1,47 @@
package stirling.software.proprietary.cluster;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
/**
* Runtime license gate for cluster mode. Cluster mode requires a SERVER or ENTERPRISE license; the
* SaaS flavor bypasses (no {@code runningProOrHigher} bean is published). Fires before any Valkey
* bean construction via {@link Ordered#HIGHEST_PRECEDENCE}.
*
* <p>There is no testing/development bypass. Live e2e tests that need cluster mode must inject a
* valid {@code stirling.premium.key} for a test-tier SERVER/ENTERPRISE license. Unit tests stub the
* {@code runningProOrHigher} bean directly.
*/
@Configuration
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
@Order(Ordered.HIGHEST_PRECEDENCE)
@Slf4j
public class ClusterLicenseGate {
@Autowired(required = false)
@Qualifier("runningProOrHigher")
private Boolean runningProOrHigher;
@PostConstruct
void verifyLicense() {
if (runningProOrHigher == null) {
return; // saas flavor - licensed via Stripe elsewhere
}
if (!runningProOrHigher) {
throw new IllegalStateException(
"Cluster mode (cluster.enabled=true) requires a SERVER or"
+ " ENTERPRISE license. Configure stirling.premium.key with a valid"
+ " license key (contact sales@stirlingpdf.com to obtain one), or set"
+ " cluster.enabled=false.");
}
log.info("Cluster license gate: SERVER/ENTERPRISE license verified, cluster mode allowed.");
}
}
@@ -0,0 +1,123 @@
package stirling.software.proprietary.cluster;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import stirling.software.common.cluster.StickyMissRecorder;
import stirling.software.common.model.ApplicationProperties;
/**
* Cluster operation metrics exposed via {@code /actuator/prometheus}. Registered only when cluster
* mode is on.
*/
@Component
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
public class ClusterMetrics implements StickyMissRecorder {
private final MeterRegistry registry;
private final ApplicationProperties applicationProperties;
private final Counter stickyMissTotal;
private final Counter rateLimitRejected;
private final Timer backplaneLatency;
private final Timer jobWaitSeconds;
// Per-lane queue depth gauges. Lanes are a fixed enum (FAST, SLOW, AI), so we register all
// three eagerly so dashboards never have a missing series.
private static final List<String> KNOWN_LANES = List.of("FAST", "SLOW", "AI");
private final ConcurrentHashMap<String, AtomicLong> queueDepth = new ConcurrentHashMap<>();
// In-flight job count for THIS node.
private final AtomicLong jobsInflight = new AtomicLong();
public ClusterMetrics(MeterRegistry registry, ApplicationProperties applicationProperties) {
this.registry = registry;
this.applicationProperties = applicationProperties;
this.stickyMissTotal =
Counter.builder("stirling_cluster_sticky_miss_total")
.description(
"Sticky-session misses: a download for a job whose result lives on"
+ " a peer node landed on this node. High sustained value means"
+ " LB affinity is broken.")
.register(registry);
this.rateLimitRejected =
Counter.builder("stirling_cluster_ratelimit_rejected_total")
.description("Cluster-wide rate limit rejections")
.register(registry);
this.backplaneLatency =
Timer.builder("stirling_cluster_backplane_latency_seconds")
.description("Backplane round-trip latency")
.register(registry);
this.jobWaitSeconds =
Timer.builder("stirling_cluster_job_wait_seconds")
.description("Time jobs spend queued before execution")
.register(registry);
Gauge.builder("stirling_cluster_jobs_inflight", jobsInflight, AtomicLong::doubleValue)
.description("Jobs currently in flight on this node")
.tag("node", applicationProperties.getCluster().resolvedNodeId())
.register(registry);
for (String lane : KNOWN_LANES) {
ensureLaneGauge(lane);
}
}
/**
* Increment when {@code JobController} returns 410 Gone because the requested job's owner is a
* peer node. Surfaces to {@code stirling_cluster_sticky_miss_total}.
*/
@Override
public void recordStickyMiss() {
stickyMissTotal.increment();
}
public void recordRateLimitReject() {
rateLimitRejected.increment();
}
public Timer backplaneLatency() {
return backplaneLatency;
}
public Timer jobWaitSeconds() {
return jobWaitSeconds;
}
public void incrementInflight() {
jobsInflight.incrementAndGet();
}
public void decrementInflight() {
jobsInflight.decrementAndGet();
}
/**
* Publish (or update) the queue depth gauge for {@code lane}. Idempotent - safe to call hot.
* Known lanes (FAST, SLOW, AI) are pre-registered at construction; unknown lanes register on
* first call.
*/
public void setQueueDepth(String lane, long depth) {
ensureLaneGauge(lane).set(depth);
}
private AtomicLong ensureLaneGauge(String lane) {
return queueDepth.computeIfAbsent(
lane,
l -> {
AtomicLong holder = new AtomicLong();
Gauge.builder("stirling_cluster_queue_depth", holder, AtomicLong::doubleValue)
.description("Pending items in a job queue lane")
.tag("lane", l)
.register(registry);
return holder;
});
}
}
@@ -0,0 +1,201 @@
package stirling.software.proprietary.cluster;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.util.Locale;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.SmartLifecycle;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.ClusterNode;
import stirling.software.common.cluster.InstanceRegistry;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.ApplicationProperties.Cluster;
/**
* Registers the local node with {@link InstanceRegistry} on startup, refreshes the entry at 1/3 of
* the TTL, and deregisters cleanly on shutdown.
*
* <p>Implements {@link SmartLifecycle} with {@code getPhase() == Integer.MAX_VALUE} so Spring tears
* this bean down before {@code LettuceConnectionFactory} - deregister therefore runs while the
* Valkey connection is still alive.
*/
@Component
@Slf4j
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
public class ClusterNodeBootstrap implements SmartLifecycle {
/** TTL of the node entry in the registry. Set to 3x the heartbeat interval. */
private final Duration heartbeatTtl;
private final ApplicationProperties applicationProperties;
private final InstanceRegistry instanceRegistry;
@Value("${server.port:8080}")
private int serverPort;
private volatile String nodeId;
private volatile String internalAddress;
private volatile boolean running = false;
public ClusterNodeBootstrap(
ApplicationProperties applicationProperties, InstanceRegistry instanceRegistry) {
this.applicationProperties = applicationProperties;
this.instanceRegistry = instanceRegistry;
Cluster cluster = applicationProperties.getCluster();
long heartbeatMs =
cluster.getNode() == null ? 10_000L : cluster.getNode().getHeartbeatIntervalMs();
// TTL = 3x heartbeat: tolerate one missed tick before the node drops out of the registry.
this.heartbeatTtl = Duration.ofMillis(heartbeatMs * 3);
}
@EventListener(ApplicationReadyEvent.class)
public void registerOnStartup() {
nodeId = applicationProperties.getCluster().resolvedNodeId();
internalAddress = resolveInternalAddress();
registerSelf("register");
}
@Scheduled(fixedDelayString = "${cluster.node.heartbeat-interval-ms:10000}")
public void heartbeat() {
// Heartbeat-after-stop race: SmartLifecycle.stop() deregisters, but the @Scheduled
// tick keeps firing during a slow drain. Without this guard, the next tick re-registers
// the dead node and the entry resurfaces in the registry until TTL expiry.
if (!running) {
return;
}
if (nodeId == null) {
return; // not yet registered (startup race)
}
// Self-healing: register() is idempotent and re-populates every field, so a wiped
// Valkey (FLUSHALL, hash eviction) recovers on the next tick without operator action.
registerSelf("heartbeat");
}
private void registerSelf(String reason) {
try {
instanceRegistry.register(
new ClusterNode(nodeId, internalAddress, Instant.now(), role()), heartbeatTtl);
if ("register".equals(reason)) {
log.info(
"Cluster node registered: nodeId={}, internalAddress={}, role={}, ttl={}s",
nodeId,
internalAddress,
role(),
heartbeatTtl.toSeconds());
}
} catch (RuntimeException e) {
log.debug("Cluster {} failed for {}", reason, nodeId, e);
}
}
// ---------- SmartLifecycle (see class javadoc for ordering rationale) ----------
@Override
public void start() {
running = true;
}
@Override
public void stop() {
running = false;
if (nodeId == null) {
return;
}
try {
instanceRegistry.deregister(nodeId);
log.info("Cluster node deregistered: {}", nodeId);
} catch (RuntimeException e) {
// Registry entry will TTL-expire within heartbeatTtl anyway.
log.warn(
"Cluster deregister failed for {} (will TTL-expire within {}s): {}",
nodeId,
heartbeatTtl.toSeconds(),
e.getMessage());
}
}
@Override
public boolean isRunning() {
return running;
}
@Override
public int getPhase() {
return Integer.MAX_VALUE; // stopped first; LettuceConnectionFactory's phase is 0
}
@Override
public boolean isAutoStartup() {
return true;
}
/**
* Resolve the address peers should hit. Order: explicit config -> {@code POD_IP} env (K8s
* downward API) -> JDK hostname -> fail loud (never silently fall back to a loopback).
*
* <p>Scheme is taken from {@code cluster.node.scheme} (default {@code http}). Set to {@code
* https} when nodes terminate TLS themselves; leave as {@code http} when an upstream LB
* terminates TLS and intra-cluster traffic is plain HTTP.
*/
private String resolveInternalAddress() {
Cluster cluster = applicationProperties.getCluster();
String configured =
cluster.getNode() == null ? null : cluster.getNode().getInternalAddress();
if (configured != null && !configured.isBlank()) {
return ensurePort(configured);
}
String podIp = System.getenv("POD_IP");
if (podIp != null && !podIp.isBlank()) {
return scheme() + "://" + podIp + ":" + serverPort;
}
try {
return scheme()
+ "://"
+ InetAddress.getLocalHost().getHostAddress()
+ ":"
+ serverPort;
} catch (UnknownHostException e) {
throw new IllegalStateException(
"Could not resolve this host's address for cluster registration; set"
+ " cluster.node.internal-address explicitly (or set POD_IP"
+ " in the Kubernetes downward API).",
e);
}
}
private String ensurePort(String addr) {
if (addr.startsWith("http://") || addr.startsWith("https://")) {
return addr;
}
if (addr.contains(":")) {
return scheme() + "://" + addr;
}
return scheme() + "://" + addr + ":" + serverPort;
}
private String scheme() {
Cluster cluster = applicationProperties.getCluster();
if (cluster.getNode() == null
|| cluster.getNode().getScheme() == null
|| cluster.getNode().getScheme().isBlank()) {
return "http";
}
String s = cluster.getNode().getScheme().trim().toLowerCase(Locale.ROOT);
return "https".equals(s) ? "https" : "http";
}
private String role() {
Cluster.NodeRole r = applicationProperties.getCluster().resolvedRole();
return r == null ? "BOTH" : r.name();
}
}
@@ -0,0 +1,55 @@
package stirling.software.proprietary.cluster;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.session.data.redis.config.annotation.web.http.EnableRedisHttpSession;
import org.springframework.session.web.http.CookieSerializer;
import org.springframework.session.web.http.DefaultCookieSerializer;
/**
* Enables Spring Session backed by Valkey when cluster mode is on AND a Lettuce connection factory
* exists (backplane=valkey). {@link ConditionalOnBean} prevents {@code @EnableRedisHttpSession}
* from wiring its filter before the required connection factory is present; without it the bean
* graph fails with "No qualifying bean of type 'SessionRepository'".
*/
@Configuration
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
@ConditionalOnBean(LettuceConnectionFactory.class)
@EnableRedisHttpSession
public class ClusterSessionConfiguration {
/**
* The bean name {@code springSessionDefaultRedisSerializer} is the exact hook Spring Session
* uses to override JDK serialization. JDK serialization is a deserialization-gadget RCE surface
* for anyone with Valkey write access; Jackson JSON is not.
*
* <p>Migrate to {@code GenericJacksonJsonRedisSerializer} when the Spring Session reference doc
* does (https://docs.spring.io/spring-session/reference/spring-security.html#config-redis).
*/
@Bean
@SuppressWarnings(
"removal") // GenericJackson2JsonRedisSerializer is the documented recipe name; migrate
// when upstream does
public RedisSerializer<Object> springSessionDefaultRedisSerializer() {
return new GenericJackson2JsonRedisSerializer();
}
/**
* Harden the Spring Session cookie: Spring Session omits {@code Secure} and {@code SameSite} by
* default, leaving the session id susceptible to plaintext leakage and CSRF. {@code Lax} allows
* top-level navigations (login redirects) while blocking cross-site sub-resource requests.
* HttpOnly is on by default - asserted in the regression test.
*/
@Bean
public CookieSerializer cookieSerializer() {
DefaultCookieSerializer serializer = new DefaultCookieSerializer();
serializer.setUseSecureCookie(true);
serializer.setSameSite("Lax");
return serializer;
}
}
@@ -1,200 +0,0 @@
package stirling.software.proprietary.cluster.s3;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
/**
* Shared factory for {@link S3Client} and {@link S3Presigner} instances used by both {@code
* S3StorageProvider} and {@code S3FileStore}, so endpoint/region/credentials wiring lives in
* exactly one place.
*/
@Slf4j
public final class S3Clients {
private S3Clients() {}
/** Paired client and presigner with coordinated lifecycle. */
public record Bundle(S3Client client, S3Presigner presigner) implements AutoCloseable {
@Override
public void close() {
try {
presigner.close();
} catch (Exception e) {
log.warn("Error closing S3 presigner", e);
}
try {
client.close();
} catch (Exception e) {
log.warn("Error closing S3 client", e);
}
}
}
/** Build a client+presigner pair from the shared S3 config block. */
public static Bundle build(ApplicationProperties.Storage.S3 cfg, String usage) {
if (cfg == null) {
throw new IllegalStateException(
usage + " requires storage.s3.* configuration to be set");
}
if (cfg.getBucket() == null || cfg.getBucket().isBlank()) {
throw new IllegalStateException(usage + " requires storage.s3.bucket to be set");
}
String region =
cfg.getRegion() == null || cfg.getRegion().isBlank()
? "us-east-1"
: cfg.getRegion();
S3Configuration s3Configuration =
S3Configuration.builder().pathStyleAccessEnabled(cfg.isPathStyleAccess()).build();
RequestChecksumCalculation requestChecksum =
parseRequestChecksum(cfg.getRequestChecksumCalculation());
ResponseChecksumValidation responseChecksum =
parseResponseChecksum(cfg.getResponseChecksumValidation());
S3ClientBuilder clientBuilder =
S3Client.builder()
.httpClient(UrlConnectionHttpClient.create())
.region(Region.of(region))
.serviceConfiguration(s3Configuration)
.requestChecksumCalculation(requestChecksum)
.responseChecksumValidation(responseChecksum);
S3Presigner.Builder presignerBuilder =
S3Presigner.builder()
.region(Region.of(region))
.serviceConfiguration(s3Configuration);
if (cfg.getEndpoint() != null && !cfg.getEndpoint().isBlank()) {
URI endpoint;
try {
endpoint = new URI(cfg.getEndpoint());
} catch (URISyntaxException e) {
throw new IllegalStateException(
"Invalid storage.s3.endpoint: " + cfg.getEndpoint(), e);
}
validateEndpointHost(endpoint, cfg.isAllowPrivateEndpoints());
clientBuilder.endpointOverride(endpoint);
presignerBuilder.endpointOverride(endpoint);
}
boolean hasStaticCreds =
cfg.getAccessKey() != null
&& !cfg.getAccessKey().isBlank()
&& cfg.getSecretKey() != null
&& !cfg.getSecretKey().isBlank();
if (hasStaticCreds) {
AwsBasicCredentials credentials =
AwsBasicCredentials.create(cfg.getAccessKey(), cfg.getSecretKey());
StaticCredentialsProvider provider = StaticCredentialsProvider.create(credentials);
clientBuilder.credentialsProvider(provider);
presignerBuilder.credentialsProvider(provider);
} else {
clientBuilder.credentialsProvider(DefaultCredentialsProvider.create());
presignerBuilder.credentialsProvider(DefaultCredentialsProvider.create());
}
log.debug(
"Configured S3 {}: bucket={}, region={}, endpoint={}, pathStyle={}",
usage,
cfg.getBucket(),
region,
cfg.getEndpoint() == null || cfg.getEndpoint().isBlank()
? "<aws-default>"
: cfg.getEndpoint(),
cfg.isPathStyleAccess());
return new Bundle(clientBuilder.build(), presignerBuilder.build());
}
/**
* Block SSRF via the S3 endpoint setting. An admin who can edit config could otherwise point
* the SDK at the cloud metadata service (e.g. {@code http://169.254.169.254/}) and exfiltrate
* instance-role credentials. Reject any endpoint whose host resolves to a loopback, link-local,
* or RFC1918 private address unless the operator has explicitly opted in via {@code
* storage.s3.allow-private-endpoints=true}.
*/
static void validateEndpointHost(URI endpoint, boolean allowPrivate) {
if (allowPrivate) {
return;
}
String host = endpoint.getHost();
if (host == null || host.isBlank()) {
throw new IllegalStateException("storage.s3.endpoint must include a host: " + endpoint);
}
InetAddress[] addresses;
try {
addresses = InetAddress.getAllByName(host);
} catch (UnknownHostException e) {
throw new IllegalStateException(
"Unable to resolve storage.s3.endpoint host '" + host + "'", e);
}
for (InetAddress address : addresses) {
if (isPrivateOrLocal(address)) {
throw new IllegalStateException(
"storage.s3.endpoint host '"
+ host
+ "' resolves to private/link-local address "
+ address.getHostAddress()
+ "; set storage.s3.allow-private-endpoints=true to opt in"
+ " (e.g. for MinIO or in-cluster S3).");
}
}
}
private static boolean isPrivateOrLocal(InetAddress address) {
return address.isLoopbackAddress()
|| address.isLinkLocalAddress()
|| address.isSiteLocalAddress()
|| address.isAnyLocalAddress()
|| address.isMulticastAddress();
}
static RequestChecksumCalculation parseRequestChecksum(String value) {
if (value == null || value.isBlank()) {
return RequestChecksumCalculation.WHEN_SUPPORTED;
}
try {
return RequestChecksumCalculation.valueOf(
value.trim().toUpperCase(java.util.Locale.ROOT));
} catch (IllegalArgumentException ex) {
log.warn(
"Unknown storage.s3.request-checksum-calculation value '{}', falling back to WHEN_SUPPORTED",
value);
return RequestChecksumCalculation.WHEN_SUPPORTED;
}
}
static ResponseChecksumValidation parseResponseChecksum(String value) {
if (value == null || value.isBlank()) {
return ResponseChecksumValidation.WHEN_SUPPORTED;
}
try {
return ResponseChecksumValidation.valueOf(
value.trim().toUpperCase(java.util.Locale.ROOT));
} catch (IllegalArgumentException ex) {
log.warn(
"Unknown storage.s3.response-checksum-validation value '{}', falling back to WHEN_SUPPORTED",
value);
return ResponseChecksumValidation.WHEN_SUPPORTED;
}
}
}
@@ -1,226 +0,0 @@
package stirling.software.proprietary.cluster.s3;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.FileStore;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception;
/**
* S3-backed {@link FileStore} for transient job-result files. Objects are namespaced under a
* configurable key prefix (default {@code transient/}) and can coexist in the same bucket as {@code
* S3StorageProvider}.
*/
@Slf4j
public class S3FileStore implements FileStore, AutoCloseable {
public static final String DEFAULT_KEY_PREFIX = "transient/";
private final S3Client s3Client;
private final String bucket;
private final String keyPrefix;
private final boolean ownsClient;
public S3FileStore(S3Client s3Client, String bucket) {
this(s3Client, bucket, DEFAULT_KEY_PREFIX, true);
}
public S3FileStore(S3Client s3Client, String bucket, String keyPrefix) {
this(s3Client, bucket, keyPrefix, true);
}
/**
* @param ownsClient when true, {@link #close()} will close the supplied client. Set to false in
* tests that share the client with another consumer.
*/
public S3FileStore(S3Client s3Client, String bucket, String keyPrefix, boolean ownsClient) {
if (bucket == null || bucket.isBlank()) {
throw new IllegalArgumentException("S3 bucket must be configured");
}
this.s3Client = s3Client;
this.bucket = bucket;
this.keyPrefix = normalizePrefix(keyPrefix);
this.ownsClient = ownsClient;
}
@Override
public Stored store(InputStream in, String originalName) throws IOException {
String fileId = UUID.randomUUID().toString();
// S3 PUT requires a known content-length; spool to a temp file first so memory stays
// bounded for large payloads, then stream the file to S3 via RequestBody.fromFile.
Path tempFile = Files.createTempFile("s3-upload-", ".bin");
long size;
try {
try (InputStream src = in) {
Files.copy(src, tempFile, StandardCopyOption.REPLACE_EXISTING);
}
size = Files.size(tempFile);
PutObjectRequest request =
PutObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try {
s3Client.putObject(request, RequestBody.fromFile(tempFile));
} catch (SdkException e) {
throw new IOException("Failed to upload object to S3", e);
}
} finally {
try {
Files.deleteIfExists(tempFile);
} catch (IOException cleanupError) {
log.warn("Failed to delete S3 upload temp file: {}", tempFile, cleanupError);
}
}
return new Stored(fileId, size);
}
@Override
public InputStream retrieve(String fileId) throws IOException {
validateFileId(fileId);
GetObjectRequest request =
GetObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try {
ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(request);
return new BufferedInputStream(stream);
} catch (NoSuchKeyException e) {
throw new IOException("File not found with ID: " + fileId, e);
} catch (SdkException e) {
throw new IOException("Failed to load object from S3", e);
}
}
@Override
public byte[] retrieveBytes(String fileId) throws IOException {
validateFileId(fileId);
GetObjectRequest request =
GetObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try (ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(request)) {
return stream.readAllBytes();
} catch (NoSuchKeyException e) {
throw new IOException("File not found with ID: " + fileId, e);
} catch (SdkException e) {
throw new IOException("Failed to load object from S3", e);
}
}
@Override
public long size(String fileId) throws IOException {
validateFileId(fileId);
HeadObjectRequest request =
HeadObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try {
HeadObjectResponse response = s3Client.headObject(request);
return Optional.ofNullable(response.contentLength()).orElse(0L);
} catch (NoSuchKeyException e) {
throw new IOException("File not found with ID: " + fileId, e);
} catch (S3Exception e) {
if (e.statusCode() == 404) {
throw new IOException("File not found with ID: " + fileId, e);
}
throw new IOException("Failed to head object in S3", e);
} catch (SdkException e) {
throw new IOException("Failed to head object in S3", e);
}
}
@Override
public boolean delete(String fileId) {
try {
validateFileId(fileId);
} catch (IllegalArgumentException e) {
log.warn("Refusing to delete invalid file id: {}", fileId);
return false;
}
try {
s3Client.deleteObject(
DeleteObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build());
return true;
} catch (SdkException e) {
log.error("Error deleting file with ID: {}", fileId, e);
return false;
}
}
@Override
public boolean exists(String fileId) {
try {
validateFileId(fileId);
} catch (IllegalArgumentException e) {
return false;
}
HeadObjectRequest request =
HeadObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try {
s3Client.headObject(request);
return true;
} catch (NoSuchKeyException e) {
return false;
} catch (S3Exception e) {
if (e.statusCode() == 404) {
return false;
}
log.warn("Error checking existence for file ID: {}", fileId, e);
return false;
} catch (SdkException e) {
log.warn("Error checking existence for file ID: {}", fileId, e);
return false;
}
}
@Override
public void close() {
if (!ownsClient) {
return;
}
try {
s3Client.close();
} catch (Exception e) {
log.warn("Error closing S3 client", e);
}
}
String resolveKey(String fileId) {
return keyPrefix + fileId;
}
private static void validateFileId(String fileId) {
if (fileId == null || fileId.isBlank()) {
throw new IllegalArgumentException("File ID must not be blank");
}
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
throw new IllegalArgumentException("Invalid file ID");
}
}
private static String normalizePrefix(String prefix) {
if (prefix == null || prefix.isBlank()) {
return "";
}
String trimmed = prefix.trim();
if (trimmed.startsWith("/")) {
trimmed = trimmed.substring(1);
}
if (!trimmed.isEmpty() && !trimmed.endsWith("/")) {
trimmed = trimmed + "/";
}
return trimmed;
}
}
@@ -1,37 +0,0 @@
package stirling.software.proprietary.cluster.s3;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.FileStore;
import stirling.software.common.model.ApplicationProperties;
/** Activates the S3-backed transient {@link FileStore} when {@code cluster.artifactStore=s3}. */
@Slf4j
@Configuration
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "cluster", name = "artifactStore", havingValue = "s3")
public class S3FileStoreConfiguration {
private final ApplicationProperties applicationProperties;
@Bean(destroyMethod = "close")
@ConditionalOnMissingBean
public FileStore fileStore(@Value("${cluster.s3.keyPrefix:transient/}") String keyPrefix) {
ApplicationProperties.Storage.S3 cfg = applicationProperties.getStorage().getS3();
S3Clients.Bundle bundle = S3Clients.build(cfg, "cluster file store");
// FileStore has no signed-URL contract; close the unused presigner immediately.
try {
bundle.presigner().close();
} catch (Exception ignored) {
}
log.info("Cluster FileStore: s3 (bucket={}, keyPrefix={})", cfg.getBucket(), keyPrefix);
return new S3FileStore(bundle.client(), cfg.getBucket(), keyPrefix, true);
}
}
@@ -0,0 +1,25 @@
package stirling.software.proprietary.cluster.valkey;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
/**
* Composite condition: cluster mode is on AND the configured backplane is Valkey.
*
* <p>Either condition alone is insufficient to load a Valkey bean. With {@code enabled=true} but
* {@code backplane=inprocess}, loading the Valkey beans would crash at boot because there's no
* {@code StringRedisTemplate}; with {@code enabled=false} the whole cluster mode is off. Combining
* the two stops both footguns.
*
* <p>Spring's {@code @ConditionalOnProperty} cannot be applied twice on the same class, so we use
* {@code @ConditionalOnExpression} via this meta-annotation.
*/
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnExpression(
"${cluster.enabled:false} and '${cluster.backplane:inprocess}'.equals('valkey')")
public @interface ConditionalOnValkeyBackplane {}
@@ -0,0 +1,62 @@
package stirling.software.proprietary.cluster.valkey;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.model.ApplicationProperties;
@Slf4j
@Component
@RequiredArgsConstructor
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
@org.springframework.boot.autoconfigure.condition.ConditionalOnProperty(
name = "cluster.backplane",
havingValue = "valkey")
public class ValkeyClusterBackplane implements ClusterBackplane {
private final ApplicationProperties applicationProperties;
private final StringRedisTemplate template;
@Override
public boolean isHealthy() {
try {
// template.execute() borrows from the pool and returns the connection in a finally
// block - critical because isHealthy() is hit on every k8s liveness/readiness probe
// tick. Calling getConnectionFactory().getConnection() directly leaks the connection
// and exhausts the pool under monitoring load.
String pong = template.execute((RedisCallback<String>) connection -> connection.ping());
return "PONG".equalsIgnoreCase(pong);
} catch (RuntimeException ex) {
log.warn("Valkey backplane health check failed: {}", ex.getMessage());
return false;
}
}
@Override
public String backplaneType() {
return "valkey";
}
@Override
public String localNodeId() {
return applicationProperties.getCluster().resolvedNodeId();
}
/**
* Disable the local {@code TaskManager#cleanupOldJobs} loop on Valkey-backed clusters: {@link
* ValkeyJobStore} stores every entry with a TTL pExpire and the reverse-index entries share
* that TTL, so Valkey itself evicts expired job state. Running the local cleanup loop on top of
* that would only delete per-node in-memory {@code TaskManager} caches that the cluster-visible
* {@code JobStore} has already authoritative state for.
*/
@Override
public boolean shouldRunLocalCleanup() {
return false;
}
}
@@ -0,0 +1,222 @@
package stirling.software.proprietary.cluster.valkey;
import java.net.URI;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.data.redis.connection.RedisPassword;
import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import io.lettuce.core.RedisCommandExecutionException;
import io.lettuce.core.SslVerifyMode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.ApplicationProperties.Cluster;
/** Wires the LettuceConnectionFactory and StringRedisTemplate for cluster mode. */
@Slf4j
@Configuration
@RequiredArgsConstructor
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
@DependsOn("clusterLicenseGate")
public class ValkeyConnectionConfiguration {
private final ApplicationProperties applicationProperties;
@Bean(destroyMethod = "destroy")
@ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey")
public LettuceConnectionFactory valkeyConnectionFactory() {
Cluster cluster = applicationProperties.getCluster();
String url = cluster.getValkey().getUrl();
if (url == null || url.isBlank()) {
throw new IllegalStateException("cluster.valkey.url must be set when backplane=valkey");
}
URI uri = URI.create(url);
boolean tls = "rediss".equalsIgnoreCase(uri.getScheme());
int port = uri.getPort() <= 0 ? 6379 : uri.getPort();
RedisStandaloneConfiguration cfg = new RedisStandaloneConfiguration(uri.getHost(), port);
if (uri.getUserInfo() != null) {
String[] parts = uri.getUserInfo().split(":", 2);
if (parts.length == 2) {
cfg.setUsername(parts[0]);
cfg.setPassword(RedisPassword.of(parts[1]));
} else if (parts.length == 1 && !parts[0].isBlank()) {
cfg.setPassword(RedisPassword.of(parts[0]));
}
}
boolean skipCertVerification =
cluster.getValkey().getTls() != null
&& cluster.getValkey().getTls().isSkipCertVerification();
LettuceClientConfiguration clientConfig =
buildClientConfiguration(tls, skipCertVerification);
LettuceConnectionFactory factory = new LettuceConnectionFactory(cfg, clientConfig);
factory.afterPropertiesSet();
// Eager handshake with retry tolerates docker-compose DNS races; fails boot loudly
// if Valkey is genuinely unreachable.
eagerHandshake(factory, uri.getHost(), port, tls);
log.info(
"Valkey connection configured: {}:{} tls={} verifyPeer={}",
uri.getHost(),
port,
tls,
tls ? clientConfig.getVerifyMode() : "n/a");
return factory;
}
/**
* Builds the Lettuce client configuration with TLS verification pinned. Package-private so unit
* tests can verify the {@code verifyPeer} mode without standing up a real Valkey.
*
* <p>{@code verifyPeer(FULL)} is pinned explicitly so a future Spring Data Redis default change
* cannot silently weaken our TLS handshake. {@code FULL} = X.509 chain + hostname check (per
* Lettuce's {@link SslVerifyMode}). The {@code skipCertVerification} opt-out is for local dev
* with self-signed certs only; production deployments MUST leave it false.
*/
static LettuceClientConfiguration buildClientConfiguration(
boolean tls, boolean skipCertVerification) {
LettuceClientConfiguration.LettuceClientConfigurationBuilder clientBuilder =
LettuceClientConfiguration.builder();
if (tls) {
clientBuilder
.useSsl()
.verifyPeer(skipCertVerification ? SslVerifyMode.NONE : SslVerifyMode.FULL);
if (skipCertVerification) {
log.warn(
"Valkey TLS hostname/chain verification DISABLED via"
+ " cluster.valkey.tls.skip-cert-verification=true"
+ " - insecure, dev-only");
}
}
return clientBuilder.build();
}
/**
* 10 x 3s = 30s of retry. Boot-time only.
*
* <p>Auth-class failures (WRONGPASS / NOAUTH / NOPERM) are unrecoverable and surfaced
* immediately on the first attempt; only transport-level errors (connection refused, timeout,
* host unreachable) get the retry loop.
*
* <p>Package-private so unit tests can drive it with a mocked connection factory.
*/
static void eagerHandshake(
LettuceConnectionFactory factory, String host, int port, boolean tls) {
RuntimeException last = null;
for (int attempt = 1; attempt <= 10; attempt++) {
try {
String pong = factory.getConnection().ping();
if (!"PONG".equalsIgnoreCase(pong)) {
throw new IllegalStateException(
"Valkey PING returned '" + pong + "' (expected PONG)");
}
if (attempt > 1) {
log.info("Valkey reachable after {} attempts", attempt);
}
return;
} catch (RuntimeException ex) {
if (isAuthFailure(ex)) {
factory.destroy();
throw new IllegalStateException(
"Valkey authentication failed for "
+ host
+ ":"
+ port
+ " (tls="
+ tls
+ "): "
+ rootAuthMessage(ex)
+ ". Check cluster.valkey.url credentials"
+ " (user/password and ACL permissions).",
ex);
}
last = ex;
log.warn(
"Valkey PING attempt {}/10 failed ({}:{}, tls={}): {}",
attempt,
host,
port,
tls,
ex.getMessage());
try {
Thread.sleep(3000);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
break;
}
}
}
factory.destroy();
throw new IllegalStateException(
"Valkey unreachable at boot after 10 attempts ("
+ host
+ ":"
+ port
+ ", tls="
+ tls
+ "): "
+ (last == null ? "no detail" : last.getMessage()),
last);
}
/**
* Walks the cause chain for a Lettuce {@link RedisCommandExecutionException} whose message
* starts with an auth-class server reply (WRONGPASS, NOAUTH, NOPERM). Spring Data Redis wraps
* Lettuce errors in a {@code RedisSystemException}, so the auth signal usually lives one level
* down from the thrown exception.
*
* <p>Checked for a typed alternative: neither Spring Data Redis 4.0.5 nor Lettuce 6.8.2 ships a
* {@code RedisAuthenticationException} on the classpath, so we keep the message-prefix match.
* Revisit when upgrading Spring Data Redis if a typed exception lands upstream.
*/
static boolean isAuthFailure(Throwable t) {
for (Throwable cur = t; cur != null; cur = cur.getCause()) {
if (cur instanceof RedisCommandExecutionException && hasAuthPrefix(cur.getMessage())) {
return true;
}
// Defensive: some translations preserve the original message on the wrapper itself.
if (hasAuthPrefix(cur.getMessage())) {
return true;
}
if (cur.getCause() == cur) {
break;
}
}
return false;
}
private static boolean hasAuthPrefix(String message) {
if (message == null) {
return false;
}
String upper = message.toUpperCase(java.util.Locale.ROOT).stripLeading();
return upper.startsWith("WRONGPASS")
|| upper.startsWith("NOAUTH")
|| upper.startsWith("NOPERM");
}
private static String rootAuthMessage(Throwable t) {
for (Throwable cur = t; cur != null; cur = cur.getCause()) {
if (cur instanceof RedisCommandExecutionException && cur.getMessage() != null) {
return cur.getMessage();
}
if (cur.getCause() == cur) {
break;
}
}
return t.getMessage();
}
@Bean
@ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey")
public StringRedisTemplate valkeyTemplate(LettuceConnectionFactory factory) {
return new StringRedisTemplate(factory);
}
}
@@ -0,0 +1,92 @@
package stirling.software.proprietary.cluster.valkey;
import java.time.Duration;
import java.util.Collections;
import java.util.Optional;
import java.util.UUID;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.script.DefaultRedisScript;
import org.springframework.data.redis.core.script.RedisScript;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.DistributedLock;
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
@Slf4j
public class ValkeyDistributedLock implements DistributedLock {
private static final String PREFIX = "stirling:lock:";
private static final RedisScript<Long> RELEASE_SCRIPT =
new DefaultRedisScript<>(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
Long.class);
private static final RedisScript<Long> RENEW_SCRIPT =
new DefaultRedisScript<>(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end",
Long.class);
private final StringRedisTemplate template;
@Override
public Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime) {
String key = PREFIX + lockKey;
String value = UUID.randomUUID().toString();
Boolean ok = template.opsForValue().setIfAbsent(key, value, leaseTime);
if (Boolean.TRUE.equals(ok)) {
return Optional.of(new ValkeyHandle(template, key, value));
}
return Optional.empty();
}
private static final class ValkeyHandle implements LockHandle {
private final StringRedisTemplate template;
private final String key;
private final String value;
private boolean released;
ValkeyHandle(StringRedisTemplate template, String key, String value) {
this.template = template;
this.key = key;
this.value = value;
}
@Override
public synchronized void release() {
if (released) {
return;
}
released = true;
template.execute(RELEASE_SCRIPT, Collections.singletonList(key), value);
}
@Override
public synchronized boolean renew(Duration leaseTime) {
if (released) {
return false;
}
try {
Long result =
template.execute(
RENEW_SCRIPT,
Collections.singletonList(key),
value,
Long.toString(leaseTime.toMillis()));
return result != null && result == 1L;
} catch (RuntimeException ex) {
log.warn(
"Lock renew failed for {} (treated as lost lease): {}",
key,
ex.getMessage());
return false;
}
}
}
}
@@ -0,0 +1,116 @@
package stirling.software.proprietary.cluster.valkey;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import stirling.software.common.cluster.ClusterNode;
import stirling.software.common.cluster.InstanceRegistry;
/**
* Valkey-backed {@link InstanceRegistry}. Each node is stored as a hash with a TTL equal to the
* configured heartbeat TTL; the heartbeat re-arms the TTL.
*/
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
public class ValkeyInstanceRegistry implements InstanceRegistry {
private static final String PREFIX = "stirling:nodes:";
private final StringRedisTemplate template;
@Override
public void register(ClusterNode node, Duration heartbeatTtl) {
String key = PREFIX + node.nodeId();
long ttlMs = heartbeatTtl.toMillis();
Map<String, String> fields = new LinkedHashMap<>();
fields.put("nodeId", node.nodeId());
fields.put("internalAddress", node.internalAddress());
fields.put("role", node.role());
fields.put("lastHeartbeat", node.lastHeartbeat().toString());
// MULTI/EXEC so the hash fields and the TTL commit together. Without this, a crash
// between HSET and EXPIRE leaves the hash with no TTL: it never expires, masks the
// dead node as alive, and only a subsequent successful register() would re-arm it.
template.execute(
(RedisCallback<Object>)
connection -> {
connection.multi();
byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
Map<byte[], byte[]> hashBytes = new LinkedHashMap<>();
for (Map.Entry<String, String> f : fields.entrySet()) {
hashBytes.put(
f.getKey().getBytes(StandardCharsets.UTF_8),
f.getValue().getBytes(StandardCharsets.UTF_8));
}
connection.hashCommands().hMSet(keyBytes, hashBytes);
connection.keyCommands().pExpire(keyBytes, ttlMs);
connection.exec();
return null;
});
}
@Override
public Optional<ClusterNode> lookup(String nodeId) {
return readNode(PREFIX + nodeId);
}
@Override
public Collection<ClusterNode> activeNodes() {
// SCAN, not KEYS - KEYS blocks the Valkey server for the duration of the walk.
ScanOptions options = ScanOptions.scanOptions().match(PREFIX + "*").count(256).build();
List<ClusterNode> nodes = new ArrayList<>();
try (Cursor<String> cursor = template.scan(options)) {
while (cursor.hasNext()) {
readNode(cursor.next()).ifPresent(nodes::add);
}
}
return nodes;
}
@Override
public void deregister(String nodeId) {
template.delete(PREFIX + nodeId);
}
private Optional<ClusterNode> readNode(String key) {
Map<Object, Object> entries = template.opsForHash().entries(key);
if (entries == null || entries.isEmpty()) {
return Optional.empty();
}
Object nodeId = entries.get("nodeId");
if (nodeId == null) {
return Optional.empty();
}
Instant heartbeat = Instant.now();
Object hb = entries.get("lastHeartbeat");
if (hb != null) {
try {
heartbeat = Instant.parse(hb.toString());
} catch (RuntimeException ignored) {
// keep default
}
}
return Optional.of(
new ClusterNode(
nodeId.toString(),
String.valueOf(entries.getOrDefault("internalAddress", "")),
heartbeat,
String.valueOf(entries.getOrDefault("role", "BOTH"))));
}
}
@@ -0,0 +1,297 @@
package stirling.software.proprietary.cluster.valkey;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.cluster.JobStoreEntry;
/**
* Valkey-backed {@link JobStore}. Each job is one hash; a reverse index maps fileId to jobId.
*
* <p><b>put() atomicity:</b> the hash fields, the per-job TTL, and the reverse-index entries are
* issued inside a single pipelined Redis transaction (MULTI/EXEC). A partial failure cannot leave
* the hash without a TTL or with half the filejob index entries written.
*/
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
@Slf4j
public class ValkeyJobStore implements JobStore {
private static final String JOB_PREFIX = "stirling:job:";
private static final String FILE_INDEX_PREFIX = "stirling:file2job:";
private static final ObjectMapper MAPPER = new ObjectMapper();
private static final TypeReference<List<String>> LIST_STRING = new TypeReference<>() {};
private static final TypeReference<Map<String, String>> MAP_STRING = new TypeReference<>() {};
private final StringRedisTemplate template;
@Override
public void put(JobStoreEntry entry, Duration ttl) {
String key = JOB_PREFIX + entry.jobId();
long ttlMs = ttl.toMillis();
Map<String, String> fields = new LinkedHashMap<>();
fields.put("jobId", entry.jobId());
fields.put("state", entry.state().name());
fields.put("owningNodeId", entry.owningNodeId() == null ? "" : entry.owningNodeId());
if (entry.createdAt() != null) {
fields.put("createdAt", entry.createdAt().toString());
}
if (entry.completedAt() != null) {
fields.put("completedAt", entry.completedAt().toString());
}
if (entry.error() != null) {
fields.put("error", entry.error());
}
fields.put("fileIds", writeJson(entry.fileIds() == null ? List.of() : entry.fileIds()));
fields.put(
"resultMeta",
writeJson(entry.resultMeta() == null ? Map.of() : entry.resultMeta()));
// Build pipelined MULTI/EXEC so the hash, its TTL, and every reverse-index entry
// commit atomically.
template.execute(
(RedisCallback<Object>)
connection -> {
connection.multi();
byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
Map<byte[], byte[]> hashBytes = new LinkedHashMap<>();
for (Map.Entry<String, String> f : fields.entrySet()) {
hashBytes.put(
f.getKey().getBytes(StandardCharsets.UTF_8),
f.getValue().getBytes(StandardCharsets.UTF_8));
}
connection.hashCommands().hMSet(keyBytes, hashBytes);
connection.keyCommands().pExpire(keyBytes, ttlMs);
if (entry.fileIds() != null) {
for (String fileId : entry.fileIds()) {
byte[] idxKey =
(FILE_INDEX_PREFIX + fileId)
.getBytes(StandardCharsets.UTF_8);
connection
.stringCommands()
.set(
idxKey,
entry.jobId().getBytes(StandardCharsets.UTF_8));
connection.keyCommands().pExpire(idxKey, ttlMs);
}
}
connection.exec();
return null;
});
}
@Override
public Optional<JobStoreEntry> get(String jobId) {
return readEntry(JOB_PREFIX + jobId);
}
@Override
public void delete(String jobId) {
// WATCH/MULTI/EXEC: read fileIds INSIDE the watched scope so a concurrent put() that
// adds new fileIds between our read and EXEC aborts the transaction. Without this guard,
// an interleaved put() that grows fileIds would leave orphaned reverse-index entries
// pointing at the deleted jobId until their TTL expires. One retry handles the common
// case; further contention falls through to lazy TTL cleanup (acceptable - this is an
// eviction path, not a correctness primitive).
String jobKey = JOB_PREFIX + jobId;
byte[] jobKeyBytes = jobKey.getBytes(StandardCharsets.UTF_8);
for (int attempt = 0; attempt < 2; attempt++) {
Boolean committed =
template.execute(
(RedisCallback<Boolean>)
connection -> {
connection.watch(jobKeyBytes);
Map<byte[], byte[]> hash =
connection.hashCommands().hGetAll(jobKeyBytes);
List<byte[]> keysToDelete = new ArrayList<>();
keysToDelete.add(jobKeyBytes);
if (hash != null) {
byte[] fileIdsBytes =
hash.get(
"fileIds"
.getBytes(
StandardCharsets
.UTF_8));
if (fileIdsBytes != null) {
List<String> fileIds =
readJsonList(
new String(
fileIdsBytes,
StandardCharsets.UTF_8),
jobKey);
for (String fileId : fileIds) {
keysToDelete.add(
(FILE_INDEX_PREFIX + fileId)
.getBytes(
StandardCharsets
.UTF_8));
}
}
}
connection.multi();
for (byte[] key : keysToDelete) {
connection.keyCommands().del(key);
}
List<Object> results = connection.exec();
// exec() returns null when WATCH detected a concurrent
// write; spring-data-redis surfaces this as either null
// or empty depending on the driver path.
return results != null && !results.isEmpty();
});
if (Boolean.TRUE.equals(committed)) {
return;
}
}
log.warn(
"JobStore.delete({}) lost two WATCH races to concurrent put(); reverse-index"
+ " entries may linger until TTL expiry",
jobId);
}
@Override
public boolean exists(String jobId) {
Boolean exists = template.hasKey(JOB_PREFIX + jobId);
return Boolean.TRUE.equals(exists);
}
@Override
public Optional<String> findJobIdByFileId(String fileId) {
return Optional.ofNullable(template.opsForValue().get(FILE_INDEX_PREFIX + fileId));
}
@Override
public Collection<JobStoreEntry> all() {
// SCAN, not KEYS - KEYS blocks the Valkey server for the duration of the walk.
ScanOptions options = ScanOptions.scanOptions().match(JOB_PREFIX + "*").count(256).build();
List<JobStoreEntry> result = new ArrayList<>();
try (Cursor<String> cursor = template.scan(options)) {
while (cursor.hasNext()) {
readEntry(cursor.next()).ifPresent(result::add);
}
}
return result;
}
private Optional<JobStoreEntry> readEntry(String key) {
Map<Object, Object> entries = template.opsForHash().entries(key);
if (entries == null || entries.isEmpty()) {
return Optional.empty();
}
Object jobId = entries.get("jobId");
if (jobId == null) {
return Optional.empty();
}
Instant createdAt = parseInstant(entries.get("createdAt"), key, "createdAt");
Instant completedAt = parseInstant(entries.get("completedAt"), key, "completedAt");
List<String> fileIds = parseList(entries.get("fileIds"), key);
Map<String, String> resultMeta = parseMap(entries.get("resultMeta"), key);
String stateName =
String.valueOf(
entries.getOrDefault("state", JobStoreEntry.JobState.PENDING.name()));
JobStoreEntry.JobState state;
try {
state = JobStoreEntry.JobState.valueOf(stateName);
} catch (IllegalArgumentException ex) {
log.warn("Unrecognised job state '{}' in {}, defaulting to PENDING", stateName, key);
state = JobStoreEntry.JobState.PENDING;
}
String owningNodeId = String.valueOf(entries.getOrDefault("owningNodeId", ""));
String error = entries.get("error") == null ? null : entries.get("error").toString();
return Optional.of(
new JobStoreEntry(
jobId.toString(),
state,
owningNodeId,
createdAt,
completedAt,
error,
fileIds,
resultMeta));
}
private Instant parseInstant(Object v, String key, String field) {
if (v == null) {
return null;
}
try {
return Instant.parse(v.toString());
} catch (RuntimeException e) {
log.warn(
"JobStore {} field '{}' has malformed timestamp '{}' - treating as missing",
key,
field,
v);
return null;
}
}
private List<String> parseList(Object v, String key) {
if (v == null) {
return new ArrayList<>();
}
return readJsonList(v.toString(), key);
}
private Map<String, String> parseMap(Object v, String key) {
if (v == null) {
return new HashMap<>();
}
try {
return MAPPER.readValue(v.toString(), MAP_STRING);
} catch (JsonProcessingException e) {
log.warn(
"JobStore {} field 'resultMeta' is not valid JSON '{}' - treating as empty",
key,
v);
return new HashMap<>();
}
}
private static String writeJson(Object value) {
try {
return MAPPER.writeValueAsString(value);
} catch (JsonProcessingException e) {
// The shapes we serialize are simple List<String> / Map<String,String>; Jackson
// can encode these without escapes that fail. Surface anything unexpected loud and
// early rather than persisting a half-serialized field that would re-throw on read.
throw new IllegalStateException("Failed to JSON-serialize JobStore field", e);
}
}
private List<String> readJsonList(String json, String key) {
try {
List<String> parsed = MAPPER.readValue(json, LIST_STRING);
return parsed == null ? new ArrayList<>() : parsed;
} catch (JsonProcessingException e) {
log.warn(
"JobStore {} field 'fileIds' is not valid JSON '{}' - treating as empty",
key,
json);
return new ArrayList<>();
}
}
}
@@ -0,0 +1,62 @@
package stirling.software.proprietary.cluster.valkey;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import stirling.software.common.cluster.KeyValueCache;
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
public class ValkeyKeyValueCache implements KeyValueCache {
private static final String PREFIX = "stirling:kv:";
private final StringRedisTemplate template;
@Override
public void put(String namespace, String key, String value, Duration ttl) {
template.opsForValue()
.set(buildKey(namespace, key), value, ttl.toMillis(), TimeUnit.MILLISECONDS);
}
@Override
public Optional<String> get(String namespace, String key) {
return Optional.ofNullable(template.opsForValue().get(buildKey(namespace, key)));
}
@Override
public void evict(String namespace, String key) {
template.delete(buildKey(namespace, key));
}
@Override
public void evictNamespace(String namespace) {
// SCAN, not KEYS: KEYS blocks the server until it has walked the entire keyspace.
ScanOptions options =
ScanOptions.scanOptions().match(PREFIX + namespace + ":*").count(256).build();
List<String> keys = new ArrayList<>();
try (Cursor<String> cursor = template.scan(options)) {
while (cursor.hasNext()) {
keys.add(cursor.next());
}
}
if (!keys.isEmpty()) {
template.delete(keys);
}
}
private String buildKey(String namespace, String key) {
return PREFIX + namespace + ":" + key;
}
}
@@ -0,0 +1,82 @@
package stirling.software.proprietary.cluster.valkey;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.stereotype.Component;
import io.github.bucket4j.BucketConfiguration;
import io.github.bucket4j.ConsumptionProbe;
import io.github.bucket4j.distributed.BucketProxy;
import io.github.bucket4j.distributed.proxy.ProxyManager;
import io.github.bucket4j.redis.lettuce.Bucket4jLettuce;
import io.lettuce.core.AbstractRedisClient;
import io.lettuce.core.RedisClient;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import stirling.software.common.cluster.RateLimitStore;
/**
* Valkey-backed token-bucket rate limiting via Bucket4j's Lettuce ProxyManager.
*
* <p>Replaces the earlier hand-rolled INCR+EXPIRE Lua fixed-window script. The fixed-window impl
* could allow a caller to spend the full bucket at second 59 of one window and the full bucket
* again at second 1 of the next window (effective burst of 2x capacity at boundaries). The Bucket4j
* token bucket refills continuously and removes that boundary doubling, giving cross-node parity
* with the in-process {@code InProcessRateLimitStore} which already uses Bucket4j.
*/
@Component
@ConditionalOnValkeyBackplane
public class ValkeyRateLimitStore implements RateLimitStore {
private static final String PREFIX = "stirling:rl:";
private final LettuceConnectionFactory connectionFactory;
private ProxyManager<byte[]> proxyManager;
public ValkeyRateLimitStore(LettuceConnectionFactory connectionFactory) {
this.connectionFactory = connectionFactory;
}
@PostConstruct
void initProxyManager() {
AbstractRedisClient client = connectionFactory.getNativeClient();
if (!(client instanceof RedisClient redisClient)) {
throw new IllegalStateException(
"ValkeyRateLimitStore requires a standalone Lettuce RedisClient; got "
+ (client == null ? "null" : client.getClass().getName())
+ " (cluster client not yet supported by this rate limit impl)");
}
this.proxyManager = Bucket4jLettuce.casBasedBuilder(redisClient).build();
}
@PreDestroy
void shutdown() {
// Lettuce client lifecycle is owned by Spring (LettuceConnectionFactory#destroy), so we
// only drop the proxy reference. No explicit close needed.
proxyManager = null;
}
@Override
public RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod) {
byte[] key = (PREFIX + bucketKey).getBytes(StandardCharsets.UTF_8);
// Greedy refill of capacity tokens per refillPeriod, matching InProcessRateLimitStore
// semantics (continuously refilling, no fixed-window boundary doubling).
BucketConfiguration cfg =
BucketConfiguration.builder()
.addLimit(
stage ->
stage.capacity(capacity)
.refillGreedy(capacity, refillPeriod))
.build();
BucketProxy bucket = proxyManager.builder().build(key, () -> cfg);
ConsumptionProbe probe = bucket.tryConsumeAndReturnRemaining(1);
if (probe.isConsumed()) {
return new RateLimitDecision(true, probe.getRemainingTokens(), 0L);
}
return new RateLimitDecision(false, 0L, probe.getNanosToWaitForRefill());
}
}
@@ -32,10 +32,7 @@ public enum AiPdfContentType {
// Heavy content
COMPLIANCE("compliance"),
IMAGES("images"),
// PyMuPDF worker pre-rendered Markdown
PYMUPDF_MARKDOWN("pymupdf_markdown");
IMAGES("images");
private final String value;
@@ -11,7 +11,7 @@ import jakarta.validation.constraints.NotNull;
import lombok.Data;
@Data
@Schema(description = "Run an AI workflow")
@Schema(description = "Run an AI workflow against one or more PDF files")
public class AiWorkflowRequest {
@NotNull
@@ -49,11 +49,7 @@ public class EEAppConfig {
@Profile("security & !saas")
@Bean(name = "SSOAutoLogin")
public boolean ssoAutoLogin() {
boolean enabled = applicationProperties.getPremium().getProFeatures().isSsoAutoLogin();
if (enabled) {
licenseKeyChecker.requireProOrEnterprise("premium.proFeatures.ssoAutoLogin=true");
}
return enabled;
return applicationProperties.getPremium().getProFeatures().isSsoAutoLogin();
}
// TODO: Remove post migration
@@ -32,10 +32,7 @@ public class LicenseKeyChecker {
private final UserLicenseSettingsService licenseSettingsService;
// volatile: written by evaluateLicense() on the @Scheduled refresh thread, read by request
// threads via getPremiumLicenseEnabledResult() / requireProOrEnterprise(). Ensures readers see
// the latest tier rather than a stale cached value.
private volatile License premiumEnabledResult = License.NORMAL;
private License premiumEnabledResult = License.NORMAL;
public LicenseKeyChecker(
KeygenLicenseVerifier licenseService,
@@ -136,16 +133,4 @@ public class LicenseKeyChecker {
public License getPremiumLicenseEnabledResult() {
return premiumEnabledResult;
}
/**
* Throws {@link IllegalStateException} if the current license is not Pro or Enterprise. Used by
* boot-time gates to fail fast when an operator enables a premium-only setting without a valid
* license. {@code configuredAs} is the human-readable property path (e.g. {@code
* "storage.provider=s3"}) and appears in the exception message.
*/
public void requireProOrEnterprise(String configuredAs) {
if (premiumEnabledResult != License.SERVER && premiumEnabledResult != License.ENTERPRISE) {
throw new IllegalStateException(configuredAs + " requires a Pro or Enterprise license");
}
}
}
@@ -17,7 +17,6 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.GeneralApi;
import stirling.software.common.enumeration.ResourceWeight;
import stirling.software.proprietary.security.model.api.Email;
import stirling.software.proprietary.security.service.EmailService;
@@ -40,10 +39,7 @@ public class EmailController {
* attachment.
* @return ResponseEntity with success or error message.
*/
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
value = "/send-email",
resourceWeight = ResourceWeight.SMALL_WEIGHT)
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/send-email")
@Operation(
summary = "Send an email with an attachment",
description =
@@ -2,9 +2,9 @@ package stirling.software.proprietary.security.filter;
import java.io.IOException;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Profile;
import org.springframework.http.HttpStatus;
@@ -15,9 +15,6 @@ import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
import io.github.bucket4j.Bandwidth;
import io.github.bucket4j.Bucket;
import io.github.bucket4j.ConsumptionProbe;
import io.github.pixee.security.Newlines;
import jakarta.servlet.FilterChain;
@@ -25,22 +22,28 @@ import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import stirling.software.common.cluster.RateLimitStore;
import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
import stirling.software.common.model.enumeration.Role;
import stirling.software.common.util.RegexPatternUtils;
import stirling.software.proprietary.cluster.ClusterMetrics;
@Component
@Profile("!saas")
public class UserBasedRateLimitingFilter extends OncePerRequestFilter {
private final Map<String, Bucket> apiBuckets = new ConcurrentHashMap<>();
private final Map<String, Bucket> webBuckets = new ConcurrentHashMap<>();
private final RateLimitStore rateLimitStore;
@Qualifier("rateLimit")
private final boolean rateLimit;
public UserBasedRateLimitingFilter(@Qualifier("rateLimit") boolean rateLimit) {
@Autowired(required = false)
private ClusterMetrics clusterMetrics;
public UserBasedRateLimitingFilter(
@Qualifier("rateLimit") boolean rateLimit, RateLimitStore rateLimitStore) {
this.rateLimit = rateLimit;
this.rateLimitStore = rateLimitStore;
}
@Override
@@ -48,54 +51,44 @@ public class UserBasedRateLimitingFilter extends OncePerRequestFilter {
HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
throws ServletException, IOException {
if (!rateLimit) {
// If rateLimit is not enabled, just pass all requests without rate limiting
filterChain.doFilter(request, response);
return;
}
String method = request.getMethod();
if (!"POST".equalsIgnoreCase(method)) {
// If the request is not a POST, just pass it through without rate limiting
filterChain.doFilter(request, response);
return;
}
String identifier = null;
// Check for API key in the request headers
String identifier;
String apiKey = request.getHeader("X-API-KEY");
if (apiKey != null && !apiKey.trim().isEmpty()) {
identifier = // Prefix to distinguish between API keys and usernames
"API_KEY_" + apiKey;
// Hash the API key so the raw value never appears in any Valkey rate-limit bucket key.
identifier = "API_KEY_" + DigestUtils.sha256Hex(apiKey);
} else {
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
if (authentication != null && authentication.isAuthenticated()) {
UserDetails userDetails = (UserDetails) authentication.getPrincipal();
// AnonymousAuthenticationToken.isAuthenticated() == true but principal is
// "anonymousUser";
// guard the cast so anonymous requests fall through to the remote-addr branch.
if (authentication != null
&& authentication.isAuthenticated()
&& authentication.getPrincipal() instanceof UserDetails userDetails) {
identifier = userDetails.getUsername();
} else {
identifier = request.getRemoteAddr();
}
}
// If neither API key nor an authenticated user is present, use IP address
if (identifier == null) {
identifier = request.getRemoteAddr();
}
Role userRole =
getRoleFromAuthentication(SecurityContextHolder.getContext().getAuthentication());
String scope;
int limitPerDay;
if (request.getHeader("X-API-KEY") != null) {
// It's an API call
processRequest(
userRole.getApiCallsPerDay(),
identifier,
apiBuckets,
request,
response,
filterChain);
scope = "api:";
limitPerDay = userRole.getApiCallsPerDay();
} else {
// It's a Web UI call
processRequest(
userRole.getWebCallsPerDay(),
identifier,
webBuckets,
request,
response,
filterChain);
scope = "web:";
limitPerDay = userRole.getWebCallsPerDay();
}
processRequest(limitPerDay, scope + identifier, request, response, filterChain);
}
private Role getRoleFromAuthentication(Authentication authentication) {
@@ -108,43 +101,50 @@ public class UserBasedRateLimitingFilter extends OncePerRequestFilter {
}
}
}
throw new IllegalStateException("User does not have a valid role.");
return Role.WEB_ONLY_USER; // no matching authority - use most restrictive bucket
}
private void processRequest(
int limitPerDay,
String identifier,
Map<String, Bucket> buckets,
String bucketKey,
HttpServletRequest request,
HttpServletResponse response,
FilterChain filterChain)
throws IOException, ServletException {
Bucket userBucket = buckets.computeIfAbsent(identifier, k -> createUserBucket(limitPerDay));
ConsumptionProbe probe = userBucket.tryConsumeAndReturnRemaining(1);
if (probe.isConsumed()) {
RateLimitDecision probe;
try {
probe = rateLimitStore.tryConsume(bucketKey, limitPerDay, Duration.ofDays(1));
} catch (RuntimeException ex) {
// Fail OPEN: a rate-limit backend outage (e.g. Valkey unreachable in cluster mode)
// must not turn every POST into a 500. Availability beats strict enforcement here -
// allow the request through and log so the outage stays visible. The in-process store
// never throws, so single-node behaviour is unchanged.
logger.warn(
"Rate-limit backend unavailable for "
+ bucketKey
+ "; allowing request (fail-open): "
+ ex.getMessage());
filterChain.doFilter(request, response);
return;
}
if (probe.allowed()) {
response.setHeader(
"X-Rate-Limit-Remaining",
stripNewlines(Newlines.stripAll(Long.toString(probe.getRemainingTokens()))));
stripNewlines(Newlines.stripAll(Long.toString(probe.remainingTokens()))));
filterChain.doFilter(request, response);
} else {
long waitForRefill = probe.getNanosToWaitForRefill() / 1_000_000_000;
long waitForRefill = probe.nanosToWaitForRefill() / 1_000_000_000;
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());
response.setHeader(
"X-Rate-Limit-Retry-After-Seconds",
Newlines.stripAll(String.valueOf(waitForRefill)));
response.getWriter().write("Rate limit exceeded for POST requests.");
if (clusterMetrics != null) {
clusterMetrics.recordRateLimitReject();
}
}
}
private Bucket createUserBucket(int limitPerDay) {
Bandwidth limit =
Bandwidth.builder()
.capacity(limitPerDay)
.refillIntervally(limitPerDay, Duration.ofDays(1))
.build();
return Bucket.builder().addLimit(limit).build();
}
private static String stripNewlines(final String s) {
return RegexPatternUtils.getInstance().getNewlineCharsPattern().matcher(s).replaceAll("");
}
@@ -1,10 +1,6 @@
package stirling.software.proprietary.security.service;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import org.springframework.security.authentication.LockedException;
import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserRequest;
@@ -12,8 +8,6 @@ import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserService;
import org.springframework.security.oauth2.client.userinfo.OAuth2UserService;
import org.springframework.security.oauth2.core.OAuth2AuthenticationException;
import org.springframework.security.oauth2.core.OAuth2Error;
import org.springframework.security.oauth2.core.oidc.OidcIdToken;
import org.springframework.security.oauth2.core.oidc.OidcUserInfo;
import org.springframework.security.oauth2.core.oidc.user.DefaultOidcUser;
import org.springframework.security.oauth2.core.oidc.user.OidcUser;
@@ -45,37 +39,20 @@ public class CustomOAuth2UserService implements OAuth2UserService<OidcUserReques
@Override
public OidcUser loadUser(OidcUserRequest userRequest) throws OAuth2AuthenticationException {
String registrationId = userRequest.getClientRegistration().getRegistrationId();
boolean debugLogging = Boolean.TRUE.equals(oauth2Properties.getDebugLogging());
// Resolved inside the try so a bad/null useAsUsername (IllegalArgumentException from
// valueOf, or NPE on toUpperCase) is caught and wrapped as OAuth2AuthenticationException
// by the existing handlers below, matching the pre-debugLogging behaviour.
String usernameAttributeKey = null;
try {
usernameAttributeKey =
OidcUser user = delegate.loadUser(userRequest);
String usernameAttributeKey =
UsernameAttribute.valueOf(oauth2Properties.getUseAsUsername().toUpperCase())
.getName();
OidcUser user = delegate.loadUser(userRequest);
if (debugLogging) {
logClaimDump(
"OAuth2/OIDC login claims received",
registrationId,
usernameAttributeKey,
user.getIdToken(),
user.getUserInfo(),
user.getAttributes(),
false);
}
// Extract SSO provider information
String ssoProviderId = user.getSubject(); // Standard OIDC 'sub' claim
String ssoProvider = userRequest.getClientRegistration().getRegistrationId();
String username = user.getAttribute(usernameAttributeKey);
log.debug(
"OAuth2 login - Provider: {}, ProviderId: {}, Username: {}",
registrationId,
ssoProvider,
ssoProviderId,
username);
@@ -102,154 +79,10 @@ public class CustomOAuth2UserService implements OAuth2UserService<OidcUserReques
usernameAttributeKey);
} catch (IllegalArgumentException e) {
log.error("Error loading OIDC user: {}", e.getMessage());
// Only emit the claim dump if we successfully resolved usernameAttributeKey. A null
// value here means UsernameAttribute.valueOf rejected the configured useAsUsername
// before delegate.loadUser ran that error message is self-explanatory and a claim
// dump would have no resolved-key to compare against.
if (debugLogging && usernameAttributeKey != null) {
// The DefaultOidcUser constructor (or our own checks) rejected the chosen
// username attribute. Dump the claims we DID receive so the operator can pick
// a different value for security.oauth2.useAsUsername.
logClaimDump(
"OAuth2/OIDC login FAILED - dumping received claims",
registrationId,
usernameAttributeKey,
userRequest.getIdToken(),
null,
userRequest.getIdToken() == null
? Collections.emptyMap()
: userRequest.getIdToken().getClaims(),
true);
}
throw new OAuth2AuthenticationException(new OAuth2Error(e.getMessage()), e);
} catch (Exception e) {
log.error("Unexpected error loading OIDC user", e);
if (debugLogging && usernameAttributeKey != null && userRequest.getIdToken() != null) {
logClaimDump(
"OAuth2/OIDC login FAILED (unexpected error) - dumping ID token claims",
registrationId,
usernameAttributeKey,
userRequest.getIdToken(),
null,
userRequest.getIdToken().getClaims(),
true);
}
throw new OAuth2AuthenticationException("Unexpected error during authentication");
}
}
/**
* Emits a multi-line diagnostic dump of the claims returned by the OAuth2/OIDC provider. Only
* invoked when {@code security.oauth2.debugLogging=true}.
*
* @param banner short title for the log block
* @param registrationId Spring client registration id (e.g. "demarest", "keycloak")
* @param usernameAttributeKey the claim key the application is configured to use as username
* @param idToken the decoded ID token, may be null on unexpected failures
* @param userInfo the decoded UserInfo response, may be null if the provider returned none
* @param mergedAttributes the merged attribute map Spring uses for {@code getAttribute()}
* @param failure true if logging in the error path (uses ERROR level), false for INFO
*/
private void logClaimDump(
String banner,
String registrationId,
String usernameAttributeKey,
OidcIdToken idToken,
OidcUserInfo userInfo,
Map<String, Object> mergedAttributes,
boolean failure) {
StringBuilder sb = new StringBuilder();
sb.append("\n========== [OAUTH2 DEBUG] ").append(banner).append(" ==========\n");
sb.append("Provider registrationId : ").append(registrationId).append('\n');
sb.append("Configured useAsUsername: ")
.append(oauth2Properties.getUseAsUsername())
.append(" (looks up claim key '")
.append(usernameAttributeKey)
.append("')\n");
if (idToken != null) {
Map<String, Object> idClaims = idToken.getClaims();
sb.append("\n-- ID token claims (")
.append(idClaims == null ? 0 : idClaims.size())
.append(") --\n");
appendClaims(sb, idClaims);
sb.append("ID token issued at : ").append(idToken.getIssuedAt()).append('\n');
sb.append("ID token expires at: ").append(idToken.getExpiresAt()).append('\n');
} else {
sb.append("\n-- ID token: <null> --\n");
}
if (userInfo != null && userInfo.getClaims() != null) {
sb.append("\n-- UserInfo endpoint claims (")
.append(userInfo.getClaims().size())
.append(") --\n");
appendClaims(sb, userInfo.getClaims());
} else {
sb.append("\n-- UserInfo endpoint claims: none returned --\n");
}
if (mergedAttributes != null) {
sb.append("\n-- Merged attribute keys available to useAsUsername: ")
.append(new TreeSet<>(mergedAttributes.keySet()))
.append("\n");
Object resolved = mergedAttributes.get(usernameAttributeKey);
sb.append("-- Value at '")
.append(usernameAttributeKey)
.append("' : ")
.append(resolved == null ? "<NULL — this is why login fails>" : resolved)
.append('\n');
if (resolved == null) {
Set<String> hints = suggestUsernameClaims(mergedAttributes.keySet());
if (!hints.isEmpty()) {
sb.append(
"-- Hint: the following claim(s) are present and map to a"
+ " known UsernameAttribute value — try setting"
+ " security.oauth2.useAsUsername to one of: ")
.append(hints)
.append('\n');
}
}
}
sb.append(
"\nWARNING: this block contains PII. Set security.oauth2.debugLogging=false once"
+ " troubleshooting is complete.\n");
sb.append("========== [/OAUTH2 DEBUG] ==========");
if (failure) {
log.error(sb.toString());
} else {
log.info(sb.toString());
}
}
private static void appendClaims(StringBuilder sb, Map<String, Object> claims) {
if (claims == null || claims.isEmpty()) {
sb.append(" (no claims)\n");
return;
}
// Sort for stable, scannable output
new TreeSet<>(claims.keySet())
.forEach(
key -> {
Object value = claims.get(key);
sb.append(" ").append(key).append(" = ").append(value).append('\n');
});
}
/**
* Returns the intersection of the claim keys the provider actually returned and the keys that
* {@link UsernameAttribute} accepts i.e. valid values the operator could put in {@code
* security.oauth2.useAsUsername} to make this login work.
*/
private static Set<String> suggestUsernameClaims(Set<String> availableClaimKeys) {
Set<String> supported = new TreeSet<>();
for (UsernameAttribute attr : UsernameAttribute.values()) {
if (availableClaimKeys.contains(attr.getName())) {
supported.add(attr.getName());
}
}
return supported;
}
}
@@ -210,20 +210,37 @@ public class JwtService implements JwtServiceInterface {
if (specificKeyPair.isPresent()) {
keyPair = specificKeyPair.get();
} else {
Optional<PublicKey> peerKey = keyPersistenceService.resolvePublicKey(keyId);
if (peerKey.isPresent()) {
return Jwts.parser()
.verifyWith(peerKey.get())
.clockSkewSeconds(getAllowedClockSkewSeconds())
.build()
.parseSignedClaims(token)
.getPayload();
}
log.warn(
"Key ID {} not found in keystore, token may have been signed with an expired key",
keyId);
if (keyId.equals(keyPersistenceService.getActiveKey().getKeyId())) {
JwtVerificationKey verificationKey =
keyPersistenceService.refreshActiveKeyPair();
Optional<KeyPair> refreshedKeyPair =
keyPersistenceService.getKeyPair(verificationKey.getKeyId());
if (refreshedKeyPair.isPresent()) {
keyPair = refreshedKeyPair.get();
// Re-check local store before rotating: rotating a key still on disk
// invalidates every in-flight token signed with it.
Optional<KeyPair> localActivePair = keyPersistenceService.getKeyPair(keyId);
if (localActivePair.isPresent()) {
keyPair = localActivePair.get();
} else {
throw new AuthenticationFailureException(
"Failed to retrieve refreshed key pair");
// Key missing everywhere - rotate to restore signing capability.
JwtVerificationKey verificationKey =
keyPersistenceService.refreshActiveKeyPair();
Optional<KeyPair> refreshedKeyPair =
keyPersistenceService.getKeyPair(verificationKey.getKeyId());
if (refreshedKeyPair.isPresent()) {
keyPair = refreshedKeyPair.get();
} else {
throw new AuthenticationFailureException(
"Failed to retrieve refreshed key pair");
}
}
} else {
// Try to use active key as fallback
@@ -240,7 +257,6 @@ public class JwtService implements JwtServiceInterface {
}
} else {
log.debug("No key ID in token header, trying all available keys");
// Try all available keys when no keyId is present
return tryAllKeys(token, allowExpired);
}
@@ -299,8 +315,6 @@ public class JwtService implements JwtServiceInterface {
| NoSuchAlgorithmException
| InvalidKeySpecException activeKeyException) {
log.debug("Active key failed, trying all available keys from cache");
// If active key fails, try all available keys from cache
List<JwtVerificationKey> allKeys =
keyPersistenceService.getKeysEligibleForCleanup(
LocalDateTime.now().plusDays(1));
@@ -339,13 +353,10 @@ public class JwtService implements JwtServiceInterface {
@Override
public String extractToken(HttpServletRequest request) {
// Extract from Authorization header Bearer token
String authHeader = request.getHeader("Authorization");
if (authHeader != null && authHeader.startsWith("Bearer ")) {
String token = authHeader.substring(7); // Remove "Bearer " prefix
return token;
return authHeader.substring(7);
}
return null;
}
@@ -354,22 +365,11 @@ public class JwtService implements JwtServiceInterface {
return v2Enabled;
}
/**
* Extract key ID from JWT header without validating the token.
*
* <p>Parses the Base64-encoded JWT header to retrieve the "kid" (key ID) claim. Returns null if
* the header cannot be parsed or does not contain a key ID.
*
* @param token the JWT token
* @return the key ID, or null if not found or parsing fails
*/
/** Return the {@code kid} claim from the JWT header, or null if absent or unparseable. */
private String extractKeyId(String token) {
try {
String[] tokenParts = token.split("\\.");
if (tokenParts.length < 2) {
log.debug(
"Token does not have enough parts (expected at least 2, got {})",
tokenParts.length);
return null;
}
@@ -1,9 +1,11 @@
package stirling.software.proprietary.security.service;
import java.io.IOException;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.KeyFactory;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
@@ -15,6 +17,7 @@ import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.RSAPublicKeySpec;
import java.security.spec.X509EncodedKeySpec;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
@@ -28,16 +31,21 @@ import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.caffeine.CaffeineCache;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.KeyValueCache;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.security.model.JwtVerificationKey;
/**
* SECURITY: the {@link #JWT_PUBKEY_NAMESPACE} cluster cache is trust-on-publish. Operators MUST
* restrict Valkey ACL writes to app pods, enable AUTH + TLS, and network-isolate the deployment.
* Future hardening: HMAC-signed broadcasts with a cluster master secret.
*/
@Slf4j
@Service
public class KeyPersistenceService implements KeyPersistenceServiceInterface {
@@ -45,18 +53,28 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
public static final String KEY_SUFFIX = ".key";
public static final String PUB_KEY_SUFFIX = ".pub";
private static final Duration JWT_PUBKEY_CLUSTER_TTL = Duration.ofHours(24);
/** Cluster KeyValueCache namespace used to broadcast public keys to peers. */
public static final String JWT_PUBKEY_NAMESPACE = "jwtkey";
private final ApplicationProperties.Security.Jwt jwtProperties;
private final CacheManager cacheManager;
private final Cache verifyingKeyCache;
private final KeyValueCache clusterKeyCache; // null in single-instance mode
private volatile JwtVerificationKey activeKey;
@Autowired
public KeyPersistenceService(
ApplicationProperties applicationProperties, CacheManager cacheManager) {
ApplicationProperties applicationProperties,
CacheManager cacheManager,
@Autowired(required = false) KeyValueCache clusterKeyCache) {
this.jwtProperties = applicationProperties.getSecurity().getJwt();
this.cacheManager = cacheManager;
this.verifyingKeyCache = cacheManager.getCache("verifyingKeys");
this.clusterKeyCache = clusterKeyCache;
}
@PostConstruct
@@ -74,12 +92,6 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
}
}
/**
* Load all existing JWT keys from disk into memory on startup.
*
* <p>This ensures tokens signed with previous keys remain valid after server restart. If no
* keys exist on disk, generates a new keypair.
*/
private void loadExistingKeysFromDisk() {
try {
Path keyDirectory = Paths.get(InstallationPathConfig.getPrivateKeyPath());
@@ -94,11 +106,8 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
try (var stream = Files.list(keyDirectory)) {
keyFiles =
stream.filter(path -> path.toString().endsWith(KEY_SUFFIX))
.sorted(
(a, b) ->
b.getFileName().compareTo(a.getFileName())) // Most
// recent
// first
// most recent first
.sorted((a, b) -> b.getFileName().compareTo(a.getFileName()))
.collect(Collectors.toList());
}
@@ -115,11 +124,10 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
try {
String keyId = keyFile.getFileName().toString().replace(KEY_SUFFIX, "");
// Load private key first
PrivateKey privateKey = loadPrivateKey(keyId);
// Try to load public key, or generate it from private key if missing
// (migration)
// Try to load public key; generate from private key if missing (legacy
// migration).
String encodedPublicKey;
try {
encodedPublicKey = loadPublicKey(keyId);
@@ -139,13 +147,11 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
log.info("Successfully migrated key: {}", keyId);
}
// Create verification key and add to cache
JwtVerificationKey verifyingKey =
new JwtVerificationKey(keyId, encodedPublicKey);
verifyingKeyCache.put(keyId, verifyingKey);
loadedCount++;
// Set the most recent key as active (first in sorted list)
if (activeKey == null) {
activeKey = verifyingKey;
log.info("Set active JWT signing key: {}", keyId);
@@ -179,7 +185,6 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
}
}
@Transactional
private JwtVerificationKey generateAndStoreKeypair() {
JwtVerificationKey verifyingKey = null;
@@ -188,9 +193,12 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
String keyId = generateKeyId();
storeKeyPair(keyId, keyPair);
verifyingKey = new JwtVerificationKey(keyId, encodePublicKey(keyPair.getPublic()));
String encodedPublicKey = encodePublicKey(keyPair.getPublic());
verifyingKey = new JwtVerificationKey(keyId, encodedPublicKey);
verifyingKeyCache.put(keyId, verifyingKey);
activeKey = verifyingKey;
// Broadcast so peer nodes can verify tokens we sign without waiting for restart.
publishToCluster(keyId, encodedPublicKey);
log.info("Generated and stored new JWT keypair: {}", keyId);
} catch (IOException e) {
log.error("Failed to generate and store keypair", e);
@@ -199,6 +207,23 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
return verifyingKey;
}
private void publishToCluster(String keyId, String encodedPublicKey) {
if (clusterKeyCache == null) {
return;
}
try {
clusterKeyCache.put(
JWT_PUBKEY_NAMESPACE, keyId, encodedPublicKey, JWT_PUBKEY_CLUSTER_TTL);
log.info("Broadcast JWT public key to cluster KeyValueCache: {}", keyId);
} catch (RuntimeException e) {
// Non-fatal: we still serve tokens locally; peers catch up on their next restart.
log.warn(
"Failed to broadcast JWT public key {} to cluster cache: {}",
keyId,
e.getMessage());
}
}
@Override
public JwtVerificationKey getActiveKey() {
if (activeKey == null) {
@@ -249,6 +274,17 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
condition = "#root.target.isKeystoreEnabled()")
public void removeKey(String keyId) {
verifyingKeyCache.evict(keyId);
// Evict cluster broadcast so peers don't keep serving the removed key for up to 24h.
if (clusterKeyCache != null && keyId != null) {
try {
clusterKeyCache.evict(JWT_PUBKEY_NAMESPACE, keyId);
} catch (RuntimeException e) {
log.warn(
"Failed to evict JWT public key {} from cluster cache: {}",
keyId,
e.getMessage());
}
}
}
@Override
@@ -305,33 +341,29 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
}
/**
* Store both private and public keys to disk.
* Store both private and public keys to disk using a temp-then-atomic-rename pattern.
*
* <p>Private key stored as: keyId.key
*
* <p>Public key stored as: keyId.pub
* <p>The caller broadcasts the public key to peers immediately after this returns. If a peer
* learned about the keyId before the private key was fully durable, a crash mid-write followed
* by restart would lose the key while peers still serve tokens signed with it. Writing to
* {@code <file>.tmp} and moving with {@link StandardCopyOption#ATOMIC_MOVE} guarantees the
* final path either contains the fully-written payload or does not exist at all.
*/
private void storeKeyPair(String keyId, KeyPair keyPair) throws IOException {
Path keyDirectory = Paths.get(InstallationPathConfig.getPrivateKeyPath());
// Store private key
Path privateKeyFile = keyDirectory.resolve(keyId + KEY_SUFFIX);
String encodedPrivateKey =
Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded());
Files.writeString(privateKeyFile, encodedPrivateKey);
// Set read/write to only the owner (security)
writeAtomically(privateKeyFile, encodedPrivateKey);
privateKeyFile.toFile().setReadable(true, true);
privateKeyFile.toFile().setWritable(true, true);
privateKeyFile.toFile().setExecutable(false, false);
// Store public key
Path publicKeyFile = keyDirectory.resolve(keyId + PUB_KEY_SUFFIX);
String encodedPublicKey =
Base64.getEncoder().encodeToString(keyPair.getPublic().getEncoded());
Files.writeString(publicKeyFile, encodedPublicKey);
// Public key can be more permissive but still restrict to owner
writeAtomically(publicKeyFile, encodedPublicKey);
publicKeyFile.toFile().setReadable(true, true);
publicKeyFile.toFile().setWritable(true, true);
publicKeyFile.toFile().setExecutable(false, false);
@@ -343,6 +375,31 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
publicKeyFile.getFileName());
}
/**
* Write {@code contents} to {@code finalPath} so that the final path either contains the full
* payload or does not exist. Writes to a sibling {@code .tmp} file first and renames it.
* Returns silently after falling back to a non-atomic move if the filesystem does not support
* {@link StandardCopyOption#ATOMIC_MOVE}.
*/
static void writeAtomically(Path finalPath, String contents) throws IOException {
Path tmp = finalPath.resolveSibling(finalPath.getFileName().toString() + ".tmp");
Files.writeString(tmp, contents);
try {
Files.move(
tmp,
finalPath,
StandardCopyOption.REPLACE_EXISTING,
StandardCopyOption.ATOMIC_MOVE);
} catch (AtomicMoveNotSupportedException e) {
log.warn(
"Filesystem does not support atomic move for {}; falling back to non-atomic"
+ " replace. A crash between rename and fsync may leave the key partially"
+ " written.",
finalPath);
Files.move(tmp, finalPath, StandardCopyOption.REPLACE_EXISTING);
}
}
private PrivateKey loadPrivateKey(String keyId)
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
Path keyFile =
@@ -360,13 +417,6 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
return keyFactory.generatePrivate(keySpec);
}
/**
* Load public key from disk.
*
* @param keyId the key identifier
* @return Base64-encoded public key string
* @throws IOException if the public key file is not found
*/
private String loadPublicKey(String keyId) throws IOException {
Path publicKeyFile =
Paths.get(InstallationPathConfig.getPrivateKeyPath())
@@ -379,29 +429,12 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
return Files.readString(publicKeyFile).trim();
}
/**
* Reconstruct a KeyPair from a PrivateKey.
*
* <p>For RSA keys, derives the public key from the private key.
*
* @param privateKey the RSA private key
* @return reconstructed KeyPair
* @throws NoSuchAlgorithmException if RSA algorithm is not available
* @throws InvalidKeySpecException if the key specification is invalid
*/
private KeyPair reconstructKeyPair(PrivateKey privateKey)
throws NoSuchAlgorithmException, InvalidKeySpecException {
// For RSA, we can derive the public key from the private key
KeyFactory keyFactory = KeyFactory.getInstance("RSA");
// Get the private key spec
RSAPrivateCrtKey rsaPrivateKey = (RSAPrivateCrtKey) privateKey;
// Create public key spec from private key parameters
RSAPublicKeySpec publicKeySpec =
new RSAPublicKeySpec(rsaPrivateKey.getModulus(), rsaPrivateKey.getPublicExponent());
// Generate public key
PublicKey publicKey = keyFactory.generatePublic(publicKeySpec);
return new KeyPair(publicKey, privateKey);
@@ -418,4 +451,50 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
KeyFactory keyFactory = KeyFactory.getInstance("RSA");
return keyFactory.generatePublic(keySpec);
}
@Override
public Optional<PublicKey> resolvePublicKey(String keyId) {
if (keyId == null || keyId.isBlank()) {
return Optional.empty();
}
// 1. Local in-memory cache (warm path - same node that signed, or already learnt).
JwtVerificationKey local = verifyingKeyCache.get(keyId, JwtVerificationKey.class);
if (local != null) {
return decodeQuietly(local.getVerifyingKey(), keyId);
}
// 2. Local disk (cold path on restart).
try {
String onDisk = loadPublicKey(keyId);
JwtVerificationKey rebuilt = new JwtVerificationKey(keyId, onDisk);
verifyingKeyCache.put(keyId, rebuilt);
return decodeQuietly(onDisk, keyId);
} catch (IOException ignored) {
// not on this node's disk - try the cluster cache
}
// 3. Cluster cache. NOT written to local cache: expireAfterWrite would outlive the
// broadcast TTL and serve stale keys after peer rotation. Valkey faults return empty.
if (clusterKeyCache != null) {
try {
Optional<String> remote = clusterKeyCache.get(JWT_PUBKEY_NAMESPACE, keyId);
if (remote.isPresent()) {
String encoded = remote.get();
log.debug("Resolved JWT public key {} from cluster KeyValueCache", keyId);
return decodeQuietly(encoded, keyId);
}
} catch (RuntimeException e) {
log.warn("Cluster key cache unavailable for keyId {}: {}", keyId, e.getMessage());
return Optional.empty();
}
}
return Optional.empty();
}
private Optional<PublicKey> decodeQuietly(String encoded, String keyId) {
try {
return Optional.of(decodePublicKey(encoded));
} catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
log.warn("Could not decode public key for {}: {}", keyId, e.getMessage());
return Optional.empty();
}
}
}
@@ -26,4 +26,10 @@ public interface KeyPersistenceServiceInterface {
PublicKey decodePublicKey(String encodedKey)
throws NoSuchAlgorithmException, InvalidKeySpecException;
/**
* Resolve a public key by id, consulting the cluster cache when the key is unknown locally.
* Returns empty if the keyId is unknown anywhere.
*/
Optional<PublicKey> resolvePublicKey(String keyId);
}
@@ -5,7 +5,10 @@ import static stirling.software.proprietary.security.service.MfaService.MFA_LAST
import static stirling.software.proprietary.security.service.MfaService.MFA_REQUIRED_KEY;
import static stirling.software.proprietary.security.service.MfaService.MFA_SECRET_KEY;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.sql.SQLException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -16,6 +19,7 @@ import java.util.Optional;
import java.util.UUID;
import java.util.function.Supplier;
import org.apache.commons.codec.digest.DigestUtils;
import org.slf4j.MDC;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
@@ -34,6 +38,7 @@ import org.springframework.transaction.annotation.Transactional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.KeyValueCache;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.enumeration.Role;
import stirling.software.common.model.exception.UnsupportedProviderException;
@@ -79,6 +84,13 @@ public class UserService implements UserServiceInterface {
private final ApplicationProperties.Security.OAUTH2 oAuth2;
private final KeyValueCache keyValueCache;
private static final String API_KEY_CACHE_NS = "apikey";
private static final Duration API_KEY_TTL = Duration.ofSeconds(60);
private static final Duration API_KEY_NEGATIVE_TTL = Duration.ofSeconds(10);
private static final String API_KEY_NEGATIVE_MARKER = "__none__";
private final PersistentLoginRepository persistentLoginRepository;
private final UserServerCertificateService userServerCertificateService;
private final WorkflowParticipantRepository workflowParticipantRepository;
@@ -143,11 +155,7 @@ public class UserService implements UserServiceInterface {
if (user.isEmpty()) {
throw new UsernameNotFoundException("API key is not valid");
}
// Convert the user into an Authentication object
return new UsernamePasswordAuthenticationToken( // principal (typically the user)
user, // credentials (we don't expose the password or API key here)
null, // user's authorities (roles/permissions)
getAuthorities(user.get()));
return new UsernamePasswordAuthenticationToken(user, null, getAuthorities(user.get()));
}
private Collection<? extends GrantedAuthority> getAuthorities(User user) {
@@ -176,14 +184,19 @@ public class UserService implements UserServiceInterface {
private User saveUser(Optional<User> user, String apiKey) {
if (user.isPresent()) {
String previousKey = user.get().getApiKey();
user.get().setApiKey(apiKey);
return userRepository.save(user.get());
User saved = userRepository.save(user.get());
// Evict the previously cached entry so peers see the rotation within negative TTL.
if (previousKey != null && !previousKey.isBlank()) {
evictApiKeyCache(previousKey);
}
return saved;
}
throw new UsernameNotFoundException("User not found");
}
public User refreshApiKeyForUser(String username) {
// reuse the add API key method for refreshing
return addApiKeyToUser(username);
}
@@ -208,25 +221,71 @@ public class UserService implements UserServiceInterface {
}
public boolean isValidApiKey(String apiKey) {
return userRepository.findByApiKey(apiKey).isPresent();
return getUserByApiKey(apiKey).isPresent();
}
public Optional<User> getUserByApiKey(String apiKey) {
return userRepository.findByApiKey(apiKey);
return findByApiKeyCached(apiKey);
}
public Optional<User> loadUserByApiKey(String apiKey) {
Optional<User> user = userRepository.findByApiKey(apiKey);
Optional<User> user = findByApiKeyCached(apiKey);
if (user.isPresent()) {
return user;
}
// or throw an exception
return null;
}
private Optional<User> findByApiKeyCached(String apiKey) {
if (apiKey == null || apiKey.isBlank() || keyValueCache == null) {
return userRepository.findByApiKey(apiKey);
}
String keyHash = DigestUtils.sha256Hex(apiKey);
Optional<String> cached = keyValueCache.get(API_KEY_CACHE_NS, keyHash);
if (cached.isPresent()) {
String value = cached.get();
if (API_KEY_NEGATIVE_MARKER.equals(value)) {
return Optional.empty();
}
Optional<User> user = userRepository.findByUsernameIgnoreCase(value);
if (user.isPresent() && constantTimeEquals(apiKey, user.get().getApiKey())) {
return user;
}
keyValueCache.evict(API_KEY_CACHE_NS, keyHash);
}
Optional<User> user = userRepository.findByApiKey(apiKey);
if (user.isPresent()) {
keyValueCache.put(API_KEY_CACHE_NS, keyHash, user.get().getUsername(), API_KEY_TTL);
} else {
keyValueCache.put(
API_KEY_CACHE_NS, keyHash, API_KEY_NEGATIVE_MARKER, API_KEY_NEGATIVE_TTL);
}
return user;
}
/** Invalidate the cached API-key entry on rotation. */
public void evictApiKeyCache(String apiKey) {
if (apiKey != null && !apiKey.isBlank() && keyValueCache != null) {
keyValueCache.evict(API_KEY_CACHE_NS, DigestUtils.sha256Hex(apiKey));
}
}
public boolean validateApiKeyForUser(String username, String apiKey) {
Optional<User> userOpt = findByUsernameIgnoreCase(username);
return userOpt.isPresent() && apiKey.equals(userOpt.get().getApiKey());
return userOpt.isPresent() && constantTimeEquals(apiKey, userOpt.get().getApiKey());
}
/**
* Constant-time comparison of two API keys. {@link MessageDigest#isEqual} short-circuits only
* on null/empty inputs; for equal-length and unequal-length non-empty strings it scans every
* byte, so a remote attacker cannot infer the stored key via response-time differences.
*/
private static boolean constantTimeEquals(String provided, String stored) {
if (provided == null || stored == null) {
return false;
}
return MessageDigest.isEqual(
provided.getBytes(StandardCharsets.UTF_8), stored.getBytes(StandardCharsets.UTF_8));
}
@Transactional
@@ -706,8 +765,13 @@ public class UserService implements UserServiceInterface {
User updatedUser = existingUser.get();
if (!customApiKey.equals(updatedUser.getApiKey())) {
// Capture before mutation so we can evict the prior cache entry.
String previousKey = updatedUser.getApiKey();
updatedUser.setApiKey(customApiKey);
userRepository.save(updatedUser);
if (previousKey != null && !previousKey.isBlank()) {
evictApiKeyCache(previousKey);
}
}
},
() -> {
@@ -31,7 +31,6 @@ import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.InternalApiClient;
import stirling.software.common.service.InternalApiTimeoutException;
import stirling.software.common.service.PyMuPdfConverter;
import stirling.software.common.service.ToolMetadataService;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.TempFile;
@@ -75,7 +74,6 @@ public class AiWorkflowService {
private final TempFileManager tempFileManager;
private final FileIdStrategy fileIdStrategy;
private final AiEngineEndpointResolver endpointResolver;
private final PyMuPdfConverter pyMuPdfConverter;
@FunctionalInterface
public interface ProgressListener {
@@ -139,9 +137,6 @@ public class AiWorkflowService {
? new ArrayList<>()
: new ArrayList<>(request.getConversationHistory()));
initialRequest.setEnabledEndpoints(endpointResolver.getEnabledEndpointUrls());
boolean workerAvailable = pyMuPdfConverter.isAvailable();
initialRequest.setPymupdfWorkerAvailable(workerAvailable);
log.info("[pymupdf-convert] available={}", workerAvailable);
listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.ANALYZING));
@@ -188,13 +183,6 @@ public class AiWorkflowService {
cannotContinue("AI engine requested content extraction more than once."));
}
// Fast path: when the engine identifies a pdf-to-markdown task and pymupdf-convert is
// available, skip feeding content back to the engine and convert directly.
if ("pdf_to_markdown".equals(response.getResumeWith())
&& request.isPymupdfWorkerAvailable()) {
return runPyMuPdfConversion(filesById, listener);
}
List<AiWorkflowFileRequest> requestedFiles = response.getFiles();
// Validate requested file ids before loading anything
@@ -377,31 +365,6 @@ public class AiWorkflowService {
return new WorkflowState.Terminal(response);
}
private WorkflowState runPyMuPdfConversion(
Map<String, MultipartFile> filesById, ProgressListener listener) throws IOException {
listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.PROCESSING));
List<Resource> outputs = new ArrayList<>();
for (MultipartFile file : filesById.values()) {
String baseName =
file.getOriginalFilename() != null
? file.getOriginalFilename().replaceFirst("\\.[^.]+$", "")
: "document";
String markdown =
pyMuPdfConverter.convertToMarkdown(file.getBytes(), file.getOriginalFilename());
String safeFilename = Filenames.toSimpleFileName(baseName + ".md");
byte[] bytes = markdown.getBytes(java.nio.charset.StandardCharsets.UTF_8);
outputs.add(
new org.springframework.core.io.ByteArrayResource(bytes) {
@Override
public String getFilename() {
return safeFilename;
}
});
}
return new WorkflowState.Terminal(
buildCompletedResponse("Converted PDF to Markdown.", outputs, List.of(), null));
}
private WorkflowState onGenerateFile(AiWorkflowResponse response, ProgressListener listener)
throws IOException {
String content = response.getGeneratedContent();
@@ -782,6 +745,5 @@ public class AiWorkflowService {
private List<WorkflowArtifact> artifacts = new ArrayList<>();
private String resumeWith;
private List<String> enabledEndpoints = new ArrayList<>();
private boolean pymupdfWorkerAvailable;
}
}
@@ -30,7 +30,6 @@ import stirling.software.SPDF.pdf.parser.PdfModels.RawLine;
import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.common.service.PyMuPdfConverter;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.PdfUtils;
import stirling.software.proprietary.model.api.ai.AiPdfContentType;
@@ -45,7 +44,6 @@ public class PdfContentExtractor {
private final TabulaTableParser tabulaTableParser;
private final PdfIngester pdfIngester;
private final PyMuPdfConverter pyMuPdfConverter;
private static final int MAX_CHARACTERS_PER_PAGE = 4_000;
@@ -192,8 +190,6 @@ public class PdfContentExtractor {
extractText(lf, fileReq, remainingPages, remainingCharacters));
case PAGE_LAYOUT ->
Optional.<PdfContentResult>ofNullable(extractPageLayout(lf, remainingPages));
case PYMUPDF_MARKDOWN ->
Optional.<PdfContentResult>ofNullable(extractPyMuPdfMarkdown(lf));
default -> {
log.warn(
"Content type {} not yet implemented, skipping for {}",
@@ -259,11 +255,6 @@ public class PdfContentExtractor {
artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList());
yield artifact;
}
case PYMUPDF_MARKDOWN -> {
PyMuPdfMarkdownArtifact artifact = new PyMuPdfMarkdownArtifact();
artifact.setFiles(results.stream().map(PyMuPdfMarkdownResult.class::cast).toList());
yield artifact;
}
case TOOL_REPORT ->
throw new IllegalArgumentException(
"TOOL_REPORT artifacts are not produced by PdfContentExtractor");
@@ -379,8 +370,7 @@ public class PdfContentExtractor {
enum ArtifactKind {
EXTRACTED_TEXT("extracted_text"),
PAGE_LAYOUT("page_layout"),
TOOL_REPORT("tool_report"),
PYMUPDF_MARKDOWN("pymupdf_markdown");
TOOL_REPORT("tool_report");
private final String value;
@@ -479,46 +469,4 @@ public class PdfContentExtractor {
private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT;
private List<PageLayoutFileResult> files = new ArrayList<>();
}
private PyMuPdfMarkdownResult extractPyMuPdfMarkdown(LoadedFile lf) {
try {
log.info("[pymupdf-convert] converting file={}", lf.fileName());
java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream();
lf.document().save(baos);
String markdown = pyMuPdfConverter.convertToMarkdown(baos.toByteArray(), lf.fileName());
log.info(
"[pymupdf-convert] success file={} markdown-chars={}",
lf.fileName(),
markdown.length());
PyMuPdfMarkdownResult result = new PyMuPdfMarkdownResult();
result.setFileName(lf.fileName());
result.setMarkdown(markdown);
return result;
} catch (Exception e) {
log.warn(
"[pymupdf-convert] failed for file={}, falling back to page layout: {}",
lf.fileName(),
e.getMessage());
return null;
}
}
/** PyMuPDF worker pre-rendered Markdown for one file. */
@Data
static final class PyMuPdfMarkdownResult implements PdfContentResult {
private String fileName;
private String markdown;
@Override
public ArtifactKind getArtifactKind() {
return ArtifactKind.PYMUPDF_MARKDOWN;
}
}
/** Artifact carrying PyMuPDF-rendered Markdown for all input files. */
@Data
static final class PyMuPdfMarkdownArtifact implements WorkflowArtifact {
private final ArtifactKind kind = ArtifactKind.PYMUPDF_MARKDOWN;
private List<PyMuPdfMarkdownResult> files = new ArrayList<>();
}
}
@@ -1,95 +0,0 @@
package stirling.software.proprietary.storage.config;
import java.util.Locale;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.security.configuration.ee.LicenseKeyChecker;
/**
* Fails fast at boot if cluster mode is enabled with node-local storage. Validates both {@code
* storage.provider} (persistent uploads) and {@code cluster.artifactStore} (transient job-result
* files): neither may be {@code local} when {@code cluster.enabled=true}. Additionally enforces
* that any S3-backed configuration ({@code storage.provider=s3} or {@code
* cluster.artifactStore=s3}) is accompanied by a valid Pro / Enterprise license.
*/
@Configuration
@RequiredArgsConstructor
@Slf4j
public class ClusterStorageGate {
private final ApplicationProperties applicationProperties;
private final LicenseKeyChecker licenseKeyChecker;
@Value("${cluster.enabled:false}")
private boolean clusterEnabled;
@Value("${cluster.artifactStore:local}")
private String clusterArtifactStore;
@PostConstruct
void validate() {
// License enforcement runs regardless of cluster.enabled: even a single-node setup that
// selects a remote backend must hold a Pro or higher license.
ApplicationProperties.Storage storage = applicationProperties.getStorage();
if (storage != null && storage.isEnabled()) {
String provider = normalize(storage.getProvider());
if ("s3".equals(provider) || "database".equals(provider)) {
licenseKeyChecker.requireProOrEnterprise("storage.provider=" + provider);
}
}
if ("s3".equals(normalize(clusterArtifactStore))) {
licenseKeyChecker.requireProOrEnterprise("cluster.artifactStore=s3");
}
if (!clusterEnabled) {
return;
}
if (storage != null && storage.isEnabled()) {
validate(
"storage.provider",
storage.getProvider(),
"Local filesystem storage cannot be shared across cluster nodes."
+ " Configure storage.provider=s3 (with storage.s3.bucket /"
+ " endpoint / credentials) or storage.provider=database before"
+ " enabling clustering.");
}
validate(
"cluster.artifactStore",
clusterArtifactStore,
"Per-node disk cannot back transient job-result files in a multi-node"
+ " deployment; downloads would 404 whenever the load balancer routes"
+ " a follow-up request to a different node. Configure"
+ " cluster.artifactStore=s3 (reuses storage.s3.* config)"
+ " before enabling clustering.");
}
private static String normalize(String value) {
return Optional.ofNullable(value).orElse("local").trim().toLowerCase(Locale.ROOT);
}
private static void validate(String propertyName, String configuredValue, String remediation) {
String normalized =
Optional.ofNullable(configuredValue)
.orElse("local")
.trim()
.toLowerCase(Locale.ROOT);
if ("local".equals(normalized)) {
throw new IllegalStateException(
"Cluster mode (cluster.enabled=true) is incompatible with "
+ propertyName
+ "=local. "
+ remediation);
}
log.info(
"Cluster storage gate: clusterEnabled=true, {}={} -> OK", propertyName, normalized);
}
}
@@ -15,11 +15,8 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.cluster.s3.S3Clients;
import stirling.software.proprietary.security.configuration.ee.LicenseKeyChecker;
import stirling.software.proprietary.storage.provider.DatabaseStorageProvider;
import stirling.software.proprietary.storage.provider.LocalStorageProvider;
import stirling.software.proprietary.storage.provider.S3StorageProvider;
import stirling.software.proprietary.storage.provider.StorageProvider;
import stirling.software.proprietary.storage.repository.StoredFileBlobRepository;
@@ -30,9 +27,8 @@ public class StorageProviderConfig {
private final ApplicationProperties applicationProperties;
private final StoredFileBlobRepository storedFileBlobRepository;
private final LicenseKeyChecker licenseKeyChecker;
@Bean(destroyMethod = "close")
@Bean
public StorageProvider storageProvider() {
boolean storageEnabled = applicationProperties.getStorage().isEnabled();
String providerName =
@@ -41,13 +37,8 @@ public class StorageProviderConfig {
.trim()
.toLowerCase(Locale.ROOT);
if ("database".equals(providerName)) {
licenseKeyChecker.requireProOrEnterprise("storage.provider=database");
return new DatabaseStorageProvider(storedFileBlobRepository);
}
if ("s3".equals(providerName)) {
licenseKeyChecker.requireProOrEnterprise("storage.provider=s3");
return buildS3Provider(applicationProperties.getStorage().getS3());
}
if (!"local".equals(providerName)) {
throw new IllegalStateException("Storage provider not supported: " + providerName);
}
@@ -80,9 +71,4 @@ public class StorageProviderConfig {
}
return new LocalStorageProvider(basePath);
}
private S3StorageProvider buildS3Provider(ApplicationProperties.Storage.S3 cfg) {
S3Clients.Bundle bundle = S3Clients.build(cfg, "storage provider");
return new S3StorageProvider(bundle.client(), bundle.presigner(), cfg.getBucket());
}
}
@@ -1,91 +0,0 @@
package stirling.software.proprietary.storage.controller;
import java.util.List;
import java.util.UUID;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PatchMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import stirling.software.proprietary.storage.service.FolderService;
/**
* Folder placement endpoints for existing stored files. Thin adapter: validates the request shape,
* delegates the transaction to {@link FolderService}, then maps the result onto the HTTP status.
* Authentication, storage-gate, ownership checks, and the bulk cap all live on the service (where
* {@code @Transactional} also lives) so the JDBC connection isn't held through JSON serialization.
*/
@RestController
@RequestMapping("/api/v1/storage/files")
@RequiredArgsConstructor
public class FileFolderPlacementController {
private static final int BULK_MOVE_MAX_FILES = 1000;
private final FolderService folderService;
/** Move a single file to a folder (or to root when folderId is null). */
@PatchMapping("/{fileId}/folder")
public ResponseEntity<Void> moveFileToFolder(
@PathVariable Long fileId, @Valid @RequestBody FolderPlacement body) {
folderService.moveFileToFolder(fileId, body.getFolderId());
return ResponseEntity.noContent().build();
}
/**
* Bulk move - fewer round-trips than calling the single endpoint N times. Returns 200 on full
* success, 207 (Multi-Status) when some files were skipped (typically because they don't belong
* to the caller).
*/
@PatchMapping("/folder")
public ResponseEntity<BulkMoveResponse> bulkMove(@Valid @RequestBody BulkMoveRequest body) {
FolderService.BulkMoveResult result =
folderService.bulkMoveFilesToFolder(body.getFolderId(), body.getFileIds());
HttpStatus status =
result.skippedFileIds().isEmpty() ? HttpStatus.OK : HttpStatus.MULTI_STATUS;
return ResponseEntity.status(status)
.body(new BulkMoveResponse(result.movedFileIds(), result.skippedFileIds()));
}
@Data
@NoArgsConstructor
@AllArgsConstructor
public static class FolderPlacement {
private UUID folderId;
}
@Data
@NoArgsConstructor
@AllArgsConstructor
public static class BulkMoveRequest {
private UUID folderId;
@NotNull
@Size(
min = 1,
max = BULK_MOVE_MAX_FILES,
message = "fileIds must contain between 1 and 1000 entries")
private List<Long> fileIds;
}
@Data
@NoArgsConstructor
@AllArgsConstructor
public static class BulkMoveResponse {
private List<Long> movedFileIds;
private List<Long> skippedFileIds;
}
}
@@ -1,11 +1,7 @@
package stirling.software.proprietary.storage.controller;
import java.io.IOException;
import java.net.URI;
import java.time.Duration;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import org.springframework.http.ContentDisposition;
import org.springframework.http.HttpHeaders;
@@ -29,7 +25,6 @@ import org.springframework.web.server.ResponseStatusException;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.security.model.User;
import stirling.software.proprietary.storage.model.FileShare;
@@ -40,22 +35,17 @@ import stirling.software.proprietary.storage.model.api.ShareLinkMetadataResponse
import stirling.software.proprietary.storage.model.api.ShareLinkResponse;
import stirling.software.proprietary.storage.model.api.ShareWithUserRequest;
import stirling.software.proprietary.storage.model.api.StoredFileResponse;
import stirling.software.proprietary.storage.provider.StorageProvider;
import stirling.software.proprietary.storage.service.FileStorageService;
@RestController
@RequestMapping("/api/v1/storage")
@RequiredArgsConstructor
@Slf4j
@Tag(
name = "File Storage",
description = "Stored file management, sharing, and share link operations")
public class FileStorageController {
private static final Duration SIGNED_URL_TTL = Duration.ofMinutes(5);
private final FileStorageService fileStorageService;
private final StorageProvider storageProvider;
@PostMapping(
value = "/files",
@@ -101,9 +91,7 @@ public class FileStorageController {
User user = fileStorageService.requireAuthenticatedUser();
StoredFile file = fileStorageService.getAccessibleFile(user, fileId);
fileStorageService.requireReadAccess(user, file);
Optional<ResponseEntity<org.springframework.core.io.Resource>> redirect =
tryRedirectToSignedUrl(file, inline);
return redirect.orElseGet(() -> buildFileResponse(file, inline));
return buildFileResponse(file, inline);
}
@DeleteMapping("/files/{fileId}")
@@ -201,9 +189,7 @@ public class FileStorageController {
fileStorageService.requireReadAccess(share);
fileStorageService.recordShareAccess(share, authentication, inline);
StoredFile file = share.getFile();
Optional<ResponseEntity<org.springframework.core.io.Resource>> redirect =
tryRedirectToSignedUrl(file, inline);
return redirect.orElseGet(() -> buildFileResponse(file, inline));
return buildFileResponse(file, inline);
}
@GetMapping("/share-links/{token}/metadata")
@@ -286,34 +272,4 @@ public class FileStorageController {
&& authentication.isAuthenticated()
&& !"anonymousUser".equals(authentication.getPrincipal());
}
private Optional<ResponseEntity<org.springframework.core.io.Resource>> tryRedirectToSignedUrl(
StoredFile file, boolean inline) {
if (file == null || file.getStorageKey() == null || file.getStorageKey().isBlank()) {
return Optional.empty();
}
try {
Optional<URI> signed =
storageProvider.signedDownloadUrl(
file.getStorageKey(),
SIGNED_URL_TTL,
inline,
file.getOriginalFilename());
if (signed.isEmpty()) {
return Optional.empty();
}
HttpHeaders headers = new HttpHeaders();
headers.setLocation(signed.get());
ResponseEntity<org.springframework.core.io.Resource> response =
ResponseEntity.status(HttpStatus.FOUND).headers(headers).build();
return Optional.of(response);
} catch (IOException e) {
log.warn(
"Failed to create signed download URL for file {} (key: {}), falling back to streaming",
file.getId(),
file.getStorageKey(),
e);
return Optional.empty();
}
}
}
@@ -1,70 +0,0 @@
package stirling.software.proprietary.storage.controller;
import java.net.URI;
import java.util.List;
import java.util.UUID;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PatchMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import stirling.software.proprietary.storage.model.api.CreateFolderRequest;
import stirling.software.proprietary.storage.model.api.FolderResponse;
import stirling.software.proprietary.storage.model.api.UpdateFolderRequest;
import stirling.software.proprietary.storage.service.FolderService;
/**
* REST endpoints for user-owned folders. Phase A - no folder-level sharing yet (Phase 3).
*
* <p>All operations are scoped to the authenticated user; existing single-file storage endpoints in
* {@link FileStorageController} are left alone so the cert-signing and standard upload flows are
* unaffected.
*/
@RestController
@RequestMapping("/api/v1/storage/folders")
@RequiredArgsConstructor
public class FolderController {
private final FolderService folderService;
@GetMapping
public List<FolderResponse> listFolders() {
return folderService.listFolders();
}
@PostMapping
public ResponseEntity<FolderResponse> createFolder(
@Valid @RequestBody CreateFolderRequest request) {
FolderResponse response = folderService.createFolder(request);
// 201 Created with Location header - conventional REST. The idempotent re-return path
// (same id resubmitted) also lands here; treating it as 201 keeps wire semantics simple.
return ResponseEntity.status(HttpStatus.CREATED)
.location(URI.create("/api/v1/storage/folders/" + response.id()))
.body(response);
}
@PatchMapping("/{folderId}")
public ResponseEntity<FolderResponse> updateFolder(
@PathVariable UUID folderId, @Valid @RequestBody UpdateFolderRequest request) {
return ResponseEntity.ok(folderService.updateFolder(folderId, request));
}
@DeleteMapping("/{folderId}")
public ResponseEntity<DeleteFolderResponse> deleteFolder(@PathVariable UUID folderId) {
List<UUID> removed = folderService.deleteFolder(folderId);
return ResponseEntity.ok(new DeleteFolderResponse(removed));
}
public record DeleteFolderResponse(List<UUID> removedFolderIds) {}
}
@@ -1,104 +0,0 @@
package stirling.software.proprietary.storage.model;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.UUID;
import org.hibernate.annotations.CreationTimestamp;
import org.hibernate.annotations.OnDelete;
import org.hibernate.annotations.OnDeleteAction;
import org.hibernate.annotations.UpdateTimestamp;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.Id;
import jakarta.persistence.Index;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import jakarta.persistence.Version;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import stirling.software.proprietary.security.model.User;
/**
* A user-owned folder used by the file manager UI to organise stored files. Phase A entity - no
* folder-level sharing yet (Phase 3).
*
* <p>The id is a UUID rather than a numeric auto-increment so it round-trips with the
* client-generated {@code FolderId} and survives cross-device sync without re-keying.
*/
@Entity
@Table(
name = "folders",
indexes = {
@Index(name = "idx_folders_owner", columnList = "owner_id"),
@Index(name = "idx_folders_parent", columnList = "parent_folder_id"),
@Index(name = "idx_folders_owner_parent", columnList = "owner_id, parent_folder_id")
})
@NoArgsConstructor
@Getter
@Setter
public class Folder implements Serializable {
private static final long serialVersionUID = 1L;
/**
* Dialect-portable UUID column. The previous {@code columnDefinition = "uuid"} was
* Postgres-specific and broke on H2/MariaDB. Hibernate's {@code UUID} mapping picks the right
* native type per dialect (BINARY(16) on H2/MariaDB, uuid on Postgres) when no explicit
* columnDefinition is set.
*/
@Id
@Column(name = "folder_id", nullable = false)
private UUID id;
/**
* {@code OnDeleteAction.CASCADE} so deleting the owning {@code User} cascades to this row at
* the DB level - UserService.deleteUserRelatedData doesn't enumerate folders today, and leaving
* the FK without an action throws a constraint violation on user delete.
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "owner_id", nullable = false)
@OnDelete(action = OnDeleteAction.CASCADE)
private User owner;
/**
* Parent folder; null = root. {@code OnDeleteAction.CASCADE} so a backend-side parent delete
* cleans children automatically, matching the service-layer recursive-delete contract.
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "parent_folder_id")
@OnDelete(action = OnDeleteAction.CASCADE)
private Folder parent;
@Column(name = "name", nullable = false, length = 255)
private String name;
@Column(name = "color", length = 32)
private String color;
@Column(name = "icon", length = 64)
private String icon;
/**
* Optimistic-locking version. Cross-PC sync without this lets last-write-win silently. The
* column is nullable so existing rows from a pre-version deployment can be backfilled by
* Hibernate's update-on-write rather than failing the ddl-auto upgrade.
*/
@Version
@Column(name = "version")
private Long version;
@CreationTimestamp
@Column(name = "created_at", updatable = false)
private LocalDateTime createdAt;
@UpdateTimestamp
@Column(name = "updated_at")
private LocalDateTime updatedAt;
}
@@ -6,8 +6,6 @@ import java.util.HashSet;
import java.util.Set;
import org.hibernate.annotations.CreationTimestamp;
import org.hibernate.annotations.OnDelete;
import org.hibernate.annotations.OnDeleteAction;
import org.hibernate.annotations.UpdateTimestamp;
import jakarta.persistence.CascadeType;
@@ -37,8 +35,7 @@ import stirling.software.proprietary.workflow.model.WorkflowSession;
name = "stored_files",
indexes = {
@Index(name = "idx_stored_files_owner", columnList = "owner_id"),
@Index(name = "idx_stored_files_workflow", columnList = "workflow_session_id"),
@Index(name = "idx_stored_files_folder", columnList = "folder_id")
@Index(name = "idx_stored_files_workflow", columnList = "workflow_session_id")
})
@NoArgsConstructor
@Getter
@@ -109,20 +106,6 @@ public class StoredFile implements Serializable {
orphanRemoval = true)
private Set<FileShare> shares = new HashSet<>();
/**
* Optional folder placement for the file manager UI. Null = root. Hibernate ddl-auto will add
* this as a nullable column on upgrade so existing records continue to work untouched.
*
* <p>{@code OnDeleteAction.SET_NULL} so any backend that drops a folder row (admin script,
* future cleanup job, cascading user delete) cleanly orphans files to root rather than leaving
* dangling FK references. The application path ({@code FolderRepository.clearFolderForFiles})
* still runs first as a belt-and-braces.
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "folder_id")
@OnDelete(action = OnDeleteAction.SET_NULL)
private Folder folder;
@CreationTimestamp
@Column(name = "created_at", updatable = false)
private LocalDateTime createdAt;
@@ -1,43 +0,0 @@
package stirling.software.proprietary.storage.model.api;
import java.util.UUID;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Pattern;
import jakarta.validation.constraints.Size;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class CreateFolderRequest {
/**
* Client-generated UUID - lets the caller round-trip the same id it stored locally. Optional;
* the server generates one when missing.
*/
private UUID id;
@NotBlank
@Size(max = 255)
private String name;
private UUID parentFolderId;
/** Hex colour string (#rrggbb or #rrggbbaa) - matches the frontend palette format. */
@Size(max = 32)
@Pattern(
regexp = "^#[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$",
message = "color must be a #RRGGBB or #RRGGBBAA hex value")
private String color;
/** Icon identifier - lowercase alphanumerics, hyphens, underscores only. */
@Size(max = 64)
@Pattern(
regexp = "^[a-z0-9_-]+$",
message = "icon must be a lowercase id (a-z, 0-9, '-' or '_')")
private String icon;
}
@@ -1,38 +0,0 @@
package stirling.software.proprietary.storage.model.api;
import java.time.LocalDateTime;
import java.util.UUID;
import stirling.software.proprietary.storage.model.Folder;
/**
* Outbound DTO for folder responses. Records are immutable, value-equality-based, and far less
* accident-prone than a {@code @Data} class with public setters.
*/
public record FolderResponse(
UUID id,
String name,
UUID parentFolderId,
String color,
String icon,
Long version,
LocalDateTime createdAt,
LocalDateTime updatedAt) {
public static FolderResponse from(Folder folder) {
// {@code folder.getParent().getId()} on a lazy proxy returns the FK value cached at the
// join column WITHOUT initialising the proxy under standard Hibernate, so this does
// not N+1. If a future Hibernate update changes that, switch the JPQL list query to a
// constructor projection.
UUID parentId = folder.getParent() == null ? null : folder.getParent().getId();
return new FolderResponse(
folder.getId(),
folder.getName(),
parentId,
folder.getColor(),
folder.getIcon(),
folder.getVersion(),
folder.getCreatedAt(),
folder.getUpdatedAt());
}
}
@@ -2,7 +2,6 @@ package stirling.software.proprietary.storage.model.api;
import java.time.LocalDateTime;
import java.util.List;
import java.util.UUID;
import lombok.Builder;
import lombok.Getter;
@@ -23,10 +22,4 @@ public class StoredFileResponse {
private final List<SharedUserResponse> sharedUsers;
private final List<ShareLinkResponse> shareLinks;
private final String filePurpose;
/**
* Optional folder placement (Phase A). Null when the file lives at the root or when the server
* build doesn't have the folders feature enabled - existing clients should treat null as root.
*/
private final UUID folderId;
}
@@ -1,58 +0,0 @@
package stirling.software.proprietary.storage.model.api;
import java.util.UUID;
import jakarta.validation.constraints.Pattern;
import jakarta.validation.constraints.Size;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* PATCH-style update - every field is optional. Send only the fields you want to change.
*
* <p>The {@code reparent} flag distinguishes "do not change parent" from "move to root" since
* {@code parentFolderId == null} alone is ambiguous in a sparse body. We use a boxed {@link
* Boolean} so a missing field deserialises to {@code null} (= "do not reparent") rather than to
* primitive {@code false}, removing a class of "I PATCHed only the name but the server reset my
* parent" footguns.
*
* <p>When the trimmed name is empty (e.g. {@code " "}) the service rejects the request with HTTP
* 400 - silent drops are too easy to mistake for a successful rename.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFolderRequest {
/** When provided, must contain at least one non-whitespace character. */
@Size(max = 255)
@Pattern(regexp = "\\S.*", message = "name must not be blank")
private String name;
private Boolean reparent;
private UUID parentFolderId;
@Size(max = 32)
@Pattern(
regexp = "^(|#[0-9a-fA-F]{6}([0-9a-fA-F]{2})?)$",
message = "color must be empty or a #RRGGBB / #RRGGBBAA hex value")
private String color;
@Size(max = 64)
@Pattern(
regexp = "^([a-z0-9_-]+)?$",
message = "icon must be a lowercase id (a-z, 0-9, '-' or '_') or empty")
private String icon;
/**
* Convenience accessor - treats null as "do not reparent". Named differently from the
* Lombok-generated {@code getReparent()} so callers don't accidentally use one for the other
* (the getter is nullable {@code Boolean}; this method collapses to primitive).
*/
@com.fasterxml.jackson.annotation.JsonIgnore
public boolean shouldReparent() {
return Boolean.TRUE.equals(reparent);
}
}
@@ -24,9 +24,6 @@ public class LocalStorageProvider implements StorageProvider {
@Override
public StoredObject store(User owner, MultipartFile file) throws IOException {
if (owner == null || owner.getId() == null) {
throw new IllegalArgumentException("owner.id is required for local storage key");
}
String originalFilename = sanitizeFilename(file.getOriginalFilename());
String storageKey =
owner.getId()
@@ -80,7 +77,6 @@ public class LocalStorageProvider implements StorageProvider {
if (filename == null || filename.isBlank()) {
return "file";
}
String stripped = Paths.get(filename).getFileName().toString().replaceAll("\\p{Cntrl}", "");
return stripped.isBlank() ? "file" : stripped;
return Paths.get(filename).getFileName().toString();
}
}
@@ -1,190 +0,0 @@
package stirling.software.proprietary.storage.provider;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.Optional;
import java.util.UUID;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.web.multipart.MultipartFile;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.security.model.User;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
/** {@link StorageProvider} backed by an S3-compatible object store. */
@Slf4j
public class S3StorageProvider implements StorageProvider, AutoCloseable {
private final S3Client s3Client;
private final S3Presigner s3Presigner;
private final String bucket;
public S3StorageProvider(S3Client s3Client, S3Presigner s3Presigner, String bucket) {
if (bucket == null || bucket.isBlank()) {
throw new IllegalArgumentException("S3 bucket must be configured");
}
this.s3Client = s3Client;
this.s3Presigner = s3Presigner;
this.bucket = bucket;
}
@Override
public StoredObject store(User owner, MultipartFile file) throws IOException {
if (owner == null || owner.getId() == null) {
throw new IllegalArgumentException("owner.id is required for S3 storage key");
}
String originalFilename = sanitizeFilename(file.getOriginalFilename());
// Key is opaque ({ownerId}/{uuid}) so non-ASCII filenames don't break vendors that
// restrict key charset (e.g. Supabase Storage returns 400 Invalid key on unicode).
// The display name is preserved in StoredObject.originalFilename and the DB row.
String storageKey = owner.getId() + "/" + UUID.randomUUID();
PutObjectRequest.Builder request =
PutObjectRequest.builder().bucket(bucket).key(storageKey);
if (file.getContentType() != null && !file.getContentType().isBlank()) {
request.contentType(file.getContentType());
}
try (InputStream inputStream = file.getInputStream()) {
s3Client.putObject(
request.build(), RequestBody.fromInputStream(inputStream, file.getSize()));
} catch (SdkException e) {
throw new IOException("Failed to upload object to S3", e);
}
return StoredObject.builder()
.storageKey(storageKey)
.originalFilename(originalFilename)
.contentType(file.getContentType())
.sizeBytes(file.getSize())
.build();
}
@Override
public Resource load(String storageKey) throws IOException {
GetObjectRequest request =
GetObjectRequest.builder().bucket(bucket).key(storageKey).build();
try {
ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(request);
long contentLength =
stream.response().contentLength() != null
? stream.response().contentLength()
: -1;
return new InputStreamResource(stream) {
@Override
public long contentLength() {
return contentLength;
}
};
} catch (NoSuchKeyException e) {
throw new IOException("File not found", e);
} catch (SdkException e) {
throw new IOException("Failed to load object from S3", e);
}
}
@Override
public void delete(String storageKey) throws IOException {
try {
s3Client.deleteObject(
DeleteObjectRequest.builder().bucket(bucket).key(storageKey).build());
} catch (SdkException e) {
throw new IOException("Failed to delete object from S3", e);
}
}
@Override
public Optional<URI> signedDownloadUrl(String storageKey, Duration ttl) throws IOException {
return signedDownloadUrl(storageKey, ttl, false, null);
}
@Override
public Optional<URI> signedDownloadUrl(
String storageKey, Duration ttl, boolean inline, String originalFilename)
throws IOException {
if (storageKey == null || storageKey.isBlank()) {
return Optional.empty();
}
Duration effectiveTtl =
ttl == null || ttl.isZero() || ttl.isNegative() ? Duration.ofMinutes(5) : ttl;
try {
GetObjectRequest.Builder getBuilder =
GetObjectRequest.builder().bucket(bucket).key(storageKey);
String disposition = buildContentDisposition(inline, originalFilename);
if (disposition != null) {
getBuilder.responseContentDisposition(disposition);
}
GetObjectPresignRequest presignRequest =
GetObjectPresignRequest.builder()
.signatureDuration(effectiveTtl)
.getObjectRequest(getBuilder.build())
.build();
PresignedGetObjectRequest presigned = s3Presigner.presignGetObject(presignRequest);
return Optional.of(presigned.url().toURI());
} catch (SdkException | URISyntaxException e) {
log.warn("Failed to create presigned S3 GET URL for key {}", storageKey, e);
return Optional.empty();
}
}
// Returns null when originalFilename is blank; S3 falls back to its own default in that case.
static String buildContentDisposition(boolean inline, String originalFilename) {
if (originalFilename == null || originalFilename.isBlank()) {
return null;
}
// Strip CR/LF and other control chars before path parsing (Paths.get throws on them on
// Windows, and they defeat header parsers).
String stripped = originalFilename.replaceAll("\\p{Cntrl}", "");
// Use only the basename to avoid leaking directory structure into the header.
int lastSeparator = Math.max(stripped.lastIndexOf('/'), stripped.lastIndexOf('\\'));
if (lastSeparator >= 0) {
stripped = stripped.substring(lastSeparator + 1);
}
if (stripped.isBlank()) {
return null;
}
// Escape per RFC 6266 quoted-string rules.
String escaped = stripped.replace("\\", "\\\\").replace("\"", "\\\"");
return (inline ? "inline" : "attachment") + "; filename=\"" + escaped + "\"";
}
@Override
public void close() {
try {
s3Presigner.close();
} catch (Exception e) {
log.warn("Error closing S3 presigner", e);
}
try {
s3Client.close();
} catch (Exception e) {
log.warn("Error closing S3 client", e);
}
}
private String sanitizeFilename(String filename) {
if (filename == null || filename.isBlank()) {
return "file";
}
String stripped = Paths.get(filename).getFileName().toString().replaceAll("\\p{Cntrl}", "");
return stripped.isBlank() ? "file" : stripped;
}
}
@@ -1,45 +1,16 @@
package stirling.software.proprietary.storage.provider;
import java.io.IOException;
import java.net.URI;
import java.time.Duration;
import java.util.Optional;
import org.springframework.core.io.Resource;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.proprietary.security.model.User;
public interface StorageProvider extends AutoCloseable {
public interface StorageProvider {
StoredObject store(User owner, MultipartFile file) throws IOException;
Resource load(String storageKey) throws IOException;
void delete(String storageKey) throws IOException;
/**
* Releases any backend-specific resources. Default no-op so {@link LocalStorageProvider} and
* {@link DatabaseStorageProvider} (which hold no closeable handles) satisfy Spring's
* {@code @Bean(destroyMethod = "close")} signature requirement without ceremony. {@code
* S3StorageProvider} overrides this to close the underlying SDK client + presigner.
*/
@Override
default void close() {}
/**
* Returns a presigned download URL valid for {@code ttl}, or {@link Optional#empty()} if the
* provider does not support signed URLs (callers fall back to {@link #load(String)}).
*/
default Optional<URI> signedDownloadUrl(String storageKey, Duration ttl) throws IOException {
return signedDownloadUrl(storageKey, ttl, false, null);
}
/**
* Like {@link #signedDownloadUrl(String, Duration)} with explicit Content-Disposition control.
*/
default Optional<URI> signedDownloadUrl(
String storageKey, Duration ttl, boolean inline, String originalFilename)
throws IOException {
return Optional.empty();
}
}

Some files were not shown because too many files have changed in this diff Show More