Compare commits
50
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df5774e8dd | ||
|
|
0dff192281 | ||
|
|
9866d6e12d | ||
|
|
e79f4a044f | ||
|
|
9ab404b2e6 | ||
|
|
a61fe012d7 | ||
|
|
c93776e297 | ||
|
|
1698769928 | ||
|
|
bd9ef0586b | ||
|
|
cb687fbf99 | ||
|
|
69ee39fa6e | ||
|
|
353b5c807c | ||
|
|
22dacbed01 | ||
|
|
3807cdfbc6 | ||
|
|
35a712a278 | ||
|
|
7f3ca7ea70 | ||
|
|
895dcbbafd | ||
|
|
b705c5b84c | ||
|
|
1264f4cfed | ||
|
|
71633861d0 | ||
|
|
e6974d52f7 | ||
|
|
256d1a86d2 | ||
|
|
919f0ade99 | ||
|
|
b355ccec9e | ||
|
|
de9d6ad3f5 | ||
|
|
2c0ebc28a7 | ||
|
|
d1486c7762 | ||
|
|
2ccff6f73f | ||
|
|
78da227eba | ||
|
|
30e782e29c | ||
|
|
2b0905887b | ||
|
|
28b81828b5 | ||
|
|
2c01f41142 | ||
|
|
4d5eeb103f | ||
|
|
83ea07ed6a | ||
|
|
61ebe977d3 | ||
|
|
763595a5a3 | ||
|
|
398617391b | ||
|
|
a0e0e88f07 | ||
|
|
c80a5db5f5 | ||
|
|
4fa67afc3d | ||
|
|
8bd78d2624 | ||
|
|
b3c4b8b463 | ||
|
|
57af5b9dc2 | ||
|
|
44fbf8c587 | ||
|
|
76840d8a57 | ||
|
|
d459ded168 | ||
|
|
43b67d213d | ||
|
|
d42b779644 | ||
|
|
4564ed5bec |
+3
-2
@@ -26,8 +26,9 @@ version_builds/
|
||||
node_modules/
|
||||
**/node_modules/
|
||||
frontend/node_modules/
|
||||
frontend/dist/
|
||||
frontend/playwright-report/
|
||||
frontend/editor/dist/
|
||||
frontend/dist-portal/
|
||||
frontend/editor/playwright-report/
|
||||
.npm/
|
||||
.yarn/
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ runs:
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
|
||||
with:
|
||||
app-id: ${{ inputs.app-id }}
|
||||
client-id: ${{ inputs.app-id }}
|
||||
private-key: ${{ inputs.private-key }}
|
||||
- name: Configure Git
|
||||
run: |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-desktop
|
||||
pkgver=2.11.0
|
||||
pkgver=2.12.0
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-server-bin
|
||||
pkgver=2.11.0
|
||||
pkgver=2.12.0
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
|
||||
arch=('any')
|
||||
|
||||
@@ -38,6 +38,8 @@ project: &project
|
||||
- frontend/**
|
||||
- docker/**
|
||||
- scripts/RestartHelper.java
|
||||
- scripts/db-migration/**
|
||||
- .github/workflows/db-migration-test.yml
|
||||
|
||||
frontend: &frontend
|
||||
- frontend/**
|
||||
|
||||
@@ -13,7 +13,7 @@ Usage:
|
||||
"""
|
||||
|
||||
# Sample for Windows:
|
||||
# python .github/scripts/check_language_toml.py --reference-file frontend/public/locales/en-GB/translation.toml --branch "" --files frontend/public/locales/de-DE/translation.toml frontend/public/locales/fr-FR/translation.toml
|
||||
# python .github/scripts/check_language_toml.py --reference-file frontend/editor/public/locales/en-GB/translation.toml --branch "" --files frontend/editor/public/locales/de-DE/translation.toml frontend/editor/public/locales/fr-FR/translation.toml
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
@@ -184,7 +184,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
|
||||
if len(file_list) == 1:
|
||||
file_arr = file_list[0].split()
|
||||
|
||||
base_dir = Path.cwd() / "frontend" / "public" / "locales"
|
||||
base_dir = Path.cwd() / "frontend" / "editor" / "public" / "locales"
|
||||
|
||||
for file_path in file_arr:
|
||||
file_path = Path(file_path)
|
||||
@@ -308,7 +308,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
|
||||
report.append("## ❌ Overall Check Status: **_Failed_**")
|
||||
report.append("")
|
||||
report.append(
|
||||
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [en-GB/translation.toml](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/frontend/public/locales/en-GB/translation.toml)"
|
||||
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [en-GB/translation.toml](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/frontend/editor/public/locales/en-GB/translation.toml)"
|
||||
)
|
||||
else:
|
||||
report.append("## ✅ Overall Check Status: **_Success_**")
|
||||
@@ -372,6 +372,7 @@ if __name__ == "__main__":
|
||||
os.path.join(
|
||||
os.getcwd(),
|
||||
"frontend",
|
||||
"editor",
|
||||
"public",
|
||||
"locales",
|
||||
"*",
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify Tauri updater .sig files against plugins.updater.pubkey in tauri.conf.json.
|
||||
|
||||
Usage: verify-updater-signatures.py <dir-to-scan> [tauri.conf.json]
|
||||
"""
|
||||
|
||||
import binascii
|
||||
import sys
|
||||
import json
|
||||
import base64
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
|
||||
ART_ROOT = Path(sys.argv[1])
|
||||
CONF = Path(
|
||||
sys.argv[2] if len(sys.argv) > 2 else "frontend/editor/src-tauri/tauri.conf.json"
|
||||
)
|
||||
|
||||
|
||||
def load_pubkey():
|
||||
# tauri pubkey = base64 of a minisign .pub file; last line is base64 of
|
||||
# [2 algo][8 key-id][32 ed25519 public key].
|
||||
raw = json.loads(CONF.read_text())["plugins"]["updater"]["pubkey"]
|
||||
blob = base64.b64decode(base64.b64decode(raw).decode().splitlines()[-1])
|
||||
return blob[2:10], Ed25519PublicKey.from_public_bytes(blob[10:])
|
||||
|
||||
|
||||
def hash_file(path: Path) -> bytes:
|
||||
h = hashlib.blake2b(digest_size=64)
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 16), b""):
|
||||
h.update(chunk)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def verify(artifact: Path, sig_file: Path, keyid_pub, pub) -> str:
|
||||
# tauri .sig = base64 of a minisign signature file (4 lines).
|
||||
try:
|
||||
lines = base64.b64decode(sig_file.read_text()).decode().splitlines()
|
||||
sig_blob = base64.b64decode(lines[1])
|
||||
except (binascii.Error, IndexError, UnicodeDecodeError) as e:
|
||||
return f"FAIL malformed sig ({type(e).__name__})"
|
||||
algo, keyid, sig = sig_blob[:2], sig_blob[2:10], sig_blob[10:74]
|
||||
if keyid != keyid_pub:
|
||||
return f"FAIL key-id mismatch (sig {keyid.hex()} vs pub {keyid_pub.hex()})"
|
||||
# 'ED' = prehashed (BLAKE2b-512), 'Ed' = legacy (raw message).
|
||||
msg = hash_file(artifact) if algo == b"ED" else artifact.read_bytes()
|
||||
try:
|
||||
pub.verify(sig, msg)
|
||||
except InvalidSignature:
|
||||
return f"FAIL signature invalid (algo={algo.decode()})"
|
||||
# Global signature covers sig + trusted_comment.
|
||||
gc = "global-sig FAIL"
|
||||
try:
|
||||
tc = lines[2].split("trusted comment: ", 1)[1]
|
||||
pub.verify(base64.b64decode(lines[3]), sig + tc.encode())
|
||||
gc = "global-sig OK"
|
||||
except (InvalidSignature, IndexError, binascii.Error):
|
||||
pass
|
||||
return f"VALID (algo={algo.decode()}, keyid={keyid.hex()}, {gc})"
|
||||
|
||||
|
||||
keyid_pub, pub = load_pubkey()
|
||||
print(f"updater pubkey keyid={keyid_pub.hex()}\n")
|
||||
sigs = sorted(ART_ROOT.rglob("*.sig"))
|
||||
if not sigs:
|
||||
print(f"WARN: no .sig files under {ART_ROOT} - nothing to verify")
|
||||
sys.exit(0)
|
||||
bad = 0
|
||||
for sig_file in sigs:
|
||||
artifact = sig_file.with_suffix("")
|
||||
if not artifact.exists():
|
||||
print(f" ? {sig_file.name}: artifact missing")
|
||||
bad += 1
|
||||
continue
|
||||
res = verify(artifact, sig_file, keyid_pub, pub)
|
||||
print(f" {artifact.name}: {res}")
|
||||
if not res.startswith("VALID") or "global-sig FAIL" in res:
|
||||
bad += 1
|
||||
print(f"\n{'ALL SIGNATURES VALID' if bad == 0 else f'{bad} SIGNATURE(S) FAILED'}")
|
||||
sys.exit(1 if bad else 0)
|
||||
@@ -287,6 +287,7 @@ jobs:
|
||||
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/data:/usr/share/tessdata:rw
|
||||
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/config:/configs:rw
|
||||
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/logs:/logs:rw
|
||||
- /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/storage:/storage:rw
|
||||
environment:
|
||||
DISABLE_ADDITIONAL_FEATURES: "false"
|
||||
SECURITY_ENABLELOGIN: "true"
|
||||
@@ -309,7 +310,7 @@ jobs:
|
||||
|
||||
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << ENDSSH
|
||||
# Create V2 PR-specific directories
|
||||
mkdir -p /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/{data,config,logs}
|
||||
mkdir -p /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/{data,config,logs,storage}
|
||||
|
||||
# Move docker-compose file to correct location
|
||||
mv /tmp/docker-compose-v2.yml /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/docker-compose.yml
|
||||
|
||||
+124
-27
@@ -1,8 +1,9 @@
|
||||
name: AI Engine CI
|
||||
|
||||
# Validates the Python AI engine: regenerates tool models, runs fixers,
|
||||
# lint, type-check, and tests. Called from build.yml on PRs and merge_group;
|
||||
# also runs directly on push to main as a post-merge safety net.
|
||||
# Validates the Python AI engine: regenerates tool models and runs the
|
||||
# engine quality gate (lint, type-check, format-check, tests). Called from
|
||||
# build.yml on PRs and merge_group; also runs directly on push to main as
|
||||
# a post-merge safety net.
|
||||
on:
|
||||
workflow_call:
|
||||
push:
|
||||
@@ -51,27 +52,95 @@ jobs:
|
||||
run: task engine:tool-models
|
||||
|
||||
- name: Verify tool models are up to date
|
||||
id: tool-models-check
|
||||
continue-on-error: true
|
||||
run: git diff --exit-code engine/src/stirling/models/tool_models.py
|
||||
|
||||
- name: Comment on tool models check failure
|
||||
# Only post a comment on PRs. github-script's PR helpers need an
|
||||
# issue/PR number, which doesn't exist on merge_group runs.
|
||||
if: steps.tool-models-check.outcome == 'failure' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- tool-models-check -->';
|
||||
const body = [
|
||||
marker,
|
||||
'### Tool Models Check Failed',
|
||||
'',
|
||||
'The generated `engine/src/stirling/models/tool_models.py` is out of date with the Java OpenAPI spec and will need to be regenerated before it can be merged in.',
|
||||
'',
|
||||
'Run `task engine:tool-models` to regenerate, then commit the updated file.',
|
||||
].join('\n');
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Fail if tool models check failed
|
||||
if: steps.tool-models-check.outcome == 'failure'
|
||||
run: |
|
||||
if ! git diff --exit-code engine/src/stirling/models/tool_models.py; then
|
||||
echo "tool_models.py is out of date."
|
||||
echo "Run 'task engine:tool-models' locally and commit the updated file."
|
||||
exit 1
|
||||
fi
|
||||
echo "============================================"
|
||||
echo " Tool Models Check Failed"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "The generated engine/src/stirling/models/tool_models.py"
|
||||
echo "is out of date with the Java OpenAPI spec and will"
|
||||
echo "need to be regenerated before it can be merged in."
|
||||
echo ""
|
||||
echo "Run 'task engine:tool-models' to regenerate, then"
|
||||
echo "commit the updated file."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
|
||||
- name: Run fixers
|
||||
run: task engine:fix
|
||||
- name: Remove tool models check comment on success
|
||||
if: steps.tool-models-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- tool-models-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Verify fixes are committed
|
||||
id: fixer_changes
|
||||
run: |
|
||||
if ! git diff --quiet; then
|
||||
git --no-pager diff --stat
|
||||
echo "::error::There are issues with your Python code that will need to be fixed before they can be merged in. Run 'task engine:fix' to auto-fix what can be fixed automatically, then run 'task engine:check' to see what still needs fixing manually."
|
||||
exit 1
|
||||
fi
|
||||
- name: Quality-check engine
|
||||
id: engine-check
|
||||
run: task engine:check
|
||||
continue-on-error: true
|
||||
|
||||
- name: Comment on fixer failures
|
||||
if: steps.fixer_changes.outcome == 'failure' && github.event_name == 'pull_request'
|
||||
- name: Comment on engine check failure
|
||||
# Only post a comment on PRs. github-script's PR helpers need an
|
||||
# issue/PR number, which doesn't exist on merge_group runs.
|
||||
if: steps.engine-check.outcome == 'failure' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
@@ -107,11 +176,39 @@ jobs:
|
||||
});
|
||||
}
|
||||
|
||||
- name: Run linting
|
||||
run: task engine:lint
|
||||
- name: Fail if engine check failed
|
||||
if: steps.engine-check.outcome == 'failure'
|
||||
run: |
|
||||
echo "============================================"
|
||||
echo " Engine Check Failed"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "There are issues with your Python code that"
|
||||
echo "will need to be fixed before they can be merged in."
|
||||
echo ""
|
||||
echo "Run 'task engine:fix' to auto-fix what can be"
|
||||
echo "fixed automatically, then run 'task engine:check'"
|
||||
echo "to see what still needs fixing manually."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
|
||||
- name: Run type checking
|
||||
run: task engine:typecheck
|
||||
|
||||
- name: Run tests
|
||||
run: task engine:test
|
||||
- name: Remove engine check comment on success
|
||||
if: steps.engine-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- engine-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
name: Backend build, format check, and coverage
|
||||
|
||||
# Reusable workflow called from build.yml. Runs the backend build matrix
|
||||
# (JDK 25 × spring-security on/off), Spotless formatting check, JUnit, and
|
||||
# (JDK 25 × every flavor), Spotless formatting check, JUnit, and
|
||||
# posts Jacoco coverage to PRs.
|
||||
#
|
||||
# Flavor axis (maps to STIRLING_FLAVOR in settings.gradle):
|
||||
# core - DISABLE_ADDITIONAL_FEATURES=true, no proprietary, no saas
|
||||
# proprietary - default build, no saas
|
||||
# saas - proprietary + the saas subproject (build + JUnit only,
|
||||
# never any runtime/integration testing)
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
@@ -25,7 +31,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk-version: [25]
|
||||
spring-security: [true, false]
|
||||
flavor: [core, proprietary, saas]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
@@ -58,7 +64,10 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Check Java formatting (Spotless)
|
||||
if: matrix.jdk-version == 25 && matrix.spring-security == false
|
||||
# Runs once per matrix combination - pick the cheapest leg
|
||||
# (core - no proprietary, no saas) so we don't wait for the
|
||||
# heavier flavors just to fail formatting.
|
||||
if: matrix.jdk-version == 25 && matrix.flavor == 'core'
|
||||
id: spotless-check
|
||||
run: task backend:format:check
|
||||
continue-on-error: true
|
||||
@@ -67,7 +76,7 @@ jobs:
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
|
||||
- name: Comment on Java formatting failure
|
||||
- name: Comment on backend format check failure
|
||||
# Only post a comment on PRs. github-script's PR helpers need an
|
||||
# issue/PR number, which doesn't exist on merge_group runs.
|
||||
if: steps.spotless-check.outcome == 'failure' && github.event_name == 'pull_request'
|
||||
@@ -78,15 +87,11 @@ jobs:
|
||||
const marker = '<!-- java-formatting-check -->';
|
||||
const body = [
|
||||
marker,
|
||||
'### Java Formatting Check Failed',
|
||||
'### Backend Format Check Failed',
|
||||
'',
|
||||
'Your code has formatting issues. Run the following command to fix them:',
|
||||
'There are formatting issues in your Java code that will need to be fixed before they can be merged in.',
|
||||
'',
|
||||
'```bash',
|
||||
'task backend:format',
|
||||
'```',
|
||||
'',
|
||||
'Then commit and push the changes.',
|
||||
'Run `task backend:format` to auto-fix, then commit and push the changes.',
|
||||
].join('\n');
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
@@ -110,33 +115,61 @@ jobs:
|
||||
});
|
||||
}
|
||||
|
||||
- name: Fail if Java formatting issues found
|
||||
- name: Fail if backend format check failed
|
||||
if: steps.spotless-check.outcome == 'failure'
|
||||
run: |
|
||||
echo "============================================"
|
||||
echo " Java Formatting Check Failed"
|
||||
echo " Backend Format Check Failed"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "Your code has formatting issues."
|
||||
echo "Run the following command to fix them:"
|
||||
echo "There are formatting issues in your Java code"
|
||||
echo "that will need to be fixed before they can be"
|
||||
echo "merged in."
|
||||
echo ""
|
||||
echo " task backend:format"
|
||||
echo ""
|
||||
echo "Then commit and push the changes."
|
||||
echo "Run 'task backend:format' to auto-fix, then"
|
||||
echo "commit and push the changes."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
|
||||
- name: Build with Gradle and spring security ${{ matrix.spring-security }}
|
||||
- name: Remove backend format check comment on success
|
||||
if: steps.spotless-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- java-formatting-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Build with Gradle (flavor=${{ matrix.flavor }})
|
||||
# STIRLING_FLAVOR is read by settings.gradle and expands into the
|
||||
# right combination of DISABLE_ADDITIONAL_FEATURES + ENABLE_SAAS
|
||||
# so we don't have to set them by hand. The saas flavor pulls in
|
||||
# the app/saas subproject (unit tests only - no runtime tests).
|
||||
run: task backend:build:ci
|
||||
env:
|
||||
MAVEN_USER: ${{ secrets.MAVEN_USER }}
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
DISABLE_ADDITIONAL_FEATURES: ${{ matrix.spring-security }}
|
||||
STIRLING_FLAVOR: ${{ matrix.flavor }}
|
||||
|
||||
- name: Check Test Reports Exist
|
||||
if: always()
|
||||
run: |
|
||||
# Common + core + proprietary always build (proprietary is
|
||||
# excluded only at runtime, not from the gradle subproject
|
||||
# graph). Saas builds add a fourth report dir.
|
||||
declare -a dirs=(
|
||||
"app/core/build/reports/tests/"
|
||||
"app/core/build/test-results/"
|
||||
@@ -145,6 +178,9 @@ jobs:
|
||||
"app/proprietary/build/reports/tests/"
|
||||
"app/proprietary/build/test-results/"
|
||||
)
|
||||
if [ "${{ matrix.flavor }}" = "saas" ]; then
|
||||
dirs+=("app/saas/build/reports/tests/" "app/saas/build/test-results/")
|
||||
fi
|
||||
for dir in "${dirs[@]}"; do
|
||||
if [ ! -d "$dir" ]; then
|
||||
echo "Missing $dir"
|
||||
@@ -156,7 +192,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: test-reports-jdk-${{ matrix.jdk-version }}-spring-security-${{ matrix.spring-security }}
|
||||
name: test-reports-jdk-${{ matrix.jdk-version }}-flavor-${{ matrix.flavor }}
|
||||
path: |
|
||||
app/**/build/reports/jacoco/test
|
||||
app/**/build/reports/tests/
|
||||
@@ -166,7 +202,47 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Add coverage to PR with spring security ${{ matrix.spring-security }} and JDK ${{ matrix.jdk-version }}
|
||||
- name: Install defusedxml for coverage summary
|
||||
# coverage-summary.py parses JaCoCo XML through defusedxml to
|
||||
# silence security scanners that pattern-match on the stdlib
|
||||
# xml.etree.ElementTree.parse call.
|
||||
if: always() && matrix.flavor == 'saas'
|
||||
run: python -m pip install --quiet defusedxml
|
||||
|
||||
- name: JaCoCo coverage step summary
|
||||
# Only the saas leg posts the JUnit summary - it's a strict
|
||||
# superset of the core + proprietary legs (same .exec files plus
|
||||
# the saas subproject). Posting from all three would mean three
|
||||
# near-identical tables crowding out the aggregate report.
|
||||
if: always() && matrix.flavor == 'saas'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Backend JUnit coverage (JDK ${{ matrix.jdk-version }})" \
|
||||
--jacoco "common=app/common/build/reports/jacoco/test/jacocoTestReport.xml" \
|
||||
--jacoco "core=app/core/build/reports/jacoco/test/jacocoTestReport.xml" \
|
||||
--jacoco "proprietary=app/proprietary/build/reports/jacoco/test/jacocoTestReport.xml" \
|
||||
--jacoco "saas=app/saas/build/reports/jacoco/test/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
|
||||
- name: Upload raw JUnit .exec for aggregate merge
|
||||
# Same dedup rationale as the summary step: upload from the saas
|
||||
# leg only (the most complete set, includes app/saas/.../test.exec)
|
||||
# so the aggregate workflow merges the union rather than three
|
||||
# overlapping subsets.
|
||||
#
|
||||
# Separate artifact from the HTML reports so the aggregate
|
||||
# workflow can grab just the .exec files with a name pattern
|
||||
# (`jacoco-exec-*`) instead of unpacking the whole test-reports
|
||||
# tarball.
|
||||
if: always() && matrix.flavor == 'saas'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-exec-junit-jdk-${{ matrix.jdk-version }}
|
||||
path: app/*/build/jacoco/*.exec
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Add coverage to PR (flavor=${{ matrix.flavor }}, JDK=${{ matrix.jdk-version }})
|
||||
# The action only supports the pull_request event (it posts a PR comment),
|
||||
# so skip it for merge_group runs and workflow_dispatch.
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
@@ -68,6 +68,18 @@ jobs:
|
||||
uses: ./.github/workflows/backend-build.yml
|
||||
secrets: inherit
|
||||
|
||||
db-migration-test:
|
||||
# Boots the current bootJar against H2 fixtures captured from past
|
||||
# releases (v2.0.0 / v2.5.0 / v2.10.0) and verifies admin login still
|
||||
# works after Hibernate's ddl-auto=update migrates the schema. Gated on
|
||||
# the `project` filter so doc-only PRs skip this ~5-minute job.
|
||||
if: needs.files-changed.outputs.project == 'true'
|
||||
needs: [files-changed]
|
||||
permissions:
|
||||
contents: read
|
||||
uses: ./.github/workflows/db-migration-test.yml
|
||||
secrets: inherit
|
||||
|
||||
check-generateOpenApiDocs:
|
||||
if: needs.files-changed.outputs.openapi == 'true'
|
||||
needs: [files-changed]
|
||||
@@ -173,6 +185,24 @@ jobs:
|
||||
uses: ./.github/workflows/dependency-review.yml
|
||||
secrets: inherit
|
||||
|
||||
# Coverage aggregate: merges the JUnit + e2e:live + cucumber .exec
|
||||
# artifacts produced by the jobs above into one report, plus pulls
|
||||
# in vitest + Playwright frontend coverage for the per-area matrix.
|
||||
# `if: always()` so a producer failing partway still gets credit
|
||||
# for whatever did record. Advisory only - intentionally NOT in
|
||||
# all-checks-passed, so a flaky aggregate run never blocks merging.
|
||||
coverage-aggregate:
|
||||
if: always()
|
||||
needs:
|
||||
- build
|
||||
- playwright-e2e-live
|
||||
- docker-compose-tests
|
||||
- frontend-validation
|
||||
permissions:
|
||||
contents: read
|
||||
uses: ./.github/workflows/coverage-aggregate.yml
|
||||
secrets: inherit
|
||||
|
||||
# Single status check that branch protection should mark as required.
|
||||
# Succeeds when every upstream job is either `success` or `skipped` (path-
|
||||
# gated jobs that didn't apply this run). Any `failure` or `cancelled`
|
||||
@@ -184,6 +214,7 @@ jobs:
|
||||
needs:
|
||||
- files-changed
|
||||
- build
|
||||
- db-migration-test
|
||||
- check-generateOpenApiDocs
|
||||
- frontend-validation
|
||||
- playwright-e2e
|
||||
@@ -208,6 +239,7 @@ jobs:
|
||||
RESULTS: |
|
||||
files-changed=${{ needs.files-changed.result }}
|
||||
build=${{ needs.build.result }}
|
||||
db-migration-test=${{ needs.db-migration-test.result }}
|
||||
check-generateOpenApiDocs=${{ needs.check-generateOpenApiDocs.result }}
|
||||
frontend-validation=${{ needs.frontend-validation.result }}
|
||||
playwright-e2e=${{ needs.playwright-e2e.result }}
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
name: Aggregate backend coverage
|
||||
|
||||
# Reusable workflow called from build.yml after every backend coverage
|
||||
# producer (JUnit, e2e:live, cucumber) has run. Downloads each job's raw
|
||||
# .exec, merges them into one JaCoCo report, and posts a combined step
|
||||
# summary alongside the per-source ones.
|
||||
#
|
||||
# Kept separate from the per-source jobs so:
|
||||
# - the per-source jobs stay fast and independent (no cross-job waits)
|
||||
# - this job can `if: always()` and still produce something useful when
|
||||
# one of the producers fails partway through
|
||||
# - frontend producers can be added later without touching the
|
||||
# producers themselves
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
aggregate:
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up JDK 25
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
with:
|
||||
java-version: "25"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
~/.gradle/caches/modules-2/files-2.1
|
||||
~/.gradle/caches/modules-2/metadata-2.*
|
||||
key: gradle-deps-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }}
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install defusedxml for coverage scripts
|
||||
# Both coverage-summary.py and coverage-matrix.py parse JaCoCo
|
||||
# XML through defusedxml - see the script headers for context.
|
||||
run: python -m pip install --quiet defusedxml
|
||||
|
||||
# Pattern matches every artifact this PR's producers might upload:
|
||||
# jacoco-exec-junit-jdk-25 (uploaded only by the saas
|
||||
# leg of backend-build, which
|
||||
# is a strict superset of the
|
||||
# core + proprietary legs)
|
||||
# jacoco-exec-e2e-live
|
||||
# jacoco-exec-cucumber
|
||||
# Each lands as a sibling dir under coverage-execs/, with the .exec
|
||||
# files preserving their original relative paths.
|
||||
- name: Download all .exec artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
|
||||
with:
|
||||
pattern: jacoco-exec-*
|
||||
path: coverage-execs/
|
||||
merge-multiple: false
|
||||
continue-on-error: true
|
||||
|
||||
- name: Inventory .exec files
|
||||
id: inventory
|
||||
# Splits the downloaded artifacts into two buckets:
|
||||
# * e2e-only = cucumber + Playwright live (user-flow coverage)
|
||||
# * all = the above plus JUnit (everything we test)
|
||||
#
|
||||
# Bucketing is by artifact-name prefix: download-artifact preserves
|
||||
# the artifact name as the top-level dir, so JUnit's `.exec`s live
|
||||
# under coverage-execs/jacoco-exec-junit-*/... while the others
|
||||
# are under coverage-execs/jacoco-exec-{e2e-live,cucumber}/...
|
||||
#
|
||||
# If nothing was uploaded (e.g. all producers crashed before
|
||||
# writing) we exit gracefully so this advisory job never fails CI.
|
||||
run: |
|
||||
mapfile -t all_execs < <(find coverage-execs -name '*.exec' -type f | sort)
|
||||
mapfile -t e2e_execs < <(find coverage-execs -name '*.exec' -type f -not -path '*/jacoco-exec-junit-*' | sort)
|
||||
if [ "${#all_execs[@]}" -eq 0 ]; then
|
||||
echo "::warning::No .exec artifacts found - skipping aggregate report"
|
||||
echo "found_all=false" >> "$GITHUB_OUTPUT"
|
||||
echo "found_e2e=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
printf 'All %d .exec files:\n' "${#all_execs[@]}"
|
||||
printf ' %s\n' "${all_execs[@]}"
|
||||
IFS=','; all_joined="${all_execs[*]}"
|
||||
echo "files_all=$all_joined" >> "$GITHUB_OUTPUT"
|
||||
echo "found_all=true" >> "$GITHUB_OUTPUT"
|
||||
if [ "${#e2e_execs[@]}" -eq 0 ]; then
|
||||
echo "::notice::No e2e/cucumber .exec files - e2e-only report will be skipped"
|
||||
echo "found_e2e=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
printf 'E2E-only %d .exec files:\n' "${#e2e_execs[@]}"
|
||||
printf ' %s\n' "${e2e_execs[@]}"
|
||||
unset IFS
|
||||
IFS=','; e2e_joined="${e2e_execs[*]}"
|
||||
echo "files_e2e=$e2e_joined" >> "$GITHUB_OUTPUT"
|
||||
echo "found_e2e=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Compile classes for JaCoCo class lookup
|
||||
# jacocoReportFromExec only needs the compiled .class files
|
||||
# under each subproject's build/classes/java/main/. `classes`
|
||||
# (compileJava + processResources) is enough; we skipped the
|
||||
# heavier `assemble` to avoid building bootJar / fat jars that
|
||||
# add 60+ seconds per run for no gain to the report.
|
||||
if: steps.inventory.outputs.found_all == 'true'
|
||||
run: ./gradlew classes -PnoSpotless
|
||||
|
||||
- name: Generate e2e-only JaCoCo report
|
||||
# "Real user-flow" coverage: only counts code reached by an actual
|
||||
# HTTP request from cucumber or live Playwright. Useful for
|
||||
# questions like "how much of our backend does a user actually
|
||||
# hit?". Skipped when neither producer uploaded a .exec.
|
||||
if: steps.inventory.outputs.found_e2e == 'true'
|
||||
run: |
|
||||
./gradlew jacocoReportFromExec \
|
||||
-PexecFile="${{ steps.inventory.outputs.files_e2e }}" \
|
||||
-PreportDir=build/reports/jacoco/aggregate-e2e \
|
||||
-PnoSpotless
|
||||
|
||||
- name: Generate combined JaCoCo report (everything)
|
||||
if: steps.inventory.outputs.found_all == 'true'
|
||||
run: |
|
||||
./gradlew jacocoReportFromExec \
|
||||
-PexecFile="${{ steps.inventory.outputs.files_all }}" \
|
||||
-PreportDir=build/reports/jacoco/aggregate-all \
|
||||
-PnoSpotless
|
||||
|
||||
- name: E2E-only step summary
|
||||
# Rendered first so it gets prime real estate in the Summary
|
||||
# tab - this is the number most readers actually want
|
||||
# ("how much of the backend do real user flows cover?").
|
||||
if: steps.inventory.outputs.found_e2e == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Real user-flow backend coverage (e2e:live + cucumber)" \
|
||||
--jacoco "merged=build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
|
||||
- name: ALL-sources step summary
|
||||
# Separate call (not a multi-input one) because the helper's
|
||||
# rightmost "Aggregate" column would sum the two reports - which
|
||||
# is meaningless when one is a strict superset of the other.
|
||||
if: steps.inventory.outputs.found_all == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Combined backend coverage (JUnit + e2e:live + cucumber)" \
|
||||
--jacoco "merged=build/reports/jacoco/aggregate-all/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
|
||||
- name: Upload combined aggregate report
|
||||
if: steps.inventory.outputs.found_all == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-aggregate-all-${{ github.run_id }}
|
||||
path: build/reports/jacoco/aggregate-all/
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload e2e-only aggregate report
|
||||
if: steps.inventory.outputs.found_e2e == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-aggregate-e2e-${{ github.run_id }}
|
||||
path: build/reports/jacoco/aggregate-e2e/
|
||||
retention-days: 14
|
||||
|
||||
# --------------------------------------------------------------
|
||||
# Per-area matrix: rolls backend + frontend coverage into one
|
||||
# table indexed by core/proprietary/saas/desktop. Pulls the
|
||||
# frontend artifacts now (after the JaCoCo step has done its
|
||||
# work) so the per-source backend summaries still render first
|
||||
# even if the matrix step fails.
|
||||
# --------------------------------------------------------------
|
||||
- name: Download vitest coverage artifact
|
||||
# frontend-validation uploads as `frontend-coverage`. Tolerate
|
||||
# absence so a backend-only PR still produces the matrix with
|
||||
# just backend rows populated.
|
||||
if: always()
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
|
||||
with:
|
||||
name: frontend-coverage
|
||||
path: matrix-inputs/vitest/
|
||||
continue-on-error: true
|
||||
|
||||
- name: Download Playwright frontend coverage artifact
|
||||
# e2e-live uploads as `playwright-frontend-coverage-<run_id>`.
|
||||
# Same tolerance as vitest - matrix script handles missing inputs.
|
||||
if: always()
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
|
||||
with:
|
||||
name: playwright-frontend-coverage-${{ github.run_id }}
|
||||
path: matrix-inputs/playwright/
|
||||
continue-on-error: true
|
||||
|
||||
- name: Coverage matrix step summary
|
||||
if: always()
|
||||
# Matrix references the two aggregate JaCoCo XMLs (already
|
||||
# generated above) plus whichever frontend artifacts landed.
|
||||
# Every input is optional; missing ones render as "-".
|
||||
run: |
|
||||
python scripts/coverage-matrix.py \
|
||||
${{ steps.inventory.outputs.found_all == 'true' && '--jacoco-all build/reports/jacoco/aggregate-all/jacocoTestReport.xml' || '' }} \
|
||||
${{ steps.inventory.outputs.found_e2e == 'true' && '--jacoco-e2e build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml' || '' }} \
|
||||
--vitest matrix-inputs/vitest/coverage-summary.json \
|
||||
--playwright-frontend matrix-inputs/playwright/coverage-pw-summary/coverage-summary.json \
|
||||
--title "Coverage matrix (per-area, e2e vs all)" \
|
||||
--github-step-summary
|
||||
@@ -0,0 +1,93 @@
|
||||
name: DB migration smoke test
|
||||
|
||||
# Boots the current Stirling-PDF JAR against H2 fixtures captured from past
|
||||
# releases (v2.0.0 / v2.5.0 / v2.10.0) and verifies admin login still works.
|
||||
# Catches schema changes that would break existing user databases under
|
||||
# Hibernate's `ddl-auto=update` upgrade path.
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
migration-test:
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-8' }}
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up JDK 25
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
with:
|
||||
java-version: 25
|
||||
distribution: temurin
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
~/.gradle/caches/modules-2/files-2.1
|
||||
~/.gradle/caches/modules-2/metadata-2.*
|
||||
key: gradle-deps-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }}
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
# No `-PnoSpotless` here yet because the upstream cache layer matches the
|
||||
# backend build's; reuse keeps cold-cache cost identical.
|
||||
- name: Build Stirling-PDF JAR
|
||||
env:
|
||||
MAVEN_USER: ${{ secrets.MAVEN_USER }}
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
run: ./gradlew :stirling-pdf:bootJar -PnoSpotless --no-daemon
|
||||
|
||||
- name: Locate built JAR
|
||||
id: jar
|
||||
run: |
|
||||
jar=$(find app/core/build/libs -maxdepth 1 -name 'Stirling-PDF*.jar' -o -name 'stirling-pdf*.jar' 2>/dev/null \
|
||||
| grep -vE '(-plain|-sources)\.jar$' | head -n 1)
|
||||
if [[ -z "$jar" ]]; then
|
||||
echo "::error::No JAR under app/core/build/libs"
|
||||
ls -lah app/core/build/libs || true
|
||||
exit 1
|
||||
fi
|
||||
# Absolute path - the migration script pushd's into a temp workdir
|
||||
# before invoking java, which would dangle a relative path.
|
||||
jar=$(realpath "$jar")
|
||||
echo "path=$jar" >> "$GITHUB_OUTPUT"
|
||||
echo "Built JAR: $jar"
|
||||
|
||||
- name: Run migration smoke test
|
||||
env:
|
||||
STIRLING_JAR: ${{ steps.jar.outputs.path }}
|
||||
run: bash scripts/db-migration/run-migration-test.sh
|
||||
|
||||
- name: Upload app logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: db-migration-app-logs
|
||||
# Path matches the preserved workdir in run-migration-test.sh -
|
||||
# only failing fixtures leave a directory behind.
|
||||
path: /tmp/stirling-migration-failed-*/app.log
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
@@ -87,6 +87,12 @@ jobs:
|
||||
run: |
|
||||
pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt
|
||||
|
||||
- name: Extract JaCoCo agent for cucumber coverage
|
||||
# Stages build/jacoco/jacocoagent.jar where the coverage override
|
||||
# file bind-mounts it into the cucumber container. The agent jar
|
||||
# never goes into the published image - this is host-only.
|
||||
run: ./gradlew copyJacocoAgent -PnoSpotless
|
||||
|
||||
- name: Run Docker Compose Tests
|
||||
run: |
|
||||
chmod +x ./testing/test_webpages.sh
|
||||
@@ -98,6 +104,62 @@ jobs:
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
DOCKER_BASE_CHANGED: ${{ inputs.docker-base-changed }}
|
||||
# Tells test.sh to layer testing/compose/docker-compose-coverage.override.yml
|
||||
# over the cucumber compose so the container starts with the
|
||||
# JaCoCo agent attached via JAVA_CUSTOM_OPTS.
|
||||
STIRLING_PDF_TEST_COVERAGE: "1"
|
||||
|
||||
- name: Generate cucumber JaCoCo report
|
||||
# `if: always()` so a behave failure still produces partial
|
||||
# coverage from whatever endpoints did run. The exec file only
|
||||
# exists when the container shut down cleanly - guard so the step
|
||||
# is silent on the (rare) crash path.
|
||||
if: always()
|
||||
id: cucumber-coverage
|
||||
run: |
|
||||
if [ -s testing/cucumber-coverage/cucumber.exec ]; then
|
||||
./gradlew jacocoReportFromExec \
|
||||
-PexecFile=testing/cucumber-coverage/cucumber.exec \
|
||||
-PreportDir=build/reports/jacoco/cucumber \
|
||||
-PnoSpotless
|
||||
echo "report=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::warning::No cucumber .exec at testing/cucumber-coverage/cucumber.exec (container may have crashed before flushing)"
|
||||
echo "report=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Install defusedxml for coverage summary
|
||||
# coverage-summary.py parses JaCoCo XML through defusedxml -
|
||||
# see the script header for context.
|
||||
if: always() && steps.cucumber-coverage.outputs.report == 'true'
|
||||
run: python -m pip install --quiet defusedxml
|
||||
|
||||
- name: Cucumber coverage step summary
|
||||
if: always() && steps.cucumber-coverage.outputs.report == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Cucumber (docker) JaCoCo coverage" \
|
||||
--jacoco "cucumber=build/reports/jacoco/cucumber/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
|
||||
- name: Upload cucumber JaCoCo report
|
||||
if: always() && steps.cucumber-coverage.outputs.report == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-cucumber-${{ github.run_id }}
|
||||
path: build/reports/jacoco/cucumber/
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload raw cucumber .exec for aggregate merge
|
||||
# Picked up by the coverage-aggregate workflow via the
|
||||
# `jacoco-exec-*` artifact name pattern.
|
||||
if: always() && steps.cucumber-coverage.outputs.report == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-exec-cucumber
|
||||
path: testing/cucumber-coverage/cucumber.exec
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload Cucumber Report
|
||||
if: always()
|
||||
|
||||
@@ -49,9 +49,132 @@ jobs:
|
||||
env:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
- name: Run live E2E tests (chromium)
|
||||
- name: Run live E2E tests (chromium) with coverage
|
||||
id: live-tests
|
||||
env:
|
||||
# Attaches the JaCoCo agent to the bootRun JVM (see
|
||||
# .taskfiles/e2e.yml live:backend). The .exec gets flushed on
|
||||
# graceful shutdown when the runner traps EXIT/INT/TERM, so the
|
||||
# report step below sees a populated file.
|
||||
COVERAGE: "1"
|
||||
# Tells the Playwright fixture (test-base.ts) to capture per-test
|
||||
# V8 JS coverage. Raw dumps land under
|
||||
# .test-state/playwright/coverage-pw/ for the post-process step
|
||||
# to aggregate. Chromium-only - other engines silently skip.
|
||||
PW_COVERAGE: "1"
|
||||
run: task e2e:live
|
||||
- name: Generate JaCoCo report from e2e:live .exec
|
||||
if: always()
|
||||
id: live-coverage
|
||||
# `if: always()` so even a failed test run still produces a
|
||||
# report from whatever flows did exercise the backend before
|
||||
# the failure. The task itself tolerates a missing .exec
|
||||
# (jacoco emits an empty report rather than crashing) but we
|
||||
# guard with `test -s` to keep the job log clean.
|
||||
run: |
|
||||
if [ -s .test-state/playwright/jacoco.exec ]; then
|
||||
./gradlew jacocoReportFromExec \
|
||||
-PexecFile=.test-state/playwright/jacoco.exec \
|
||||
-PreportDir=build/reports/jacoco/e2e-live \
|
||||
-PnoSpotless
|
||||
echo "report=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::warning::No e2e:live .exec found at .test-state/playwright/jacoco.exec; skipping report"
|
||||
echo "report=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- name: Set up Python for coverage summary
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install defusedxml for coverage summary
|
||||
# coverage-summary.py uses defusedxml instead of stdlib xml.etree
|
||||
# to dodge XXE / billion-laughs scanner findings.
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
run: python -m pip install --quiet defusedxml
|
||||
- name: e2e:live coverage step summary
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Playwright (live backend) JaCoCo coverage" \
|
||||
--jacoco "e2e-live=build/reports/jacoco/e2e-live/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
- name: Upload e2e:live JaCoCo report
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-e2e-live-${{ github.run_id }}
|
||||
path: build/reports/jacoco/e2e-live/
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload raw e2e:live .exec for aggregate merge
|
||||
# Picked up by the coverage-aggregate workflow via the
|
||||
# `jacoco-exec-*` artifact name pattern.
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-exec-e2e-live
|
||||
path: .test-state/playwright/jacoco.exec
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Set up Python for frontend coverage summary
|
||||
# Separate from the backend-coverage python step because the
|
||||
# frontend path doesn't depend on a JaCoCo report - it produces
|
||||
# a summary even on backend failure, as long as some Playwright
|
||||
# tests ran far enough to dump V8 coverage.
|
||||
if: always()
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install defusedxml for frontend coverage summary
|
||||
# Idempotent re-install: the backend-coverage step may have
|
||||
# installed it already, but this leg can run on its own when the
|
||||
# backend report step skips (e.g. .exec missing).
|
||||
if: always()
|
||||
run: python -m pip install --quiet defusedxml
|
||||
|
||||
- name: Aggregate Playwright frontend (V8) coverage
|
||||
# Rolls per-test V8 dumps from the test-base fixture into one
|
||||
# vitest-shaped coverage-summary.json. Tolerates a missing dump
|
||||
# dir (firefox/webkit runs, or a failure before any test got
|
||||
# far enough to dump).
|
||||
if: always()
|
||||
id: pw-frontend-coverage
|
||||
run: |
|
||||
if [ -d .test-state/playwright/coverage-pw ] && \
|
||||
find .test-state/playwright/coverage-pw -name '*.json' -type f | grep -q .; then
|
||||
python scripts/playwright-coverage-summary.py \
|
||||
.test-state/playwright/coverage-pw \
|
||||
--out .test-state/playwright/coverage-pw-summary/coverage-summary.json
|
||||
echo "summary=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::notice::No Playwright frontend coverage dumps found (chromium-only feature)"
|
||||
echo "summary=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Playwright frontend coverage step summary
|
||||
if: always() && steps.pw-frontend-coverage.outputs.summary == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Playwright (live) frontend coverage" \
|
||||
--vitest .test-state/playwright/coverage-pw-summary/coverage-summary.json \
|
||||
--github-step-summary
|
||||
|
||||
- name: Upload Playwright frontend coverage
|
||||
# Bundle both the aggregated summary and the raw V8 dumps so
|
||||
# someone debugging "why is this function showing as covered"
|
||||
# can trace it back to the source dump.
|
||||
if: always() && steps.pw-frontend-coverage.outputs.summary == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-frontend-coverage-${{ github.run_id }}
|
||||
path: |
|
||||
.test-state/playwright/coverage-pw-summary/
|
||||
.test-state/playwright/coverage-pw/
|
||||
retention-days: 7
|
||||
|
||||
- name: Print backend log on failure
|
||||
if: failure() && steps.live-tests.conclusion == 'failure'
|
||||
run: |
|
||||
|
||||
@@ -110,8 +110,8 @@ jobs:
|
||||
NPM_CONFIG_IGNORE_SCRIPTS: "true"
|
||||
working-directory: frontend
|
||||
run: |
|
||||
mkdir -p src/assets
|
||||
npx --yes license-report --only=prod --output=json > src/assets/3rdPartyLicenses.json
|
||||
mkdir -p editor/src/assets
|
||||
npx --yes license-report --only=prod --output=json > editor/src/assets/3rdPartyLicenses.json
|
||||
|
||||
- name: Postprocess with project script (BASE version)
|
||||
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true
|
||||
|
||||
@@ -109,6 +109,41 @@ jobs:
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
- name: Vitest coverage
|
||||
# Separate from `frontend:check:all` so the quality-gate run stays
|
||||
# uninstrumented (faster signal) and coverage stays an informational
|
||||
# follow-up. Continue-on-error keeps the workflow green even when
|
||||
# a handful of test files refuse to import (e.g. missing icon
|
||||
# specifiers) - the summary still gets posted with whatever
|
||||
# vitest managed to instrument.
|
||||
id: frontend-coverage
|
||||
continue-on-error: true
|
||||
run: task frontend:test:coverage
|
||||
- name: Set up Python for coverage summary
|
||||
if: always()
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install defusedxml for coverage summary
|
||||
# See coverage-summary.py header - it parses XML through defusedxml
|
||||
# to dodge the stdlib parser's exposure to XXE / billion-laughs.
|
||||
if: always()
|
||||
run: python -m pip install --quiet defusedxml
|
||||
- name: Vitest coverage step summary
|
||||
if: always()
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Frontend Vitest coverage" \
|
||||
--vitest frontend/editor/coverage/coverage-summary.json \
|
||||
--github-step-summary
|
||||
- name: Upload vitest coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: frontend-coverage
|
||||
path: frontend/editor/coverage/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
- name: Upload frontend build artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
|
||||
@@ -442,10 +442,6 @@ jobs:
|
||||
echo "Generated tauri.windows.conf.json (alias masked):"
|
||||
sed "s/${KEYPAIR_ALIAS}/***/g" ./frontend/editor/src-tauri/tauri.windows.conf.json
|
||||
|
||||
- name: Sign JPDFium dylibs inside bootJar (macOS only)
|
||||
if: matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
run: bash frontend/scripts/sign-jpdfium-dylibs-in-bootjar.sh
|
||||
|
||||
- name: Import release GPG signing key (Linux)
|
||||
if: matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master')
|
||||
run: |
|
||||
@@ -495,6 +491,7 @@ jobs:
|
||||
projectPath: ./frontend/editor
|
||||
tauriScript: npx tauri
|
||||
args: ${{ matrix.args }}
|
||||
updaterJsonKeepUniversal: true
|
||||
|
||||
- name: Clear release GPG key from runner keyring (Linux)
|
||||
if: always() && matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master')
|
||||
@@ -586,21 +583,37 @@ jobs:
|
||||
if: always() && steps.digicert-setup.conclusion != 'failure'
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p ./dist
|
||||
# Absolute dist path so the cd below can't break the copy targets.
|
||||
DIST="$GITHUB_WORKSPACE/dist"
|
||||
mkdir -p "$DIST"
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
|
||||
# Find and rename artifacts based on platform
|
||||
echo "=== tauri bundle artifacts ==="
|
||||
find . -path "*/bundle/*" \( -name "*.msi" -o -name "*.deb" \
|
||||
-o -name "*.rpm" -o -name "*.AppImage" -o -name "*.dmg" \
|
||||
-o -name "*.app.tar.gz" -o -name "*.sig" \) 2>/dev/null | sort || true
|
||||
echo "=============================="
|
||||
|
||||
# createUpdaterArtifacts:true signs the native installers in place;
|
||||
# each <bundle> ships with a sibling <bundle>.sig consumed by latest.json.
|
||||
if [ "${{ matrix.platform }}" = "windows-latest" ]; then
|
||||
# Only ship the MSI installer on Windows. The loose exe and WiX toolset exes
|
||||
# are not the user-facing installer - the MSI contains the signed inner exe.
|
||||
find . -name "*.msi" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.msi" \;
|
||||
find . -name "*.msi" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.msi" \;
|
||||
find . -name "*.msi.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.msi.sig" \;
|
||||
elif [ "${{ matrix.platform }}" = "macos-15" ]; then
|
||||
find . -name "*.dmg" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.dmg" \;
|
||||
find . -name "*.app" -exec cp -r {} "../../../dist/Stirling-PDF-${{ matrix.name }}.app" \;
|
||||
# DMG = manual install; .app.tar.gz (+ .sig) = updater payload.
|
||||
# Raw .app is intentionally not shipped (hundreds of MB of uncompressed input).
|
||||
find . -name "*.dmg" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.dmg" \;
|
||||
find . -name "*.app.tar.gz" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.app.tar.gz" \;
|
||||
find . -name "*.app.tar.gz.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.app.tar.gz.sig" \;
|
||||
else
|
||||
find . -name "*.deb" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.deb" \;
|
||||
find . -name "*.rpm" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.rpm" \;
|
||||
find . -name "*.AppImage" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.AppImage" \;
|
||||
# The raw .AppImage IS its updater payload (signed -> .AppImage.sig),
|
||||
# not a .tar.gz wrapper - that's only produced under v1Compatible.
|
||||
find . -name "*.deb" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.deb" \;
|
||||
find . -name "*.deb.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.deb.sig" \;
|
||||
find . -name "*.rpm" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.rpm" \;
|
||||
find . -name "*.rpm.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.rpm.sig" \;
|
||||
find . -name "*.AppImage" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.AppImage" \;
|
||||
find . -name "*.AppImage.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.AppImage.sig" \;
|
||||
fi
|
||||
|
||||
- name: Upload build artifacts
|
||||
@@ -611,8 +624,7 @@ jobs:
|
||||
path: ./dist/*
|
||||
retention-days: 1
|
||||
|
||||
create-release:
|
||||
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.test_mode != 'true') || github.event_name == 'release' || github.ref == 'refs/heads/V2-master'
|
||||
collect-and-release:
|
||||
needs: [pick, determine-matrix, build, build-jars]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
@@ -623,6 +635,16 @@ jobs:
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
# Sparse-check out the verifier + pubkey before the artifact downloads
|
||||
# so the checkout cannot clobber ./artifacts.
|
||||
- name: Checkout updater verifier
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github/scripts/verify-updater-signatures.py
|
||||
frontend/editor/src-tauri/tauri.conf.json
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Download all Tauri artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
@@ -650,17 +672,146 @@ jobs:
|
||||
- name: Display structure of downloaded files
|
||||
run: ls -R ./artifacts
|
||||
|
||||
# tauri-action only emits latest.json when it also publishes the release
|
||||
# (tagName/releaseId set). We publish separately via action-gh-release,
|
||||
# so build latest.json here from the per-platform .sig files.
|
||||
- name: Generate updater latest.json
|
||||
env:
|
||||
VERSION: ${{ needs.determine-matrix.outputs.version }}
|
||||
TAG: v${{ needs.determine-matrix.outputs.version }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
python3 - << 'PYEOF'
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
VERSION = os.environ['VERSION']
|
||||
TAG = os.environ['TAG']
|
||||
REPO = os.environ['REPO']
|
||||
|
||||
ART = Path('./artifacts/tauri')
|
||||
|
||||
# Tauri updater looks up {os}-{arch}-{installer} (e.g. linux-x86_64-deb)
|
||||
# before bare {os}-{arch}, so per-format Linux keys let deb/rpm/appimage
|
||||
# each self-update from their matching file. macOS universal serves both
|
||||
# arches from the one .app.tar.gz.
|
||||
PLATFORM_MAP = [
|
||||
{
|
||||
'bundles': ['Stirling-PDF-linux-x86_64.deb'],
|
||||
'targets': ['linux-x86_64-deb'],
|
||||
},
|
||||
{
|
||||
'bundles': ['Stirling-PDF-linux-x86_64.rpm'],
|
||||
'targets': ['linux-x86_64-rpm'],
|
||||
},
|
||||
{
|
||||
'bundles': ['Stirling-PDF-linux-x86_64.AppImage'],
|
||||
'targets': ['linux-x86_64-appimage'],
|
||||
},
|
||||
{
|
||||
'bundles': ['Stirling-PDF-windows-x86_64.msi'],
|
||||
'targets': ['windows-x86_64-msi', 'windows-x86_64'],
|
||||
},
|
||||
{
|
||||
'bundles': ['Stirling-PDF-macos-universal.app.tar.gz'],
|
||||
'targets': ['darwin-x86_64', 'darwin-aarch64'],
|
||||
},
|
||||
]
|
||||
|
||||
# rglob() because download-artifact varies layout: one artifact -> flat,
|
||||
# many -> nested under <artifact-name>/.
|
||||
def find_signed(name):
|
||||
for bundle_path in sorted(ART.rglob(name)):
|
||||
sig_path = bundle_path.with_name(bundle_path.name + '.sig')
|
||||
if sig_path.exists():
|
||||
return bundle_path, sig_path
|
||||
return None
|
||||
|
||||
platforms = {}
|
||||
skipped = []
|
||||
for entry in PLATFORM_MAP:
|
||||
picked = None
|
||||
for name in entry['bundles']:
|
||||
picked = find_signed(name)
|
||||
if picked:
|
||||
break
|
||||
if not picked:
|
||||
skipped.append(
|
||||
f"{entry['targets']} (no signed bundle among "
|
||||
f"{entry['bundles']} - TAURI_SIGNING_PRIVATE_KEY unset "
|
||||
f"or createUpdaterArtifacts disabled?)"
|
||||
)
|
||||
continue
|
||||
bundle_path, sig_path = picked
|
||||
signature = sig_path.read_text(encoding='utf-8').strip()
|
||||
url = f"https://github.com/{REPO}/releases/download/{TAG}/{bundle_path.name}"
|
||||
for target in entry['targets']:
|
||||
platforms[target] = {'signature': signature, 'url': url}
|
||||
print(f"Added {entry['targets']} from {bundle_path.name}")
|
||||
|
||||
if skipped:
|
||||
print("Skipped platforms:")
|
||||
for s in skipped:
|
||||
print(f" - {s}")
|
||||
|
||||
if not platforms:
|
||||
print(
|
||||
"WARN: no signed updater bundles found - "
|
||||
"skipping latest.json generation"
|
||||
)
|
||||
sys.exit(0)
|
||||
|
||||
manifest = {
|
||||
'version': VERSION,
|
||||
'notes': f"See https://github.com/{REPO}/releases/tag/{TAG}",
|
||||
'pub_date': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'platforms': platforms,
|
||||
}
|
||||
|
||||
out = Path('./artifacts/latest.json')
|
||||
out.write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8')
|
||||
print(f"Generated {out} with platforms: {sorted(platforms.keys())}")
|
||||
PYEOF
|
||||
|
||||
- name: Upload merged artifacts for review
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: release-artifacts
|
||||
path: ./artifacts/
|
||||
retention-days: 7
|
||||
|
||||
# Gate publish on valid updater sigs. Runs after the review upload (so
|
||||
# artifacts survive for debugging) and before action-gh-release.
|
||||
- name: Verify updater signatures
|
||||
run: |
|
||||
python3 -m pip install --quiet 'cryptography==44.0.0'
|
||||
python3 .github/scripts/verify-updater-signatures.py \
|
||||
./artifacts/tauri frontend/editor/src-tauri/tauri.conf.json
|
||||
|
||||
# workflow_dispatch path requires platform=='all' so a single-platform
|
||||
# dispatch can't overwrite an existing release's full latest.json with a
|
||||
# partial one (action-gh-release defaults overwrite_files:true).
|
||||
# release / V2-master always build the full matrix so no extra guard needed.
|
||||
# fail_on_unmatched_files makes a missing latest.json or installer fail loudly
|
||||
# instead of silently shipping a broken auto-update.
|
||||
- name: Upload binaries to Release
|
||||
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.test_mode != 'true' && github.event.inputs.platform == 'all') || github.event_name == 'release' || github.ref == 'refs/heads/V2-master'
|
||||
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0
|
||||
with:
|
||||
tag_name: v${{ needs.determine-matrix.outputs.version }}
|
||||
generate_release_notes: true
|
||||
fail_on_unmatched_files: true
|
||||
# Installers + updater payloads + manifest. .sig contents are embedded
|
||||
# in latest.json so the .sig files themselves are not uploaded.
|
||||
files: |
|
||||
./artifacts/**/*.jar
|
||||
./artifacts/**/*.msi
|
||||
./artifacts/**/*.dmg
|
||||
./artifacts/**/*.app.tar.gz
|
||||
./artifacts/**/*.deb
|
||||
./artifacts/**/*.rpm
|
||||
./artifacts/**/*.AppImage
|
||||
./artifacts/latest.json
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
@@ -157,6 +157,9 @@ jobs:
|
||||
JPDFIUM_PLATFORMS: ${{ matrix.jpdfium_platforms }}
|
||||
run: task desktop:prepare
|
||||
|
||||
- name: Run Tauri/Cargo tests
|
||||
run: task desktop:test
|
||||
|
||||
# DigiCert KeyLocker Setup (Cloud HSM)
|
||||
- name: Setup DigiCert KeyLocker
|
||||
id: digicert-setup
|
||||
@@ -268,10 +271,6 @@ jobs:
|
||||
echo "APPLE_SIGNING_IDENTITY=$CERT_ID" >> $GITHUB_ENV
|
||||
echo "Certificate imported successfully."
|
||||
|
||||
- name: Sign JPDFium dylibs inside bootJar (macOS only)
|
||||
if: matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
run: bash frontend/scripts/sign-jpdfium-dylibs-in-bootjar.sh
|
||||
|
||||
- name: Check DMG creation dependencies (macOS only)
|
||||
if: matrix.platform == 'macos-15'
|
||||
run: |
|
||||
@@ -417,20 +416,22 @@ jobs:
|
||||
- name: Rename artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p ./dist
|
||||
# Absolute dist path so the cd below can't break the copy targets.
|
||||
DIST="$GITHUB_WORKSPACE/dist"
|
||||
mkdir -p "$DIST"
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
|
||||
# Find and rename artifacts based on platform
|
||||
if [ "${{ matrix.platform }}" = "windows-latest" ]; then
|
||||
# Only ship the MSI installer. The loose exe and WiX toolset exes
|
||||
# are not the user-facing installer - the MSI contains the signed inner exe.
|
||||
find . -name "*.msi" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.msi" \;
|
||||
find . -name "*.msi" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.msi" \;
|
||||
elif [ "${{ matrix.platform }}" = "macos-15" ]; then
|
||||
find . -name "*.dmg" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.dmg" \;
|
||||
find . -name "*.dmg" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.dmg" \;
|
||||
else
|
||||
find . -name "*.deb" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.deb" \;
|
||||
find . -name "*.rpm" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.rpm" \;
|
||||
find . -name "*.AppImage" -exec cp {} "../../../dist/Stirling-PDF-${{ matrix.name }}.AppImage" \;
|
||||
find . -name "*.deb" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.deb" \;
|
||||
find . -name "*.rpm" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.rpm" \;
|
||||
find . -name "*.AppImage" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.AppImage" \;
|
||||
fi
|
||||
|
||||
# Verify the MSI AND the inner exe extracted from it are signed.
|
||||
|
||||
+11
-3
@@ -23,6 +23,10 @@ customFiles/
|
||||
configs/
|
||||
watchedFolders/
|
||||
clientWebUI/
|
||||
# Scratch dir used by local fixture-regeneration runs (see
|
||||
# app/proprietary/src/test/resources/db-migration-fixtures/README.md).
|
||||
# Holds downloaded JARs and disposable workdirs. Never committed.
|
||||
.alpha-local/
|
||||
!cucumber/
|
||||
!cucumber/exampleFiles/
|
||||
!cucumber/exampleFiles/example_html.zip
|
||||
@@ -53,6 +57,8 @@ app/core/src/main/resources/static/robots.txt
|
||||
app/core/src/main/resources/static/pdfium/
|
||||
app/core/src/main/resources/static/pdfjs/
|
||||
app/core/src/main/resources/static/vendor/
|
||||
app/core/src/main/resources/static/**/*.gz
|
||||
app/core/src/main/resources/static/**/*.br
|
||||
# Note: Keep backend-managed files like fonts/, css/, js/, pdfjs/, etc.
|
||||
|
||||
# Gradle
|
||||
@@ -174,7 +180,6 @@ venv.bak/
|
||||
|
||||
# Env files (secrets / local overrides). Subproject .gitignore files whitelist any committed defaults.
|
||||
.env*
|
||||
!.env.saas.example
|
||||
|
||||
# VS Code
|
||||
/.vscode/**/*
|
||||
@@ -209,7 +214,7 @@ out/
|
||||
*.asc
|
||||
|
||||
# Allow test fixture certificates (synthetic, no real credentials)
|
||||
!frontend/src/core/tests/test-fixtures/certs/**
|
||||
!frontend/editor/src/core/tests/test-fixtures/certs/**
|
||||
|
||||
# SSH Keys
|
||||
*.pub
|
||||
@@ -251,7 +256,7 @@ node_modules/
|
||||
*compact*.json
|
||||
test_batch.json
|
||||
*.backup.*.json
|
||||
frontend/public/locales/*/translation.backup*.json
|
||||
frontend/editor/public/locales/*/translation.backup*.json
|
||||
|
||||
# Development/build artifacts
|
||||
.gradle-cache/
|
||||
@@ -274,3 +279,6 @@ docs/type3/signatures/
|
||||
# Playwright MCP screenshots / traces
|
||||
.playwright-mcp/
|
||||
*.playwright-mcp.png
|
||||
|
||||
# Local screenshot artifacts from *-screenshots.spec.ts
|
||||
frontend/editor/screenshots/
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
# PostHog project-level key — phc_ prefix keys are public/client-side by design
|
||||
# (PostHog client-side tracking embeds them in the browser bundle). Committed
|
||||
# intentionally in #6150 so engine/.env has a working default, with real
|
||||
# credentials overridden via engine/.env.local.
|
||||
engine/.env:generic-api-key:41
|
||||
@@ -22,12 +22,13 @@ tasks:
|
||||
vars:
|
||||
PORT: '{{.PORT | default "8080"}}'
|
||||
AIENGINE_URL: '{{.AIENGINE_URL | default ""}}'
|
||||
AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS | default "120"}}'
|
||||
env:
|
||||
SERVER_PORT: '{{.PORT}}'
|
||||
cmds:
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}cmd /c ".\gradlew.bat :stirling-pdf:bootRun"'
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}cmd /c ".\gradlew.bat :stirling-pdf:bootRun"'
|
||||
platforms: [windows]
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}./gradlew :stirling-pdf:bootRun'
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}./gradlew :stirling-pdf:bootRun'
|
||||
platforms: [linux, darwin]
|
||||
|
||||
dev:bundled:
|
||||
@@ -40,10 +41,10 @@ tasks:
|
||||
platforms: [linux, darwin]
|
||||
|
||||
dev:saas:
|
||||
desc: "Start backend in SaaS flavor against Supabase (loads .env.saas.local)"
|
||||
desc: "Start backend in SaaS flavor against Supabase"
|
||||
# `dotenv:` reads from the root Taskfile's directory (".") because this
|
||||
# subtaskfile is included with `dir: .`. Drop the file at the repo root.
|
||||
dotenv: ['.env.saas.local']
|
||||
# subtaskfile is included with `dir: .`.
|
||||
dotenv: ['app/.env.saas.local', 'app/.env.saas']
|
||||
ignore_error: true
|
||||
vars:
|
||||
PORT: '{{.PORT | default "8080"}}'
|
||||
|
||||
+29
-1
@@ -78,6 +78,13 @@ tasks:
|
||||
cmds:
|
||||
- npx tauri build --bundles appimage
|
||||
|
||||
test:
|
||||
desc: "Run Tauri/Cargo tests"
|
||||
deps: [prepare]
|
||||
dir: editor/src-tauri
|
||||
cmds:
|
||||
- cargo test
|
||||
|
||||
clean:
|
||||
desc: "Clean Tauri/Cargo build artifacts"
|
||||
dir: editor
|
||||
@@ -126,8 +133,29 @@ tasks:
|
||||
--no-header-files
|
||||
--no-man-pages
|
||||
--output runtime/jre
|
||||
# jlink emits its files mode 444 (read-only). Tauri's build-script
|
||||
# resource copier preserves source permissions when staging
|
||||
# `runtime/jre/**/*` into `target/<profile>/runtime/jre/...`, so the
|
||||
# staged copies are read-only too. On any subsequent incremental
|
||||
# build the copier tries to overwrite them and fails with a bare
|
||||
# `Permission denied (os error 13)` (Rust's io::Error Display drops
|
||||
# the path, so the failure is opaque). Make the source writable here
|
||||
# so the staged destinations are writable and can be overwritten.
|
||||
#
|
||||
# Trade-off: this task runs for both `task desktop:dev` and
|
||||
# `task desktop:build`, so production bundles also ship mode-644
|
||||
# JRE files instead of 444. Functionally harmless on POSIX (the
|
||||
# `other` bit is `r--` either way, and on macOS code signing is the
|
||||
# real integrity check) and on Windows the DOS read-only attribute
|
||||
# isn't load-bearing for the bundled JDK. If we ever need strict
|
||||
# 444 in production, split the chmod into a dev-only step and have
|
||||
# `desktop:build` run `jlink:clean` first to force a fresh build.
|
||||
- cmd: chmod -R u+w runtime/jre
|
||||
platforms: [linux, darwin]
|
||||
- cmd: powershell -NoProfile -Command "Get-ChildItem -Recurse runtime/jre | ForEach-Object { $_.IsReadOnly = $false }"
|
||||
platforms: [windows]
|
||||
status:
|
||||
- test -d editor/src-tauri/runtime/jre
|
||||
- test -f runtime/jre/release
|
||||
|
||||
jlink:clean:
|
||||
desc: "Remove JLink runtime and bundled JARs"
|
||||
|
||||
+11
-1
@@ -34,6 +34,9 @@ tasks:
|
||||
ignore_error: true
|
||||
vars:
|
||||
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
|
||||
# COVERAGE=1 in the calling environment attaches the JaCoCo agent to
|
||||
# the bootRun JVM and writes to BASE_DIR/jacoco.exec on shutdown.
|
||||
# Off by default to keep local dev runs uninstrumented; CI flips it.
|
||||
env:
|
||||
STIRLING_BASE_PATH: '{{.BASE_DIR}}'
|
||||
# Suppress the analytics opt-in modal that fires on first admin login.
|
||||
@@ -58,12 +61,19 @@ tasks:
|
||||
set -e
|
||||
rm -rf "{{.BASE_DIR}}"
|
||||
mkdir -p "{{.BASE_DIR}}"
|
||||
GRADLE_ARGS=":stirling-pdf:bootRun"
|
||||
if [ -n "${COVERAGE:-}" ]; then
|
||||
# copyJacocoAgent is wired as a dependency of bootRun when
|
||||
# -PjacocoAgent=true, so we do not need to invoke it separately.
|
||||
GRADLE_ARGS="$GRADLE_ARGS -PjacocoAgent=true -PjacocoExec={{.BASE_DIR}}/jacoco.exec"
|
||||
echo "JaCoCo coverage enabled, writing to {{.BASE_DIR}}/jacoco.exec"
|
||||
fi
|
||||
# Background gradle and record its PID so the runner can clean up
|
||||
# the exact process tree (wrapper + forked Spring Boot JVM) without
|
||||
# resorting to fuzzy `pkill -f` patterns. `wait` keeps this script
|
||||
# alive for the lifetime of gradle so Task'"'"'s parallel deps stay
|
||||
# synchronised.
|
||||
bash gradlew :stirling-pdf:bootRun > "{{.BASE_DIR}}/backend.log" 2>&1 &
|
||||
bash gradlew $GRADLE_ARGS > "{{.BASE_DIR}}/backend.log" 2>&1 &
|
||||
GRADLE_PID=$!
|
||||
echo $GRADLE_PID > "{{.BASE_DIR}}/backend.pid"
|
||||
wait $GRADLE_PID
|
||||
|
||||
+72
-3
@@ -112,6 +112,12 @@ tasks:
|
||||
- task: dev:_run
|
||||
vars: { MODE: prototypes, PORT: '{{.PORT}}', BACKEND_URL: '{{.BACKEND_URL}}', OPEN: '{{.OPEN}}' }
|
||||
|
||||
dev:portal:
|
||||
desc: "Start developer portal dev server"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx vite portal --port {{.PORT | default "5173"}}{{if .OPEN}} --open{{end}}
|
||||
|
||||
# ============================================================
|
||||
# Build
|
||||
# ============================================================
|
||||
@@ -156,6 +162,24 @@ tasks:
|
||||
cmds:
|
||||
- npx vite build editor --mode prototypes
|
||||
|
||||
build:portal:
|
||||
desc: "Build developer portal"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx vite build portal
|
||||
|
||||
storybook:
|
||||
desc: "Start Storybook dev server"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx storybook dev -p 6006 {{.CLI_ARGS}}
|
||||
|
||||
storybook:build:
|
||||
desc: "Build static Storybook"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx storybook build {{.CLI_ARGS}}
|
||||
|
||||
# ============================================================
|
||||
# Code quality
|
||||
# ============================================================
|
||||
@@ -163,9 +187,23 @@ tasks:
|
||||
lint:
|
||||
desc: "Run linting"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- task: lint:eslint
|
||||
- task: lint:dpdm
|
||||
|
||||
lint:eslint:
|
||||
desc: "Run ESLint linting"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx eslint --max-warnings=0
|
||||
- npx dpdm editor/src --circular --no-warning --no-tree --exit-code circular:1
|
||||
|
||||
lint:dpdm:
|
||||
desc: "Run circular import linting"
|
||||
deps: [install]
|
||||
cmds:
|
||||
# Globs so dpdm walks the whole tree. dpdm expands the braces itself, so this is
|
||||
# shell-agnostic. Covers editor, portal, and the shared design system.
|
||||
- npx dpdm "editor/src/**/*.{ts,tsx}" "portal/src/**/*.{ts,tsx}" "shared/**/*.{ts,tsx}" --circular --no-warning --no-tree --exit-code circular:1
|
||||
|
||||
lint:fix:
|
||||
desc: "Auto-fix lint issues"
|
||||
@@ -236,6 +274,18 @@ tasks:
|
||||
cmds:
|
||||
- npx tsc --noEmit --project editor/src/prototypes/tsconfig.json
|
||||
|
||||
typecheck:portal:
|
||||
desc: "Typecheck developer portal build variant"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx tsc --noEmit --project portal/tsconfig.json
|
||||
|
||||
typecheck:shared:
|
||||
desc: "Typecheck the shared design system"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx tsc --noEmit --project shared/tsconfig.json
|
||||
|
||||
typecheck:all:
|
||||
desc: "Typecheck all build variants"
|
||||
cmds:
|
||||
@@ -245,6 +295,8 @@ tasks:
|
||||
- task: typecheck:desktop
|
||||
- task: typecheck:scripts
|
||||
- task: typecheck:prototypes
|
||||
- task: typecheck:portal
|
||||
- task: typecheck:shared
|
||||
|
||||
# ============================================================
|
||||
# Quality Gate
|
||||
@@ -265,7 +317,9 @@ tasks:
|
||||
- task: lint
|
||||
- task: format:check
|
||||
- task: build
|
||||
- task: build:portal
|
||||
- task: test
|
||||
- task: storybook:build
|
||||
|
||||
# ============================================================
|
||||
# Test
|
||||
@@ -284,10 +338,25 @@ tasks:
|
||||
- npx vitest --watch --root editor
|
||||
|
||||
test:coverage:
|
||||
desc: "Run tests with coverage"
|
||||
desc: "Run tests with coverage (one-shot; CI-friendly)."
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx vitest --coverage --root editor
|
||||
# `vitest run` makes this CI-safe (the bare `vitest` form enters watch
|
||||
# mode). Explicit reporter list because v8 + json-summary is what the
|
||||
# coverage-summary.py helper consumes; html/text are kept for humans.
|
||||
#
|
||||
# reportsDirectory is pinned to ./coverage relative to vitest's root
|
||||
# (--root editor), so output lands at frontend/editor/coverage/. The
|
||||
# CI upload step reads from that path. An earlier attempt with
|
||||
# `./editor/coverage` double-nested into frontend/editor/editor/coverage;
|
||||
# pinning future-proofs against vitest changing the default.
|
||||
- >
|
||||
npx vitest run --root editor --coverage
|
||||
--coverage.provider=v8
|
||||
--coverage.reporter=text-summary
|
||||
--coverage.reporter=json-summary
|
||||
--coverage.reporter=html
|
||||
--coverage.reportsDirectory=./coverage
|
||||
|
||||
# ============================================================
|
||||
# Code Generation
|
||||
|
||||
@@ -10,14 +10,16 @@ if that directory exists, is licensed under the license defined in "app/propriet
|
||||
if that directory exists, is licensed under the license defined in "app/saas/LICENSE".
|
||||
* All content that resides under the "engine/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "engine/LICENSE".
|
||||
* All content that resides under the "frontend/src/proprietary/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/src/proprietary/LICENSE".
|
||||
* All content that resides under the "frontend/src/desktop/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/src/desktop/LICENSE".
|
||||
* All content that resides under the "frontend/src/saas/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/src/saas/LICENSE".
|
||||
* All content that resides under the "frontend/src/prototypes/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/src/prototypes/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/proprietary/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/proprietary/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/desktop/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/desktop/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/saas/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/saas/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/prototypes/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/prototypes/LICENSE".
|
||||
* All content that resides under the "frontend/portal/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/portal/LICENSE".
|
||||
* Content outside of the above mentioned directories or restrictions above is
|
||||
available under the MIT License as defined below.
|
||||
|
||||
|
||||
+18
-1
@@ -58,6 +58,23 @@ tasks:
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
OPEN: "true"
|
||||
|
||||
dev:saas:
|
||||
desc: "Start SaaS backend + frontend concurrently on free ports"
|
||||
vars:
|
||||
PORTS:
|
||||
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 5173{{else}}{{.FIND_FREE_PORT_SH}} 8080 5173{{end}}'
|
||||
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
|
||||
FRONTEND_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
deps:
|
||||
- task: backend:dev:saas
|
||||
vars:
|
||||
PORT: '{{.BACKEND_PORT}}'
|
||||
- task: frontend:dev:saas
|
||||
vars:
|
||||
PORT: '{{.FRONTEND_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
OPEN: "true"
|
||||
|
||||
dev:all:
|
||||
desc: "Start backend + frontend + engine concurrently on free ports"
|
||||
vars:
|
||||
@@ -74,7 +91,7 @@ tasks:
|
||||
vars:
|
||||
PORT: '{{.BACKEND_PORT}}'
|
||||
AIENGINE_URL: 'http://localhost:{{.ENGINE_PORT}}'
|
||||
- task: frontend:dev:prototypes
|
||||
- task: frontend:dev
|
||||
vars:
|
||||
PORT: '{{.FRONTEND_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
###############################################################################
|
||||
# Stirling-PDF SaaS local environment template.
|
||||
# Stirling-PDF SaaS environment defaults.
|
||||
#
|
||||
# Copy this file to `.env.saas.local` (gitignored) and fill in real values.
|
||||
# Loaded by `task backend:dev:saas` via Taskfile's `dotenv:` directive, then
|
||||
# read by Spring Boot's `${...}` placeholders in application-saas.properties
|
||||
# and application-dev.properties.
|
||||
# This file is committed and provides non-secret defaults loaded by
|
||||
# `task backend:dev:saas`. Put real values for secrets (passwords, project
|
||||
# refs, edge function secrets) in `.env.saas.local` - any variable set there
|
||||
# takes precedence over what's defined here.
|
||||
#
|
||||
# DO NOT commit `.env.saas.local`. Only `.env.saas.example` is checked in.
|
||||
# DO NOT commit `.env.saas.local`. Only `.env.saas` is checked in.
|
||||
###############################################################################
|
||||
|
||||
# ---------- Supabase project ----------
|
||||
# Project reference (the subdomain part of <ref>.supabase.co). Required.
|
||||
# Example dev project:
|
||||
# Set in .env.saas.local.
|
||||
SAAS_DB_PROJECT_REF=
|
||||
|
||||
# Edge function secret used by billing/license rollup calls.
|
||||
# Edge function secret used by billing/license rollup calls. Set in .env.saas.local.
|
||||
SUPABASE_EDGE_FUNCTION_SECRET=
|
||||
|
||||
# ---------- Database (saas profile) ----------
|
||||
@@ -28,7 +28,7 @@ SAAS_DB_PASSWORD=
|
||||
# ---------- Database (dev profile overrides) ----------
|
||||
# Used when `--spring.profiles.include=dev` is active. The dev profile
|
||||
# defaults the URL/username to the shared dev Supabase project, but the
|
||||
# password must still be provided here.
|
||||
# password must still be provided in .env.saas.local.
|
||||
SAAS_DEV_DB_URL=
|
||||
SAAS_DEV_DB_USERNAME=postgres
|
||||
SAAS_DEV_DB_PASSWORD=
|
||||
@@ -0,0 +1,3 @@
|
||||
# Whitelist committed env defaults. `.env.saas.local` (and any other .env*)
|
||||
# stays ignored via the root .gitignore.
|
||||
!.env.saas
|
||||
@@ -44,6 +44,10 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "The MIT License"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "MIT-0"
|
||||
},
|
||||
{
|
||||
"moduleName": "com.github.jai-imageio:jai-imageio-core",
|
||||
"moduleLicense": "LICENSE.txt"
|
||||
|
||||
@@ -60,7 +60,7 @@ dependencies {
|
||||
exclude group: 'com.google.code.gson', module: 'gson'
|
||||
}
|
||||
|
||||
api 'com.stirling:jpdfium:1.0.1'
|
||||
api 'com.stirling:jpdfium:1.0.2'
|
||||
|
||||
// -PjpdfiumPlatforms=all|<csv of linux-x64,linux-arm64,darwin-x64,darwin-arm64,windows-x64>
|
||||
def jpdfiumPlatformsProp = (project.findProperty('jpdfiumPlatforms') ?: 'all').toString().trim()
|
||||
@@ -75,9 +75,12 @@ dependencies {
|
||||
}
|
||||
logger.lifecycle("JPDFium native platforms: ${jpdfiumPlatforms.join(', ')}")
|
||||
jpdfiumPlatforms.each { platform ->
|
||||
runtimeOnly "com.stirling:jpdfium-natives-${platform}:1.0.1"
|
||||
runtimeOnly "com.stirling:jpdfium-natives-${platform}:1.0.2"
|
||||
}
|
||||
|
||||
// Bucket4j (local in-process token bucket for RateLimitStore default impl)
|
||||
implementation 'com.bucket4j:bucket4j_jdk17-core:8.19.0'
|
||||
|
||||
// ArchUnit: enforces module dependency direction (see ArchitectureTest)
|
||||
testImplementation 'com.tngtech.archunit:archunit-junit5:1.4.2'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Detects whether a page is one- or two-column from per-line bounding boxes, and classifies an
|
||||
* X-span into the column it belongs to. Detection is a midpoint vote at {@code pageWidth / 2}.
|
||||
*
|
||||
* <p>Capped at two columns by design — sufficient for the redaction target set (single-column
|
||||
* documents and IEEE-style two-column papers). 3+ column layouts (newspapers, magazines) and
|
||||
* off-centre gutters (asymmetric two-column) would need a histogram or clustering approach to
|
||||
* detect the actual gutter X. (future work)
|
||||
*
|
||||
* <p>Coordinates are PDFTextStripper screen space (top-left origin, Y increases downward).
|
||||
*/
|
||||
public final class PageColumnLayout {
|
||||
|
||||
/**
|
||||
* Slack when checking "crosses a gutter" so single-pixel overshoots don't mark a line as
|
||||
* spanning.
|
||||
*/
|
||||
public static final float SPAN_SLACK_PT = 2f;
|
||||
|
||||
/**
|
||||
* Slack on each side of the page midpoint inside which a line is considered "spanning"
|
||||
* (covering both columns) rather than belonging to one side.
|
||||
*/
|
||||
private static final float MIDPOINT_SLACK_PT = 30f;
|
||||
|
||||
/**
|
||||
* Minimum line width (points) for a line to count toward the two-column tally. Avoids false
|
||||
* positives where right-aligned dates, page numbers, or short "Link" fragments next to a
|
||||
* heading look like a second column when they're really just inline metadata.
|
||||
*/
|
||||
private static final float MIN_COLUMN_LINE_WIDTH_PT = 100f;
|
||||
|
||||
/**
|
||||
* Minimum number of clearly leftish AND clearly rightish lines (each of width ≥ {@link
|
||||
* #MIN_COLUMN_LINE_WIDTH_PT}) required to call the page two-column. Anything below this falls
|
||||
* back to single-column.
|
||||
*/
|
||||
private static final int MIN_SIDE_LINES = 3;
|
||||
|
||||
private final List<float[]> columns;
|
||||
private final List<float[]> gutters;
|
||||
|
||||
private PageColumnLayout(List<float[]> columns, List<float[]> gutters) {
|
||||
this.columns = columns;
|
||||
this.gutters = gutters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines column layout from per-line bounding boxes ({@code [x1, _, x2, _]}). Counts lines
|
||||
* whose X-midpoint sits clearly left of, or clearly right of, the page midpoint (with {@link
|
||||
* #MIDPOINT_SLACK_PT} slack each side). If both sides have at least {@link #MIN_SIDE_LINES}
|
||||
* lines, the page is treated as two-column with the gutter at the page midpoint. Otherwise it's
|
||||
* single-column.
|
||||
*
|
||||
* <p>Cross-column lines must already be split: callers should feed boxes from a line extractor
|
||||
* that splits same-Y glyphs at large X gaps (see {@code AllTextLineExtractor}). Without that
|
||||
* split, IEEE-style aligned-baseline 2-column PDFs produce one wide merged box per row and the
|
||||
* side tallies all end up classified as "spanning", falling to single-column.
|
||||
*/
|
||||
public static PageColumnLayout fromLineBoxes(List<float[]> lineBoxes, float pageWidth) {
|
||||
if (lineBoxes == null || lineBoxes.isEmpty()) {
|
||||
return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of());
|
||||
}
|
||||
float pageMid = pageWidth / 2f;
|
||||
int left = 0, right = 0;
|
||||
for (float[] lb : lineBoxes) {
|
||||
if (lb == null || lb.length < 3) continue;
|
||||
float width = lb[2] - lb[0];
|
||||
// Skip narrow lines — dates, page numbers, "Link" labels next to a heading should
|
||||
// not, on their own, make a single-column doc look two-column.
|
||||
if (width < MIN_COLUMN_LINE_WIDTH_PT) continue;
|
||||
float mid = (lb[0] + lb[2]) * 0.5f;
|
||||
if (mid < pageMid - MIDPOINT_SLACK_PT) left++;
|
||||
else if (mid > pageMid + MIDPOINT_SLACK_PT) right++;
|
||||
}
|
||||
if (left < MIN_SIDE_LINES || right < MIN_SIDE_LINES) {
|
||||
return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of());
|
||||
}
|
||||
float gutterL = pageMid - MIDPOINT_SLACK_PT;
|
||||
float gutterR = pageMid + MIDPOINT_SLACK_PT;
|
||||
return new PageColumnLayout(
|
||||
List.of(new float[] {0f, gutterL}, new float[] {gutterR, pageWidth}),
|
||||
List.of(new float[] {gutterL, gutterR}));
|
||||
}
|
||||
|
||||
/** All columns, left-to-right, as {@code [leftX, rightX]} pairs. Never empty. */
|
||||
public List<float[]> columns() {
|
||||
return columns;
|
||||
}
|
||||
|
||||
/** Gutters between columns, left-to-right, as {@code [leftX, rightX]} pairs. */
|
||||
public List<float[]> gutters() {
|
||||
return gutters;
|
||||
}
|
||||
|
||||
public int columnCount() {
|
||||
return columns.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the column index containing the X-midpoint of {@code [x1, x2]}, falling back to the
|
||||
* closest column if the midpoint sits inside a gutter.
|
||||
*/
|
||||
public int columnOf(float x1, float x2) {
|
||||
float mid = (x1 + x2) * 0.5f;
|
||||
int best = 0;
|
||||
float bestDist = Float.MAX_VALUE;
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
float[] c = columns.get(i);
|
||||
if (mid >= c[0] && mid <= c[1]) return i;
|
||||
float dist = mid < c[0] ? c[0] - mid : mid - c[1];
|
||||
if (dist < bestDist) {
|
||||
bestDist = dist;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns every column index whose X-range overlaps {@code [x1, x2]} with at least {@link
|
||||
* #SPAN_SLACK_PT} of intrusion. A normal in-column line returns one index; a line crossing a
|
||||
* gutter returns two or more.
|
||||
*/
|
||||
public int[] columnsCrossing(float x1, float x2) {
|
||||
List<Integer> hits = new ArrayList<>();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
float[] c = columns.get(i);
|
||||
float overlap = Math.min(x2, c[1]) - Math.max(x1, c[0]);
|
||||
if (overlap > SPAN_SLACK_PT) hits.add(i);
|
||||
}
|
||||
if (hits.isEmpty()) hits.add(columnOf(x1, x2));
|
||||
int[] out = new int[hits.size()];
|
||||
for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.contentstream.PDFGraphicsStreamEngine;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImage;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
|
||||
/**
|
||||
* PDFGraphicsStreamEngine that intercepts {@code drawImage} calls and records each image's bounding
|
||||
* box in PDF user-space (origin bottom-left, Y up) by transforming the unit square through the
|
||||
* current transformation matrix (CTM).
|
||||
*
|
||||
* <p>Usage:
|
||||
*
|
||||
* <pre>{@code
|
||||
* PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
* locator.processPage(page);
|
||||
* List<ImageBox> boxes = locator.getImageBoxes();
|
||||
* }</pre>
|
||||
*
|
||||
* <p>Each {@link ImageBox} carries the 0-based page index and the axis-aligned bounding box {@code
|
||||
* (x1, y1, x2, y2)} in PDF user-space coordinates.
|
||||
*/
|
||||
public final class PageImageLocator extends PDFGraphicsStreamEngine {
|
||||
|
||||
/**
|
||||
* Bounding box of a raster or vector image found on a PDF page.
|
||||
*
|
||||
* @param pageIndex 0-based page index
|
||||
* @param x1 left edge in PDF user-space (origin bottom-left)
|
||||
* @param y1 bottom edge in PDF user-space
|
||||
* @param x2 right edge
|
||||
* @param y2 top edge
|
||||
*/
|
||||
public record ImageBox(int pageIndex, float x1, float y1, float x2, float y2) {}
|
||||
|
||||
private final int pageIndex;
|
||||
private final List<ImageBox> imageBoxes = new ArrayList<>();
|
||||
private final Point2D.Float currentPoint = new Point2D.Float();
|
||||
|
||||
/**
|
||||
* @param page the PDPage to process
|
||||
* @param pageIndex 0-based index of this page in the document (stored on each returned {@link
|
||||
* ImageBox})
|
||||
*/
|
||||
public PageImageLocator(PDPage page, int pageIndex) {
|
||||
super(page);
|
||||
this.pageIndex = pageIndex;
|
||||
}
|
||||
|
||||
/** Returns all image bounding boxes collected during {@link #processPage}. */
|
||||
public List<ImageBox> getImageBoxes() {
|
||||
return imageBoxes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void drawImage(PDImage pdImage) throws IOException {
|
||||
Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
|
||||
// An image occupies the unit square (0,0)→(1,1) in image space.
|
||||
// Transform all four corners through the CTM to get the page-space bounding box.
|
||||
float a = ctm.getScaleX();
|
||||
float b = ctm.getShearY();
|
||||
float c = ctm.getShearX();
|
||||
float d = ctm.getScaleY();
|
||||
float e = ctm.getTranslateX();
|
||||
float f = ctm.getTranslateY();
|
||||
float[] xs = {e, a + e, c + e, a + c + e};
|
||||
float[] ys = {f, b + f, d + f, b + d + f};
|
||||
float x1 = Float.MAX_VALUE, y1 = Float.MAX_VALUE;
|
||||
float x2 = -Float.MAX_VALUE, y2 = -Float.MAX_VALUE;
|
||||
for (float x : xs) {
|
||||
x1 = Math.min(x1, x);
|
||||
x2 = Math.max(x2, x);
|
||||
}
|
||||
for (float y : ys) {
|
||||
y1 = Math.min(y1, y);
|
||||
y2 = Math.max(y2, y);
|
||||
}
|
||||
imageBoxes.add(new ImageBox(pageIndex, x1, y1, x2, y2));
|
||||
}
|
||||
|
||||
// ---------- required abstract methods (no-op for path operations) ----------
|
||||
|
||||
@Override
|
||||
public void appendRectangle(Point2D p0, Point2D p1, Point2D p2, Point2D p3) {}
|
||||
|
||||
@Override
|
||||
public void clip(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void moveTo(float x, float y) {
|
||||
currentPoint.setLocation(x, y);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void lineTo(float x, float y) {
|
||||
currentPoint.setLocation(x, y);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void curveTo(float x1, float y1, float x2, float y2, float x3, float y3) {
|
||||
currentPoint.setLocation(x3, y3);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Point2D getCurrentPoint() {
|
||||
return currentPoint;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closePath() {}
|
||||
|
||||
@Override
|
||||
public void endPath() {}
|
||||
|
||||
@Override
|
||||
public void strokePath() {}
|
||||
|
||||
@Override
|
||||
public void fillPath(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void fillAndStrokePath(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void shadingFill(COSName shadingName) {}
|
||||
}
|
||||
+5
-1
@@ -77,6 +77,10 @@ public @interface AutoJobPostMapping {
|
||||
/**
|
||||
* Relative resource weight (1-100). See {@link
|
||||
* stirling.software.common.enumeration.ResourceWeight} for the standard tiers.
|
||||
*
|
||||
* <p>The default is a sentinel ({@link Integer#MIN_VALUE}); {@code
|
||||
* AutoJobPostMappingWeightTest} fails the build if any endpoint leaves it unset. Runtime
|
||||
* readers clamp the value into {@code [1, 100]}.
|
||||
*/
|
||||
int resourceWeight() default 1;
|
||||
int resourceWeight() default Integer.MIN_VALUE;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
/** Health and identity facade for the active cluster backplane. */
|
||||
public interface ClusterBackplane {
|
||||
|
||||
/** Returns {@code true} when the backplane is reachable; used for health endpoints. */
|
||||
boolean isHealthy();
|
||||
|
||||
/** Returns {@code "inprocess"} or {@code "valkey"}. */
|
||||
String backplaneType();
|
||||
|
||||
/** Returns this JVM's stable node id (matches {@code Cluster.resolvedNodeId()}). */
|
||||
String localNodeId();
|
||||
|
||||
/**
|
||||
* Whether this JVM should run the local {@link
|
||||
* stirling.software.common.service.TaskManager#cleanupOldJobs()} loop. Distributed backplanes
|
||||
* own job expiry via their own TTL, so they should override this to return {@code false}.
|
||||
* Defaults to {@code true} so in-process behavior is preserved without an explicit override.
|
||||
*/
|
||||
default boolean shouldRunLocalCleanup() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.ApplicationProperties.Cluster;
|
||||
|
||||
/**
|
||||
* Validates that cluster mode is internally consistent.
|
||||
*
|
||||
* <p>Cluster settings are bound on the central {@link ApplicationProperties} under {@code
|
||||
* cluster.*}; this class reads {@link ApplicationProperties#getCluster()} and runs guards in {@link
|
||||
* PostConstruct}. When {@code cluster.enabled=false} (the default) all checks are skipped so a
|
||||
* single-instance install needs no new config.
|
||||
*/
|
||||
@Slf4j
|
||||
@Configuration
|
||||
@RequiredArgsConstructor
|
||||
public class ClusterConfig {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@PostConstruct
|
||||
void validate() {
|
||||
Cluster cluster = applicationProperties.getCluster();
|
||||
if (!cluster.isEnabled()) {
|
||||
return;
|
||||
}
|
||||
String backplane = cluster.getBackplane();
|
||||
if ("valkey".equalsIgnoreCase(backplane)) {
|
||||
String url = cluster.getValkey() == null ? null : cluster.getValkey().getUrl();
|
||||
if (url == null || url.isBlank()) {
|
||||
throw new IllegalStateException(
|
||||
"cluster.enabled=true with backplane=valkey requires"
|
||||
+ " cluster.valkey.url to be set (e.g."
|
||||
+ " redis://valkey:6379).");
|
||||
}
|
||||
} else if ("inprocess".equalsIgnoreCase(backplane)) {
|
||||
// enabled+inprocess only coordinates the local JVM; cross-node lookups will 410.
|
||||
log.warn(
|
||||
"cluster.enabled=true with backplane=inprocess - only the local"
|
||||
+ " JVM is coordinated. Cross-node lookups and the file proxy will fail."
|
||||
+ " Use backplane=valkey for real multi-node deployments.");
|
||||
} else {
|
||||
// Fail fast on typos like "valky" so Spring doesn't later report a cryptic
|
||||
// "no ClusterBackplane bean" - the operator-facing error names the bad value.
|
||||
throw new IllegalStateException(
|
||||
"cluster.enabled=true with unknown backplane '"
|
||||
+ backplane
|
||||
+ "'. Valid values: inprocess | valkey.");
|
||||
}
|
||||
log.info(
|
||||
"Cluster mode enabled (backplane={}, nodeRole={}, nodeId={}).",
|
||||
backplane,
|
||||
cluster.resolvedRole(),
|
||||
cluster.resolvedNodeId());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* Snapshot of a peer node as recorded in the {@link InstanceRegistry}.
|
||||
*
|
||||
* @param internalAddress {@code host:port} the node listens on for {@code /internal/cluster/**}
|
||||
* @param role one of {@code WEB}, {@code WORKER}, {@code BOTH}
|
||||
*/
|
||||
public record ClusterNode(
|
||||
String nodeId, String internalAddress, Instant lastHeartbeat, String role) {}
|
||||
@@ -0,0 +1,21 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Cluster-wide mutual exclusion primitive; non-reentrant by contract. */
|
||||
public interface DistributedLock {
|
||||
|
||||
Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime);
|
||||
|
||||
interface LockHandle extends AutoCloseable {
|
||||
void release();
|
||||
|
||||
boolean renew(Duration leaseTime);
|
||||
|
||||
@Override
|
||||
default void close() {
|
||||
release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/** Low-level storage seam for result/job files. */
|
||||
public interface FileStore {
|
||||
|
||||
/** Stored file record. */
|
||||
record Stored(String fileId, long size) {}
|
||||
|
||||
/** Store the given stream and return a generated file id and total bytes written. */
|
||||
Stored store(InputStream in, String originalName) throws IOException;
|
||||
|
||||
/**
|
||||
* Store the file at {@code source} and return a generated file id and total bytes written.
|
||||
*
|
||||
* <p>Default implementation opens {@code source} as a stream and delegates to {@link
|
||||
* #store(InputStream, String)}. Local-disk implementations should override to use a direct
|
||||
* file-to-file copy ({@code Files.copy(source, dest)} can use {@code sendfile(2)} on Linux),
|
||||
* which avoids the two-memory-copy hit of streaming a disk-backed upload through the JVM heap.
|
||||
*/
|
||||
default Stored store(Path source, String originalName) throws IOException {
|
||||
try (InputStream in = Files.newInputStream(source)) {
|
||||
return store(in, originalName);
|
||||
}
|
||||
}
|
||||
|
||||
/** Open the stored file for streaming reads. Caller closes. */
|
||||
InputStream retrieve(String fileId) throws IOException;
|
||||
|
||||
/** Load the stored file into a byte array. */
|
||||
byte[] retrieveBytes(String fileId) throws IOException;
|
||||
|
||||
/** Size of the stored file in bytes. */
|
||||
long size(String fileId) throws IOException;
|
||||
|
||||
/** Delete the stored file. Returns true if a file was removed. */
|
||||
boolean delete(String fileId);
|
||||
|
||||
/** Whether the file id exists in the store. */
|
||||
boolean exists(String fileId);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Maps {@code nodeId} to its internal cluster address, with TTL'd heartbeats. */
|
||||
public interface InstanceRegistry {
|
||||
|
||||
/** Register or refresh this node. Idempotent so a wiped backplane self-heals on next tick. */
|
||||
void register(ClusterNode node, Duration heartbeatTtl);
|
||||
|
||||
Optional<ClusterNode> lookup(String nodeId);
|
||||
|
||||
Collection<ClusterNode> activeNodes();
|
||||
|
||||
void deregister(String nodeId);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Cluster-visible storage for job status and result metadata, with TTL'd entries. */
|
||||
public interface JobStore {
|
||||
|
||||
/** Persist or overwrite a job entry. {@code ttl} sets the lifetime of the entry. */
|
||||
void put(JobStoreEntry entry, Duration ttl);
|
||||
|
||||
Optional<JobStoreEntry> get(String jobId);
|
||||
|
||||
void delete(String jobId);
|
||||
|
||||
boolean exists(String jobId);
|
||||
|
||||
/** Reverse lookup: which job owns this result file id? */
|
||||
Optional<String> findJobIdByFileId(String fileId);
|
||||
|
||||
/** Snapshot of every active entry. Used by admin/stats endpoints; may be O(n). */
|
||||
Collection<JobStoreEntry> all();
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Cluster-visible projection of a job's status and result metadata, as persisted in {@link
|
||||
* JobStore}.
|
||||
*
|
||||
* @param owningNodeId the node id that originally executed the job
|
||||
*/
|
||||
public record JobStoreEntry(
|
||||
String jobId,
|
||||
JobState state,
|
||||
String owningNodeId,
|
||||
Instant createdAt,
|
||||
Instant completedAt,
|
||||
String error,
|
||||
List<String> fileIds,
|
||||
Map<String, String> resultMeta) {
|
||||
|
||||
/** Lifecycle states for a job as observed by the cluster. */
|
||||
public enum JobState {
|
||||
PENDING,
|
||||
RUNNING,
|
||||
COMPLETE,
|
||||
FAILED
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Short-TTL namespaced key/value cache backed by the cluster backplane. */
|
||||
public interface KeyValueCache {
|
||||
|
||||
void put(String namespace, String key, String value, Duration ttl);
|
||||
|
||||
Optional<String> get(String namespace, String key);
|
||||
|
||||
void evict(String namespace, String key);
|
||||
|
||||
void evictNamespace(String namespace);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Token-bucket rate limiting backed by the cluster backplane.
|
||||
*
|
||||
* <p>In-process implementations enforce a per-JVM limit; distributed implementations enforce a
|
||||
* single global limit across every node. Both use a Bucket4j greedy-refill token bucket so the
|
||||
* semantics match across single-node and cluster deployments.
|
||||
*/
|
||||
public interface RateLimitStore {
|
||||
|
||||
/**
|
||||
* Attempt to consume one token from the bucket identified by {@code bucketKey}.
|
||||
*
|
||||
* @param bucketKey opaque key identifying the bucket (e.g. {@code api:user:123})
|
||||
* @param capacity bucket capacity
|
||||
* @param refillPeriod time window over which {@code capacity} tokens refill
|
||||
*/
|
||||
RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod);
|
||||
|
||||
record RateLimitDecision(boolean allowed, long remainingTokens, long nanosToWaitForRefill) {}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
/** Records one increment per sticky-session miss (a 410 Gone for a job owned by another node). */
|
||||
@FunctionalInterface
|
||||
public interface StickyMissRecorder {
|
||||
void recordStickyMiss();
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
@Slf4j
|
||||
public class InProcessClusterBackplane implements ClusterBackplane {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
public InProcessClusterBackplane(ApplicationProperties applicationProperties) {
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isHealthy() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String backplaneType() {
|
||||
return "inprocess";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String localNodeId() {
|
||||
return applicationProperties.getCluster().resolvedNodeId();
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.cluster.DistributedLock;
|
||||
import stirling.software.common.cluster.InstanceRegistry;
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.KeyValueCache;
|
||||
import stirling.software.common.cluster.RateLimitStore;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
/**
|
||||
* Default cluster backplane wiring: every interface gets an {@code InProcess*} bean. Active when
|
||||
* cluster mode is off or {@code cluster.backplane=inprocess}.
|
||||
*/
|
||||
@Slf4j
|
||||
@Configuration
|
||||
@ConditionalOnExpression(
|
||||
"!${cluster.enabled:false} ||"
|
||||
+ " '${cluster.backplane:inprocess}'.equalsIgnoreCase('inprocess')")
|
||||
public class InProcessClusterConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public ClusterBackplane clusterBackplane(ApplicationProperties applicationProperties) {
|
||||
log.info("Cluster backplane: in-process (single node)");
|
||||
return new InProcessClusterBackplane(applicationProperties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public JobStore jobStore() {
|
||||
return new InProcessJobStore();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public RateLimitStore rateLimitStore() {
|
||||
return new InProcessRateLimitStore();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public DistributedLock distributedLock() {
|
||||
return new InProcessDistributedLock();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public KeyValueCache keyValueCache() {
|
||||
return new InProcessKeyValueCache();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public InstanceRegistry instanceRegistry() {
|
||||
return new InProcessInstanceRegistry();
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import stirling.software.common.cluster.DistributedLock;
|
||||
|
||||
/**
|
||||
* In-process {@link DistributedLock}, non-reentrant per the interface contract, with lease-expiry
|
||||
* semantics that mirror a SET-NX-EX style distributed backend.
|
||||
*
|
||||
* <p>Each lock state carries a per-acquire {@code ownerToken} and an {@code expiryNanos}; another
|
||||
* caller can take over once the lease has elapsed even if the original holder never called {@link
|
||||
* LockHandle#release()}. This matters mostly for parity with the Valkey-backed implementation
|
||||
* (Redis {@code SETEX} auto-expires the key); within a single JVM a crashed holder takes its lock
|
||||
* state with it, but tests and code that rely on the {@code leaseTime} parameter still need it to
|
||||
* be honored.
|
||||
*
|
||||
* <p>Expiry is lazy: an expired lock state lingers in the map until the next acquire attempt for
|
||||
* the same key replaces it. Per-key cleanup also happens on explicit {@link LockHandle#release()},
|
||||
* so a balanced acquire/release workload keeps the map size bounded.
|
||||
*/
|
||||
public class InProcessDistributedLock implements DistributedLock {
|
||||
|
||||
private final ConcurrentHashMap<String, LockState> locks = new ConcurrentHashMap<>();
|
||||
private final AtomicLong tokenSeq = new AtomicLong();
|
||||
|
||||
/**
|
||||
* Lease state for a single acquired lock. {@code ownerToken} prevents a former holder from
|
||||
* releasing or renewing a lock now owned by someone else after lease expiry; {@code
|
||||
* expiryNanos} is read/written only inside {@link ConcurrentHashMap#compute} so the bin lock
|
||||
* provides the necessary happens-before guarantee.
|
||||
*/
|
||||
private static final class LockState {
|
||||
final long ownerToken;
|
||||
long expiryNanos;
|
||||
|
||||
LockState(long ownerToken, long expiryNanos) {
|
||||
this.ownerToken = ownerToken;
|
||||
this.expiryNanos = expiryNanos;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime) {
|
||||
long token = tokenSeq.incrementAndGet();
|
||||
long nowNanos = System.nanoTime();
|
||||
long expiryNanos = nowNanos + leaseTime.toNanos();
|
||||
boolean[] acquired = {false};
|
||||
locks.compute(
|
||||
lockKey,
|
||||
(k, existing) -> {
|
||||
if (existing == null || existing.expiryNanos - nowNanos <= 0L) {
|
||||
// No lock, or the previous lease has expired - we take it. Subtraction
|
||||
// form avoids the long-overflow trap that would bite a naive
|
||||
// expiryNanos <= nowNanos comparison around System.nanoTime() rollover.
|
||||
acquired[0] = true;
|
||||
return new LockState(token, expiryNanos);
|
||||
}
|
||||
return existing;
|
||||
});
|
||||
if (!acquired[0]) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(new InProcessHandle(lockKey, token));
|
||||
}
|
||||
|
||||
private void releaseInternal(String lockKey, long token) {
|
||||
locks.compute(
|
||||
lockKey,
|
||||
(k, existing) -> {
|
||||
if (existing == null || existing.ownerToken != token) {
|
||||
// Already removed, expired-and-replaced, or never ours.
|
||||
return existing;
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private boolean renewInternal(String lockKey, long token, Duration leaseTime) {
|
||||
long nowNanos = System.nanoTime();
|
||||
boolean[] renewed = {false};
|
||||
locks.compute(
|
||||
lockKey,
|
||||
(k, existing) -> {
|
||||
if (existing == null
|
||||
|| existing.ownerToken != token
|
||||
|| existing.expiryNanos - nowNanos <= 0L) {
|
||||
// Lock is gone or expired; renewal is a no-op so the caller can detect it.
|
||||
return existing;
|
||||
}
|
||||
existing.expiryNanos = nowNanos + leaseTime.toNanos();
|
||||
renewed[0] = true;
|
||||
return existing;
|
||||
});
|
||||
return renewed[0];
|
||||
}
|
||||
|
||||
private final class InProcessHandle implements LockHandle {
|
||||
private final String lockKey;
|
||||
private final long token;
|
||||
private boolean released;
|
||||
|
||||
InProcessHandle(String lockKey, long token) {
|
||||
this.lockKey = lockKey;
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void release() {
|
||||
if (released) {
|
||||
return;
|
||||
}
|
||||
released = true;
|
||||
releaseInternal(lockKey, token);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized boolean renew(Duration leaseTime) {
|
||||
if (released) {
|
||||
return false;
|
||||
}
|
||||
return renewInternal(lockKey, token, leaseTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import stirling.software.common.cluster.ClusterNode;
|
||||
import stirling.software.common.cluster.InstanceRegistry;
|
||||
|
||||
public class InProcessInstanceRegistry implements InstanceRegistry {
|
||||
|
||||
private final AtomicReference<ClusterNode> self = new AtomicReference<>();
|
||||
|
||||
@Override
|
||||
public void register(ClusterNode node, Duration heartbeatTtl) {
|
||||
self.set(node);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ClusterNode> lookup(String nodeId) {
|
||||
ClusterNode current = self.get();
|
||||
return current != null && current.nodeId().equals(nodeId)
|
||||
? Optional.of(current)
|
||||
: Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<ClusterNode> activeNodes() {
|
||||
ClusterNode current = self.get();
|
||||
return current == null ? Collections.emptyList() : Collections.singletonList(current);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deregister(String nodeId) {
|
||||
ClusterNode current = self.get();
|
||||
if (current != null && current.nodeId().equals(nodeId)) {
|
||||
self.set(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
|
||||
@Slf4j
|
||||
public class InProcessJobStore implements JobStore {
|
||||
|
||||
private final ConcurrentHashMap<String, Holder> entries = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public void put(JobStoreEntry entry, Duration ttl) {
|
||||
Instant expiry = ttl == null ? Instant.MAX : Instant.now().plus(ttl);
|
||||
entries.put(entry.jobId(), new Holder(entry, expiry));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<JobStoreEntry> get(String jobId) {
|
||||
Holder h = entries.get(jobId);
|
||||
if (h == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
if (h.isExpired()) {
|
||||
entries.remove(jobId, h);
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(h.entry);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(String jobId) {
|
||||
entries.remove(jobId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exists(String jobId) {
|
||||
return get(jobId).isPresent();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> findJobIdByFileId(String fileId) {
|
||||
for (Map.Entry<String, Holder> e : entries.entrySet()) {
|
||||
Holder h = e.getValue();
|
||||
if (h.isExpired()) {
|
||||
continue;
|
||||
}
|
||||
List<String> fileIds = h.entry.fileIds();
|
||||
if (fileIds != null && fileIds.contains(fileId)) {
|
||||
return Optional.of(e.getKey());
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<JobStoreEntry> all() {
|
||||
List<JobStoreEntry> result = new ArrayList<>(entries.size());
|
||||
for (Holder h : entries.values()) {
|
||||
if (!h.isExpired()) {
|
||||
result.add(h.entry);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Drop entries whose TTL has elapsed. Called by the {@code TaskManager} cleanup scheduler. */
|
||||
public int purgeExpired() {
|
||||
int removed = 0;
|
||||
Instant now = Instant.now();
|
||||
for (Map.Entry<String, Holder> e : entries.entrySet()) {
|
||||
if (!e.getValue().expiry.equals(Instant.MAX) && e.getValue().expiry.isBefore(now)) {
|
||||
if (entries.remove(e.getKey(), e.getValue())) {
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
private record Holder(JobStoreEntry entry, Instant expiry) {
|
||||
boolean isExpired() {
|
||||
return !expiry.equals(Instant.MAX) && expiry.isBefore(Instant.now());
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import stirling.software.common.cluster.KeyValueCache;
|
||||
|
||||
public class InProcessKeyValueCache implements KeyValueCache {
|
||||
|
||||
private final ConcurrentHashMap<String, ConcurrentHashMap<String, Expiring>> namespaces =
|
||||
new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public void put(String namespace, String key, String value, Duration ttl) {
|
||||
Instant expiry = ttl == null ? Instant.MAX : Instant.now().plus(ttl);
|
||||
namespaces
|
||||
.computeIfAbsent(namespace, n -> new ConcurrentHashMap<>())
|
||||
.put(key, new Expiring(value, expiry));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> get(String namespace, String key) {
|
||||
Map<String, Expiring> ns = namespaces.get(namespace);
|
||||
if (ns == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
Expiring e = ns.get(key);
|
||||
if (e == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
if (e.expiry.isBefore(Instant.now())) {
|
||||
ns.remove(key, e);
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(e.value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void evict(String namespace, String key) {
|
||||
Map<String, Expiring> ns = namespaces.get(namespace);
|
||||
if (ns != null) {
|
||||
ns.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void evictNamespace(String namespace) {
|
||||
namespaces.remove(namespace);
|
||||
}
|
||||
|
||||
private record Expiring(String value, Instant expiry) {}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import io.github.bucket4j.Bandwidth;
|
||||
import io.github.bucket4j.Bucket;
|
||||
import io.github.bucket4j.ConsumptionProbe;
|
||||
import io.github.bucket4j.local.LocalBucketBuilder;
|
||||
|
||||
import stirling.software.common.cluster.RateLimitStore;
|
||||
|
||||
/** Bucket4j-backed token bucket implementation of {@link RateLimitStore}. */
|
||||
public class InProcessRateLimitStore implements RateLimitStore {
|
||||
|
||||
/** Cap to bound memory; oldest accessed buckets are evicted. */
|
||||
private static final int MAX_BUCKETS = 10_000;
|
||||
|
||||
private final Map<String, Bucket> buckets =
|
||||
Collections.synchronizedMap(
|
||||
new LinkedHashMap<String, Bucket>(256, 0.75f, true) {
|
||||
@Override
|
||||
protected boolean removeEldestEntry(Map.Entry<String, Bucket> eldest) {
|
||||
return size() > MAX_BUCKETS;
|
||||
}
|
||||
});
|
||||
|
||||
@Override
|
||||
public RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod) {
|
||||
String compositeKey = bucketKey + "|" + capacity + "|" + refillPeriod.toNanos();
|
||||
Bucket bucket =
|
||||
buckets.computeIfAbsent(compositeKey, k -> buildBucket(capacity, refillPeriod));
|
||||
ConsumptionProbe probe = bucket.tryConsumeAndReturnRemaining(1);
|
||||
return new RateLimitDecision(
|
||||
probe.isConsumed(),
|
||||
probe.getRemainingTokens(),
|
||||
probe.isConsumed() ? 0L : probe.getNanosToWaitForRefill());
|
||||
}
|
||||
|
||||
private static Bucket buildBucket(long capacity, Duration refillPeriod) {
|
||||
Bandwidth limit =
|
||||
Bandwidth.builder().capacity(capacity).refillGreedy(capacity, refillPeriod).build();
|
||||
LocalBucketBuilder builder = Bucket.builder();
|
||||
builder.addLimit(limit);
|
||||
return builder.build();
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.UUID;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
|
||||
/** Local-disk {@link FileStore} storing files under a base directory keyed by a UUID file id. */
|
||||
@Slf4j
|
||||
public class LocalDiskFileStore implements FileStore {
|
||||
|
||||
private final String baseDirPath;
|
||||
|
||||
public LocalDiskFileStore(String baseDirPath) {
|
||||
this.baseDirPath = baseDirPath;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stored store(InputStream in, String originalName) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = resolve(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
boolean success = false;
|
||||
try {
|
||||
long size = Files.copy(in, filePath);
|
||||
success = true;
|
||||
return new Stored(fileId, size);
|
||||
} finally {
|
||||
if (!success) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* File-to-file copy. {@link Files#copy(Path, Path, java.nio.file.CopyOption...)} can use {@code
|
||||
* sendfile(2)} on Linux for a zero-copy kernel transfer when source and destination share a
|
||||
* filesystem, avoiding the streaming overhead of pulling the bytes through the JVM heap. Reads
|
||||
* the source size before copying so the post-copy stat is unnecessary.
|
||||
*/
|
||||
@Override
|
||||
public Stored store(Path source, String originalName) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = resolve(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
long size = Files.size(source);
|
||||
boolean success = false;
|
||||
try {
|
||||
Files.copy(source, filePath);
|
||||
success = true;
|
||||
return new Stored(fileId, size);
|
||||
} finally {
|
||||
if (!success) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream retrieve(String fileId) throws IOException {
|
||||
return new BufferedInputStream(Files.newInputStream(resolve(fileId)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] retrieveBytes(String fileId) throws IOException {
|
||||
Path filePath = resolve(fileId);
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
return Files.readAllBytes(filePath);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long size(String fileId) throws IOException {
|
||||
Path filePath = resolve(fileId);
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
return Files.size(filePath);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(String fileId) {
|
||||
try {
|
||||
return Files.deleteIfExists(resolve(fileId));
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exists(String fileId) {
|
||||
return Files.exists(resolve(fileId));
|
||||
}
|
||||
|
||||
public Path resolve(String fileId) {
|
||||
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
throw new IllegalArgumentException("Invalid file ID");
|
||||
}
|
||||
Path basePath = Path.of(baseDirPath).normalize().toAbsolutePath();
|
||||
Path resolvedPath = basePath.resolve(fileId).normalize();
|
||||
if (!resolvedPath.startsWith(basePath)) {
|
||||
throw new IllegalArgumentException("File ID resolves to an invalid path");
|
||||
}
|
||||
return resolvedPath;
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
|
||||
/**
|
||||
* Always-on wiring for the per-node local-disk {@link FileStore}. Active when {@code
|
||||
* cluster.artifactStore=local} (the default; {@code matchIfMissing=true}). The S3 artifact-store
|
||||
* supplies its own bean when {@code cluster.artifactStore=s3}.
|
||||
*/
|
||||
@Configuration
|
||||
@ConditionalOnProperty(
|
||||
prefix = "cluster",
|
||||
name = "artifactStore",
|
||||
havingValue = "local",
|
||||
matchIfMissing = true)
|
||||
public class LocalDiskFileStoreConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public FileStore fileStore(@Value("${stirling.tempDir:/tmp/stirling-files}") String tempDir) {
|
||||
return new LocalDiskFileStore(tempDir);
|
||||
}
|
||||
}
|
||||
+27
-5
@@ -80,6 +80,7 @@ public class ConfigInitializer {
|
||||
YamlHelper settingsFile = new YamlHelper(settingTempPath);
|
||||
|
||||
migrateEnterpriseEditionToPremium(settingsFile, settingsTemplateFile);
|
||||
migrateProFeaturesKeyCasing(settingsFile, settingsTemplateFile);
|
||||
|
||||
boolean changesMade =
|
||||
settingsTemplateFile.updateValuesFromYaml(settingsFile, settingsTemplateFile);
|
||||
@@ -116,31 +117,52 @@ public class ConfigInitializer {
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "SSOAutoLogin") != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "SSOAutoLogin"),
|
||||
List.of("premium", "proFeatures", "ssoAutoLogin"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "SSOAutoLogin"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "autoUpdateMetadata")
|
||||
!= null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "CustomMetadata", "autoUpdateMetadata"),
|
||||
List.of("premium", "proFeatures", "customMetadata", "autoUpdateMetadata"),
|
||||
yaml.getValueByExactKeyPath(
|
||||
"enterpriseEdition", "CustomMetadata", "autoUpdateMetadata"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "author") != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "CustomMetadata", "author"),
|
||||
List.of("premium", "proFeatures", "customMetadata", "author"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "author"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "creator") != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "CustomMetadata", "creator"),
|
||||
List.of("premium", "proFeatures", "customMetadata", "creator"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "creator"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "producer")
|
||||
!= null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "CustomMetadata", "producer"),
|
||||
List.of("premium", "proFeatures", "customMetadata", "producer"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "producer"));
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Remove post migration
|
||||
// settings.yml.template renamed the two non-camelCase proFeatures keys
|
||||
// ("SSOAutoLogin" -> "ssoAutoLogin", "CustomMetadata" -> "customMetadata") so the whole
|
||||
// settings pipeline is consistent camelCase. The save path (YamlHelper.updateValue) matches
|
||||
// keys case-sensitively, so without this carry-forward an existing install's values written
|
||||
// under the old PascalCase keys would be dropped on upgrade and reset to template defaults.
|
||||
void migrateProFeaturesKeyCasing(YamlHelper yaml, YamlHelper template) {
|
||||
Object ssoAutoLogin = yaml.getValueByExactKeyPath("premium", "proFeatures", "SSOAutoLogin");
|
||||
if (ssoAutoLogin != null) {
|
||||
template.updateValue(List.of("premium", "proFeatures", "ssoAutoLogin"), ssoAutoLogin);
|
||||
}
|
||||
for (String field : List.of("autoUpdateMetadata", "author", "creator", "producer")) {
|
||||
Object value =
|
||||
yaml.getValueByExactKeyPath("premium", "proFeatures", "CustomMetadata", field);
|
||||
if (value != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "customMetadata", field), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
@@ -77,6 +78,7 @@ public class ApplicationProperties {
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
private Cluster cluster = new Cluster();
|
||||
|
||||
@Bean
|
||||
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
|
||||
@@ -254,6 +256,106 @@ public class ApplicationProperties {
|
||||
private int longRunningTimeoutSeconds = 600;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cluster backplane configuration. All keys live under the top-level {@code cluster.*} prefix
|
||||
* (e.g. env var {@code CLUSTER_ENABLED}). The master switch is {@link #enabled} and defaults to
|
||||
* off; when off the in-process backplane is wired and no other cluster keys are required.
|
||||
*/
|
||||
@Data
|
||||
public static class Cluster {
|
||||
|
||||
/** Master switch. When {@code false} (default) the in-process backplane is wired. */
|
||||
private boolean enabled = false;
|
||||
|
||||
/** Backplane implementation selector. Valid values: {@code inprocess} | {@code valkey}. */
|
||||
private String backplane = "inprocess";
|
||||
|
||||
/**
|
||||
* Transient cluster job-artifact store selector. Valid values: {@code local} | {@code s3}.
|
||||
*
|
||||
* <p>This is distinct from {@code storage.provider}, which selects the backend for
|
||||
* persistent user-uploaded files. The two switches exist because the user-facing storage
|
||||
* feature is optional ({@code storage.enabled=false} is common) but every multi-node
|
||||
* cluster still needs a shared artifact store to serve cross-node downloads. Both
|
||||
* implementations share credentials from {@code storage.s3.*} when set to {@code s3}.
|
||||
*/
|
||||
private String artifactStore = "local";
|
||||
|
||||
private Valkey valkey = new Valkey();
|
||||
private Node node = new Node();
|
||||
|
||||
private transient String cachedNodeId;
|
||||
|
||||
public NodeRole resolvedRole() {
|
||||
if (node == null || node.getRole() == null) {
|
||||
return NodeRole.BOTH;
|
||||
}
|
||||
String value = node.getRole().trim().toUpperCase(Locale.ROOT);
|
||||
try {
|
||||
return NodeRole.valueOf(value);
|
||||
} catch (IllegalArgumentException ex) {
|
||||
return NodeRole.BOTH;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized String resolvedNodeId() {
|
||||
if (node != null && node.getId() != null && !node.getId().isBlank()) {
|
||||
return node.getId();
|
||||
}
|
||||
if (cachedNodeId == null) {
|
||||
cachedNodeId = UUID.randomUUID().toString();
|
||||
}
|
||||
return cachedNodeId;
|
||||
}
|
||||
|
||||
public enum NodeRole {
|
||||
WEB,
|
||||
WORKER,
|
||||
BOTH
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Valkey {
|
||||
/**
|
||||
* {@code redis://host:6379} or {@code rediss://...} for TLS. Required when cluster mode
|
||||
* is on and backplane is valkey.
|
||||
*/
|
||||
private String url = "";
|
||||
|
||||
private Tls tls = new Tls();
|
||||
|
||||
@Data
|
||||
public static class Tls {
|
||||
/**
|
||||
* When {@code true}, skip Valkey/Redis TLS certificate verification (dev/test
|
||||
* only). Leave {@code false} in production.
|
||||
*/
|
||||
private boolean skipCertVerification = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Node {
|
||||
/** Optional explicit node id. Blank = auto-generated UUID at startup. */
|
||||
private String id = "";
|
||||
|
||||
/** {@code web} | {@code worker} | {@code both}. */
|
||||
private String role = "both";
|
||||
|
||||
/**
|
||||
* Internal cluster address advertised in the instance registry (host:port). Blank =
|
||||
* derived at startup.
|
||||
*/
|
||||
private String internalAddress = "";
|
||||
|
||||
/** {@code http} | {@code https} - scheme used when peers call this node. */
|
||||
private String scheme = "http";
|
||||
|
||||
/** Heartbeat publish interval for the instance registry, in milliseconds. */
|
||||
private long heartbeatIntervalMs = 5000;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP timeouts for loopback calls to internal Stirling API endpoints, used by the AI workflow
|
||||
* executor and the pipeline processor. A bounded read timeout prevents a hung tool (e.g. an
|
||||
@@ -426,6 +528,16 @@ public class ApplicationProperties {
|
||||
private String provider;
|
||||
private Client client = new Client();
|
||||
|
||||
/**
|
||||
* When true, the OAuth2/OIDC login flow logs the full set of ID token and UserInfo
|
||||
* claims at INFO level (and again at ERROR level if the username attribute cannot be
|
||||
* resolved). Used to diagnose provider misconfiguration (for example ADFS not returning
|
||||
* an {@code email} claim). WARNING: writes PII (sub, email, name) to application logs.
|
||||
* Leave disabled in production; enable only while actively troubleshooting and disable
|
||||
* again afterwards.
|
||||
*/
|
||||
private Boolean debugLogging = false;
|
||||
|
||||
public void setScopes(String scopes) {
|
||||
List<String> scopesList =
|
||||
Arrays.stream(scopes.split(",")).map(String::trim).toList();
|
||||
@@ -676,6 +788,7 @@ public class ApplicationProperties {
|
||||
private boolean enabled = false;
|
||||
private String provider = "local";
|
||||
private Local local = new Local();
|
||||
private S3 s3 = new S3();
|
||||
private Quotas quotas = new Quotas();
|
||||
private Sharing sharing = new Sharing();
|
||||
private Signing signing = new Signing();
|
||||
@@ -685,6 +798,57 @@ public class ApplicationProperties {
|
||||
private String basePath = InstallationPathConfig.getPath() + "storage";
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class S3 {
|
||||
/**
|
||||
* Optional custom endpoint (e.g. {@code https://<account>.r2.cloudflarestorage.com},
|
||||
* {@code https://<project>.supabase.co/storage/v1/s3}, or {@code http://localhost:9000}
|
||||
* for MinIO). Blank = use AWS regional default.
|
||||
*/
|
||||
private String endpoint = "";
|
||||
|
||||
private String bucket = "";
|
||||
|
||||
private String region = "us-east-1";
|
||||
|
||||
private String accessKey = "";
|
||||
private String secretKey = "";
|
||||
|
||||
/**
|
||||
* When {@code true} use path-style URLs ({@code <endpoint>/<bucket>/<key>}) instead of
|
||||
* virtual-hosted ({@code <bucket>.<endpoint>/<key>}). MinIO and most S3-compatible
|
||||
* gateways require path-style; AWS S3 prefers virtual-hosted.
|
||||
*/
|
||||
private boolean pathStyleAccess = false;
|
||||
|
||||
/**
|
||||
* When {@code false} (default), {@code endpoint} hostnames that resolve to private,
|
||||
* loopback, or link-local addresses are rejected at startup to block SSRF attacks via
|
||||
* the cloud metadata service (e.g. {@code http://169.254.169.254/}). Set to {@code
|
||||
* true} to opt in for MinIO / in-cluster S3 endpoints on private networks.
|
||||
*/
|
||||
private boolean allowPrivateEndpoints = false;
|
||||
|
||||
/**
|
||||
* Controls when the SDK adds an {@code x-amz-checksum-*} header on PUT/UploadPart.
|
||||
* Default {@code WHEN_SUPPORTED} (the SDK default since 2.30) makes the SDK send a
|
||||
* CRC32 checksum on every upload - this works on AWS S3, MinIO, current Supabase,
|
||||
* Backblaze B2 (post-July-2025), and modern R2. Set to {@code WHEN_REQUIRED} to
|
||||
* suppress the auto-checksum on vendors that reject unknown {@code x-amz-checksum-*}
|
||||
* headers (older Backblaze B2, some R2 corner cases, GCS S3 endpoint). Invalid values
|
||||
* fall back to {@code WHEN_SUPPORTED}.
|
||||
*/
|
||||
private String requestChecksumCalculation = "WHEN_SUPPORTED";
|
||||
|
||||
/**
|
||||
* Controls when the SDK validates returned {@code x-amz-checksum-*} headers on GET
|
||||
* responses. Default {@code WHEN_SUPPORTED}. Set to {@code WHEN_REQUIRED} if your
|
||||
* vendor never returns these headers and you see false-positive checksum-mismatch
|
||||
* errors. Invalid values fall back to {@code WHEN_SUPPORTED}.
|
||||
*/
|
||||
private String responseChecksumValidation = "WHEN_SUPPORTED";
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Sharing {
|
||||
private boolean enabled = false;
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.UUID;
|
||||
import java.io.PipedInputStream;
|
||||
import java.io.PipedOutputStream;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
@@ -18,9 +16,11 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBo
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
|
||||
/**
|
||||
* Service for storing and retrieving files with unique file IDs. Used by the AutoJobPostMapping
|
||||
* system to handle file references.
|
||||
* system to handle file references. Disk I/O is delegated to the injected {@link FileStore} bean.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -30,251 +30,150 @@ public class FileStorage {
|
||||
/** Holds the result of a stream-to-disk store operation: the file ID and the bytes written. */
|
||||
public record StoredFile(String fileId, long size) {}
|
||||
|
||||
@Value("${stirling.tempDir:/tmp/stirling-files}")
|
||||
private String tempDirPath;
|
||||
|
||||
private final FileOrUploadService fileOrUploadService;
|
||||
private final FileStore fileStore;
|
||||
|
||||
/**
|
||||
* Store a file and return its unique ID
|
||||
*
|
||||
* @param file The file to store
|
||||
* @return The unique ID assigned to the file
|
||||
* @throws IOException If there is an error storing the file
|
||||
*/
|
||||
public String storeFile(MultipartFile file) throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
// Ensure the directory exists
|
||||
Files.createDirectories(filePath.getParent());
|
||||
|
||||
// Transfer the file to the storage location
|
||||
file.transferTo(filePath.toFile());
|
||||
|
||||
log.debug("Stored file with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a byte array as a file and return its unique ID
|
||||
*
|
||||
* @param bytes The byte array to store
|
||||
* @param originalName The original name of the file (for extension)
|
||||
* @return The unique ID assigned to the file
|
||||
* @throws IOException If there is an error storing the file
|
||||
*/
|
||||
public String storeBytes(byte[] bytes, String originalName) throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
// Ensure the directory exists
|
||||
Files.createDirectories(filePath.getParent());
|
||||
|
||||
// Write the bytes to the file
|
||||
Files.write(filePath, bytes);
|
||||
|
||||
log.debug("Stored byte array with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file by its ID as a MultipartFile
|
||||
*
|
||||
* @param fileId The ID of the file to retrieve
|
||||
* @return The file as a MultipartFile
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public MultipartFile retrieveFile(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
// Fast path: when Spring buffered the multipart to disk (typical for large uploads), the
|
||||
// backing Resource exposes a real File. Hand the Path to the FileStore so it can do a
|
||||
// file-to-file copy (Linux sendfile, no copy through Java heap) rather than streaming
|
||||
// the bytes through an 8K buffer. Falls back to the InputStream path for in-memory
|
||||
// multiparts, exotic Resource impls, and anything that does not back onto a File.
|
||||
Resource res;
|
||||
try {
|
||||
res = file.getResource();
|
||||
} catch (RuntimeException ignored) {
|
||||
res = null;
|
||||
}
|
||||
if (res != null && res.isFile()) {
|
||||
try {
|
||||
FileStore.Stored stored =
|
||||
fileStore.store(res.getFile().toPath(), file.getOriginalFilename());
|
||||
log.debug("Stored file with ID: {} (fast path)", stored.fileId());
|
||||
return stored.fileId();
|
||||
} catch (IOException ex) {
|
||||
// Some Resource impls advertise isFile()=true but throw on getFile(); fall through.
|
||||
log.debug("Resource fast path failed, falling back to stream copy", ex);
|
||||
}
|
||||
}
|
||||
try (InputStream in = file.getInputStream()) {
|
||||
FileStore.Stored stored = fileStore.store(in, file.getOriginalFilename());
|
||||
log.debug("Stored file with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
}
|
||||
|
||||
byte[] fileData = Files.readAllBytes(filePath);
|
||||
public String storeBytes(byte[] bytes, String originalName) throws IOException {
|
||||
FileStore.Stored stored = fileStore.store(new ByteArrayInputStream(bytes), originalName);
|
||||
log.debug("Stored byte array with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
|
||||
public MultipartFile retrieveFile(String fileId) throws IOException {
|
||||
byte[] fileData = fileStore.retrieveBytes(fileId);
|
||||
return fileOrUploadService.toMockMultipartFile(fileId, fileData);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file by its ID as a byte array
|
||||
*
|
||||
* @param fileId The ID of the file to retrieve
|
||||
* @return The file as a byte array
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public byte[] retrieveBytes(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
|
||||
return Files.readAllBytes(filePath);
|
||||
return fileStore.retrieveBytes(fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file by its ID as a streaming InputStream. The caller is responsible for closing
|
||||
* the returned stream.
|
||||
*
|
||||
* @param fileId The ID of the file to retrieve
|
||||
* @return A buffered InputStream for the file
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public InputStream retrieveInputStream(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
// Let Files.newInputStream throw NoSuchFileException naturally — avoids TOCTOU race
|
||||
// between exists-check and open when another thread may delete concurrently.
|
||||
return new BufferedInputStream(Files.newInputStream(filePath));
|
||||
return fileStore.retrieve(fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store data from an InputStream as a file and return its unique ID and byte count. Streams
|
||||
* directly to disk without buffering the entire content in heap.
|
||||
*
|
||||
* @param inputStream The input stream to read from
|
||||
* @param originalName The original name of the file (unused, kept for API symmetry)
|
||||
* @return A {@link StoredFile} containing the file ID and the number of bytes written
|
||||
* @throws IOException If there is an error storing the file
|
||||
*/
|
||||
public StoredFile storeInputStream(InputStream inputStream, String originalName)
|
||||
throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
long size = Files.copy(inputStream, filePath);
|
||||
log.debug("Stored input stream with ID: {}", fileId);
|
||||
return new StoredFile(fileId, size);
|
||||
FileStore.Stored stored = fileStore.store(inputStream, originalName);
|
||||
log.debug("Stored input stream with ID: {}", stored.fileId());
|
||||
return new StoredFile(stored.fileId(), stored.size());
|
||||
}
|
||||
|
||||
public String storeFromStreamingBody(StreamingResponseBody body, String originalName)
|
||||
throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
boolean success = false;
|
||||
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(filePath))) {
|
||||
body.writeTo(os);
|
||||
success = true;
|
||||
} finally {
|
||||
if (!success) {
|
||||
// Hold Throwable not IOException: an unchecked failure (NPE, IllegalState, OOM, etc.)
|
||||
// from the body writer would otherwise close the pipe with EOF and the consumer would
|
||||
// return a truncated file with no error surfaced to the caller.
|
||||
AtomicReference<Throwable> bodyError = new AtomicReference<>();
|
||||
try (PipedOutputStream out = new PipedOutputStream();
|
||||
PipedInputStream in = new PipedInputStream(out, 8192)) {
|
||||
var executor = Executors.newSingleThreadExecutor(Thread.ofVirtual().factory());
|
||||
java.util.concurrent.Future<?> task = null;
|
||||
try {
|
||||
task =
|
||||
executor.submit(
|
||||
() -> {
|
||||
try {
|
||||
body.writeTo(out);
|
||||
} catch (Throwable ex) {
|
||||
bodyError.set(ex);
|
||||
} finally {
|
||||
try {
|
||||
out.close();
|
||||
} catch (IOException ignored) {
|
||||
// closed on the consumer side too
|
||||
}
|
||||
}
|
||||
});
|
||||
FileStore.Stored stored = fileStore.store(in, originalName);
|
||||
Throwable writerErr = bodyError.get();
|
||||
if (writerErr != null) {
|
||||
// Body failed mid-write: the FileStore persisted a truncated entry.
|
||||
// Best-effort delete so we don't leak partial files; never let cleanup
|
||||
// mask the original writer error.
|
||||
try {
|
||||
fileStore.delete(stored.fileId());
|
||||
} catch (RuntimeException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to delete partial file {} after writer error: {}",
|
||||
stored.fileId(),
|
||||
cleanupEx.getMessage());
|
||||
}
|
||||
if (writerErr instanceof IOException ioe) {
|
||||
throw ioe;
|
||||
}
|
||||
throw new IOException(
|
||||
"StreamingResponseBody writer failed: " + writerErr.getMessage(),
|
||||
writerErr);
|
||||
}
|
||||
log.debug("Stored StreamingResponseBody with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
} finally {
|
||||
// Interrupt and join the writer task: shutdown() alone returns immediately and a
|
||||
// failed store leaves the writer running, leaking a thread per failed upload.
|
||||
if (task != null && !task.isDone()) {
|
||||
task.cancel(true);
|
||||
}
|
||||
executor.shutdown();
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
if (!executor.awaitTermination(5, java.util.concurrent.TimeUnit.SECONDS)) {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException ie) {
|
||||
executor.shutdownNow();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
log.debug("Stored StreamingResponseBody with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a {@link Resource} body to disk, returning the generated file ID. Used by the async
|
||||
* job pipeline to capture {@code ResponseEntity<Resource>} results produced by controllers.
|
||||
*/
|
||||
public String storeFromResource(Resource resource, String originalName) throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
boolean success = false;
|
||||
try (InputStream in = resource.getInputStream()) {
|
||||
Files.copy(in, filePath);
|
||||
success = true;
|
||||
} finally {
|
||||
if (!success) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
}
|
||||
FileStore.Stored stored = fileStore.store(in, originalName);
|
||||
log.debug("Stored Resource with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
log.debug("Stored Resource with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file by its ID
|
||||
*
|
||||
* @param fileId The ID of the file to delete
|
||||
* @return true if the file was deleted, false otherwise
|
||||
*/
|
||||
public boolean deleteFile(String fileId) {
|
||||
try {
|
||||
Path filePath = getFilePath(fileId);
|
||||
return Files.deleteIfExists(filePath);
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
}
|
||||
return fileStore.delete(fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file exists by its ID
|
||||
*
|
||||
* @param fileId The ID of the file to check
|
||||
* @return true if the file exists, false otherwise
|
||||
*/
|
||||
public boolean fileExists(String fileId) {
|
||||
Path filePath = getFilePath(fileId);
|
||||
return Files.exists(filePath);
|
||||
return fileStore.exists(fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the size of a file by its ID without loading the content into memory
|
||||
*
|
||||
* @param fileId The ID of the file
|
||||
* @return The size of the file in bytes
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public long getFileSize(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
|
||||
return Files.size(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path for a file ID
|
||||
*
|
||||
* @param fileId The ID of the file
|
||||
* @return The path to the file
|
||||
* @throws IllegalArgumentException if fileId contains path traversal characters or resolves
|
||||
* outside base directory
|
||||
*/
|
||||
private Path getFilePath(String fileId) {
|
||||
// Validate fileId to prevent path traversal
|
||||
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
throw new IllegalArgumentException("Invalid file ID");
|
||||
}
|
||||
|
||||
Path basePath = Path.of(tempDirPath).normalize().toAbsolutePath();
|
||||
Path resolvedPath = basePath.resolve(fileId).normalize();
|
||||
|
||||
// Ensure resolved path is within the base directory
|
||||
if (!resolvedPath.startsWith(basePath)) {
|
||||
throw new IllegalArgumentException("File ID resolves to an invalid path");
|
||||
}
|
||||
|
||||
return resolvedPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique file ID
|
||||
*
|
||||
* @return A unique file ID
|
||||
*/
|
||||
private String generateFileId() {
|
||||
return UUID.randomUUID().toString();
|
||||
return fileStore.size(fileId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,85 +57,38 @@ public class JobExecutorService {
|
||||
this.resourceMonitor = resourceMonitor;
|
||||
this.jobQueue = jobQueue;
|
||||
|
||||
// Parse session timeout and calculate effective timeout once during initialization
|
||||
long sessionTimeoutMs = parseSessionTimeout(sessionTimeout);
|
||||
this.effectiveTimeoutMs = Math.min(asyncRequestTimeoutMs, sessionTimeoutMs);
|
||||
log.debug(
|
||||
"Job executor configured with effective timeout of {} ms", this.effectiveTimeoutMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(boolean async, Supplier<Object> work) {
|
||||
return runJobGeneric(async, work, -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously with a custom timeout
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(
|
||||
boolean async, Supplier<Object> work, long customTimeoutMs) {
|
||||
return runJobGeneric(async, work, customTimeoutMs, false, 50);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously with custom parameters
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
|
||||
* @param queueable Whether this job can be queued when system resources are limited
|
||||
* @param resourceWeight The resource weight of this job (1-100)
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(
|
||||
boolean async,
|
||||
Supplier<Object> work,
|
||||
long customTimeoutMs,
|
||||
boolean queueable,
|
||||
int resourceWeight) {
|
||||
// Generate base UUID
|
||||
String baseJobId = UUID.randomUUID().toString();
|
||||
|
||||
// Scope job to authenticated user if security is enabled
|
||||
String scopedJobKey = getScopedJobKey(baseJobId);
|
||||
|
||||
log.debug("Generated jobId: {} (base: {})", scopedJobKey, baseJobId);
|
||||
|
||||
// Store the scoped job ID in the request for potential use by other components
|
||||
if (request != null) {
|
||||
request.setAttribute("jobId", scopedJobKey);
|
||||
|
||||
// Also track this job ID in the user's session for authorization purposes
|
||||
// This ensures users can only cancel their own jobs
|
||||
if (request.getSession() != null) {
|
||||
@SuppressWarnings("unchecked")
|
||||
java.util.Set<String> userJobIds =
|
||||
(java.util.Set<String>) request.getSession().getAttribute("userJobIds");
|
||||
|
||||
if (userJobIds == null) {
|
||||
userJobIds = new java.util.concurrent.ConcurrentSkipListSet<>();
|
||||
request.getSession().setAttribute("userJobIds", userJobIds);
|
||||
}
|
||||
|
||||
userJobIds.add(scopedJobKey);
|
||||
log.debug("Added scoped job ID {} to user session", scopedJobKey);
|
||||
}
|
||||
}
|
||||
|
||||
String jobId = scopedJobKey;
|
||||
|
||||
// Determine which timeout to use
|
||||
long timeoutToUse = customTimeoutMs > 0 ? customTimeoutMs : effectiveTimeoutMs;
|
||||
|
||||
log.debug(
|
||||
@@ -146,7 +99,6 @@ public class JobExecutorService {
|
||||
queueable,
|
||||
resourceWeight);
|
||||
|
||||
// Check if we need to queue this job based on resource availability
|
||||
boolean shouldQueue =
|
||||
queueable
|
||||
&& async
|
||||
@@ -154,7 +106,6 @@ public class JobExecutorService {
|
||||
resourceMonitor.shouldQueueJob(resourceWeight);
|
||||
|
||||
if (shouldQueue) {
|
||||
// Queue the job instead of executing immediately
|
||||
log.debug(
|
||||
"Queueing job {} due to resource constraints (weight: {})",
|
||||
jobId,
|
||||
@@ -162,18 +113,12 @@ public class JobExecutorService {
|
||||
|
||||
taskManager.createTask(jobId);
|
||||
|
||||
// Create a specialized wrapper that updates the TaskManager
|
||||
final String capturedJobIdForQueue = jobId;
|
||||
Supplier<Object> wrappedWork =
|
||||
() -> {
|
||||
try {
|
||||
// Set jobId in ThreadLocal context for the queued job
|
||||
stirling.software.common.util.JobContext.setJobId(
|
||||
capturedJobIdForQueue);
|
||||
log.debug(
|
||||
"Set jobId {} in JobContext for queued job execution",
|
||||
capturedJobIdForQueue);
|
||||
|
||||
Object result = work.get();
|
||||
processJobResult(capturedJobIdForQueue, result);
|
||||
return result;
|
||||
@@ -186,21 +131,17 @@ public class JobExecutorService {
|
||||
taskManager.setError(capturedJobIdForQueue, e.getMessage());
|
||||
throw e;
|
||||
} finally {
|
||||
// Clean up ThreadLocal to avoid memory leaks
|
||||
stirling.software.common.util.JobContext.clear();
|
||||
}
|
||||
};
|
||||
|
||||
// Queue the job and get the future
|
||||
CompletableFuture<ResponseEntity<?>> future =
|
||||
jobQueue.queueJob(jobId, resourceWeight, wrappedWork, timeoutToUse);
|
||||
|
||||
// Return immediately with job ID
|
||||
return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
|
||||
} else if (async) {
|
||||
taskManager.createTask(jobId);
|
||||
|
||||
// Capture the jobId for the async thread
|
||||
final String capturedJobId = jobId;
|
||||
|
||||
executor.execute(
|
||||
@@ -211,13 +152,7 @@ public class JobExecutorService {
|
||||
capturedJobId,
|
||||
timeoutToUse);
|
||||
|
||||
// Set jobId in ThreadLocal context for the async thread
|
||||
stirling.software.common.util.JobContext.setJobId(capturedJobId);
|
||||
log.debug(
|
||||
"Set jobId {} in JobContext for async execution",
|
||||
capturedJobId);
|
||||
|
||||
// Execute with timeout
|
||||
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
|
||||
processJobResult(capturedJobId, result);
|
||||
} catch (TimeoutException te) {
|
||||
@@ -227,7 +162,6 @@ public class JobExecutorService {
|
||||
log.error("Error executing job {}: {}", jobId, e.getMessage(), e);
|
||||
taskManager.setError(jobId, e.getMessage());
|
||||
} finally {
|
||||
// Clean up ThreadLocal to avoid memory leaks
|
||||
stirling.software.common.util.JobContext.clear();
|
||||
}
|
||||
});
|
||||
@@ -237,27 +171,19 @@ public class JobExecutorService {
|
||||
try {
|
||||
log.debug("Running sync job with timeout {} ms", timeoutToUse);
|
||||
|
||||
// Make jobId available to downstream components on the worker thread
|
||||
stirling.software.common.util.JobContext.setJobId(jobId);
|
||||
log.debug("Set jobId {} in JobContext for sync execution", jobId);
|
||||
|
||||
// Execute with timeout
|
||||
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
|
||||
|
||||
// If the result is already a ResponseEntity, return it directly
|
||||
if (result instanceof ResponseEntity) {
|
||||
return (ResponseEntity<?>) result;
|
||||
}
|
||||
|
||||
// Process different result types
|
||||
return handleResultForSyncJob(result);
|
||||
} catch (TimeoutException te) {
|
||||
log.error("Synchronous job timed out after {} ms", timeoutToUse);
|
||||
return ResponseEntity.internalServerError()
|
||||
.body(Map.of("error", "Job timed out after " + timeoutToUse + " ms"));
|
||||
} catch (RuntimeException e) {
|
||||
// Check if this is a typed exception that should be handled by
|
||||
// GlobalExceptionHandler (either directly or wrapped)
|
||||
Throwable cause = e.getCause();
|
||||
if (e instanceof IllegalArgumentException
|
||||
|| cause
|
||||
@@ -267,16 +193,13 @@ public class JobExecutorService {
|
||||
instanceof
|
||||
stirling.software.common.util.ExceptionUtils
|
||||
.BaseValidationException) {
|
||||
// Rethrow so GlobalExceptionHandler can handle with proper HTTP status codes
|
||||
throw e;
|
||||
}
|
||||
// Handle other RuntimeExceptions as generic errors
|
||||
log.error("Error executing synchronous job: {}", e.getMessage(), e);
|
||||
return ResponseEntity.internalServerError()
|
||||
.body(Map.of("error", "Job failed: " + e.getMessage()));
|
||||
} catch (Exception e) {
|
||||
log.error("Error executing synchronous job: {}", e.getMessage(), e);
|
||||
// Construct a JSON error response
|
||||
return ResponseEntity.internalServerError()
|
||||
.body(Map.of("error", "Job failed: " + e.getMessage()));
|
||||
} finally {
|
||||
@@ -285,23 +208,13 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the result of an asynchronous job
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param result The result
|
||||
*/
|
||||
private void processJobResult(String jobId, Object result) {
|
||||
try {
|
||||
if (result instanceof byte[]) {
|
||||
// Store byte array directly to disk to avoid double memory consumption
|
||||
String fileId = fileStorage.storeBytes((byte[]) result, "result.pdf");
|
||||
taskManager.setFileResult(
|
||||
jobId, fileId, "result.pdf", MediaType.APPLICATION_PDF_VALUE);
|
||||
log.debug("Stored byte[] result with fileId: {}", fileId);
|
||||
|
||||
// Let the byte array get collected naturally in the next GC cycle
|
||||
// We don't need to force System.gc() which can be harmful
|
||||
} else if (result instanceof ResponseEntity) {
|
||||
ResponseEntity<?> response = (ResponseEntity<?>) result;
|
||||
Object body = response.getBody();
|
||||
@@ -330,16 +243,13 @@ public class JobExecutorService {
|
||||
taskManager.setFileResult(jobId, fileId, filename, contentType);
|
||||
log.debug("Stored ResponseEntity<Resource> result with fileId: {}", fileId);
|
||||
} else {
|
||||
// Check if the response body contains a fileId
|
||||
if (body != null && body.toString().contains("fileId")) {
|
||||
try {
|
||||
// Try to extract fileId using reflection
|
||||
java.lang.reflect.Method getFileId =
|
||||
body.getClass().getMethod("getFileId");
|
||||
String fileId = (String) getFileId.invoke(body);
|
||||
|
||||
if (fileId != null && !fileId.isEmpty()) {
|
||||
// Try to get filename and content type
|
||||
String filename = "result.pdf";
|
||||
String contentType = MediaType.APPLICATION_PDF_VALUE;
|
||||
|
||||
@@ -379,7 +289,6 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
// Store generic result
|
||||
taskManager.setResult(jobId, body);
|
||||
}
|
||||
} else if (result instanceof MultipartFile file) {
|
||||
@@ -388,16 +297,13 @@ public class JobExecutorService {
|
||||
jobId, fileId, file.getOriginalFilename(), file.getContentType());
|
||||
log.debug("Stored MultipartFile result with fileId: {}", fileId);
|
||||
} else {
|
||||
// Check if result has a fileId field
|
||||
if (result != null) {
|
||||
try {
|
||||
// Try to extract fileId using reflection
|
||||
java.lang.reflect.Method getFileId =
|
||||
result.getClass().getMethod("getFileId");
|
||||
String fileId = (String) getFileId.invoke(result);
|
||||
|
||||
if (fileId != null && !fileId.isEmpty()) {
|
||||
// Try to get filename and content type
|
||||
String filename = "result.pdf";
|
||||
String contentType = MediaType.APPLICATION_PDF_VALUE;
|
||||
|
||||
@@ -435,7 +341,6 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
// Default case: store the result as is
|
||||
taskManager.setResult(jobId, result);
|
||||
}
|
||||
|
||||
@@ -446,16 +351,8 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle different result types for synchronous jobs
|
||||
*
|
||||
* @param result The result object
|
||||
* @return The appropriate ResponseEntity
|
||||
* @throws IOException If there is an error processing the result
|
||||
*/
|
||||
private ResponseEntity<?> handleResultForSyncJob(Object result) throws IOException {
|
||||
if (result instanceof byte[]) {
|
||||
// Return byte array as PDF
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_PDF)
|
||||
.header(
|
||||
@@ -463,7 +360,6 @@ public class JobExecutorService {
|
||||
"form-data; name=\"attachment\"; filename=\"result.pdf\"")
|
||||
.body(result);
|
||||
} else if (result instanceof MultipartFile file) {
|
||||
// Return MultipartFile content
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.parseMediaType(file.getContentType()))
|
||||
.header(
|
||||
@@ -473,7 +369,6 @@ public class JobExecutorService {
|
||||
+ "\"")
|
||||
.body(file.getBytes());
|
||||
} else {
|
||||
// Default case: return as JSON
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
}
|
||||
@@ -493,15 +388,9 @@ public class JobExecutorService {
|
||||
return mediaType != null ? mediaType.toString() : MediaType.APPLICATION_PDF_VALUE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse session timeout string (e.g., "30m", "1h") to milliseconds
|
||||
*
|
||||
* @param timeout The timeout string
|
||||
* @return The timeout in milliseconds
|
||||
*/
|
||||
private long parseSessionTimeout(String timeout) {
|
||||
if (timeout == null || timeout.isEmpty()) {
|
||||
return 30 * 60 * 1000; // Default: 30 minutes
|
||||
return 30 * 60 * 1000;
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -523,27 +412,16 @@ public class JobExecutorService {
|
||||
case "m" -> (long) (numericValue * 60 * 1000);
|
||||
case "h" -> (long) (numericValue * 60 * 60 * 1000);
|
||||
case "d" -> (long) (numericValue * 24 * 60 * 60 * 1000);
|
||||
default -> (long) (numericValue * 60 * 1000); // Default to minutes
|
||||
default -> (long) (numericValue * 60 * 1000);
|
||||
};
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not parse session timeout '{}', using default", timeout);
|
||||
return 30 * 60 * 1000; // Default: 30 minutes
|
||||
return 30 * 60 * 1000;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a supplier with a timeout
|
||||
*
|
||||
* @param supplier The supplier to execute
|
||||
* @param timeoutMs The timeout in milliseconds
|
||||
* @return The result from the supplier
|
||||
* @throws TimeoutException If the execution times out
|
||||
* @throws Exception If the supplier throws an exception
|
||||
*/
|
||||
private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs)
|
||||
throws TimeoutException, Exception {
|
||||
// Use the same executor as other async jobs for consistency
|
||||
// This ensures all operations run on the same thread pool
|
||||
String currentJobId = stirling.software.common.util.JobContext.getJobId();
|
||||
|
||||
java.util.concurrent.CompletableFuture<T> future =
|
||||
@@ -577,17 +455,10 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a scoped job key that includes user ownership when security is enabled.
|
||||
*
|
||||
* @param baseJobId the base job identifier
|
||||
* @return scoped job key, or just baseJobId if no ownership service available
|
||||
*/
|
||||
private String getScopedJobKey(String baseJobId) {
|
||||
if (jobOwnershipService != null) {
|
||||
return jobOwnershipService.createScopedJobKey(baseJobId);
|
||||
}
|
||||
// Security disabled, return unsecured job key
|
||||
return baseJobId;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,13 @@ package stirling.software.common.service;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
@@ -16,6 +20,7 @@ import java.util.concurrent.TimeUnit;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -26,6 +31,10 @@ import jakarta.annotation.PreDestroy;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
import stirling.software.common.cluster.JobStoreEntry.JobState;
|
||||
import stirling.software.common.model.job.JobResult;
|
||||
import stirling.software.common.model.job.JobStats;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
@@ -40,20 +49,20 @@ public class TaskManager {
|
||||
private int jobResultExpiryMinutes = 30;
|
||||
|
||||
private final FileStorage fileStorage;
|
||||
private final JobStore jobStore;
|
||||
private final ClusterBackplane clusterBackplane;
|
||||
private final ScheduledExecutorService cleanupExecutor =
|
||||
Executors.newSingleThreadScheduledExecutor(
|
||||
Thread.ofVirtual().name("task-cleanup-", 0).factory());
|
||||
|
||||
/** Initialize the task manager and start the cleanup scheduler */
|
||||
public TaskManager(FileStorage fileStorage) {
|
||||
@Autowired
|
||||
public TaskManager(
|
||||
FileStorage fileStorage, JobStore jobStore, ClusterBackplane clusterBackplane) {
|
||||
this.fileStorage = fileStorage;
|
||||
this.jobStore = jobStore;
|
||||
this.clusterBackplane = clusterBackplane;
|
||||
|
||||
// Schedule periodic cleanup of old job results
|
||||
cleanupExecutor.scheduleAtFixedRate(
|
||||
this::cleanupOldJobs,
|
||||
10, // Initial delay
|
||||
10, // Interval
|
||||
TimeUnit.MINUTES);
|
||||
cleanupExecutor.scheduleAtFixedRate(this::cleanupOldJobs, 10, 10, TimeUnit.MINUTES);
|
||||
|
||||
log.debug(
|
||||
"Task manager initialized with job result expiry of {} minutes",
|
||||
@@ -66,7 +75,9 @@ public class TaskManager {
|
||||
* @param jobId The job ID
|
||||
*/
|
||||
public void createTask(String jobId) {
|
||||
jobResults.put(jobId, JobResult.createNew(jobId));
|
||||
JobResult result = JobResult.createNew(jobId);
|
||||
jobResults.put(jobId, result);
|
||||
writeThrough(jobId, result);
|
||||
log.debug("Created task with job ID: {}", jobId);
|
||||
}
|
||||
|
||||
@@ -79,6 +90,7 @@ public class TaskManager {
|
||||
public void setResult(String jobId, Object result) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.completeWithResult(result);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug("Set result for job ID: {}", jobId);
|
||||
}
|
||||
|
||||
@@ -101,6 +113,7 @@ public class TaskManager {
|
||||
extractZipToIndividualFiles(fileId, originalFileName);
|
||||
if (!extractedFiles.isEmpty()) {
|
||||
jobResult.completeWithFiles(extractedFiles);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug(
|
||||
"Set multiple file results for job ID: {} with {} files extracted from"
|
||||
+ " ZIP",
|
||||
@@ -127,6 +140,7 @@ public class TaskManager {
|
||||
"Failed to get file size for job {}: {}. Using size 0.", jobId, e.getMessage());
|
||||
jobResult.completeWithSingleFile(fileId, originalFileName, contentType, 0);
|
||||
}
|
||||
writeThrough(jobId, jobResult);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -138,6 +152,7 @@ public class TaskManager {
|
||||
public void setMultipleFileResults(String jobId, List<ResultFile> resultFiles) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.completeWithFiles(resultFiles);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug(
|
||||
"Set multiple file results for job ID: {} with {} files",
|
||||
jobId,
|
||||
@@ -153,6 +168,7 @@ public class TaskManager {
|
||||
public void setError(String jobId, String error) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.failWithError(error);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug("Set error for job ID: {}: {}", jobId, error);
|
||||
}
|
||||
|
||||
@@ -169,6 +185,7 @@ public class TaskManager {
|
||||
// If no result or error has been set, mark it as complete with an empty result
|
||||
jobResult.completeWithResult("Task completed successfully");
|
||||
}
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug("Marked job ID: {} as complete", jobId);
|
||||
}
|
||||
|
||||
@@ -205,6 +222,7 @@ public class TaskManager {
|
||||
JobResult jobResult = jobResults.get(jobId);
|
||||
if (jobResult != null) {
|
||||
jobResult.addNote(note);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug("Added note to job ID: {}: {}", jobId, note);
|
||||
return true;
|
||||
}
|
||||
@@ -295,8 +313,11 @@ public class TaskManager {
|
||||
return jobResults.computeIfAbsent(jobId, JobResult::createNew);
|
||||
}
|
||||
|
||||
/** Clean up old completed job results */
|
||||
/** Clean up old completed job results. No-op in cluster mode; the backplane TTL owns expiry. */
|
||||
public void cleanupOldJobs() {
|
||||
if (clusterBackplane != null && !clusterBackplane.shouldRunLocalCleanup()) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime expiryThreshold =
|
||||
LocalDateTime.now().minus(jobResultExpiryMinutes, ChronoUnit.MINUTES);
|
||||
int removedCount = 0;
|
||||
@@ -315,6 +336,9 @@ public class TaskManager {
|
||||
|
||||
// Remove the job result
|
||||
jobResults.remove(entry.getKey());
|
||||
if (jobStore != null) {
|
||||
jobStore.delete(entry.getKey());
|
||||
}
|
||||
removedCount++;
|
||||
}
|
||||
}
|
||||
@@ -327,6 +351,53 @@ public class TaskManager {
|
||||
}
|
||||
}
|
||||
|
||||
/** Mirror the in-memory {@code JobResult} into the cluster-visible {@link JobStore}. */
|
||||
private void writeThrough(String jobId, JobResult result) {
|
||||
if (jobStore == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
jobStore.put(toEntry(jobId, result), Duration.ofMinutes(jobResultExpiryMinutes));
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("JobStore write-through failed for job {}: {}", jobId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private JobStoreEntry toEntry(String jobId, JobResult result) {
|
||||
JobState state;
|
||||
if (result.isComplete()) {
|
||||
state = result.getError() != null ? JobState.FAILED : JobState.COMPLETE;
|
||||
} else {
|
||||
state = JobState.PENDING;
|
||||
}
|
||||
Instant createdAt = toInstant(result.getCreatedAt());
|
||||
Instant completedAt = toInstant(result.getCompletedAt());
|
||||
List<String> fileIds = new ArrayList<>();
|
||||
if (result.hasFiles()) {
|
||||
for (ResultFile rf : result.getAllResultFiles()) {
|
||||
fileIds.add(rf.getFileId());
|
||||
}
|
||||
}
|
||||
Map<String, String> meta = new HashMap<>();
|
||||
if (result.getNotes() != null && !result.getNotes().isEmpty()) {
|
||||
meta.put("notesCount", Integer.toString(result.getNotes().size()));
|
||||
}
|
||||
String owningNodeId = clusterBackplane == null ? "local" : clusterBackplane.localNodeId();
|
||||
return new JobStoreEntry(
|
||||
jobId,
|
||||
state,
|
||||
owningNodeId,
|
||||
createdAt,
|
||||
completedAt,
|
||||
result.getError(),
|
||||
fileIds,
|
||||
meta);
|
||||
}
|
||||
|
||||
private Instant toInstant(LocalDateTime ldt) {
|
||||
return ldt == null ? null : ldt.atZone(ZoneId.systemDefault()).toInstant();
|
||||
}
|
||||
|
||||
/** Shutdown the cleanup executor */
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
@@ -370,7 +441,7 @@ public class TaskManager {
|
||||
while ((entry = zipIn.getNextEntry()) != null) {
|
||||
if (!entry.isDirectory()) {
|
||||
String contentType = determineContentType(entry.getName());
|
||||
// storeInputStream returns the fileId and byte count — no extra stat needed
|
||||
// storeInputStream returns the fileId and byte count - no extra stat needed
|
||||
FileStorage.StoredFile stored =
|
||||
fileStorage.storeInputStream(zipIn, entry.getName());
|
||||
|
||||
@@ -458,7 +529,8 @@ public class TaskManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the job key that owns a given file ID.
|
||||
* Find the job key that owns a given file ID. Checks the local in-memory map first, then falls
|
||||
* back to the cluster-visible {@link JobStore}.
|
||||
*
|
||||
* @param fileId file identifier to look up
|
||||
* @return scoped job key if found, otherwise null
|
||||
@@ -474,6 +546,18 @@ public class TaskManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (jobStore != null) {
|
||||
// Propagate JobStore failures: returning null on a backplane outage would conflate
|
||||
// "no such file" with "lookup unavailable" and the caller would respond 404 to a
|
||||
// transient blip that should be retried. Let Spring's exception handler surface a
|
||||
// 5xx so clients know to retry.
|
||||
try {
|
||||
return jobStore.findJobIdByFileId(fileId).orElse(null);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("JobStore findJobIdByFileId failed for {}: {}", fileId, e.getMessage());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +83,16 @@ public class RequestUriUtils {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Blocklist of backend/non-frontend paths that should still go through filters
|
||||
// Blocklist of backend/non-frontend paths that should still go through filters.
|
||||
//
|
||||
// `/files` was historically a backend route; it is now a frontend route
|
||||
// owned by HomePage / FileManagerView. Direct-nav or refresh on /files
|
||||
// (or /files/<folder-uuid>) was returning the Spring auth filter's 401
|
||||
// JSON instead of serving index.html, so the SPA never got a chance to
|
||||
// mount and the user saw a raw error response. There are no `/files`
|
||||
// backend mappings at the servlet root - the real storage endpoints
|
||||
// live under `/api/v1/storage/files`, which is filtered out a few lines
|
||||
// up by the `startsWith("/api/")` guard.
|
||||
String[] backendOnlyPrefixes = {
|
||||
"/register",
|
||||
"/pipeline",
|
||||
@@ -91,7 +100,6 @@ public class RequestUriUtils {
|
||||
"/pdfjs-legacy",
|
||||
"/fonts",
|
||||
"/images",
|
||||
"/files",
|
||||
"/css",
|
||||
"/js",
|
||||
"/swagger",
|
||||
@@ -181,7 +189,7 @@ public class RequestUriUtils {
|
||||
|| trimmedUri.startsWith(
|
||||
"/api/v1/mobile-scanner/") // Mobile scanner endpoints (no auth)
|
||||
|| trimmedUri.startsWith("/v1/api-docs")
|
||||
// Workflow participant endpoints — access controlled by share tokens, not login
|
||||
// Workflow participant endpoints - access controlled by share tokens, not login
|
||||
|| trimmedUri.startsWith("/api/v1/workflow/participant/")
|
||||
// Share-link SPA bootstrap; data APIs remain protected
|
||||
|| trimmedUri.matches("^/share/[^/]+/?$");
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Binds a multipart form value containing a JSON array into a typed {@code List<T>}. Used for
|
||||
* endpoints that accept structured list parameters via {@code @ModelAttribute} — the form field
|
||||
* carries the full JSON array as its value and the editor parses it once.
|
||||
*/
|
||||
@Slf4j
|
||||
public class JsonListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER =
|
||||
JsonMapper.builder()
|
||||
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final TypeReference<? extends List<T>> typeRef;
|
||||
|
||||
public JsonListPropertyEditor(TypeReference<? extends List<T>> typeRef) {
|
||||
this.typeRef = typeRef;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(new ArrayList<T>());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setValue(OBJECT_MAPPER.readValue(text, typeRef));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to parse JSON list value", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Expected a JSON array but could not parse: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Binds a multipart form value containing a JSON object into a typed {@code T}. Companion to {@link
|
||||
* JsonListPropertyEditor} for single-object nested fields on {@code @ModelAttribute} endpoints.
|
||||
*/
|
||||
@Slf4j
|
||||
public class JsonObjectPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER =
|
||||
JsonMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build();
|
||||
|
||||
private final Class<T> type;
|
||||
|
||||
public JsonObjectPropertyEditor(Class<T> type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setValue(OBJECT_MAPPER.readValue(text, type));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to parse JSON object value", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Expected a JSON object but could not parse: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
-53
@@ -1,53 +0,0 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.JavaType;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Spring property editor that decodes a JSON string into a typed {@link ArrayList}. Used to bind
|
||||
* complex list parameters (e.g. {@code List<RedactionArea>}, {@code List<EditTextOperation>}) from
|
||||
* multipart form fields, where Spring's default binding cannot deserialize a JSON array.
|
||||
*/
|
||||
@Slf4j
|
||||
public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private final ObjectMapper objectMapper =
|
||||
JsonMapper.builder()
|
||||
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final Class<T> elementType;
|
||||
|
||||
public StringToArrayListPropertyEditor(Class<T> elementType) {
|
||||
this.elementType = elementType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(new ArrayList<>());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
JavaType listType =
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(ArrayList.class, elementType);
|
||||
List<T> list = objectMapper.readValue(text, listType);
|
||||
setValue(list);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception while converting {}", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Failed to convert java.lang.String to java.util.List");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Unit tests for {@link PageColumnLayout} gutter detection and column classification. */
|
||||
class PageColumnLayoutTest {
|
||||
|
||||
private static final float PAGE_WIDTH = 612f; // Letter portrait
|
||||
|
||||
// ── single-column ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void singleColumn_oneColumnNoGutters() {
|
||||
List<float[]> lines = List.of(lineBox(72f, 396f));
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
assertThat(layout.gutters()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void singleColumn_classifyAnchor_returnsZero() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(List.of(lineBox(72f, 396f)), PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnOf(100f, 200f)).isEqualTo(0);
|
||||
}
|
||||
|
||||
// ── two-column ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void twoColumn_detectsGutter() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(2);
|
||||
assertThat(layout.gutters()).hasSize(1);
|
||||
float[] gutter = layout.gutters().get(0);
|
||||
// Gutter is centered on pageWidth/2 with PageColumnLayout.MIDPOINT_SLACK_PT slack each
|
||||
// side.
|
||||
float pageMid = PAGE_WIDTH / 2f;
|
||||
assertThat(gutter[0]).isBetween(pageMid - 40f, pageMid - 20f);
|
||||
assertThat(gutter[1]).isBetween(pageMid + 20f, pageMid + 40f);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_classifyLeftAndRightAnchors() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
assertThat(layout.columnOf(100f, 200f)).isEqualTo(0);
|
||||
assertThat(layout.columnOf(380f, 460f)).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_columnsCrossing_leftLineOnlyHitsLeft() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0);
|
||||
assertThat(layout.columnsCrossing(320f, 540f)).containsExactly(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_spanningLine_returnsBothColumns() {
|
||||
List<float[]> lines = new ArrayList<>(buildTwoColumnLines(3));
|
||||
// Full-width header that crosses pageWidth/2.
|
||||
lines.add(lineBox(72f, 396f));
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnsCrossing(72f, 540f)).containsExactly(0, 1);
|
||||
assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0);
|
||||
}
|
||||
|
||||
private static List<float[]> buildTwoColumnLines(int rowsPerColumn) {
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
for (int i = 0; i < rowsPerColumn; i++) {
|
||||
lines.add(lineBox(72f, 136f)); // left column body (72..208)
|
||||
lines.add(lineBox(320f, 220f)); // right column body (320..540)
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// ── three-column ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void threeColumn_collapsesToLeftRightSplit() {
|
||||
// The midpoint-based detector splits the page at pageWidth/2 and treats anything else as
|
||||
// single-column or spanning. A genuine 3-column layout collapses to 2 columns; the middle
|
||||
// column's content ends up classified by midpoint as left or right of pageMid.
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
for (int i = 0; i < 6; i++) {
|
||||
lines.add(lineBox(72f, 150f)); // 72..222
|
||||
lines.add(lineBox(252f, 150f)); // 252..402
|
||||
lines.add(lineBox(432f, 150f)); // 432..582
|
||||
}
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(2);
|
||||
assertThat(layout.gutters()).hasSize(1);
|
||||
}
|
||||
|
||||
// ── empty page ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void emptyPage_singleColumnFallback() {
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(List.of(), PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
assertThat(layout.gutters()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void onlyShortFragments_singleColumnFallback() {
|
||||
// Page numbers / decorations — too narrow to vote either side.
|
||||
List<float[]> lines = List.of(lineBox(300f, 6f));
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ── narrow gap should not be confused for a gutter ───────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void narrowInternalGap_doesNotProduceGutter() {
|
||||
// Both halves sit left of the page midpoint, so no line votes for a right column and
|
||||
// detection falls back to single-column.
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
lines.add(lineBox(72f, 100f));
|
||||
lines.add(lineBox(180f, 100f));
|
||||
for (int i = 0; i < 5; i++) {
|
||||
lines.add(lineBox(72f, 208f));
|
||||
}
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Builds a line bounding box {@code [x1, 0, x1+width, 0]}; Y is unused by detection. */
|
||||
private static float[] lineBox(float x1, float width) {
|
||||
return new float[] {x1, 0f, x1 + width, 0f};
|
||||
}
|
||||
}
|
||||
@@ -56,4 +56,17 @@ class ArchitectureTest {
|
||||
.resideInAPackage("stirling.software.saas..");
|
||||
rule.check(commonClasses);
|
||||
}
|
||||
|
||||
@Test
|
||||
void clusterInterfacesHaveNoImplementationDependencies() {
|
||||
ArchRule rule =
|
||||
noClasses()
|
||||
.that()
|
||||
.resideInAPackage("stirling.software.common.cluster..")
|
||||
.should()
|
||||
.dependOnClassesThat()
|
||||
.resideInAnyPackage(
|
||||
"stirling.software.proprietary..", "stirling.software.saas..");
|
||||
rule.check(commonClasses);
|
||||
}
|
||||
}
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class BackplaneContractCompilationTest {
|
||||
|
||||
@Test
|
||||
void jobStoreEntryRecordRoundTrips() {
|
||||
Instant now = Instant.now();
|
||||
JobStoreEntry entry =
|
||||
new JobStoreEntry(
|
||||
"job-1",
|
||||
JobStoreEntry.JobState.PENDING,
|
||||
"node-a",
|
||||
now,
|
||||
null,
|
||||
null,
|
||||
List.of("file-1"),
|
||||
Map.of("k", "v"));
|
||||
assertEquals("job-1", entry.jobId());
|
||||
assertEquals(JobStoreEntry.JobState.PENDING, entry.state());
|
||||
assertEquals("node-a", entry.owningNodeId());
|
||||
assertEquals(now, entry.createdAt());
|
||||
assertEquals(List.of("file-1"), entry.fileIds());
|
||||
assertEquals("v", entry.resultMeta().get("k"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void clusterNodeRecordRoundTrips() {
|
||||
Instant heartbeat = Instant.now();
|
||||
ClusterNode node = new ClusterNode("node-a", "10.0.0.1:8080", heartbeat, "BOTH");
|
||||
assertEquals("node-a", node.nodeId());
|
||||
assertEquals("10.0.0.1:8080", node.internalAddress());
|
||||
assertEquals(heartbeat, node.lastHeartbeat());
|
||||
assertEquals("BOTH", node.role());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rateLimitDecisionRecordRoundTrips() {
|
||||
RateLimitStore.RateLimitDecision d = new RateLimitStore.RateLimitDecision(true, 7, 0L);
|
||||
assertEquals(true, d.allowed());
|
||||
assertEquals(7, d.remainingTokens());
|
||||
assertEquals(0L, d.nanosToWaitForRefill());
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.ApplicationProperties.Cluster;
|
||||
|
||||
class ClusterConfigValidationTest {
|
||||
|
||||
@Test
|
||||
void validationPassesWhenDisabled() {
|
||||
ApplicationProperties props = new ApplicationProperties();
|
||||
ClusterConfig config = new ClusterConfig(props);
|
||||
assertDoesNotThrow(() -> invokeValidate(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validationFailsWhenValkeyEnabledWithoutUrl() {
|
||||
ApplicationProperties props = new ApplicationProperties();
|
||||
Cluster cluster = props.getCluster();
|
||||
cluster.setEnabled(true);
|
||||
cluster.setBackplane("valkey");
|
||||
ClusterConfig config = new ClusterConfig(props);
|
||||
assertThrows(IllegalStateException.class, () -> invokeValidate(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validationPassesWhenValkeyEnabledWithUrl() {
|
||||
ApplicationProperties props = new ApplicationProperties();
|
||||
Cluster cluster = props.getCluster();
|
||||
cluster.setEnabled(true);
|
||||
cluster.setBackplane("valkey");
|
||||
cluster.getValkey().setUrl("redis://localhost:6379");
|
||||
ClusterConfig config = new ClusterConfig(props);
|
||||
assertDoesNotThrow(() -> invokeValidate(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validationPassesWhenInProcessEnabled() {
|
||||
ApplicationProperties props = new ApplicationProperties();
|
||||
Cluster cluster = props.getCluster();
|
||||
cluster.setEnabled(true);
|
||||
cluster.setBackplane("inprocess");
|
||||
ClusterConfig config = new ClusterConfig(props);
|
||||
assertDoesNotThrow(() -> invokeValidate(config));
|
||||
}
|
||||
|
||||
private void invokeValidate(ClusterConfig config) throws Exception {
|
||||
Method m = ClusterConfig.class.getDeclaredMethod("validate");
|
||||
m.setAccessible(true);
|
||||
try {
|
||||
m.invoke(config);
|
||||
} catch (java.lang.reflect.InvocationTargetException ex) {
|
||||
if (ex.getCause() instanceof RuntimeException re) {
|
||||
throw re;
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.ApplicationProperties.Cluster;
|
||||
|
||||
class ClusterPropertiesTest {
|
||||
|
||||
@Test
|
||||
void defaultsAreDisabledAndInprocess() {
|
||||
Cluster props = new ApplicationProperties().getCluster();
|
||||
assertFalse(props.isEnabled());
|
||||
assertEquals("inprocess", props.getBackplane());
|
||||
assertEquals("local", props.getArtifactStore());
|
||||
assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
|
||||
assertEquals("", props.getValkey().getUrl());
|
||||
assertFalse(props.getValkey().getTls().isSkipCertVerification());
|
||||
assertEquals("both", props.getNode().getRole());
|
||||
assertEquals("http", props.getNode().getScheme());
|
||||
assertEquals(5000L, props.getNode().getHeartbeatIntervalMs());
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvedRoleParsesCaseInsensitively() {
|
||||
Cluster props = new ApplicationProperties().getCluster();
|
||||
props.getNode().setRole("WEB");
|
||||
assertEquals(Cluster.NodeRole.WEB, props.resolvedRole());
|
||||
|
||||
props.getNode().setRole("web");
|
||||
assertEquals(Cluster.NodeRole.WEB, props.resolvedRole());
|
||||
|
||||
props.getNode().setRole("Worker");
|
||||
assertEquals(Cluster.NodeRole.WORKER, props.resolvedRole());
|
||||
|
||||
props.getNode().setRole("garbage");
|
||||
assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
|
||||
|
||||
props.getNode().setRole(null);
|
||||
assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvedNodeIdIsStableAcrossCalls() {
|
||||
Cluster props = new ApplicationProperties().getCluster();
|
||||
String first = props.resolvedNodeId();
|
||||
String second = props.resolvedNodeId();
|
||||
assertNotNull(first);
|
||||
assertEquals(first, second);
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvedNodeIdHonoursExplicitId() {
|
||||
Cluster props = new ApplicationProperties().getCluster();
|
||||
props.getNode().setId("abc");
|
||||
assertEquals("abc", props.resolvedNodeId());
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import stirling.software.common.cluster.inprocess.InProcessClusterConfiguration;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
/**
|
||||
* Verifies the {@link InProcessClusterConfiguration} conditional wiring: in-process beans wire when
|
||||
* cluster mode is off or {@code backplane=inprocess}, and are skipped when {@code
|
||||
* backplane=valkey}.
|
||||
*/
|
||||
class InProcessConfigurationConditionalTest {
|
||||
|
||||
private final ApplicationContextRunner runner =
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(
|
||||
org.springframework.boot.autoconfigure.AutoConfigurations.of(
|
||||
PropertyPlaceholderAutoConfiguration.class))
|
||||
.withUserConfiguration(
|
||||
TestAppPropertiesConfig.class,
|
||||
ClusterConfig.class,
|
||||
InProcessClusterConfiguration.class);
|
||||
|
||||
@Test
|
||||
void inProcessBeansWireWhenClusterDisabled() {
|
||||
runner.run(
|
||||
context ->
|
||||
assertThat(context)
|
||||
.hasNotFailed()
|
||||
.hasSingleBean(ClusterBackplane.class)
|
||||
.hasSingleBean(JobStore.class)
|
||||
.hasSingleBean(RateLimitStore.class)
|
||||
.hasSingleBean(DistributedLock.class)
|
||||
.hasSingleBean(KeyValueCache.class)
|
||||
.hasSingleBean(InstanceRegistry.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void inProcessBeansWireWhenEnabledWithInProcessBackplane() {
|
||||
runner.withPropertyValues("cluster.enabled=true", "cluster.backplane=inprocess")
|
||||
.run(
|
||||
context ->
|
||||
assertThat(context)
|
||||
.hasNotFailed()
|
||||
.hasSingleBean(ClusterBackplane.class)
|
||||
.hasSingleBean(JobStore.class)
|
||||
.hasSingleBean(RateLimitStore.class)
|
||||
.hasSingleBean(DistributedLock.class)
|
||||
.hasSingleBean(KeyValueCache.class)
|
||||
.hasSingleBean(InstanceRegistry.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void inProcessBeansSkippedWhenEnabledWithDistributedBackplane() {
|
||||
runner.withPropertyValues(
|
||||
"cluster.enabled=true",
|
||||
"cluster.backplane=valkey",
|
||||
"cluster.valkey.url=redis://localhost:6379")
|
||||
.run(
|
||||
context ->
|
||||
assertThat(context)
|
||||
.hasNotFailed()
|
||||
.doesNotHaveBean(ClusterBackplane.class)
|
||||
.doesNotHaveBean(JobStore.class)
|
||||
.doesNotHaveBean(RateLimitStore.class)
|
||||
.doesNotHaveBean(DistributedLock.class)
|
||||
.doesNotHaveBean(KeyValueCache.class)
|
||||
.doesNotHaveBean(InstanceRegistry.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* Hand-rolled {@link ApplicationProperties} bean: the production class loads YAML at startup
|
||||
* via a {@code @PostConstruct} hook that isn't appropriate for the slice runner, so we wire a
|
||||
* defaults-only instance here.
|
||||
*/
|
||||
@Configuration
|
||||
static class TestAppPropertiesConfig {
|
||||
@Bean
|
||||
ApplicationProperties applicationProperties() {
|
||||
return new ApplicationProperties();
|
||||
}
|
||||
}
|
||||
}
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.DistributedLock;
|
||||
|
||||
class InProcessDistributedLockTest {
|
||||
|
||||
@Test
|
||||
void acquireReleaseAcquire() {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
h1.release();
|
||||
assertTrue(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void reentryFromSameThreadFails_parityWithValkey() {
|
||||
// The Valkey impl refuses reentry (SET NX semantics); the in-process impl must match,
|
||||
// otherwise code working in single-instance silently breaks in cluster mode.
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
Optional<DistributedLock.LockHandle> reentry = lock.tryAcquire("k", Duration.ofSeconds(30));
|
||||
assertFalse(reentry.isPresent(), "in-process lock must be non-reentrant");
|
||||
h1.release();
|
||||
// After release, anyone can acquire again.
|
||||
assertTrue(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void secondAcquireFromAnotherThreadFails() throws InterruptedException {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
|
||||
CountDownLatch done = new CountDownLatch(1);
|
||||
AtomicBoolean acquired = new AtomicBoolean(true);
|
||||
Thread t =
|
||||
new Thread(
|
||||
() -> {
|
||||
Optional<DistributedLock.LockHandle> attempt =
|
||||
lock.tryAcquire("k", Duration.ofSeconds(30));
|
||||
acquired.set(attempt.isPresent());
|
||||
attempt.ifPresent(DistributedLock.LockHandle::release);
|
||||
done.countDown();
|
||||
});
|
||||
t.start();
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS));
|
||||
assertFalse(acquired.get());
|
||||
h1.release();
|
||||
}
|
||||
|
||||
@Test
|
||||
void leaseExpiryAllowsTakeoverEvenWithoutRelease() throws InterruptedException {
|
||||
// Acquire with a short lease, never call release, then try to acquire again after the
|
||||
// lease has elapsed. Matches Redis SET-NX-EX semantics - the second caller gets the lock
|
||||
// because the first lease auto-expired. 250ms lease + 350ms wait gives CI generous slack.
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofMillis(250)).orElseThrow();
|
||||
Thread.sleep(350);
|
||||
Optional<DistributedLock.LockHandle> takeover =
|
||||
lock.tryAcquire("k", Duration.ofSeconds(30));
|
||||
assertTrue(
|
||||
takeover.isPresent(),
|
||||
"expired lease must release the lock so a new caller can take over");
|
||||
// Calling release() on the original handle after takeover must be a no-op (token check).
|
||||
h1.release();
|
||||
// The takeover holder is still the legitimate owner.
|
||||
assertFalse(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
|
||||
takeover.get().release();
|
||||
}
|
||||
|
||||
@Test
|
||||
void renewExtendsLease() throws InterruptedException {
|
||||
// Acquire with a short lease, renew it before it expires, then verify the lock is still
|
||||
// held past the original expiry point. 200ms initial + renew to 2s + wait 350ms.
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofMillis(200)).orElseThrow();
|
||||
assertTrue(h1.renew(Duration.ofSeconds(2)), "renew on a held lease must succeed");
|
||||
Thread.sleep(350);
|
||||
assertFalse(
|
||||
lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent(),
|
||||
"renew should have pushed expiry well past the original 200ms");
|
||||
h1.release();
|
||||
}
|
||||
|
||||
@Test
|
||||
void renewAfterReleaseFails() {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
h1.release();
|
||||
assertFalse(h1.renew(Duration.ofSeconds(30)), "renew on a released handle must fail");
|
||||
}
|
||||
|
||||
/**
|
||||
* Concurrency stress: many threads contending on the same key with each holder respecting the
|
||||
* lease (hold << lease). The lock behaves as a strict mutex in this regime so asserting
|
||||
* mutual exclusion is meaningful. A separate test ({@link
|
||||
* #leaseExpiryAllowsTakeoverEvenWithoutRelease}) covers the takeover-across-expiry branch,
|
||||
* which legitimately allows two holders momentarily and is split-brain behaviour inherent to
|
||||
* any lease-based lock.
|
||||
*/
|
||||
@Test
|
||||
void concurrentContentionPreservesMutualExclusion() throws InterruptedException {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
int threads = 16;
|
||||
int attemptsPerThread = 200;
|
||||
// Lease far exceeds any plausible hold time, so the takeover branch never triggers in
|
||||
// this test and the lock acts as a strict mutex.
|
||||
Duration lease = Duration.ofSeconds(5);
|
||||
java.util.concurrent.atomic.AtomicInteger concurrentHolders =
|
||||
new java.util.concurrent.atomic.AtomicInteger();
|
||||
java.util.concurrent.atomic.AtomicInteger maxConcurrent =
|
||||
new java.util.concurrent.atomic.AtomicInteger();
|
||||
java.util.concurrent.atomic.AtomicInteger acquires =
|
||||
new java.util.concurrent.atomic.AtomicInteger();
|
||||
java.util.concurrent.atomic.AtomicReference<Throwable> firstFailure =
|
||||
new java.util.concurrent.atomic.AtomicReference<>();
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
CountDownLatch done = new CountDownLatch(threads);
|
||||
|
||||
for (int i = 0; i < threads; i++) {
|
||||
new Thread(
|
||||
() -> {
|
||||
try {
|
||||
start.await();
|
||||
for (int j = 0; j < attemptsPerThread; j++) {
|
||||
Optional<DistributedLock.LockHandle> h =
|
||||
lock.tryAcquire("hot", lease);
|
||||
if (h.isPresent()) {
|
||||
int now = concurrentHolders.incrementAndGet();
|
||||
maxConcurrent.accumulateAndGet(now, Math::max);
|
||||
acquires.incrementAndGet();
|
||||
// Trivial critical section; well within lease.
|
||||
concurrentHolders.decrementAndGet();
|
||||
h.get().release();
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
firstFailure.compareAndSet(null, t);
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
},
|
||||
"lock-stress-" + i)
|
||||
.start();
|
||||
}
|
||||
start.countDown();
|
||||
assertTrue(done.await(30, TimeUnit.SECONDS), "stress workers must finish in time");
|
||||
org.junit.jupiter.api.Assertions.assertNull(firstFailure.get(), "no worker may throw");
|
||||
org.junit.jupiter.api.Assertions.assertEquals(
|
||||
1,
|
||||
maxConcurrent.get(),
|
||||
"mutual exclusion violated: more than one holder observed simultaneously");
|
||||
assertTrue(
|
||||
acquires.get() > 0,
|
||||
"at least some acquires must succeed under contention (saw "
|
||||
+ acquires.get()
|
||||
+ ")");
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.ClusterNode;
|
||||
|
||||
class InProcessInstanceRegistryTest {
|
||||
|
||||
@Test
|
||||
void registerThenLookupAndActiveNodes() {
|
||||
InProcessInstanceRegistry registry = new InProcessInstanceRegistry();
|
||||
ClusterNode node = new ClusterNode("node-1", "127.0.0.1:8080", Instant.now(), "BOTH");
|
||||
registry.register(node, Duration.ofSeconds(30));
|
||||
|
||||
assertTrue(registry.lookup("node-1").isPresent());
|
||||
assertEquals("node-1", registry.lookup("node-1").get().nodeId());
|
||||
assertEquals(1, registry.activeNodes().size());
|
||||
|
||||
registry.deregister("node-1");
|
||||
assertTrue(registry.lookup("node-1").isEmpty());
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
|
||||
class InProcessJobStoreTest {
|
||||
|
||||
private final InProcessJobStore store = new InProcessJobStore();
|
||||
|
||||
@Test
|
||||
void putGetDeleteExistsRoundTrip() {
|
||||
JobStoreEntry entry = entry("job-1");
|
||||
store.put(entry, Duration.ofMinutes(30));
|
||||
|
||||
assertTrue(store.exists("job-1"));
|
||||
assertEquals(entry, store.get("job-1").orElseThrow());
|
||||
|
||||
store.delete("job-1");
|
||||
assertFalse(store.exists("job-1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ttlExpiry() throws InterruptedException {
|
||||
store.put(entry("job-2"), Duration.ofMillis(50));
|
||||
Thread.sleep(100);
|
||||
assertFalse(store.get("job-2").isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void purgeExpiredRemovesOnlyStaleEntries() throws InterruptedException {
|
||||
store.put(entry("job-fresh"), Duration.ofMinutes(30));
|
||||
store.put(entry("job-stale"), Duration.ofMillis(20));
|
||||
Thread.sleep(80);
|
||||
|
||||
int removed = store.purgeExpired();
|
||||
assertEquals(1, removed);
|
||||
assertTrue(store.exists("job-fresh"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void findJobIdByFileIdReturnsTheRightJob() {
|
||||
store.put(
|
||||
new JobStoreEntry(
|
||||
"job-a",
|
||||
JobStoreEntry.JobState.COMPLETE,
|
||||
"node-1",
|
||||
Instant.now(),
|
||||
Instant.now(),
|
||||
null,
|
||||
List.of("file-1", "file-2"),
|
||||
Map.of()),
|
||||
Duration.ofMinutes(30));
|
||||
store.put(
|
||||
new JobStoreEntry(
|
||||
"job-b",
|
||||
JobStoreEntry.JobState.COMPLETE,
|
||||
"node-1",
|
||||
Instant.now(),
|
||||
Instant.now(),
|
||||
null,
|
||||
List.of("file-3"),
|
||||
Map.of()),
|
||||
Duration.ofMinutes(30));
|
||||
|
||||
assertEquals("job-a", store.findJobIdByFileId("file-1").orElseThrow());
|
||||
assertEquals("job-b", store.findJobIdByFileId("file-3").orElseThrow());
|
||||
assertFalse(store.findJobIdByFileId("missing").isPresent());
|
||||
}
|
||||
|
||||
private JobStoreEntry entry(String id) {
|
||||
return new JobStoreEntry(
|
||||
id,
|
||||
JobStoreEntry.JobState.PENDING,
|
||||
"node-1",
|
||||
Instant.now(),
|
||||
null,
|
||||
null,
|
||||
List.of(),
|
||||
Map.of());
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.KeyValueCache;
|
||||
|
||||
class InProcessKeyValueCacheTest {
|
||||
|
||||
@Test
|
||||
void putGetEvict() {
|
||||
KeyValueCache cache = new InProcessKeyValueCache();
|
||||
cache.put("apikey", "a", "userA", Duration.ofMinutes(1));
|
||||
assertEquals("userA", cache.get("apikey", "a").orElseThrow());
|
||||
|
||||
cache.evict("apikey", "a");
|
||||
assertFalse(cache.get("apikey", "a").isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void ttlExpiry() throws InterruptedException {
|
||||
KeyValueCache cache = new InProcessKeyValueCache();
|
||||
cache.put("ns", "k", "v", Duration.ofMillis(40));
|
||||
Thread.sleep(80);
|
||||
assertFalse(cache.get("ns", "k").isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void evictNamespace() {
|
||||
KeyValueCache cache = new InProcessKeyValueCache();
|
||||
cache.put("ns", "a", "1", Duration.ofMinutes(1));
|
||||
cache.put("ns", "b", "2", Duration.ofMinutes(1));
|
||||
cache.evictNamespace("ns");
|
||||
assertFalse(cache.get("ns", "a").isPresent());
|
||||
assertFalse(cache.get("ns", "b").isPresent());
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.RateLimitStore;
|
||||
import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
|
||||
|
||||
class InProcessRateLimitStoreTest {
|
||||
|
||||
@Test
|
||||
void firstNConsumesAllowed() {
|
||||
RateLimitStore store = new InProcessRateLimitStore();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
assertTrue(store.tryConsume("k", 5, Duration.ofSeconds(60)).allowed(), "i=" + i);
|
||||
}
|
||||
assertFalse(store.tryConsume("k", 5, Duration.ofSeconds(60)).allowed());
|
||||
}
|
||||
|
||||
@Test
|
||||
void remainingTokensDecrements() {
|
||||
RateLimitStore store = new InProcessRateLimitStore();
|
||||
RateLimitDecision d1 = store.tryConsume("k", 5, Duration.ofSeconds(60));
|
||||
RateLimitDecision d2 = store.tryConsume("k", 5, Duration.ofSeconds(60));
|
||||
assertTrue(d1.allowed());
|
||||
assertTrue(d2.allowed());
|
||||
assertEquals(4, d1.remainingTokens());
|
||||
assertEquals(3, d2.remainingTokens());
|
||||
}
|
||||
|
||||
@Test
|
||||
void refillRestoresTokens() throws InterruptedException {
|
||||
RateLimitStore store = new InProcessRateLimitStore();
|
||||
// Capacity 2 with smooth refill over 100 ms -> ~1 token per 50 ms.
|
||||
for (int i = 0; i < 2; i++) {
|
||||
assertTrue(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
|
||||
}
|
||||
assertFalse(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
|
||||
Thread.sleep(150);
|
||||
assertTrue(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deniedConsumeReportsWaitNanos() {
|
||||
RateLimitStore store = new InProcessRateLimitStore();
|
||||
assertTrue(store.tryConsume("wait", 1, Duration.ofSeconds(10)).allowed());
|
||||
RateLimitDecision denied = store.tryConsume("wait", 1, Duration.ofSeconds(10));
|
||||
assertFalse(denied.allowed());
|
||||
assertTrue(denied.nanosToWaitForRefill() > 0L);
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
|
||||
class LocalDiskFileStoreTest {
|
||||
|
||||
@Test
|
||||
void storeRetrieveSizeDeleteExistsRoundTrip(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
byte[] payload = "hello-bytes".getBytes();
|
||||
|
||||
FileStore.Stored stored = store.store(new ByteArrayInputStream(payload), "x.txt");
|
||||
assertEquals(payload.length, stored.size());
|
||||
assertTrue(store.exists(stored.fileId()));
|
||||
assertEquals(payload.length, store.size(stored.fileId()));
|
||||
assertArrayEquals(payload, store.retrieveBytes(stored.fileId()));
|
||||
|
||||
assertTrue(store.delete(stored.fileId()));
|
||||
assertFalse(store.exists(stored.fileId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void traversalIdsAreRejected(@TempDir Path dir) {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("../foo"));
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("a/b"));
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("a\\b"));
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
package stirling.software.common.configuration;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.snakeyaml.engine.v2.api.LoadSettings;
|
||||
|
||||
import stirling.software.common.util.YamlHelper;
|
||||
|
||||
class ConfigInitializerTest {
|
||||
|
||||
private static final LoadSettings LOAD_SETTINGS =
|
||||
LoadSettings.builder()
|
||||
.setUseMarks(true)
|
||||
.setMaxAliasesForCollections(Integer.MAX_VALUE)
|
||||
.setAllowRecursiveKeys(true)
|
||||
.setParseComments(true)
|
||||
.build();
|
||||
|
||||
// Mirrors the proFeatures block of settings.yml.template after the camelCase rename.
|
||||
private static final String CAMEL_CASE_TEMPLATE =
|
||||
"""
|
||||
premium:
|
||||
proFeatures:
|
||||
ssoAutoLogin: false
|
||||
customMetadata:
|
||||
autoUpdateMetadata: false
|
||||
author: username
|
||||
creator: Stirling-PDF
|
||||
producer: Stirling-PDF
|
||||
""";
|
||||
|
||||
@Test
|
||||
void migrateProFeaturesKeyCasing_carriesForwardLegacyPascalCaseValues() {
|
||||
// An existing install whose settings.yml still uses the old PascalCase keys.
|
||||
String legacy =
|
||||
"""
|
||||
premium:
|
||||
proFeatures:
|
||||
SSOAutoLogin: true
|
||||
CustomMetadata:
|
||||
autoUpdateMetadata: true
|
||||
author: alice
|
||||
creator: bob
|
||||
producer: carol
|
||||
""";
|
||||
YamlHelper template = new YamlHelper(LOAD_SETTINGS, CAMEL_CASE_TEMPLATE);
|
||||
YamlHelper existing = new YamlHelper(LOAD_SETTINGS, legacy);
|
||||
|
||||
new ConfigInitializer().migrateProFeaturesKeyCasing(existing, template);
|
||||
|
||||
assertEquals(
|
||||
"true", template.getValueByExactKeyPath("premium", "proFeatures", "ssoAutoLogin"));
|
||||
assertEquals(
|
||||
"true",
|
||||
template.getValueByExactKeyPath(
|
||||
"premium", "proFeatures", "customMetadata", "autoUpdateMetadata"));
|
||||
assertEquals(
|
||||
"alice",
|
||||
template.getValueByExactKeyPath(
|
||||
"premium", "proFeatures", "customMetadata", "author"));
|
||||
assertEquals(
|
||||
"bob",
|
||||
template.getValueByExactKeyPath(
|
||||
"premium", "proFeatures", "customMetadata", "creator"));
|
||||
assertEquals(
|
||||
"carol",
|
||||
template.getValueByExactKeyPath(
|
||||
"premium", "proFeatures", "customMetadata", "producer"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void migrateProFeaturesKeyCasing_withoutLegacyKeys_keepsTemplateDefaults() {
|
||||
// No PascalCase keys present -> this migration step must be a no-op.
|
||||
String alreadyCamel =
|
||||
"""
|
||||
premium:
|
||||
proFeatures:
|
||||
ssoAutoLogin: true
|
||||
customMetadata:
|
||||
author: dave
|
||||
""";
|
||||
YamlHelper template = new YamlHelper(LOAD_SETTINGS, CAMEL_CASE_TEMPLATE);
|
||||
YamlHelper existing = new YamlHelper(LOAD_SETTINGS, alreadyCamel);
|
||||
|
||||
new ConfigInitializer().migrateProFeaturesKeyCasing(existing, template);
|
||||
|
||||
assertEquals(
|
||||
"false", template.getValueByExactKeyPath("premium", "proFeatures", "ssoAutoLogin"));
|
||||
assertEquals(
|
||||
"username",
|
||||
template.getValueByExactKeyPath(
|
||||
"premium", "proFeatures", "customMetadata", "author"));
|
||||
}
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import stirling.software.common.cluster.inprocess.LocalDiskFileStore;
|
||||
|
||||
class FileStorageDelegationTest {
|
||||
|
||||
@Test
|
||||
void storeBytesThenRetrieveBytesRoundTripsThroughFileStore(@TempDir Path tempDir)
|
||||
throws IOException {
|
||||
FileStorage fs =
|
||||
new FileStorage(
|
||||
mock(FileOrUploadService.class),
|
||||
new LocalDiskFileStore(tempDir.toString()));
|
||||
byte[] payload = "round-trip".getBytes();
|
||||
String id = fs.storeBytes(payload, "x.bin");
|
||||
assertArrayEquals(payload, fs.retrieveBytes(id));
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package stirling.software.common.service;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -13,29 +14,30 @@ import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.common.cluster.inprocess.LocalDiskFileStore;
|
||||
|
||||
class FileStorageTest {
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
@Mock private FileOrUploadService fileOrUploadService;
|
||||
|
||||
@InjectMocks private FileStorage fileStorage;
|
||||
private FileStorage fileStorage;
|
||||
|
||||
private MultipartFile mockFile;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
void setUp() throws IOException {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
ReflectionTestUtils.setField(fileStorage, "tempDirPath", tempDir.toString());
|
||||
fileStorage =
|
||||
new FileStorage(fileOrUploadService, new LocalDiskFileStore(tempDir.toString()));
|
||||
|
||||
// Create a mock MultipartFile
|
||||
mockFile = mock(MultipartFile.class);
|
||||
@@ -47,17 +49,7 @@ class FileStorageTest {
|
||||
void testStoreFile() throws IOException {
|
||||
// Arrange
|
||||
byte[] fileContent = "Test PDF content".getBytes();
|
||||
when(mockFile.getBytes()).thenReturn(fileContent);
|
||||
|
||||
// Set up mock to handle transferTo by writing the file
|
||||
doAnswer(
|
||||
invocation -> {
|
||||
java.io.File file = invocation.getArgument(0);
|
||||
Files.write(file.toPath(), fileContent);
|
||||
return null;
|
||||
})
|
||||
.when(mockFile)
|
||||
.transferTo(any(java.io.File.class));
|
||||
when(mockFile.getInputStream()).thenReturn(new ByteArrayInputStream(fileContent));
|
||||
|
||||
// Act
|
||||
String fileId = fileStorage.storeFile(mockFile);
|
||||
@@ -65,7 +57,7 @@ class FileStorageTest {
|
||||
// Assert
|
||||
assertNotNull(fileId);
|
||||
assertTrue(Files.exists(tempDir.resolve(fileId)));
|
||||
verify(mockFile).transferTo(any(java.io.File.class));
|
||||
assertArrayEquals(fileContent, Files.readAllBytes(tempDir.resolve(fileId)));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -247,11 +239,11 @@ class FileStorageTest {
|
||||
filesBefore = s.count();
|
||||
}
|
||||
|
||||
// Act + Assert: IOException must propagate out — not be swallowed.
|
||||
// Act + Assert: IOException must propagate out - not be swallowed.
|
||||
assertThrows(
|
||||
IOException.class, () -> fileStorage.storeFromResource(flakyResource, "n.pdf"));
|
||||
|
||||
// Assert: no partial file lingers under the storage directory — the finally
|
||||
// Assert: no partial file lingers under the storage directory - the finally
|
||||
// branch's deleteIfExists must have cleaned it up.
|
||||
long filesAfter;
|
||||
try (Stream<Path> s = Files.list(tempDir)) {
|
||||
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
import stirling.software.common.cluster.JobStoreEntry.JobState;
|
||||
import stirling.software.common.cluster.inprocess.InProcessClusterBackplane;
|
||||
import stirling.software.common.cluster.inprocess.InProcessJobStore;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.job.JobResult;
|
||||
|
||||
class TaskManagerJobStoreDelegationTest {
|
||||
|
||||
@Mock private FileStorage fileStorage;
|
||||
|
||||
private InProcessJobStore jobStore;
|
||||
private ClusterBackplane backplane;
|
||||
private TaskManager taskManager;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
jobStore = spy(new InProcessJobStore());
|
||||
backplane = new InProcessClusterBackplane(new ApplicationProperties());
|
||||
taskManager = new TaskManager(fileStorage, jobStore, backplane);
|
||||
ReflectionTestUtils.setField(taskManager, "jobResultExpiryMinutes", 30);
|
||||
}
|
||||
|
||||
@Test
|
||||
void createTaskWritesPendingEntry() {
|
||||
taskManager.createTask("job-1");
|
||||
JobStoreEntry entry = jobStore.get("job-1").orElseThrow();
|
||||
assertEquals(JobState.PENDING, entry.state());
|
||||
assertEquals(backplane.localNodeId(), entry.owningNodeId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void setCompleteFlipsToComplete() {
|
||||
taskManager.createTask("job-2");
|
||||
taskManager.setResult("job-2", "ok");
|
||||
taskManager.setComplete("job-2");
|
||||
JobStoreEntry entry = jobStore.get("job-2").orElseThrow();
|
||||
assertEquals(JobState.COMPLETE, entry.state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void setErrorFlipsToFailed() {
|
||||
taskManager.createTask("job-3");
|
||||
taskManager.setError("job-3", "boom");
|
||||
JobStoreEntry entry = jobStore.get("job-3").orElseThrow();
|
||||
assertEquals(JobState.FAILED, entry.state());
|
||||
assertEquals("boom", entry.error());
|
||||
}
|
||||
|
||||
@Test
|
||||
void cleanupOldJobsIsNoopWhenBackplaneIsNotInProcess() {
|
||||
ClusterBackplane mockedValkeyBackplane =
|
||||
new ClusterBackplane() {
|
||||
@Override
|
||||
public boolean isHealthy() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String backplaneType() {
|
||||
return "valkey";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String localNodeId() {
|
||||
return "node-1";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean shouldRunLocalCleanup() {
|
||||
// Distributed backplanes own job TTL eviction themselves; this mock
|
||||
// mirrors the real ValkeyClusterBackplane override of the default true.
|
||||
return false;
|
||||
}
|
||||
};
|
||||
TaskManager tm = new TaskManager(fileStorage, jobStore, mockedValkeyBackplane);
|
||||
ReflectionTestUtils.setField(tm, "jobResultExpiryMinutes", 30);
|
||||
tm.createTask("job-4");
|
||||
tm.setComplete("job-4");
|
||||
ageJobPastExpiry(tm, "job-4");
|
||||
tm.cleanupOldJobs();
|
||||
// cleanup must short-circuit before touching jobStore in cluster mode; the backplane
|
||||
// TTL owns expiry there. If the gate fired correctly, delete is never called.
|
||||
verify(jobStore, never()).delete(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void cleanupOldJobsDeletesFromJobStoreWhenBackplaneIsInProcess() {
|
||||
taskManager.createTask("job-5");
|
||||
taskManager.setComplete("job-5");
|
||||
ageJobPastExpiry(taskManager, "job-5");
|
||||
taskManager.cleanupOldJobs();
|
||||
verify(jobStore).delete("job-5");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static void ageJobPastExpiry(TaskManager tm, String jobId) {
|
||||
var jobResults =
|
||||
(java.util.Map<String, JobResult>) ReflectionTestUtils.getField(tm, "jobResults");
|
||||
JobResult result = jobResults.get(jobId);
|
||||
ReflectionTestUtils.setField(result, "completedAt", LocalDateTime.now().minusHours(2));
|
||||
ReflectionTestUtils.setField(result, "complete", true);
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,28 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
import stirling.software.common.cluster.JobStoreEntry.JobState;
|
||||
import stirling.software.common.model.job.JobResult;
|
||||
import stirling.software.common.model.job.JobStats;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
@@ -22,6 +30,8 @@ import stirling.software.common.model.job.ResultFile;
|
||||
class TaskManagerTest {
|
||||
|
||||
@Mock private FileStorage fileStorage;
|
||||
@Mock private JobStore jobStore;
|
||||
@Mock private ClusterBackplane clusterBackplane;
|
||||
|
||||
@InjectMocks private TaskManager taskManager;
|
||||
|
||||
@@ -30,6 +40,10 @@ class TaskManagerTest {
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
closeable = MockitoAnnotations.openMocks(this);
|
||||
// Treat the backplane as in-process so cleanupOldJobs is not short-circuited.
|
||||
lenient().when(clusterBackplane.backplaneType()).thenReturn("inprocess");
|
||||
lenient().when(clusterBackplane.localNodeId()).thenReturn("test-node");
|
||||
lenient().when(clusterBackplane.shouldRunLocalCleanup()).thenReturn(true);
|
||||
ReflectionTestUtils.setField(taskManager, "jobResultExpiryMinutes", 30);
|
||||
}
|
||||
|
||||
@@ -270,6 +284,33 @@ class TaskManagerTest {
|
||||
verify(fileStorage).deleteFile("file-id");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCleanupOldJobs_NoOpWhenBackplaneOwnsExpiry() {
|
||||
// When the backplane reports it should NOT run local cleanup (e.g. a distributed
|
||||
// backplane with its own TTL), the cleanup loop must leave local state untouched.
|
||||
when(clusterBackplane.shouldRunLocalCleanup()).thenReturn(false);
|
||||
|
||||
// Seed an old completed job that would normally be removed.
|
||||
String oldJobId = "old-job-distributed";
|
||||
taskManager.createTask(oldJobId);
|
||||
JobResult oldJob = taskManager.getJobResult(oldJobId);
|
||||
ReflectionTestUtils.setField(oldJob, "completedAt", LocalDateTime.now().minusHours(1));
|
||||
ReflectionTestUtils.setField(oldJob, "complete", true);
|
||||
|
||||
Map<String, JobResult> jobResultsMap =
|
||||
(Map<String, JobResult>) ReflectionTestUtils.getField(taskManager, "jobResults");
|
||||
assertNotNull(jobResultsMap);
|
||||
assertTrue(jobResultsMap.containsKey(oldJobId));
|
||||
|
||||
// Act
|
||||
taskManager.cleanupOldJobs();
|
||||
|
||||
// Assert: nothing was removed locally, and no jobStore.delete was issued.
|
||||
assertTrue(jobResultsMap.containsKey(oldJobId));
|
||||
verify(jobStore, never()).delete(anyString());
|
||||
verify(fileStorage, never()).deleteFile(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testShutdown() {
|
||||
// This mainly tests that the shutdown method doesn't throw exceptions
|
||||
@@ -310,4 +351,33 @@ class TaskManagerTest {
|
||||
// Assert
|
||||
assertFalse(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWriteThroughOnUpdate() {
|
||||
// Mutating calls must write through to the injected JobStore.
|
||||
String jobId = "write-through-job";
|
||||
taskManager.createTask(jobId);
|
||||
taskManager.setResult(jobId, "done");
|
||||
|
||||
ArgumentCaptor<JobStoreEntry> captor = ArgumentCaptor.forClass(JobStoreEntry.class);
|
||||
verify(jobStore, atLeast(2)).put(captor.capture(), any());
|
||||
|
||||
JobStoreEntry last = captor.getValue();
|
||||
assertEquals(jobId, last.jobId());
|
||||
assertEquals(JobState.COMPLETE, last.state());
|
||||
assertEquals("test-node", last.owningNodeId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFindJobKeyByFileId_FallsBackToJobStore() {
|
||||
// When the file id is not in the local map, TaskManager delegates to JobStore.
|
||||
String fileId = "remote-file-id";
|
||||
String expectedJobKey = "remote-job-key";
|
||||
when(jobStore.findJobIdByFileId(fileId)).thenReturn(Optional.of(expectedJobKey));
|
||||
|
||||
String actual = taskManager.findJobKeyByFileId(fileId);
|
||||
|
||||
assertEquals(expectedJobKey, actual);
|
||||
verify(jobStore).findJobIdByFileId(fileId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ package stirling.software.common.util;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
@@ -12,9 +14,47 @@ import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import stirling.software.common.configuration.InstallationPathConfig;
|
||||
|
||||
public class GeneralUtilsTest {
|
||||
|
||||
// Regression guard for the SSO auto-login persistence bug: the admin UI writes camelCase
|
||||
// proFeatures keys, so saveKeyToSettings must match (and persist) them against the camelCase
|
||||
// settings.yml.template. A case mismatch makes YamlHelper.updateValue silently no-op.
|
||||
@Test
|
||||
void saveKeyToSettings_persistsCamelCaseProFeatureKeys(@TempDir Path tempDir) throws Exception {
|
||||
Path settings = tempDir.resolve("settings.yml");
|
||||
Files.writeString(
|
||||
settings,
|
||||
"""
|
||||
premium:
|
||||
proFeatures:
|
||||
ssoAutoLogin: false
|
||||
customMetadata:
|
||||
author: username
|
||||
""");
|
||||
|
||||
try (MockedStatic<InstallationPathConfig> mocked =
|
||||
Mockito.mockStatic(InstallationPathConfig.class)) {
|
||||
mocked.when(InstallationPathConfig::getSettingsPath).thenReturn(settings.toString());
|
||||
|
||||
GeneralUtils.saveKeyToSettings("premium.proFeatures.ssoAutoLogin", true);
|
||||
GeneralUtils.saveKeyToSettings("premium.proFeatures.customMetadata.author", "alice");
|
||||
}
|
||||
|
||||
YamlHelper reloaded = new YamlHelper(settings);
|
||||
assertEquals(
|
||||
"true", reloaded.getValueByExactKeyPath("premium", "proFeatures", "ssoAutoLogin"));
|
||||
assertEquals(
|
||||
"alice",
|
||||
reloaded.getValueByExactKeyPath(
|
||||
"premium", "proFeatures", "customMetadata", "author"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParsePageListWithAll() {
|
||||
List<Integer> result = GeneralUtils.parsePageList(new String[] {"all"}, 5, false);
|
||||
|
||||
@@ -98,6 +98,17 @@ class RequestUriUtilsTest {
|
||||
assertTrue(RequestUriUtils.isFrontendRoute("", "/split-pdf"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIsFrontendRoute_filesRouteOwnedByFrontend() {
|
||||
// /files and /files/<folder-uuid> are FileManagerView routes - they
|
||||
// must fall through to the SPA index.html, not get blocked by the
|
||||
// backend auth filter. Regression test for direct-nav/refresh on
|
||||
// the file manager returning a 401 JSON.
|
||||
assertTrue(RequestUriUtils.isFrontendRoute("", "/files"));
|
||||
assertTrue(
|
||||
RequestUriUtils.isFrontendRoute("", "/files/3331910a-4155-4f71-8111-e38c896bc458"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIsFrontendRoute_pathWithExtension() {
|
||||
assertFalse(RequestUriUtils.isFrontendRoute("", "/some/file.pdf"));
|
||||
@@ -183,7 +194,7 @@ class RequestUriUtilsTest {
|
||||
|
||||
@Test
|
||||
void testIsPublicAuthEndpoint_shareRootNotPublic() {
|
||||
// Avoid matching bare "/share" or "/share/" — must have a token segment
|
||||
// Avoid matching bare "/share" or "/share/" - must have a token segment
|
||||
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/share", ""));
|
||||
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/share/", ""));
|
||||
}
|
||||
@@ -197,7 +208,7 @@ class RequestUriUtilsTest {
|
||||
|
||||
@Test
|
||||
void testIsPublicAuthEndpoint_shareApiStillProtected() {
|
||||
// Share-link data APIs must NOT be public — they enforce auth + access checks
|
||||
// Share-link data APIs must NOT be public - they enforce auth + access checks
|
||||
assertFalse(RequestUriUtils.isPublicAuthEndpoint("/api/v1/storage/share-links/abc123", ""));
|
||||
assertFalse(
|
||||
RequestUriUtils.isPublicAuthEndpoint(
|
||||
|
||||
-163
@@ -1,163 +0,0 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
|
||||
class StringToArrayListPropertyEditorTest {
|
||||
|
||||
private StringToArrayListPropertyEditor<RedactionArea> editor;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
editor = new StringToArrayListPropertyEditor<>(RedactionArea.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_ValidJson() {
|
||||
// Arrange
|
||||
String json =
|
||||
"[{\"x\":10.5,\"y\":20.5,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry");
|
||||
|
||||
RedactionArea area = list.get(0);
|
||||
assertEquals(10.5, area.getX(), "X should be 10.5");
|
||||
assertEquals(20.5, area.getY(), "Y should be 20.5");
|
||||
assertEquals(100.0, area.getWidth(), "Width should be 100.0");
|
||||
assertEquals(50.0, area.getHeight(), "Height should be 50.0");
|
||||
assertEquals(1, area.getPage(), "Page should be 1");
|
||||
assertEquals("#FF0000", area.getColor(), "Color should be #FF0000");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_MultipleItems() {
|
||||
// Arrange
|
||||
String json =
|
||||
"["
|
||||
+ "{\"x\":10.0,\"y\":20.0,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"},"
|
||||
+ "{\"x\":30.0,\"y\":40.0,\"width\":200.0,\"height\":150.0,\"page\":2,\"color\":\"#00FF00\"}"
|
||||
+ "]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(2, list.size(), "List should have 2 entries");
|
||||
|
||||
RedactionArea area1 = list.get(0);
|
||||
assertEquals(10.0, area1.getX(), "X should be 10.0");
|
||||
assertEquals(20.0, area1.getY(), "Y should be 20.0");
|
||||
assertEquals(1, area1.getPage(), "Page should be 1");
|
||||
|
||||
RedactionArea area2 = list.get(1);
|
||||
assertEquals(30.0, area2.getX(), "X should be 30.0");
|
||||
assertEquals(40.0, area2.getY(), "Y should be 40.0");
|
||||
assertEquals(2, area2.getPage(), "Page should be 2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_EmptyString() {
|
||||
// Arrange
|
||||
String json = "";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertTrue(list.isEmpty(), "List should be empty");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_NullString() {
|
||||
// Act
|
||||
editor.setAsText(null);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertTrue(list.isEmpty(), "List should be empty");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_SingleItemAsArray() {
|
||||
// Arrange - note this is a single object, not an array
|
||||
String json =
|
||||
"{\"x\":10.0,\"y\":20.0,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry");
|
||||
|
||||
RedactionArea area = list.get(0);
|
||||
assertEquals(10.0, area.getX(), "X should be 10.0");
|
||||
assertEquals(20.0, area.getY(), "Y should be 20.0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_InvalidJson() {
|
||||
// Arrange
|
||||
String json = "invalid json";
|
||||
|
||||
// Act & Assert
|
||||
assertThrows(IllegalArgumentException.class, () -> editor.setAsText(json));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_UnknownProperties() {
|
||||
// Arrange - this JSON contains properties not in RedactionArea
|
||||
// With FAIL_ON_UNKNOWN_PROPERTIES disabled, this should ignore the unknown properties
|
||||
String json = "[{\"invalid\":\"structure\"}]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry (empty object)");
|
||||
}
|
||||
}
|
||||
+17
-1
@@ -207,13 +207,29 @@ def resourcesStaticDir = file('src/main/resources/static')
|
||||
def generatedFrontendPaths = [
|
||||
'assets',
|
||||
'index.html',
|
||||
'index.html.gz',
|
||||
'index.html.br',
|
||||
'sw.js',
|
||||
'sw.js.gz',
|
||||
'sw.js.br',
|
||||
'manifest.json.gz',
|
||||
'manifest.json.br',
|
||||
'site.webmanifest.gz',
|
||||
'site.webmanifest.br',
|
||||
'browserconfig.xml.gz',
|
||||
'browserconfig.xml.br',
|
||||
'manifest-classic.json',
|
||||
'manifest-classic.json.gz',
|
||||
'manifest-classic.json.br',
|
||||
'locales',
|
||||
'Login',
|
||||
'classic-logo',
|
||||
'modern-logo',
|
||||
'og_images',
|
||||
'samples',
|
||||
'manifest-classic.json'
|
||||
'pdfium',
|
||||
'vendor',
|
||||
'pdfjs'
|
||||
]
|
||||
|
||||
tasks.register('npmInstall', Exec) {
|
||||
|
||||
@@ -21,6 +21,13 @@ public class EndpointInterceptor implements HandlerInterceptor {
|
||||
HttpServletRequest request, HttpServletResponse response, Object handler)
|
||||
throws Exception {
|
||||
String requestURI = request.getRequestURI();
|
||||
|
||||
// Prevent API responses from being stored by browsers or intermediary caches by default
|
||||
String servletPath = request.getServletPath();
|
||||
if (servletPath != null && servletPath.startsWith("/api/")) {
|
||||
response.setHeader("Cache-Control", "private, no-store");
|
||||
}
|
||||
|
||||
boolean isEnabled = endpointConfiguration.isEndpointEnabledForUri(requestURI);
|
||||
if (!isEnabled) {
|
||||
response.sendError(HttpServletResponse.SC_FORBIDDEN, "This endpoint is disabled");
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
package stirling.software.SPDF.config;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
@@ -10,6 +13,7 @@ import org.springframework.web.servlet.config.annotation.CorsRegistry;
|
||||
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
|
||||
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
|
||||
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
|
||||
import org.springframework.web.servlet.resource.EncodedResourceResolver;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -24,6 +28,10 @@ public class WebMvcConfig implements WebMvcConfigurer {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(WebMvcConfig.class);
|
||||
|
||||
private static final CacheControl NO_CACHE = CacheControl.noCache();
|
||||
private static final CacheControl IMMUTABLE_ONE_YEAR =
|
||||
CacheControl.maxAge(365, TimeUnit.DAYS).cachePublic().immutable();
|
||||
|
||||
@Override
|
||||
public void addInterceptors(InterceptorRegistry registry) {
|
||||
registry.addInterceptor(endpointInterceptor);
|
||||
@@ -31,37 +39,106 @@ public class WebMvcConfig implements WebMvcConfigurer {
|
||||
|
||||
@Override
|
||||
public void addResourceHandlers(ResourceHandlerRegistry registry) {
|
||||
// Cache hashed assets (JS/CSS with content hashes) for 1 year
|
||||
// These files have names like index-ChAS4tCC.js that change when content changes
|
||||
// Check customFiles/static first, then fall back to classpath
|
||||
String staticPath =
|
||||
"file:"
|
||||
+ stirling.software.common.configuration.InstallationPathConfig
|
||||
.getStaticPath();
|
||||
|
||||
// 1. Service worker and PWA metadata (never store)
|
||||
// Browsers revalidate SW bytes anyway; no-store is the safest for atomic updates.
|
||||
registry.addResourceHandler(
|
||||
"/sw.js", "/manifest.json", "/site.webmanifest", "/browserconfig.xml")
|
||||
.addResourceLocations(staticPath, "classpath:/static/")
|
||||
.setCacheControl(CacheControl.noStore())
|
||||
.resourceChain(true)
|
||||
.addResolver(new EncodedResourceResolver());
|
||||
|
||||
// 2. Vite fingerprinted assets (immutable)
|
||||
// These already have content hashes in filenames (e.g. index-ChAS4tCC.js)
|
||||
registry.addResourceHandler("/assets/**")
|
||||
.addResourceLocations(
|
||||
"file:"
|
||||
+ stirling.software.common.configuration.InstallationPathConfig
|
||||
.getStaticPath()
|
||||
+ "assets/",
|
||||
"classpath:/static/assets/")
|
||||
.setCacheControl(CacheControl.maxAge(365, TimeUnit.DAYS).cachePublic());
|
||||
.addResourceLocations(staticPath + "assets/", "classpath:/static/assets/")
|
||||
.setCacheControl(IMMUTABLE_ONE_YEAR)
|
||||
.resourceChain(true)
|
||||
.addResolver(new EncodedResourceResolver());
|
||||
|
||||
// Don't cache index.html - it needs to be fresh to reference latest hashed assets
|
||||
// Note: index.html is handled by ReactRoutingController for dynamic processing
|
||||
registry.addResourceHandler("/index.html")
|
||||
// 3. Media and fonts (immutable)
|
||||
registry.addResourceHandler("/images/**", "/fonts/**")
|
||||
.addResourceLocations(
|
||||
"file:"
|
||||
+ stirling.software.common.configuration.InstallationPathConfig
|
||||
.getStaticPath(),
|
||||
"classpath:/static/")
|
||||
.setCacheControl(CacheControl.noCache().mustRevalidate());
|
||||
staticPath + "images/",
|
||||
"classpath:/static/images/",
|
||||
staticPath + "fonts/",
|
||||
"classpath:/static/fonts/")
|
||||
.setCacheControl(IMMUTABLE_ONE_YEAR)
|
||||
.resourceChain(true)
|
||||
.addResolver(new EncodedResourceResolver());
|
||||
|
||||
// Handle all other static resources (js, css, images, fonts, etc.)
|
||||
// Check customFiles/static first for user overrides
|
||||
// 4. Branding and stable non-fingerprinted assets (1 day + SWR)
|
||||
// Use stale-while-revalidate to improve perceived performance.
|
||||
registry.addResourceHandler(
|
||||
"/favicon.*",
|
||||
"/apple-touch-icon.png",
|
||||
"/android-chrome-*.png",
|
||||
"/mstile-*.png",
|
||||
"/safari-pinned-tab.svg",
|
||||
"/icons/**",
|
||||
"/modern-logo/**",
|
||||
"/classic-logo/**",
|
||||
"/robots.txt",
|
||||
"/3rdPartyLicenses.json",
|
||||
"/pdfjs/**",
|
||||
"/pdfjs-legacy/**",
|
||||
"/pdfium/**",
|
||||
"/locales/**",
|
||||
"/css/**",
|
||||
"/js/**",
|
||||
"/vendor/**",
|
||||
"/samples/**",
|
||||
"/og_images/**",
|
||||
"/Login/**",
|
||||
"/manifest-classic.json")
|
||||
.addResourceLocations(
|
||||
staticPath,
|
||||
"classpath:/static/",
|
||||
staticPath + "pdfjs/",
|
||||
"classpath:/static/pdfjs/",
|
||||
staticPath + "pdfjs-legacy/",
|
||||
"classpath:/static/pdfjs-legacy/",
|
||||
staticPath + "pdfium/",
|
||||
"classpath:/static/pdfium/",
|
||||
staticPath + "locales/",
|
||||
"classpath:/static/locales/",
|
||||
staticPath + "css/",
|
||||
"classpath:/static/css/",
|
||||
staticPath + "js/",
|
||||
"classpath:/static/js/",
|
||||
staticPath + "vendor/",
|
||||
"classpath:/static/vendor/",
|
||||
staticPath + "samples/",
|
||||
"classpath:/static/samples/",
|
||||
staticPath + "og_images/",
|
||||
"classpath:/static/og_images/",
|
||||
staticPath + "Login/",
|
||||
"classpath:/static/Login/",
|
||||
staticPath + "icons/",
|
||||
"classpath:/static/icons/",
|
||||
staticPath + "modern-logo/",
|
||||
"classpath:/static/modern-logo/",
|
||||
staticPath + "classic-logo/",
|
||||
"classpath:/static/classic-logo/")
|
||||
.setCacheControl(
|
||||
CacheControl.maxAge(Duration.ofDays(1))
|
||||
.cachePublic()
|
||||
.staleWhileRevalidate(Duration.ofDays(7)))
|
||||
.resourceChain(true)
|
||||
.addResolver(new EncodedResourceResolver());
|
||||
|
||||
// 5. Catch-all (SPA fallback)
|
||||
// Must check with server to ensure index.html is always fresh.
|
||||
registry.addResourceHandler("/**")
|
||||
.addResourceLocations(
|
||||
"file:"
|
||||
+ stirling.software.common.configuration.InstallationPathConfig
|
||||
.getStaticPath(),
|
||||
"classpath:/static/")
|
||||
.setCacheControl(CacheControl.maxAge(1, TimeUnit.HOURS));
|
||||
.addResourceLocations(staticPath, "classpath:/static/")
|
||||
.setCacheControl(NO_CACHE)
|
||||
.resourceChain(true)
|
||||
.addResolver(new EncodedResourceResolver());
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -115,9 +192,8 @@ public class WebMvcConfig implements WebMvcConfigurer {
|
||||
applicationProperties.getSystem().getCorsAllowedOrigins());
|
||||
|
||||
// Combine user-configured origins with Tauri origins
|
||||
java.util.List<String> allOrigins =
|
||||
new java.util.ArrayList<>(
|
||||
applicationProperties.getSystem().getCorsAllowedOrigins());
|
||||
List<String> allOrigins =
|
||||
new ArrayList<>(applicationProperties.getSystem().getCorsAllowedOrigins());
|
||||
|
||||
// Always include Tauri origins for desktop app compatibility
|
||||
// Tauri v1 uses tauri://localhost, v2 uses http(s)://tauri.localhost
|
||||
@@ -158,7 +234,8 @@ public class WebMvcConfig implements WebMvcConfigurer {
|
||||
} else {
|
||||
// Default to allowing all origins when nothing is configured
|
||||
logger.debug(
|
||||
"No CORS allowed origins configured in settings.yml (system.corsAllowedOrigins); WebMvcConfig allowing all origins.");
|
||||
"No CORS allowed origins configured in settings.yml"
|
||||
+ " (system.corsAllowedOrigins); WebMvcConfig allowing all origins.");
|
||||
registry.addMapping("/**")
|
||||
.allowedOriginPatterns("*")
|
||||
.allowedMethods("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS")
|
||||
|
||||
+9
-3
@@ -32,13 +32,16 @@ import stirling.software.SPDF.model.json.PdfJsonTextElement;
|
||||
import stirling.software.SPDF.service.PdfJsonConversionService;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.GeneralApi;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.api.general.EditTextOperation;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.common.util.propertyeditor.StringToArrayListPropertyEditor;
|
||||
import stirling.software.common.util.propertyeditor.JsonListPropertyEditor;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
|
||||
/**
|
||||
* Find/replace text editing for PDFs. Round-trips through {@link PdfJsonConversionService}: the
|
||||
@@ -72,10 +75,13 @@ public class EditTextController {
|
||||
binder.registerCustomEditor(
|
||||
List.class,
|
||||
"edits",
|
||||
new StringToArrayListPropertyEditor<>(EditTextOperation.class));
|
||||
new JsonListPropertyEditor<>(new TypeReference<List<EditTextOperation>>() {}));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(consumes = "multipart/form-data", value = "/edit-text")
|
||||
@AutoJobPostMapping(
|
||||
consumes = "multipart/form-data",
|
||||
value = "/edit-text",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@StandardPdfResponse
|
||||
@Operation(
|
||||
summary = "Edit text in a PDF via find and replace",
|
||||
|
||||
+11
-42
@@ -40,7 +40,8 @@ public class ScalePagesController {
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
private static PDRectangle getTargetSize(String targetPDRectangle, PDDocument sourceDocument) {
|
||||
private static PDRectangle getTargetSize(
|
||||
String targetPDRectangle, String orientation, PDDocument sourceDocument) {
|
||||
if ("KEEP".equals(targetPDRectangle)) {
|
||||
if (sourceDocument.getNumberOfPages() == 0) {
|
||||
throw ExceptionUtils.createInvalidPageSizeException("KEEP");
|
||||
@@ -57,18 +58,19 @@ public class ScalePagesController {
|
||||
}
|
||||
|
||||
Map<String, PDRectangle> sizeMap = getSizeMap();
|
||||
|
||||
if (sizeMap.containsKey(targetPDRectangle)) {
|
||||
return sizeMap.get(targetPDRectangle);
|
||||
PDRectangle base = sizeMap.get(targetPDRectangle);
|
||||
if (base == null) {
|
||||
throw ExceptionUtils.createInvalidPageSizeException(targetPDRectangle);
|
||||
}
|
||||
|
||||
throw ExceptionUtils.createInvalidPageSizeException(targetPDRectangle);
|
||||
if ("LANDSCAPE".equalsIgnoreCase(orientation)) {
|
||||
return new PDRectangle(base.getHeight(), base.getWidth());
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
private static Map<String, PDRectangle> getSizeMap() {
|
||||
Map<String, PDRectangle> sizeMap = new HashMap<>();
|
||||
|
||||
// Portrait sizes (A0-A6)
|
||||
sizeMap.put("A0", PDRectangle.A0);
|
||||
sizeMap.put("A1", PDRectangle.A1);
|
||||
sizeMap.put("A2", PDRectangle.A2);
|
||||
@@ -76,42 +78,8 @@ public class ScalePagesController {
|
||||
sizeMap.put("A4", PDRectangle.A4);
|
||||
sizeMap.put("A5", PDRectangle.A5);
|
||||
sizeMap.put("A6", PDRectangle.A6);
|
||||
|
||||
// Landscape sizes (A0-A6)
|
||||
sizeMap.put(
|
||||
"A0_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.A0.getHeight(), PDRectangle.A0.getWidth()));
|
||||
sizeMap.put(
|
||||
"A1_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.A1.getHeight(), PDRectangle.A1.getWidth()));
|
||||
sizeMap.put(
|
||||
"A2_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.A2.getHeight(), PDRectangle.A2.getWidth()));
|
||||
sizeMap.put(
|
||||
"A3_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.A3.getHeight(), PDRectangle.A3.getWidth()));
|
||||
sizeMap.put(
|
||||
"A4_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.A4.getHeight(), PDRectangle.A4.getWidth()));
|
||||
sizeMap.put(
|
||||
"A5_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.A5.getHeight(), PDRectangle.A5.getWidth()));
|
||||
sizeMap.put(
|
||||
"A6_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.A6.getHeight(), PDRectangle.A6.getWidth()));
|
||||
|
||||
// Portrait US sizes
|
||||
sizeMap.put("LETTER", PDRectangle.LETTER);
|
||||
sizeMap.put("LEGAL", PDRectangle.LEGAL);
|
||||
|
||||
// Landscape US sizes
|
||||
sizeMap.put(
|
||||
"LETTER_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.LETTER.getHeight(), PDRectangle.LETTER.getWidth()));
|
||||
sizeMap.put(
|
||||
"LEGAL_LANDSCAPE",
|
||||
new PDRectangle(PDRectangle.LEGAL.getHeight(), PDRectangle.LEGAL.getWidth()));
|
||||
|
||||
return sizeMap;
|
||||
}
|
||||
|
||||
@@ -128,13 +96,14 @@ public class ScalePagesController {
|
||||
throws IOException {
|
||||
MultipartFile file = request.getFileInput();
|
||||
String targetPDRectangle = request.getPageSize();
|
||||
String orientation = request.getOrientation();
|
||||
float scaleFactor = request.getScaleFactor();
|
||||
|
||||
try (PDDocument sourceDocument = pdfDocumentFactory.load(file);
|
||||
PDDocument outputDocument =
|
||||
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument)) {
|
||||
|
||||
PDRectangle targetSize = getTargetSize(targetPDRectangle, sourceDocument);
|
||||
PDRectangle targetSize = getTargetSize(targetPDRectangle, orientation, sourceDocument);
|
||||
|
||||
// Create LayerUtility once outside the loop for better performance
|
||||
LayerUtility layerUtility = new LayerUtility(outputDocument);
|
||||
|
||||
+16
-4
@@ -275,7 +275,10 @@ public class ConvertImgPDFController {
|
||||
GeneralUtils.generateFilename(file[0].getOriginalFilename(), "_converted.pdf"));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/cbz/pdf")
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/cbz/pdf",
|
||||
resourceWeight = ResourceWeight.MEDIUM_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Convert CBZ comic book archive to PDF",
|
||||
description =
|
||||
@@ -301,7 +304,10 @@ public class ConvertImgPDFController {
|
||||
return WebResponseUtils.pdfFileToWebResponse(pdfFile, filename);
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/pdf/cbz")
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/pdf/cbz",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Convert PDF to CBZ comic book archive",
|
||||
description =
|
||||
@@ -324,7 +330,10 @@ public class ConvertImgPDFController {
|
||||
return WebResponseUtils.zipFileToWebResponse(cbzFile, filename);
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/cbr/pdf")
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/cbr/pdf",
|
||||
resourceWeight = ResourceWeight.MEDIUM_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Convert CBR comic book archive to PDF",
|
||||
description =
|
||||
@@ -350,7 +359,10 @@ public class ConvertImgPDFController {
|
||||
return WebResponseUtils.bytesToWebResponse(pdfBytes, filename);
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/pdf/cbr")
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/pdf/cbr",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Convert PDF to CBR comic book archive",
|
||||
description =
|
||||
|
||||
+10
-4
@@ -141,7 +141,8 @@ public class AttachmentController {
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/extract-attachments")
|
||||
value = "/extract-attachments",
|
||||
resourceWeight = ResourceWeight.SMALL_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Extract attachments from PDF",
|
||||
description =
|
||||
@@ -176,7 +177,10 @@ public class AttachmentController {
|
||||
}
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/list-attachments")
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/list-attachments",
|
||||
resourceWeight = ResourceWeight.SMALL_WEIGHT)
|
||||
@Operation(
|
||||
summary = "List attachments in PDF",
|
||||
description =
|
||||
@@ -193,7 +197,8 @@ public class AttachmentController {
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/rename-attachment")
|
||||
value = "/rename-attachment",
|
||||
resourceWeight = ResourceWeight.SMALL_WEIGHT)
|
||||
@StandardPdfResponse
|
||||
@Operation(
|
||||
summary = "Rename attachment in PDF",
|
||||
@@ -228,7 +233,8 @@ public class AttachmentController {
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/delete-attachment")
|
||||
value = "/delete-attachment",
|
||||
resourceWeight = ResourceWeight.SMALL_WEIGHT)
|
||||
@StandardPdfResponse
|
||||
@Operation(
|
||||
summary = "Delete attachment from PDF",
|
||||
|
||||
+65
-13
@@ -12,6 +12,8 @@ import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration;
|
||||
@@ -91,6 +93,63 @@ public class ConfigController {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the frontend URL the client should advertise to phones / share-link recipients.
|
||||
* Priority: explicit system.frontendUrl, then the Host the user is already using to reach this
|
||||
* server (works for Docker, reverse proxies, and bare-metal LANs), then a detected site-local
|
||||
* IPv4, then empty.
|
||||
*/
|
||||
// visible for testing
|
||||
String resolveFrontendUrl(HttpServletRequest request, AppConfig appConfig) {
|
||||
String configured = applicationProperties.getSystem().getFrontendUrl();
|
||||
if (configured != null && !configured.isBlank()) {
|
||||
return configured;
|
||||
}
|
||||
if (request != null) {
|
||||
String host = request.getServerName();
|
||||
if (host != null && !host.isBlank() && !isLoopbackHost(host)) {
|
||||
String scheme = request.getScheme();
|
||||
int port = request.getServerPort();
|
||||
boolean defaultPort =
|
||||
("http".equals(scheme) && port == 80)
|
||||
|| ("https".equals(scheme) && port == 443);
|
||||
return defaultPort ? scheme + "://" + host : scheme + "://" + host + ":" + port;
|
||||
}
|
||||
}
|
||||
String localIp = GeneralUtils.getLocalNetworkIp();
|
||||
if (localIp != null) {
|
||||
String scheme = appConfig.getBackendUrl().startsWith("https") ? "https" : "http";
|
||||
return scheme + "://" + localIp + ":" + resolveEffectiveServerPort(appConfig);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* The port the embedded server is actually listening on. With {@code server.port=0} (an
|
||||
* ephemeral port, which the desktop bundle uses to dodge port clashes) the configured value
|
||||
* stays {@code "0"} while Spring publishes the real bound port as {@code local.server.port}
|
||||
* once the server is up. Advertised URLs (the mobile-scanner QR, share links) must carry the
|
||||
* real port - a literal {@code :0} is unreachable and browsers reject it as ERR_UNSAFE_PORT.
|
||||
*/
|
||||
// visible for testing
|
||||
String resolveEffectiveServerPort(AppConfig appConfig) {
|
||||
String configured = appConfig.getServerPort();
|
||||
if (configured == null || "0".equals(configured.trim())) {
|
||||
String actual = applicationContext.getEnvironment().getProperty("local.server.port");
|
||||
if (actual != null && !actual.isBlank()) {
|
||||
return actual;
|
||||
}
|
||||
}
|
||||
return configured;
|
||||
}
|
||||
|
||||
private static boolean isLoopbackHost(String host) {
|
||||
return "localhost".equalsIgnoreCase(host)
|
||||
|| "127.0.0.1".equals(host)
|
||||
|| "::1".equals(host)
|
||||
|| "0:0:0:0:0:0:0:1".equals(host);
|
||||
}
|
||||
|
||||
/** Check if running Enterprise edition dynamically. */
|
||||
private Boolean isRunningEE() {
|
||||
// Use LicenseService for fresh license status if available
|
||||
@@ -107,7 +166,7 @@ public class ConfigController {
|
||||
}
|
||||
|
||||
@GetMapping("/app-config")
|
||||
public ResponseEntity<Map<String, Object>> getAppConfig() {
|
||||
public ResponseEntity<Map<String, Object>> getAppConfig(HttpServletRequest request) {
|
||||
Map<String, Object> configData = new HashMap<>();
|
||||
|
||||
try {
|
||||
@@ -121,20 +180,10 @@ public class ConfigController {
|
||||
// Note: Frontend expects "baseUrl" field name for compatibility
|
||||
configData.put("baseUrl", appConfig.getBackendUrl());
|
||||
configData.put("contextPath", appConfig.getContextPath());
|
||||
configData.put("serverPort", appConfig.getServerPort());
|
||||
configData.put("serverPort", resolveEffectiveServerPort(appConfig));
|
||||
|
||||
String frontendUrl = applicationProperties.getSystem().getFrontendUrl();
|
||||
if ((frontendUrl == null || frontendUrl.isBlank())
|
||||
&& Boolean.parseBoolean(
|
||||
System.getProperty("STIRLING_PDF_TAURI_MODE", "false"))) {
|
||||
String localIp = GeneralUtils.getLocalNetworkIp();
|
||||
if (localIp != null) {
|
||||
String scheme =
|
||||
appConfig.getBackendUrl().startsWith("https") ? "https" : "http";
|
||||
frontendUrl = scheme + "://" + localIp + ":" + appConfig.getServerPort();
|
||||
}
|
||||
}
|
||||
configData.put("frontendUrl", frontendUrl != null ? frontendUrl : "");
|
||||
configData.put("frontendUrl", resolveFrontendUrl(request, appConfig));
|
||||
|
||||
// Add mobile scanner settings
|
||||
configData.put(
|
||||
@@ -277,6 +326,9 @@ public class ConfigController {
|
||||
// Premium/Enterprise settings
|
||||
configData.put("premiumEnabled", applicationProperties.getPremium().isEnabled());
|
||||
|
||||
// AI Engine settings
|
||||
configData.put("aiEngineEnabled", applicationProperties.getAiEngine().isEnabled());
|
||||
|
||||
// Timestamp TSA settings — single source of truth for presets + admin URLs
|
||||
ApplicationProperties.Security.Timestamp tsConfig =
|
||||
applicationProperties.getSecurity().getTimestamp();
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* PDFTextStripper subclass that collects all text positions and groups them into line-level
|
||||
* bounding boxes.
|
||||
*
|
||||
* <p>Two outputs are maintained in parallel:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link #getLineBoxes()} returns {@code [x1, pdfYbottom, x2, pdfYtop]} in PDF user-space
|
||||
* (origin bottom-left, Y up). This is what existing callers expect.
|
||||
* <li>{@link #getScreenLineBoxes()} returns {@code [x1, screenYtop, x2, screenYbottom]} computed
|
||||
* directly from glyph positions without a PDF↔screen round-trip — used by column-aware
|
||||
* redaction where ulp-level drift in the round-trip caused false rejects against anchors.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Lines are flushed not only on Y jumps but also on large X gaps within the same Y row. That way
|
||||
* left-column glyphs and right-column glyphs that happen to share a baseline (common in IEEE
|
||||
* conference templates) get emitted as two distinct line boxes instead of one wide merged box.
|
||||
*/
|
||||
final class AllTextLineExtractor extends PDFTextStripper {
|
||||
|
||||
/** Min vertical jump (screen Y) before the next glyph is treated as a new line. */
|
||||
private static final float LINE_Y_TOLERANCE = 3.0f;
|
||||
|
||||
/**
|
||||
* Min horizontal gap (screen X) between consecutive glyphs on the same Y that indicates a
|
||||
* column boundary. Chosen large enough to not split normal inter-word spacing (~6–10pt for 11pt
|
||||
* text) but small enough to catch standard column gutters (typically ≥15pt).
|
||||
*/
|
||||
private static final float COLUMN_GAP_X = 14f;
|
||||
|
||||
private final float pageHeight;
|
||||
private final List<float[]> lineBoxes = new ArrayList<>();
|
||||
private final List<float[]> screenLineBoxes = new ArrayList<>();
|
||||
|
||||
private final List<TextPosition> currentLine = new ArrayList<>();
|
||||
private float lastScreenY = Float.NaN;
|
||||
private float lastGlyphRight = Float.NaN;
|
||||
|
||||
AllTextLineExtractor(int pageNumber, float pageHeight) throws IOException {
|
||||
this.pageHeight = pageHeight;
|
||||
setStartPage(pageNumber);
|
||||
setEndPage(pageNumber);
|
||||
setSortByPosition(true);
|
||||
}
|
||||
|
||||
List<float[]> getLineBoxes() {
|
||||
return lineBoxes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns line boxes as {@code [x1, screenYtop, x2, screenYbottom]}. {@code screenYtop} is the
|
||||
* minimum {@code TextPosition.getY() - getHeight()} on the line and {@code screenYbottom} is
|
||||
* the maximum {@code getY()} (the line's baseline). These values come straight from PDFBox
|
||||
* without going through {@code pageHeight - …}, so they're stable for ulp-sensitive comparisons
|
||||
* against anchor screen Ys.
|
||||
*/
|
||||
List<float[]> getScreenLineBoxes() {
|
||||
return screenLineBoxes;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> positions) {
|
||||
for (TextPosition tp : positions) {
|
||||
// Skip whitespace-only positions (spaces, newline markers, indent characters).
|
||||
// These have a TextPosition but no visible glyph; including them causes
|
||||
// space-only "lines" to produce degenerate segments that appear as thin
|
||||
// black bars after redaction.
|
||||
String unicode = tp.getUnicode();
|
||||
if (unicode == null || unicode.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
float screenY = tp.getY();
|
||||
float screenX = tp.getX();
|
||||
boolean yJump =
|
||||
!Float.isNaN(lastScreenY) && Math.abs(screenY - lastScreenY) > LINE_Y_TOLERANCE;
|
||||
boolean xJump =
|
||||
!Float.isNaN(lastGlyphRight) && (screenX - lastGlyphRight) > COLUMN_GAP_X;
|
||||
if (yJump || xJump) {
|
||||
flushLine();
|
||||
}
|
||||
lastScreenY = screenY;
|
||||
lastGlyphRight = screenX + tp.getWidth();
|
||||
currentLine.add(tp);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void endPage(PDPage page) throws IOException {
|
||||
flushLine();
|
||||
super.endPage(page);
|
||||
}
|
||||
|
||||
private void flushLine() {
|
||||
if (currentLine.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
float minX = Float.MAX_VALUE, maxX = -Float.MAX_VALUE;
|
||||
float minScreenY = Float.MAX_VALUE, maxScreenY = -Float.MAX_VALUE;
|
||||
for (TextPosition tp : currentLine) {
|
||||
minX = Math.min(minX, tp.getX());
|
||||
maxX = Math.max(maxX, tp.getX() + tp.getWidth());
|
||||
minScreenY = Math.min(minScreenY, tp.getY() - tp.getHeight());
|
||||
maxScreenY = Math.max(maxScreenY, tp.getY());
|
||||
}
|
||||
emitSegment(minX, maxX, minScreenY, maxScreenY);
|
||||
currentLine.clear();
|
||||
lastScreenY = Float.NaN;
|
||||
lastGlyphRight = Float.NaN;
|
||||
}
|
||||
|
||||
private void emitSegment(float minX, float maxX, float minScreenY, float maxScreenY) {
|
||||
float pdfY1 = pageHeight - maxScreenY; // bottom in PDF coords
|
||||
float pdfY2 = pageHeight - minScreenY; // top in PDF coords
|
||||
lineBoxes.add(new float[] {minX, pdfY1, maxX, pdfY2});
|
||||
screenLineBoxes.add(new float[] {minX, minScreenY, maxX, maxScreenY});
|
||||
}
|
||||
}
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
class ManualRedactionService {
|
||||
|
||||
private static final float DEFAULT_TEXT_PADDING_MULTIPLIER = 0.6f;
|
||||
private static final float REDACTION_WIDTH_REDUCTION_FACTOR = 0.9f;
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Area and page redaction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactAreas(List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
if (redactionAreas == null || redactionAreas.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<Integer, List<RedactionArea>> redactionsByPage = new HashMap<>();
|
||||
|
||||
for (RedactionArea redactionArea : redactionAreas) {
|
||||
if (redactionArea.getPage() == null
|
||||
|| redactionArea.getPage() <= 0
|
||||
|| redactionArea.getHeight() == null
|
||||
|| redactionArea.getHeight() <= 0.0D
|
||||
|| redactionArea.getWidth() == null
|
||||
|| redactionArea.getWidth() <= 0.0D) {
|
||||
continue;
|
||||
}
|
||||
|
||||
redactionsByPage
|
||||
.computeIfAbsent(redactionArea.getPage(), k -> new ArrayList<>())
|
||||
.add(redactionArea);
|
||||
}
|
||||
|
||||
for (Map.Entry<Integer, List<RedactionArea>> entry : redactionsByPage.entrySet()) {
|
||||
Integer pageNumber = entry.getKey();
|
||||
List<RedactionArea> areasForPage = entry.getValue();
|
||||
|
||||
if (pageNumber > allPages.getCount()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
PDPage page = allPages.get(pageNumber - 1);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
for (RedactionArea redactionArea : areasForPage) {
|
||||
Color redactColor = decodeOrDefault(redactionArea.getColor());
|
||||
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
float x = redactionArea.getX().floatValue();
|
||||
float y = redactionArea.getY().floatValue();
|
||||
float width = redactionArea.getWidth().floatValue();
|
||||
float height = redactionArea.getHeight().floatValue();
|
||||
|
||||
float pdfY = page.getBBox().getHeight() - y - height;
|
||||
|
||||
contentStream.addRect(x, pdfY, width, height);
|
||||
contentStream.fill();
|
||||
}
|
||||
contentStream.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void redactPages(ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Color redactColor = decodeOrDefault(request.getPageRedactionColor());
|
||||
List<Integer> pageNumbers = getPageNumbers(request, allPages.getCount());
|
||||
|
||||
for (Integer pageNumber : pageNumbers) {
|
||||
PDPage page = allPages.get(pageNumber);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
PDRectangle box = page.getBBox();
|
||||
contentStream.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
contentStream.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Overlay drawing
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactFoundText(
|
||||
PDDocument document,
|
||||
List<PDFText> blocks,
|
||||
float customPadding,
|
||||
Color redactColor,
|
||||
boolean isTextRemovalMode)
|
||||
throws IOException {
|
||||
|
||||
var allPages = document.getDocumentCatalog().getPages();
|
||||
|
||||
Map<Integer, List<PDFText>> blocksByPage = new HashMap<>();
|
||||
for (PDFText block : blocks) {
|
||||
blocksByPage.computeIfAbsent(block.getPageIndex(), k -> new ArrayList<>()).add(block);
|
||||
}
|
||||
|
||||
for (Map.Entry<Integer, List<PDFText>> entry : blocksByPage.entrySet()) {
|
||||
Integer pageIndex = entry.getKey();
|
||||
List<PDFText> pageBlocks = entry.getValue();
|
||||
|
||||
if (pageIndex >= allPages.getCount()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
var page = allPages.get(pageIndex);
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
|
||||
try {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
PDRectangle pageBox = page.getBBox();
|
||||
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
(block.getY2() - block.getY1()) * DEFAULT_TEXT_PADDING_MULTIPLIER
|
||||
+ customPadding;
|
||||
|
||||
float originalWidth = block.getX2() - block.getX1();
|
||||
float boxWidth;
|
||||
float boxX;
|
||||
|
||||
if (isTextRemovalMode) {
|
||||
boxWidth = originalWidth * REDACTION_WIDTH_REDUCTION_FACTOR;
|
||||
float widthReduction = originalWidth - boxWidth;
|
||||
boxX = block.getX1() + (widthReduction / 2);
|
||||
} else {
|
||||
boxWidth = originalWidth;
|
||||
boxX = block.getX1();
|
||||
}
|
||||
|
||||
contentStream.addRect(
|
||||
boxX,
|
||||
pageBox.getHeight() - block.getY2() - padding,
|
||||
boxWidth,
|
||||
block.getY2() - block.getY1() + 2 * padding);
|
||||
}
|
||||
|
||||
contentStream.fill();
|
||||
|
||||
} finally {
|
||||
contentStream.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent
|
||||
// users from hovering over redacted URLs and seeing the underlying destination.
|
||||
try {
|
||||
float pageH = page.getBBox().getHeight();
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
for (PDAnnotation ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
boolean overlaps = false;
|
||||
if (ar != null) {
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
(block.getY2() - block.getY1())
|
||||
* DEFAULT_TEXT_PADDING_MULTIPLIER
|
||||
+ customPadding;
|
||||
float bx1 = block.getX1();
|
||||
float bx2 = block.getX2();
|
||||
float by1 = pageH - block.getY2() - padding;
|
||||
float by2 = pageH - block.getY1() + padding;
|
||||
if (ar.getLowerLeftX() < bx2
|
||||
&& ar.getUpperRightX() > bx1
|
||||
&& ar.getLowerLeftY() < by2
|
||||
&& ar.getUpperRightY() > by1) {
|
||||
overlaps = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!overlaps) {
|
||||
kept.add(ann);
|
||||
}
|
||||
}
|
||||
page.setAnnotations(kept);
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"[redact] could not remove annotations on page {}: {}",
|
||||
pageIndex,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void redactImageBoxes(PDDocument document, List<float[]> imageBoxes, Color color)
|
||||
throws IOException {
|
||||
Map<Integer, List<float[]>> byPage = new HashMap<>();
|
||||
for (float[] box : imageBoxes) {
|
||||
byPage.computeIfAbsent((int) box[0], k -> new ArrayList<>()).add(box);
|
||||
}
|
||||
PDPageTree pages = document.getDocumentCatalog().getPages();
|
||||
for (Map.Entry<Integer, List<float[]>> entry : byPage.entrySet()) {
|
||||
int pageIdx = entry.getKey();
|
||||
if (pageIdx < 0 || pageIdx >= pages.getCount()) {
|
||||
log.warn("[redact/execute] image box references out-of-range page {}", pageIdx);
|
||||
continue;
|
||||
}
|
||||
PDPage page = pages.get(pageIdx);
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(color);
|
||||
for (float[] box : entry.getValue()) {
|
||||
float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4];
|
||||
cs.addRect(x1, y1, x2 - x1, y2 - y1);
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Page element extraction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns bounding boxes for every text line and image on {@code page} in PDF user-space
|
||||
* coordinates: {@code [x1, y1, x2, y2]} (origin bottom-left, Y increases upward).
|
||||
*/
|
||||
List<float[]> extractPageElementBoxes(PDDocument document, PDPage page, int pageIndex)
|
||||
throws IOException {
|
||||
List<float[]> boxes = new ArrayList<>();
|
||||
|
||||
AllTextLineExtractor textExtractor =
|
||||
new AllTextLineExtractor(pageIndex + 1, page.getBBox().getHeight());
|
||||
textExtractor.getText(document);
|
||||
boxes.addAll(textExtractor.getLineBoxes());
|
||||
|
||||
PageImageLocator imgLocator = new PageImageLocator(page, pageIndex);
|
||||
imgLocator.processPage(page);
|
||||
for (PageImageLocator.ImageBox imgBox : imgLocator.getImageBoxes()) {
|
||||
boxes.add(new float[] {imgBox.x1(), imgBox.y1(), imgBox.x2(), imgBox.y2()});
|
||||
}
|
||||
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Finalization
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> allFoundTextsByPage,
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode)
|
||||
throws IOException {
|
||||
|
||||
List<PDFText> allFoundTexts = new ArrayList<>();
|
||||
for (List<PDFText> pageTexts : allFoundTextsByPage.values()) {
|
||||
allFoundTexts.addAll(pageTexts);
|
||||
}
|
||||
|
||||
if (!allFoundTexts.isEmpty()) {
|
||||
Color redactColor = decodeOrDefault(colorString);
|
||||
redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode);
|
||||
cleanDocumentMetadata(document);
|
||||
}
|
||||
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
cleanDocumentMetadata(convertedPdf);
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
convertedPdf.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized (image mode): {} pages ➜ {} KB",
|
||||
convertedPdf.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
}
|
||||
}
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
document.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized: {} pages ➜ {} KB",
|
||||
document.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
}
|
||||
|
||||
private void cleanDocumentMetadata(PDDocument document) {
|
||||
try {
|
||||
var documentInfo = document.getDocumentInformation();
|
||||
if (documentInfo != null) {
|
||||
documentInfo.setAuthor(null);
|
||||
documentInfo.setSubject(null);
|
||||
documentInfo.setKeywords(null);
|
||||
documentInfo.setModificationDate(java.util.Calendar.getInstance());
|
||||
log.debug("Cleaned document metadata for security");
|
||||
}
|
||||
|
||||
if (document.getDocumentCatalog() != null) {
|
||||
try {
|
||||
document.getDocumentCatalog().setMetadata(null);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not clear XMP metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to clean document metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Utilities
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
static Color decodeOrDefault(String hex) {
|
||||
if (hex == null) {
|
||||
return Color.BLACK;
|
||||
}
|
||||
|
||||
String colorString = hex.startsWith("#") ? hex : "#" + hex;
|
||||
|
||||
try {
|
||||
return Color.decode(colorString);
|
||||
} catch (NumberFormatException e) {
|
||||
return Color.BLACK;
|
||||
}
|
||||
}
|
||||
|
||||
private List<Integer> getPageNumbers(ManualRedactPdfRequest request, int pagesCount) {
|
||||
String pageNumbersInput = request.getPageNumbers();
|
||||
String[] parsedPageNumbers =
|
||||
pageNumbersInput != null ? pageNumbersInput.split(",") : new String[0];
|
||||
List<Integer> pageNumbers =
|
||||
GeneralUtils.parsePageList(parsedPageNumbers, pagesCount, false);
|
||||
Collections.sort(pageNumbers);
|
||||
return pageNumbers;
|
||||
}
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
|
||||
/**
|
||||
* Scans a PDF document once and matches all provided patterns in a single pass, collecting
|
||||
* bounding-box positions for every match. Use in place of creating one {@code TextFinder} per
|
||||
* search term to avoid O(n) full-document scans.
|
||||
*/
|
||||
@Slf4j
|
||||
final class MultiPatternTextFinder extends PDFTextStripper {
|
||||
|
||||
private static final long REGEX_MATCH_TIMEOUT_SECONDS = 30;
|
||||
private static final ExecutorService REGEX_EXECUTOR =
|
||||
Executors.newVirtualThreadPerTaskExecutor();
|
||||
|
||||
private final List<Pattern> patterns;
|
||||
private final Map<Integer, List<PDFText>> foundTextsByPage = new HashMap<>();
|
||||
|
||||
private final List<TextPosition> pageTextPositions = new ArrayList<>();
|
||||
private final StringBuilder pageTextBuilder = new StringBuilder();
|
||||
|
||||
MultiPatternTextFinder(List<Pattern> patterns) throws IOException {
|
||||
this.patterns = patterns;
|
||||
this.setWordSeparator(" ");
|
||||
this.setLineSeparator("\n");
|
||||
}
|
||||
|
||||
Map<Integer, List<PDFText>> getFoundTextsByPage() {
|
||||
return foundTextsByPage;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startPage(PDPage page) throws IOException {
|
||||
super.startPage(page);
|
||||
pageTextPositions.clear();
|
||||
pageTextBuilder.setLength(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> textPositions) {
|
||||
pageTextBuilder.append(text);
|
||||
pageTextPositions.addAll(textPositions);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeWordSeparator() {
|
||||
pageTextBuilder.append(getWordSeparator());
|
||||
pageTextPositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeLineSeparator() {
|
||||
pageTextBuilder.append(getLineSeparator());
|
||||
pageTextPositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void endPage(PDPage page) throws IOException {
|
||||
String text = pageTextBuilder.toString();
|
||||
if (!text.isEmpty()) {
|
||||
int pageIndex = getCurrentPageNo() - 1;
|
||||
for (Pattern pattern : patterns) {
|
||||
Matcher matcher = pattern.matcher(text);
|
||||
while (safeFind(matcher)) {
|
||||
PDFText pdfText = resolveMatchPosition(matcher, pageIndex);
|
||||
if (pdfText != null) {
|
||||
foundTextsByPage
|
||||
.computeIfAbsent(pageIndex, k -> new ArrayList<>())
|
||||
.add(pdfText);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
super.endPage(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a single {@code matcher.find()} call with a {@value #REGEX_MATCH_TIMEOUT_SECONDS}
|
||||
* second timeout. Prevents pathological regex backtracking from blocking the request
|
||||
* indefinitely; per-match timeout so fast legitimate scans are unaffected.
|
||||
*/
|
||||
private static boolean safeFind(Matcher matcher) throws IOException {
|
||||
Future<Boolean> future =
|
||||
REGEX_EXECUTOR.submit((java.util.concurrent.Callable<Boolean>) matcher::find);
|
||||
try {
|
||||
return future.get(REGEX_MATCH_TIMEOUT_SECONDS, TimeUnit.SECONDS);
|
||||
} catch (TimeoutException e) {
|
||||
future.cancel(true);
|
||||
throw new IOException(
|
||||
"Regex match timed out after "
|
||||
+ REGEX_MATCH_TIMEOUT_SECONDS
|
||||
+ "s — pattern may cause catastrophic backtracking");
|
||||
} catch (InterruptedException e) {
|
||||
future.cancel(true);
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("Regex match interrupted", e);
|
||||
} catch (ExecutionException e) {
|
||||
Throwable cause = e.getCause();
|
||||
if (cause instanceof IOException ioEx) throw ioEx;
|
||||
throw new IOException("Regex match failed: " + cause.getMessage(), cause);
|
||||
}
|
||||
}
|
||||
|
||||
private PDFText resolveMatchPosition(Matcher matcher, int pageIndex) {
|
||||
int matchStart = matcher.start();
|
||||
int matchEnd = matcher.end();
|
||||
|
||||
float minX = Float.MAX_VALUE;
|
||||
float minY = Float.MAX_VALUE;
|
||||
float maxX = Float.MIN_VALUE;
|
||||
float maxY = Float.MIN_VALUE;
|
||||
boolean foundPosition = false;
|
||||
|
||||
for (int i = matchStart; i < matchEnd; i++) {
|
||||
if (i >= pageTextPositions.size()) break;
|
||||
TextPosition pos = pageTextPositions.get(i);
|
||||
if (pos != null) {
|
||||
foundPosition = true;
|
||||
minX = Math.min(minX, pos.getX());
|
||||
maxX = Math.max(maxX, pos.getX() + pos.getWidth());
|
||||
minY = Math.min(minY, pos.getY() - pos.getHeight());
|
||||
maxY = Math.max(maxY, pos.getY());
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundPosition && matchStart < pageTextPositions.size()) {
|
||||
for (int i = Math.max(0, matchStart - 5);
|
||||
i < Math.min(pageTextPositions.size(), matchEnd + 5);
|
||||
i++) {
|
||||
TextPosition pos = pageTextPositions.get(i);
|
||||
if (pos != null) {
|
||||
foundPosition = true;
|
||||
minX = Math.min(minX, pos.getX());
|
||||
maxX = Math.max(maxX, pos.getX() + pos.getWidth());
|
||||
minY = Math.min(minY, pos.getY() - pos.getHeight());
|
||||
maxY = Math.max(maxY, pos.getY());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundPosition) {
|
||||
log.warn(
|
||||
"Found text match '{}' but no valid position data at {}-{}",
|
||||
matcher.group(),
|
||||
matchStart,
|
||||
matchEnd);
|
||||
return null;
|
||||
}
|
||||
|
||||
return new PDFText(pageIndex, minX, minY, maxX, maxY, matcher.group());
|
||||
}
|
||||
}
|
||||
+67
-1477
File diff suppressed because it is too large
Load Diff
+850
@@ -0,0 +1,850 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
class RedactExecuteService {
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final ManualRedactionService manualRedactionService;
|
||||
private final TextRedactionService textRedactionService;
|
||||
|
||||
TempFile execute(RedactExecuteRequest request) throws IOException {
|
||||
RedactStyle style = request.getStyle() != null ? request.getStyle() : new RedactStyle();
|
||||
List<String> textValues = orEmpty(request.getTextValues());
|
||||
List<String> regexPatterns = orEmpty(request.getRegexPatterns());
|
||||
List<Integer> wipePages = orEmpty(request.getWipePages());
|
||||
List<TextRange> ranges = orEmpty(request.getRanges());
|
||||
List<ImageBox> imageBoxes = orEmpty(request.getImageBoxes());
|
||||
|
||||
boolean hasTargets =
|
||||
!textValues.isEmpty()
|
||||
|| !regexPatterns.isEmpty()
|
||||
|| !wipePages.isEmpty()
|
||||
|| !ranges.isEmpty()
|
||||
|| !imageBoxes.isEmpty()
|
||||
|| request.getRedactImagePages() != null;
|
||||
|
||||
if (!hasTargets) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.redaction.no.targets", "No redaction targets provided");
|
||||
}
|
||||
|
||||
boolean overlayOnly =
|
||||
RedactExecuteRequest.RedactionStrategy.OVERLAY_ONLY.equals(style.getStrategy());
|
||||
boolean imageFinalize =
|
||||
RedactExecuteRequest.RedactionStrategy.IMAGE_FINALIZE.equals(style.getStrategy());
|
||||
boolean convertToImage = imageFinalize || style.isConvertToImage();
|
||||
|
||||
boolean hasTextOps = !textValues.isEmpty() || !regexPatterns.isEmpty();
|
||||
|
||||
log.info(
|
||||
"[redact/execute] strategy={} textValues={} regexPatterns={} wipePages={} ranges={} imageBoxes={} imagePages={}",
|
||||
style.getStrategy(),
|
||||
textValues.size(),
|
||||
regexPatterns.size(),
|
||||
wipePages.size(),
|
||||
ranges.size(),
|
||||
imageBoxes.size(),
|
||||
request.getRedactImagePages());
|
||||
|
||||
if (request.getFileInput() == null) {
|
||||
throw ExceptionUtils.createFileNullOrEmptyException();
|
||||
}
|
||||
|
||||
PDDocument document = null;
|
||||
try {
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF
|
||||
// stripper only once across the entire execute() call rather than once per target.
|
||||
Map<Integer, List<PDFText>> foundTexts =
|
||||
hasTextOps ? collectTextMatches(document, request) : new HashMap<>();
|
||||
|
||||
int totalMatches = foundTexts.values().stream().mapToInt(List::size).sum();
|
||||
log.info(
|
||||
"[redact/execute] scan complete: {} text matches across {} pages",
|
||||
totalMatches,
|
||||
foundTexts.size());
|
||||
|
||||
// Text removal (content-stream rewriting) — skipped in overlay-only mode.
|
||||
boolean needsOverlayOnly = overlayOnly;
|
||||
if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) {
|
||||
needsOverlayOnly = applyTextRemoval(document, request);
|
||||
} else if (overlayOnly) {
|
||||
log.info(
|
||||
"[redact/execute] overlay-only mode requested — skipping content-stream rewriting");
|
||||
}
|
||||
|
||||
// Reload fresh document on fallback so we overlay onto clean content.
|
||||
if (needsOverlayOnly && !foundTexts.isEmpty()) {
|
||||
log.info("[redact/execute] reloading document for clean overlay pass");
|
||||
document.close();
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
foundTexts.clear();
|
||||
if (hasTextOps) {
|
||||
foundTexts.putAll(collectTextMatches(document, request));
|
||||
}
|
||||
}
|
||||
|
||||
// Non-text operations.
|
||||
Map<Integer, PageColumnLayout> layoutCache = new HashMap<>();
|
||||
|
||||
if (!wipePages.isEmpty()) {
|
||||
applyPageWipe(document, wipePages, style);
|
||||
}
|
||||
|
||||
for (TextRange range : ranges) {
|
||||
applyRangeRedaction(document, range, style, layoutCache);
|
||||
}
|
||||
|
||||
for (ImageBox box : imageBoxes) {
|
||||
applyImageBoxRedaction(document, box, style);
|
||||
}
|
||||
|
||||
if (request.getRedactImagePages() != null) {
|
||||
applyAllImagesRedaction(document, request.getRedactImagePages(), style);
|
||||
}
|
||||
|
||||
return manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
foundTexts,
|
||||
style.getColor(),
|
||||
style.getPadding(),
|
||||
convertToImage,
|
||||
!needsOverlayOnly);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Execute redaction failed: {}", e.getMessage(), e);
|
||||
throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e);
|
||||
} finally {
|
||||
if (document != null) {
|
||||
try {
|
||||
document.close();
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to close document: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Single-pass text scan (one stripper pass per execute() call)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit
|
||||
* map.
|
||||
*/
|
||||
private Map<Integer, List<PDFText>> collectTextMatches(
|
||||
PDDocument document, RedactExecuteRequest request) {
|
||||
Map<Integer, List<PDFText>> found = new HashMap<>();
|
||||
|
||||
String[] terms = cleanStrings(request.getTextValues());
|
||||
if (terms.length > 0) {
|
||||
textRedactionService
|
||||
.findTextToRedact(document, terms, false, false)
|
||||
.forEach(
|
||||
(page, hits) ->
|
||||
found.computeIfAbsent(page, k -> new ArrayList<>())
|
||||
.addAll(hits));
|
||||
}
|
||||
|
||||
String[] patterns = cleanStrings(request.getRegexPatterns());
|
||||
if (patterns.length > 0) {
|
||||
textRedactionService
|
||||
.findTextToRedact(document, patterns, true, false)
|
||||
.forEach(
|
||||
(page, hits) ->
|
||||
found.computeIfAbsent(page, k -> new ArrayList<>())
|
||||
.addAll(hits));
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Text removal (content-stream rewriting)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the
|
||||
* document fell back to overlay-only mode.
|
||||
*/
|
||||
private boolean applyTextRemoval(PDDocument document, RedactExecuteRequest request) {
|
||||
try {
|
||||
boolean fallback = false;
|
||||
|
||||
String[] terms = cleanStrings(request.getTextValues());
|
||||
if (terms.length > 0) {
|
||||
Map<Integer, List<PDFText>> exactFound =
|
||||
textRedactionService.findTextToRedact(document, terms, false, false);
|
||||
if (!exactFound.isEmpty()) {
|
||||
fallback |=
|
||||
textRedactionService.performTextReplacement(
|
||||
document, exactFound, terms, false, false);
|
||||
}
|
||||
}
|
||||
|
||||
String[] patterns = cleanStrings(request.getRegexPatterns());
|
||||
if (patterns.length > 0) {
|
||||
Map<Integer, List<PDFText>> regexFound =
|
||||
textRedactionService.findTextToRedact(document, patterns, true, false);
|
||||
if (!regexFound.isEmpty()) {
|
||||
fallback |=
|
||||
textRedactionService.performTextReplacement(
|
||||
document, regexFound, patterns, true, false);
|
||||
}
|
||||
}
|
||||
|
||||
if (fallback) {
|
||||
log.warn(
|
||||
"[redact/execute] font compatibility issue — falling back to overlay-only");
|
||||
} else {
|
||||
log.info("[redact/execute] content-stream text removal applied successfully");
|
||||
}
|
||||
return fallback;
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] text removal failed, falling back to overlay: {}",
|
||||
e.getMessage());
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Per-operation dispatch methods
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void applyPageWipe(PDDocument document, List<Integer> pageNumbers, RedactStyle style)
|
||||
throws IOException {
|
||||
List<Integer> pageIndices = toZeroBasedIndices(pageNumbers);
|
||||
if (pageIndices.isEmpty()) return;
|
||||
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
Color pageColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
Collections.sort(pageIndices);
|
||||
log.info("[redact/execute] full-page wipe: {} pages ({})", pageIndices.size(), pageIndices);
|
||||
|
||||
Map<Integer, List<float[]>> pageElementBoxes = new HashMap<>();
|
||||
for (Integer idx : pageIndices) {
|
||||
if (idx >= 0 && idx < allPages.getCount()) {
|
||||
try {
|
||||
pageElementBoxes.put(
|
||||
idx,
|
||||
manualRedactionService.extractPageElementBoxes(
|
||||
document, allPages.get(idx), idx));
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] element extraction failed for page {}: {}",
|
||||
idx,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Integer idx : pageIndices) {
|
||||
if (idx >= 0 && idx < allPages.getCount()) {
|
||||
PDPage page = allPages.get(idx);
|
||||
List<float[]> elementBoxes =
|
||||
pageElementBoxes.getOrDefault(idx, Collections.emptyList());
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
try (PDPageContentStream cs = new PDPageContentStream(document, page)) {
|
||||
cs.setNonStrokingColor(pageColor);
|
||||
if (elementBoxes.isEmpty()) {
|
||||
PDRectangle box = page.getBBox();
|
||||
cs.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
} else {
|
||||
log.info(
|
||||
"[redact/execute] page {}: drawing {} element boxes",
|
||||
idx + 1,
|
||||
elementBoxes.size());
|
||||
for (float[] r : elementBoxes) {
|
||||
cs.addRect(r[0], r[1], r[2] - r[0], r[3] - r[1]);
|
||||
}
|
||||
}
|
||||
cs.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void applyRangeRedaction(
|
||||
PDDocument document,
|
||||
TextRange range,
|
||||
RedactStyle style,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
String rangeStart = trimOrEmpty(range.startString());
|
||||
String rangeEnd = trimOrEmpty(range.endString());
|
||||
log.info("[redact/execute] range redaction: start='{}' end='{}'", rangeStart, rangeEnd);
|
||||
try {
|
||||
List<PDFText> blocks = collectRangeBlocks(document, rangeStart, rangeEnd, layoutCache);
|
||||
if (!blocks.isEmpty()) {
|
||||
manualRedactionService.redactFoundText(
|
||||
document,
|
||||
blocks,
|
||||
style.getPadding(),
|
||||
ManualRedactionService.decodeOrDefault(style.getColor()),
|
||||
false);
|
||||
} else {
|
||||
log.warn(
|
||||
"[redact/execute] range not found: start='{}' end='{}'",
|
||||
rangeStart,
|
||||
rangeEnd);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("[redact/execute] range redaction failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style)
|
||||
throws IOException {
|
||||
List<float[]> boxes =
|
||||
List.of(
|
||||
new float[] {
|
||||
(float) box.pageIndex(), box.x1(), box.y1(), box.x2(), box.y2()
|
||||
});
|
||||
log.info("[redact/execute] image box overlay on page {}", box.pageIndex());
|
||||
Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
manualRedactionService.redactImageBoxes(document, boxes, boxColor);
|
||||
}
|
||||
|
||||
private void applyAllImagesRedaction(
|
||||
PDDocument document, List<Integer> pageNumbers, RedactStyle style) throws IOException {
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
Color imgColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
|
||||
List<Integer> imagePageIndices = toZeroBasedIndices(pageNumbers);
|
||||
if (imagePageIndices.isEmpty()) {
|
||||
imagePageIndices = new ArrayList<>();
|
||||
for (int i = 0; i < allPages.getCount(); i++) {
|
||||
imagePageIndices.add(i);
|
||||
}
|
||||
}
|
||||
|
||||
List<float[]> detectedBoxes = new ArrayList<>();
|
||||
for (int pageIdx : imagePageIndices) {
|
||||
if (pageIdx < 0 || pageIdx >= allPages.getCount()) continue;
|
||||
try {
|
||||
PDPage page = allPages.get(pageIdx);
|
||||
PageImageLocator locator = new PageImageLocator(page, pageIdx);
|
||||
locator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) {
|
||||
detectedBoxes.add(new float[] {pageIdx, ib.x1(), ib.y1(), ib.x2(), ib.y2()});
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] image detection failed for page {}: {}",
|
||||
pageIdx + 1,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] auto image detection: {} images across {} pages",
|
||||
detectedBoxes.size(),
|
||||
imagePageIndices.size());
|
||||
|
||||
if (!detectedBoxes.isEmpty()) {
|
||||
manualRedactionService.redactImageBoxes(document, detectedBoxes, imgColor);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range collection helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text
|
||||
* line and image from that point up to (but NOT including) the line where {@code endStr}
|
||||
* begins. If {@code endStr} is blank, redacts from {@code startStr} to the end of the document.
|
||||
*
|
||||
* <p>Multi-column pages follow reading order: down the start column, jump to the top of the
|
||||
* next column, continue to the end anchor. Single-column pages reduce to a plain Y-band check.
|
||||
*/
|
||||
List<PDFText> collectRangeBlocks(
|
||||
PDDocument document,
|
||||
String startStr,
|
||||
String endStr,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
int totalPages = allPages.getCount();
|
||||
|
||||
Map<Integer, List<PDFText>> startMatchesByPage = findWithFallbacks(document, startStr);
|
||||
if (startMatchesByPage.isEmpty()) {
|
||||
log.warn("[redact/execute] range start not found: '{}'", startStr);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
List<Anchor> starts = toAnchors(document, startMatchesByPage, layoutCache);
|
||||
starts.sort(READING_ORDER);
|
||||
log.info(
|
||||
"[redact/execute] start='{}' matched {} anchor(s): {}",
|
||||
startStr,
|
||||
starts.size(),
|
||||
anchorSummary(starts));
|
||||
|
||||
boolean openEnded = (endStr == null || endStr.isBlank());
|
||||
List<Anchor> ends = new ArrayList<>();
|
||||
if (!openEnded) {
|
||||
Map<Integer, List<PDFText>> endMatchesByPage = findWithFallbacks(document, endStr);
|
||||
if (endMatchesByPage.isEmpty()) {
|
||||
log.warn(
|
||||
"[redact/execute] range end '{}' not found in document - skipping range"
|
||||
+ " (start='{}')",
|
||||
endStr,
|
||||
startStr);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
ends = toAnchors(document, endMatchesByPage, layoutCache);
|
||||
ends.sort(READING_ORDER);
|
||||
log.info(
|
||||
"[redact/execute] end='{}' matched {} anchor(s): {}",
|
||||
endStr,
|
||||
ends.size(),
|
||||
anchorSummary(ends));
|
||||
}
|
||||
|
||||
List<PDFText> blocks = new ArrayList<>();
|
||||
for (Anchor start : starts) {
|
||||
Anchor end = null;
|
||||
int endPage;
|
||||
if (openEnded) {
|
||||
endPage = totalPages - 1;
|
||||
} else {
|
||||
for (Anchor candidate : ends) {
|
||||
if (READING_ORDER.compare(candidate, start) > 0) {
|
||||
end = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (end == null) {
|
||||
log.warn(
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping",
|
||||
start.page + 1,
|
||||
start.col,
|
||||
start.y);
|
||||
continue;
|
||||
}
|
||||
endPage = end.page;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range pages {}-{}: start='{}' (col {}) end='{}'",
|
||||
start.page + 1,
|
||||
endPage + 1,
|
||||
startStr,
|
||||
start.col,
|
||||
openEnded ? "<end of document>" : endStr);
|
||||
|
||||
collectBlocksForRange(document, allPages, start, end, openEnded, blocks, layoutCache);
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range '{}'→'{}': {} total blocks",
|
||||
startStr,
|
||||
openEnded ? "<end of document>" : endStr,
|
||||
blocks.size());
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects all redactable content (text line segments and images) between two anchor positions.
|
||||
*
|
||||
* <p>Line boxes are cached per page number in {@code lineBoxCache} and reused across range
|
||||
* iterations within one execute() call, avoiding redundant {@link AllTextLineExtractor} passes.
|
||||
*/
|
||||
private void collectBlocksForRange(
|
||||
PDDocument document,
|
||||
PDPageTree allPages,
|
||||
Anchor start,
|
||||
Anchor end,
|
||||
boolean openEnded,
|
||||
List<PDFText> blocks,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
|
||||
int startPage = start.page;
|
||||
int endPage = openEnded ? allPages.getCount() - 1 : end.page;
|
||||
int endCol =
|
||||
openEnded ? layoutFor(document, endPage, layoutCache).columnCount() - 1 : end.col;
|
||||
float startY = start.y;
|
||||
// Use bottom of end anchor so the end anchor line itself is included (inclusive range).
|
||||
float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2();
|
||||
|
||||
// Line-box cache: populated lazily per page, reused across range iterations.
|
||||
// Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException.
|
||||
Map<Integer, List<float[]>> lineBoxCache = new HashMap<>();
|
||||
|
||||
for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) {
|
||||
PDPage page = allPages.get(pageIdx);
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
PageColumnLayout layout = layoutFor(document, pageIdx, layoutCache);
|
||||
|
||||
List<float[]> screenLineBoxes = lineBoxCache.get(pageIdx);
|
||||
if (screenLineBoxes == null) {
|
||||
AllTextLineExtractor textExtractor =
|
||||
new AllTextLineExtractor(pageIdx + 1, pageHeight);
|
||||
textExtractor.getText(document);
|
||||
screenLineBoxes = textExtractor.getScreenLineBoxes();
|
||||
lineBoxCache.put(pageIdx, screenLineBoxes);
|
||||
}
|
||||
|
||||
for (float[] sb : screenLineBoxes) {
|
||||
emitColumnSlices(
|
||||
pageIdx, layout, sb[0], sb[2], sb[1], sb[3], start.col, startPage, startY,
|
||||
endCol, endPage, endY, blocks);
|
||||
}
|
||||
|
||||
PageImageLocator imgLocator = new PageImageLocator(page, pageIdx);
|
||||
imgLocator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : imgLocator.getImageBoxes()) {
|
||||
// ImageBox coordinates are in PDF user-space (Y up); convert to screen-Y (Y down).
|
||||
float screenY1 = pageHeight - ib.y2();
|
||||
float screenY2 = pageHeight - ib.y1();
|
||||
emitColumnSlices(
|
||||
pageIdx, layout, ib.x1(), ib.x2(), screenY1, screenY2, start.col, startPage,
|
||||
startY, endCol, endPage, endY, blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Emits each per-column sub-box accepted by the reading-order predicate. */
|
||||
private static void emitColumnSlices(
|
||||
int pageIdx,
|
||||
PageColumnLayout layout,
|
||||
float x1,
|
||||
float x2,
|
||||
float yTop,
|
||||
float yBottom,
|
||||
int startCol,
|
||||
int startPage,
|
||||
float startY,
|
||||
int endCol,
|
||||
int endPage,
|
||||
float endY,
|
||||
List<PDFText> blocks) {
|
||||
int[] cols = layout.columnsCrossing(x1, x2);
|
||||
if (cols.length == 1) {
|
||||
if (inColumnZone(
|
||||
pageIdx, cols[0], yTop, yBottom, startPage, startCol, startY, endPage, endCol,
|
||||
endY)) {
|
||||
blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, ""));
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (int col : cols) {
|
||||
if (!inColumnZone(
|
||||
pageIdx, col, yTop, yBottom, startPage, startCol, startY, endPage, endCol,
|
||||
endY)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, ""));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reading-order predicate: true when (col, yBottom) on page {@code pageIdx} sits between the
|
||||
* start anchor (inclusive) and end anchor (inclusive).
|
||||
*/
|
||||
static boolean inColumnZone(
|
||||
int pageIdx,
|
||||
int col,
|
||||
float yTop,
|
||||
float yBottom,
|
||||
int startPage,
|
||||
int startCol,
|
||||
float startY,
|
||||
int endPage,
|
||||
int endCol,
|
||||
float endY) {
|
||||
if (pageIdx > startPage && pageIdx < endPage) return true;
|
||||
if (pageIdx == startPage && pageIdx == endPage) {
|
||||
if (startCol == endCol) {
|
||||
return col == startCol && yBottom >= startY && yBottom <= endY;
|
||||
}
|
||||
if (startCol < endCol) {
|
||||
if (col < startCol || col > endCol) return false;
|
||||
if (col == startCol) return yBottom >= startY;
|
||||
if (col == endCol) return yBottom <= endY;
|
||||
return true;
|
||||
}
|
||||
return col == startCol && yBottom >= startY;
|
||||
}
|
||||
if (pageIdx == startPage) {
|
||||
if (col == startCol) return yBottom >= startY;
|
||||
return col > startCol;
|
||||
}
|
||||
if (pageIdx == endPage) {
|
||||
if (col == endCol) return yBottom <= endY;
|
||||
return col < endCol;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Lazily builds and caches the column layout for a single page. */
|
||||
private PageColumnLayout layoutFor(
|
||||
PDDocument document, int pageIdx, Map<Integer, PageColumnLayout> cache)
|
||||
throws IOException {
|
||||
PageColumnLayout cached = cache.get(pageIdx);
|
||||
if (cached != null) return cached;
|
||||
PDPage page = document.getDocumentCatalog().getPages().get(pageIdx);
|
||||
float pageWidth = page.getBBox().getWidth();
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
AllTextLineExtractor extractor = new AllTextLineExtractor(pageIdx + 1, pageHeight);
|
||||
extractor.getText(document);
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(extractor.getLineBoxes(), pageWidth);
|
||||
if (layout.columnCount() > 1) {
|
||||
float[] g = layout.gutters().get(0);
|
||||
log.info(
|
||||
"[redact/execute] page {} layout: 2 cols, gutter x=[{}, {}]",
|
||||
pageIdx + 1,
|
||||
g[0],
|
||||
g[1]);
|
||||
} else {
|
||||
log.info("[redact/execute] page {} layout: 1 col (single-column mode)", pageIdx + 1);
|
||||
}
|
||||
cache.put(pageIdx, layout);
|
||||
return layout;
|
||||
}
|
||||
|
||||
private List<Anchor> toAnchors(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> matchesByPage,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
List<Anchor> out = new ArrayList<>();
|
||||
for (int page : matchesByPage.keySet().stream().sorted().toList()) {
|
||||
PageColumnLayout layout = layoutFor(document, page, layoutCache);
|
||||
for (PDFText hit : matchesByPage.get(page)) {
|
||||
int col = layout.columnOf(hit.getX1(), hit.getX2());
|
||||
out.add(new Anchor(page, col, hit.getY1(), hit));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Lexicographic ordering by (page, column, screenY). */
|
||||
private static final Comparator<Anchor> READING_ORDER =
|
||||
Comparator.comparingInt((Anchor a) -> a.page)
|
||||
.thenComparingInt(a -> a.col)
|
||||
.thenComparingDouble(a -> a.y);
|
||||
|
||||
private static String anchorSummary(List<Anchor> anchors) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int max = Math.min(anchors.size(), 5);
|
||||
for (int i = 0; i < max; i++) {
|
||||
Anchor a = anchors.get(i);
|
||||
if (i > 0) sb.append(", ");
|
||||
sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y));
|
||||
}
|
||||
if (anchors.size() > max) sb.append(", …");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private record Anchor(int page, int col, float y, PDFText text) {}
|
||||
|
||||
/**
|
||||
* Tries progressively more permissive variants: raw (regex then literal), letter-spacing
|
||||
* collapsed, then a punctuation-tolerant regex over alphanumeric runs.
|
||||
*/
|
||||
private Map<Integer, List<PDFText>> findWithFallbacks(PDDocument document, String raw) {
|
||||
String trimmed = raw.trim();
|
||||
String collapsed = collapseLetterSpacing(trimmed);
|
||||
String tolerant = punctuationTolerantRegex(trimmed);
|
||||
|
||||
List<Candidate> candidates = new ArrayList<>();
|
||||
candidates.add(new Candidate(trimmed, true));
|
||||
candidates.add(new Candidate(trimmed, false));
|
||||
if (!collapsed.equals(trimmed)) {
|
||||
candidates.add(new Candidate(collapsed, true));
|
||||
candidates.add(new Candidate(collapsed, false));
|
||||
}
|
||||
if (tolerant != null && !tolerant.equals(trimmed)) {
|
||||
candidates.add(new Candidate(tolerant, true));
|
||||
}
|
||||
|
||||
// If the anchor spans multiple lines (model provided entire paragraph instead of a short
|
||||
// phrase), try just the first non-empty line — it's usually sufficient to locate the
|
||||
// position and avoids mismatches from mid-paragraph text extraction artifacts.
|
||||
if (trimmed.contains("\n")) {
|
||||
String firstLine =
|
||||
Arrays.stream(trimmed.split("\n"))
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (firstLine != null && firstLine.length() >= 4) {
|
||||
String firstLineCollapsed = collapseLetterSpacing(firstLine);
|
||||
String firstLineTolerant = punctuationTolerantRegex(firstLine);
|
||||
candidates.add(new Candidate(firstLine, false));
|
||||
if (!firstLineCollapsed.equals(firstLine)) {
|
||||
candidates.add(new Candidate(firstLineCollapsed, false));
|
||||
}
|
||||
if (firstLineTolerant != null && !firstLineTolerant.equals(firstLine)) {
|
||||
candidates.add(new Candidate(firstLineTolerant, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Candidate c : candidates) {
|
||||
Map<Integer, List<PDFText>> m =
|
||||
textRedactionService.findTextToRedact(
|
||||
document, new String[] {c.pattern}, c.useRegex, false);
|
||||
if (!m.isEmpty()) {
|
||||
if (!c.pattern.equals(trimmed)) {
|
||||
log.info(
|
||||
"[redact/execute] range boundary matched via fallback: '{}' → '{}'",
|
||||
trimmed,
|
||||
c.pattern);
|
||||
}
|
||||
return m;
|
||||
}
|
||||
}
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
private record Candidate(String pattern, boolean useRegex) {}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Static helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation
|
||||
* drift. Returns {@code null} when fewer than two tokens exist.
|
||||
*/
|
||||
private static String punctuationTolerantRegex(String raw) {
|
||||
List<String> tokens = new ArrayList<>();
|
||||
StringBuilder current = new StringBuilder();
|
||||
for (int i = 0; i < raw.length(); i++) {
|
||||
char ch = raw.charAt(i);
|
||||
if (Character.isLetterOrDigit(ch)) {
|
||||
current.append(ch);
|
||||
} else if (current.length() > 0) {
|
||||
tokens.add(current.toString());
|
||||
current.setLength(0);
|
||||
}
|
||||
}
|
||||
if (current.length() > 0) tokens.add(current.toString());
|
||||
if (tokens.size() < 2) return null;
|
||||
StringBuilder out = new StringBuilder();
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
if (i > 0) out.append("\\W*");
|
||||
out.append(Pattern.quote(tokens.get(i)));
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapses letter-spaced text produced by position-sorted text extraction.
|
||||
*
|
||||
* <p>When a PDF text stripper runs with {@code setSortByPosition(true)}, letter-spaced headings
|
||||
* come out as {@code "T a b l e o f c o n t e n t s"}. This method converts the spaced form
|
||||
* back to words.
|
||||
*/
|
||||
private static String collapseLetterSpacing(String text) {
|
||||
String[] tokens = text.split(" ", -1);
|
||||
StringBuilder result = new StringBuilder();
|
||||
StringBuilder current = new StringBuilder();
|
||||
for (String token : tokens) {
|
||||
if (token.isEmpty()) {
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
current.setLength(0);
|
||||
}
|
||||
} else if (token.length() == 1) {
|
||||
current.append(token);
|
||||
} else {
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
current.setLength(0);
|
||||
}
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(token);
|
||||
}
|
||||
}
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
}
|
||||
return result.toString().trim();
|
||||
}
|
||||
|
||||
private static <T> List<T> orEmpty(List<T> list) {
|
||||
return list != null ? list : List.of();
|
||||
}
|
||||
|
||||
private static String[] cleanStrings(List<String> input) {
|
||||
if (input == null || input.isEmpty()) {
|
||||
return new String[0];
|
||||
}
|
||||
return input.stream()
|
||||
.filter(s -> s != null)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.toArray(String[]::new);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts 1-based page numbers from the request to the 0-based indices used internally.
|
||||
* Out-of-range and non-positive values are silently dropped.
|
||||
*/
|
||||
private static List<Integer> toZeroBasedIndices(List<Integer> oneBasedPageNumbers) {
|
||||
if (oneBasedPageNumbers == null || oneBasedPageNumbers.isEmpty()) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (Integer page : oneBasedPageNumbers) {
|
||||
if (page != null && page > 0) {
|
||||
result.add(page - 1);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String trimOrEmpty(String s) {
|
||||
return s == null ? "" : s.trim();
|
||||
}
|
||||
}
|
||||
+1207
File diff suppressed because it is too large
Load Diff
+20
-5
@@ -12,6 +12,7 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.CacheControl;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Controller;
|
||||
@@ -134,13 +135,22 @@ public class ReactRoutingController {
|
||||
public ResponseEntity<String> serveIndexHtml(HttpServletRequest request) {
|
||||
try {
|
||||
if (indexHtmlExists && cachedIndexHtml != null) {
|
||||
return ResponseEntity.ok().contentType(MediaType.TEXT_HTML).body(cachedIndexHtml);
|
||||
return ResponseEntity.ok()
|
||||
.cacheControl(CacheControl.noCache().mustRevalidate())
|
||||
.contentType(MediaType.TEXT_HTML)
|
||||
.body(cachedIndexHtml);
|
||||
}
|
||||
// Fallback: process on each request (dev mode or cache failed)
|
||||
return ResponseEntity.ok().contentType(MediaType.TEXT_HTML).body(processIndexHtml());
|
||||
return ResponseEntity.ok()
|
||||
.cacheControl(CacheControl.noCache().mustRevalidate())
|
||||
.contentType(MediaType.TEXT_HTML)
|
||||
.body(processIndexHtml());
|
||||
} catch (Exception ex) {
|
||||
log.error("Failed to serve index.html, returning fallback", ex);
|
||||
return ResponseEntity.ok().contentType(MediaType.TEXT_HTML).body(buildFallbackHtml());
|
||||
return ResponseEntity.ok()
|
||||
.cacheControl(CacheControl.noCache().mustRevalidate())
|
||||
.contentType(MediaType.TEXT_HTML)
|
||||
.body(buildFallbackHtml());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,14 +170,19 @@ public class ReactRoutingController {
|
||||
return ResponseEntity.ok().contentType(MediaType.TEXT_HTML).body(cachedCallbackHtml);
|
||||
}
|
||||
|
||||
// `files` was historically a backend static-asset directory and was therefore
|
||||
// in the exclusion list - removing it lets /files and /files/<folder-uuid>
|
||||
// forward to the SPA index.html, which is what FileManagerView expects.
|
||||
// (Real storage endpoints live under /api/v1/storage/files, already
|
||||
// excluded by the leading `api` token in the same regex.)
|
||||
@GetMapping(
|
||||
"/{path:^(?!api|static|robots\\.txt|favicon\\.ico|manifest.*\\.json|pipeline|pdfjs|pdfjs-legacy|pdfium|vendor|fonts|images|files|css|js|assets|locales|modern-logo|classic-logo|Login|og_images|samples)[^\\.]*$}")
|
||||
"/{path:^(?!api|static|robots\\.txt|favicon\\.ico|manifest.*\\.json|pipeline|pdfjs|pdfjs-legacy|pdfium|vendor|fonts|images|css|js|assets|locales|modern-logo|classic-logo|Login|og_images|samples)[^\\.]*$}")
|
||||
public ResponseEntity<String> forwardRootPaths(HttpServletRequest request) throws IOException {
|
||||
return serveIndexHtml(request);
|
||||
}
|
||||
|
||||
@GetMapping(
|
||||
"/{path:^(?!api|static|pipeline|pdfjs|pdfjs-legacy|pdfium|vendor|fonts|images|files|css|js|assets|locales|modern-logo|classic-logo|Login|og_images|samples)[^\\.]*}/{subpath:^(?!.*\\.).*$}")
|
||||
"/{path:^(?!api|static|pipeline|pdfjs|pdfjs-legacy|pdfium|vendor|fonts|images|css|js|assets|locales|modern-logo|classic-logo|Login|og_images|samples)[^\\.]*}/{subpath:^(?!.*\\.).*$}")
|
||||
public ResponseEntity<String> forwardNestedPaths(HttpServletRequest request)
|
||||
throws IOException {
|
||||
return serveIndexHtml(request);
|
||||
|
||||
+40
-2
@@ -22,6 +22,7 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
import org.springframework.web.multipart.MaxUploadSizeExceededException;
|
||||
import org.springframework.web.multipart.support.MissingServletRequestPartException;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import org.springframework.web.servlet.NoHandlerFoundException;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
@@ -196,12 +197,12 @@ public class GlobalExceptionHandler {
|
||||
/**
|
||||
* Checks whether the given IOException indicates that the client disconnected before the
|
||||
* response could be written (broken pipe, connection reset, etc.). When this happens there is
|
||||
* no point in serialising a {@link ProblemDetail} body because the socket is already closed —
|
||||
* no point in serialising a {@link ProblemDetail} body because the socket is already closed -
|
||||
* and attempting to do so may trigger a secondary {@code HttpMessageNotWritableException} if
|
||||
* the response Content-Type was already committed as a non-JSON type (e.g. image/png).
|
||||
*/
|
||||
private static boolean isClientDisconnectException(IOException ex) {
|
||||
// Walk the causal chain — Jetty/Tomcat may wrap the low-level SocketException
|
||||
// Walk the causal chain - Jetty/Tomcat may wrap the low-level SocketException
|
||||
Throwable current = ex;
|
||||
while (current != null) {
|
||||
String msg = current.getMessage();
|
||||
@@ -1040,6 +1041,43 @@ public class GlobalExceptionHandler {
|
||||
* @param request the HTTP servlet request
|
||||
* @return ProblemDetail with appropriate HTTP status
|
||||
*/
|
||||
/**
|
||||
* Handle ResponseStatusException explicitly so its embedded HTTP status reaches the client
|
||||
* instead of being swallowed by the {@code RuntimeException} catch-all (which would downgrade
|
||||
* every controller-thrown 400/404/409 to a generic 500). Folder/file storage controllers and
|
||||
* any other code that throws {@code ResponseStatusException} relies on this handler taking
|
||||
* precedence.
|
||||
*/
|
||||
@ExceptionHandler(ResponseStatusException.class)
|
||||
public ResponseEntity<ProblemDetail> handleResponseStatusException(
|
||||
ResponseStatusException ex, HttpServletRequest request) {
|
||||
HttpStatus status =
|
||||
HttpStatus.resolve(ex.getStatusCode().value()) != null
|
||||
? HttpStatus.valueOf(ex.getStatusCode().value())
|
||||
: HttpStatus.INTERNAL_SERVER_ERROR;
|
||||
String reason = ex.getReason() != null ? ex.getReason() : status.getReasonPhrase();
|
||||
ProblemDetail problemDetail = createBaseProblemDetail(status, reason, request);
|
||||
problemDetail.setType(URI.create("/errors/" + status.value()));
|
||||
problemDetail.setTitle(status.getReasonPhrase());
|
||||
problemDetail.setProperty("title", status.getReasonPhrase());
|
||||
// 5xx is operator-relevant; 4xx is a normal client-rejection - log at the right level.
|
||||
if (status.is5xxServerError()) {
|
||||
log.error(
|
||||
"ResponseStatusException {} at {}: {}",
|
||||
status.value(),
|
||||
request.getRequestURI(),
|
||||
reason,
|
||||
ex);
|
||||
} else {
|
||||
log.debug(
|
||||
"ResponseStatusException {} at {}: {}",
|
||||
status.value(),
|
||||
request.getRequestURI(),
|
||||
reason);
|
||||
}
|
||||
return ResponseEntity.status(status).contentType(PROBLEM_JSON).body(problemDetail);
|
||||
}
|
||||
|
||||
@ExceptionHandler(RuntimeException.class)
|
||||
public ResponseEntity<ProblemDetail> handleRuntimeException(
|
||||
RuntimeException ex, HttpServletRequest request) {
|
||||
|
||||
@@ -18,4 +18,11 @@ public class PDFWithPageSize extends PDFFile {
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
allowableValues = {"A0", "A1", "A2", "A3", "A4", "A5", "A6", "LETTER", "LEGAL", "KEEP"})
|
||||
private String pageSize;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Orientation to apply to the target page size. Ignored when pageSize is KEEP.",
|
||||
defaultValue = "PORTRAIT",
|
||||
allowableValues = {"PORTRAIT", "LANDSCAPE"})
|
||||
private String orientation = "PORTRAIT";
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user